working commit
This commit is contained in:
+21
@@ -0,0 +1,21 @@
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2023, Extism
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+365
@@ -0,0 +1,365 @@
|
||||
# Extism Go SDK
|
||||
|
||||
This repo houses the Go SDK for integrating with the [Extism](https://extism.org/) runtime. Install this library into your host Go applications to run Extism plugins.
|
||||
|
||||
Join the [Extism Discord](https://extism.org/discord) and chat with us!
|
||||
|
||||
## Installation
|
||||
|
||||
Install via `go get`:
|
||||
|
||||
```
|
||||
go get github.com/extism/go-sdk
|
||||
```
|
||||
|
||||
## Reference Docs
|
||||
|
||||
You can find the reference docs at [https://pkg.go.dev/github.com/extism/go-sdk](https://pkg.go.dev/github.com/extism/go-sdk).
|
||||
|
||||
## Getting Started
|
||||
|
||||
This guide should walk you through some of the concepts in Extism and this Go library.
|
||||
|
||||
### Creating A Plug-in
|
||||
|
||||
The primary concept in Extism is the [plug-in](https://extism.org/docs/concepts/plug-in). You can think of a plug-in as a code module stored in a `.wasm` file.
|
||||
|
||||
Plug-in code can come from a file on disk, object storage or any number of places. Since you may not have one handy let's load a demo plug-in from the web. Let's
|
||||
start by creating a main func and loading an Extism Plug-in:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/extism/go-sdk"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
manifest := extism.Manifest{
|
||||
Wasm: []extism.Wasm{
|
||||
extism.WasmUrl{
|
||||
Url: "https://github.com/extism/plugins/releases/latest/download/count_vowels.wasm",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
config := extism.PluginConfig{}
|
||||
plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{})
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to initialize plugin: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
```
|
||||
> **Note**: See [the Manifest docs](https://pkg.go.dev/github.com/extism/go-sdk#Manifest) as it has a rich schema and a lot of options.
|
||||
|
||||
### Calling A Plug-in's Exports
|
||||
|
||||
This plug-in was written in Rust and it does one thing, it counts vowels in a string. As such, it exposes one "export" function: `count_vowels`. We can call exports using [extism.Plugin.Call](https://pkg.go.dev/github.com/extism/go-sdk#Plugin.Call).
|
||||
Let's add that code to our main func:
|
||||
|
||||
```go
|
||||
func main() {
|
||||
// ...
|
||||
|
||||
data := []byte("Hello, World!")
|
||||
exit, out, err := plugin.Call("count_vowels", data)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(int(exit))
|
||||
}
|
||||
|
||||
response := string(out)
|
||||
fmt.Println(response)
|
||||
// => {"count": 3, "total": 3, "vowels": "aeiouAEIOU"}
|
||||
}
|
||||
```
|
||||
|
||||
Running this should print out the JSON vowel count report:
|
||||
|
||||
```bash
|
||||
$ go run main.go
|
||||
# => {"count":3,"total":3,"vowels":"aeiouAEIOU"}
|
||||
```
|
||||
|
||||
All exports have a simple interface of optional bytes in, and optional bytes out. This plug-in happens to take a string and return a JSON encoded string with a report of results.
|
||||
|
||||
> **Note**: If you want to pass a custom `context.Context` when calling a plugin function, you can use the [extism.Plugin.CallWithContext](https://pkg.go.dev/github.com/extism/go-sdk#Plugin.CallWithContext) method instead.
|
||||
|
||||
|
||||
### Plug-in State
|
||||
|
||||
Plug-ins may be stateful or stateless. Plug-ins can maintain state between calls by the use of variables. Our count vowels plug-in remembers the total number of vowels it's ever counted in the "total" key in the result. You can see this by making subsequent calls to the export:
|
||||
|
||||
```go
|
||||
func main () {
|
||||
// ...
|
||||
|
||||
exit, out, err := plugin.Call("count_vowels", []byte("Hello, World!"))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(int(exit))
|
||||
}
|
||||
fmt.Println(string(out))
|
||||
// => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
|
||||
|
||||
exit, out, err = plugin.Call("count_vowels", []byte("Hello, World!"))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(int(exit))
|
||||
}
|
||||
fmt.Println(string(out))
|
||||
// => {"count": 3, "total": 9, "vowels": "aeiouAEIOU"}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
These variables will persist until this plug-in is freed or you initialize a new one.
|
||||
|
||||
### Configuration
|
||||
|
||||
Plug-ins may optionally take a configuration object. This is a static way to configure the plug-in. Our count-vowels plugin takes an optional configuration to change out which characters are considered vowels. Example:
|
||||
|
||||
```go
|
||||
func main() {
|
||||
manifest := extism.Manifest{
|
||||
Wasm: []extism.Wasm{
|
||||
extism.WasmUrl{
|
||||
Url: "https://github.com/extism/plugins/releases/latest/download/count_vowels.wasm",
|
||||
},
|
||||
},
|
||||
Config: map[string]string{
|
||||
"vowels": "aeiouyAEIOUY",
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
config := extism.PluginConfig{}
|
||||
|
||||
plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{})
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to initialize plugin: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
exit, out, err := plugin.Call("count_vowels", []byte("Yellow, World!"))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(int(exit))
|
||||
}
|
||||
|
||||
fmt.Println(string(out))
|
||||
// => {"count": 4, "total": 4, "vowels": "aeiouAEIOUY"}
|
||||
}
|
||||
```
|
||||
|
||||
### Host Functions
|
||||
|
||||
Let's extend our count-vowels example a little bit: Instead of storing the `total` in an ephemeral plug-in var, let's store it in a persistent key-value store!
|
||||
|
||||
Wasm can't use our KV store on it's own. This is where [Host Functions](https://extism.org/docs/concepts/host-functions) come in.
|
||||
|
||||
[Host functions](https://extism.org/docs/concepts/host-functions) allow us to grant new capabilities to our plug-ins from our application. They are simply some Go functions you write which can be passed down and invoked from any language inside the plug-in.
|
||||
|
||||
Let's load the manifest like usual but load up this `count_vowels_kvstore` plug-in:
|
||||
|
||||
```go
|
||||
manifest := extism.Manifest{
|
||||
Wasm: []extism.Wasm{
|
||||
extism.WasmUrl{
|
||||
Url: "https://github.com/extism/plugins/releases/latest/download/count_vowels_kvstore.wasm",
|
||||
},
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
> *Note*: The source code for this is [here](https://github.com/extism/plugins/blob/main/count_vowels_kvstore/src/lib.rs) and is written in rust, but it could be written in any of our PDK languages.
|
||||
|
||||
Unlike our previous plug-in, this plug-in expects you to provide host functions that satisfy our its import interface for a KV store.
|
||||
|
||||
We want to expose two functions to our plugin, `kv_write(key string, value []bytes)` which writes a bytes value to a key and `kv_read(key string) []byte` which reads the bytes at the given `key`.
|
||||
```go
|
||||
// pretend this is Redis or something :)
|
||||
kvStore := make(map[string][]byte)
|
||||
|
||||
kvRead := extism.NewHostFunctionWithStack(
|
||||
"kv_read",
|
||||
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
||||
key, err := p.ReadString(stack[0])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
value, success := kvStore[key]
|
||||
if !success {
|
||||
value = []byte{0, 0, 0, 0}
|
||||
}
|
||||
|
||||
stack[0], err = p.WriteBytes(value)
|
||||
},
|
||||
[]ValueType{ValueTypePTR},
|
||||
[]ValueType{ValueTypePTR},
|
||||
)
|
||||
|
||||
kvWrite := extism.NewHostFunctionWithStack(
|
||||
"kv_write",
|
||||
func(ctx context.Context, p *extism.CurrentPlugin, stack []uint64) {
|
||||
key, err := p.ReadString(stack[0])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
value, err := p.ReadBytes(stack[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
kvStore[key] = value
|
||||
},
|
||||
[]ValueType{ValueTypePTR, ValueTypePTR},
|
||||
[]ValueType{},
|
||||
)
|
||||
```
|
||||
|
||||
> *Note*: In order to write host functions you should get familiar with the methods on the [extism.CurrentPlugin](https://pkg.go.dev/github.com/extism/go-sdk#CurrentPlugin) type. The `p` parameter is an instance of this type.
|
||||
|
||||
We need to pass these imports to the plug-in to create them. All imports of a plug-in must be satisfied for it to be initialized:
|
||||
|
||||
```go
|
||||
plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{kvRead, kvWrite});
|
||||
```
|
||||
|
||||
Now we can invoke the event:
|
||||
|
||||
```go
|
||||
exit, out, err := plugin.Call("count_vowels", []byte("Hello, World!"))
|
||||
// => Read from key=count-vowels"
|
||||
// => Writing value=3 from key=count-vowels"
|
||||
// => {"count": 3, "total": 3, "vowels": "aeiouAEIOU"}
|
||||
|
||||
exit, out, err = plugin.Call("count_vowels", []byte("Hello, World!"))
|
||||
// => Read from key=count-vowels"
|
||||
// => Writing value=6 from key=count-vowels"
|
||||
// => {"count": 3, "total": 6, "vowels": "aeiouAEIOU"}
|
||||
```
|
||||
|
||||
### Enabling Compilation Cache
|
||||
While Wazero (the underlying Wasm runtime) is very fast in initializing modules, you can make subsequent initializations even faster by enabling the compilation cache:
|
||||
|
||||
```go
|
||||
ctx := context.Background()
|
||||
cache := wazero.NewCompilationCache()
|
||||
defer cache.Close(ctx)
|
||||
|
||||
manifest := Manifest{Wasm: []Wasm{WasmFile{Path: "wasm/noop.wasm"}}}
|
||||
|
||||
config := PluginConfig{
|
||||
EnableWasi: true,
|
||||
ModuleConfig: wazero.NewModuleConfig(),
|
||||
RuntimeConfig: wazero.NewRuntimeConfig().WithCompilationCache(cache),
|
||||
}
|
||||
|
||||
_, err := NewPlugin(ctx, manifest, config, []HostFunction{})
|
||||
```
|
||||
|
||||
### Integrate with Dylibso Observe SDK
|
||||
Dylibso provides [observability SDKs](https://github.com/dylibso/observe-sdk) for WebAssembly (Wasm), enabling continuous monitoring of WebAssembly code as it executes within a runtime. It provides developers with the tools necessary to capture and emit telemetry data from Wasm code, including function execution and memory allocation traces, logs, and metrics.
|
||||
|
||||
While Observe SDK has adapters for many popular observability platforms, it also ships with an stdout adapter:
|
||||
|
||||
```
|
||||
ctx := context.Background()
|
||||
|
||||
adapter := stdout.NewStdoutAdapter()
|
||||
adapter.Start(ctx)
|
||||
|
||||
manifest := manifest("nested.c.instr.wasm")
|
||||
|
||||
config := PluginConfig{
|
||||
ModuleConfig: wazero.NewModuleConfig().WithSysWalltime(),
|
||||
EnableWasi: true,
|
||||
ObserveAdapter: adapter.AdapterBase,
|
||||
}
|
||||
|
||||
plugin, err := NewPlugin(ctx, manifest, config, []HostFunction{})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
meta := map[string]string{
|
||||
"http.url": "https://example.com/my-endpoint",
|
||||
"http.status_code": "200",
|
||||
"http.client_ip": "192.168.1.0",
|
||||
}
|
||||
|
||||
plugin.TraceCtx.Metadata(meta)
|
||||
|
||||
_, _, _ = plugin.Call("_start", []byte("hello world"))
|
||||
plugin.Close()
|
||||
```
|
||||
|
||||
### Enable filesystem access
|
||||
|
||||
WASM plugins can read/write files outside the runtime. To do this we add `AllowedPaths` mapping of "HOST:PLUGIN" to the `extism.Manifest` of our plugin.
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
extism "github.com/extism/go-sdk"
|
||||
)
|
||||
|
||||
func main() {
|
||||
manifest := extism.Manifest{
|
||||
AllowedPaths: map[string]string{
|
||||
// Here we specifify a host directory data to be linked
|
||||
// to the /mnt directory inside the wasm runtime
|
||||
"data": "/mnt",
|
||||
},
|
||||
Wasm: []extism.Wasm{
|
||||
extism.WasmFile{
|
||||
Path: "fs_plugin.wasm",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
config := extism.PluginConfig{
|
||||
EnableWasi: true,
|
||||
}
|
||||
plugin, err := extism.NewPlugin(ctx, manifest, config, []extism.HostFunction{})
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to initialize plugin: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
data := []byte("Hello world, this is written from within our wasm plugin.")
|
||||
exit, _, err := plugin.Call("write_file", data)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(int(exit))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> *Note*: In order for filesystem APIs to work the plugin needs to be compiled with WASI target. Source code for the plugin can be found [here](https://github.com/extism/go-pdk/blob/main/example/fs/main.go) and is written in Go, but it could be written in any of our PDK languages.
|
||||
|
||||
## Build example plugins
|
||||
|
||||
Since our [example plugins](./plugins/) are also written in Go, for compiling them we use [TinyGo](https://tinygo.org/):
|
||||
|
||||
```sh
|
||||
cd plugins/config
|
||||
tinygo build -target wasi -o ../wasm/config.wasm main.go
|
||||
```
|
||||
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
v1.7.0
|
||||
+551
@@ -0,0 +1,551 @@
|
||||
package extism
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
_ "embed"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
observe "github.com/dylibso/observe-sdk/go"
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/api"
|
||||
"github.com/tetratelabs/wazero/sys"
|
||||
)
|
||||
|
||||
type PluginCtxKey string
|
||||
type InputOffsetKey string
|
||||
|
||||
//go:embed extism-runtime.wasm
|
||||
var extismRuntimeWasm []byte
|
||||
|
||||
//go:embed extism-runtime.wasm.version
|
||||
var extismRuntimeWasmVersion string
|
||||
|
||||
func RuntimeVersion() string {
|
||||
return extismRuntimeWasmVersion
|
||||
}
|
||||
|
||||
// Runtime represents the Extism plugin's runtime environment, including the underlying Wazero runtime and modules.
|
||||
type Runtime struct {
|
||||
Wazero wazero.Runtime
|
||||
Extism api.Module
|
||||
Env api.Module
|
||||
}
|
||||
|
||||
// PluginInstanceConfig contains configuration options for the Extism plugin.
|
||||
type PluginInstanceConfig struct {
|
||||
// ModuleConfig allows the user to specify custom module configuration.
|
||||
//
|
||||
// NOTE: Module name and start functions are ignored as they are overridden by Extism, also if Manifest contains
|
||||
// non-empty AllowedPaths, then FS is also ignored. If EXTISM_ENABLE_WASI_OUTPUT is set, then stdout and stderr are
|
||||
// set to os.Stdout and os.Stderr respectively (ignoring user defined module config).
|
||||
ModuleConfig wazero.ModuleConfig
|
||||
}
|
||||
|
||||
// HttpRequest represents an HTTP request to be made by the plugin.
|
||||
type HttpRequest struct {
|
||||
Url string
|
||||
Headers map[string]string
|
||||
Method string
|
||||
}
|
||||
|
||||
// LogLevel defines different log levels.
|
||||
type LogLevel int32
|
||||
|
||||
const (
|
||||
logLevelUnset LogLevel = iota // unexporting this intentionally so its only ever the default
|
||||
LogLevelTrace
|
||||
LogLevelDebug
|
||||
LogLevelInfo
|
||||
LogLevelWarn
|
||||
LogLevelError
|
||||
|
||||
LogLevelOff LogLevel = math.MaxInt32
|
||||
)
|
||||
|
||||
func (l LogLevel) ExtismCompat() int32 {
|
||||
switch l {
|
||||
case LogLevelTrace:
|
||||
return 0
|
||||
case LogLevelDebug:
|
||||
return 1
|
||||
case LogLevelInfo:
|
||||
return 2
|
||||
case LogLevelWarn:
|
||||
return 3
|
||||
case LogLevelError:
|
||||
return 4
|
||||
default:
|
||||
return int32(LogLevelOff)
|
||||
}
|
||||
}
|
||||
|
||||
func (l LogLevel) String() string {
|
||||
s := ""
|
||||
switch l {
|
||||
case LogLevelTrace:
|
||||
s = "TRACE"
|
||||
case LogLevelDebug:
|
||||
s = "DEBUG"
|
||||
case LogLevelInfo:
|
||||
s = "INFO"
|
||||
case LogLevelWarn:
|
||||
s = "WARN"
|
||||
case LogLevelError:
|
||||
s = "ERROR"
|
||||
default:
|
||||
s = "OFF"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Plugin is used to call WASM functions
|
||||
type Plugin struct {
|
||||
close []func(ctx context.Context) error
|
||||
extism api.Module
|
||||
mainModule api.Module
|
||||
modules map[string]api.Module
|
||||
Timeout time.Duration
|
||||
Config map[string]string
|
||||
Var map[string][]byte
|
||||
AllowedHosts []string
|
||||
AllowedPaths map[string]string
|
||||
LastStatusCode int
|
||||
LastResponseHeaders map[string]string
|
||||
MaxHttpResponseBytes int64
|
||||
MaxVarBytes int64
|
||||
log func(LogLevel, string)
|
||||
hasWasi bool
|
||||
guestRuntime guestRuntime
|
||||
Adapter *observe.AdapterBase
|
||||
traceCtx *observe.TraceCtx
|
||||
}
|
||||
|
||||
func logStd(level LogLevel, message string) {
|
||||
log.Print(message)
|
||||
}
|
||||
|
||||
func (p *Plugin) Module() *Module {
|
||||
return &Module{inner: p.mainModule}
|
||||
}
|
||||
|
||||
// SetLogger sets a custom logging callback
|
||||
func (p *Plugin) SetLogger(logger func(LogLevel, string)) {
|
||||
p.log = logger
|
||||
}
|
||||
|
||||
func (p *Plugin) Log(level LogLevel, message string) {
|
||||
minimumLevel := LogLevel(pluginLogLevel.Load())
|
||||
|
||||
// If the global log level hasn't been set, use LogLevelOff as default
|
||||
if minimumLevel == logLevelUnset {
|
||||
minimumLevel = LogLevelOff
|
||||
}
|
||||
|
||||
if level >= minimumLevel {
|
||||
p.log(level, message)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Plugin) Logf(level LogLevel, format string, args ...any) {
|
||||
message := fmt.Sprintf(format, args...)
|
||||
p.Log(level, message)
|
||||
}
|
||||
|
||||
// Wasm is an interface that represents different ways of providing WebAssembly data.
|
||||
type Wasm interface {
|
||||
ToWasmData(ctx context.Context) (WasmData, error)
|
||||
}
|
||||
|
||||
// WasmData represents in-memory WebAssembly data, including its content, hash, and name.
|
||||
type WasmData struct {
|
||||
Data []byte `json:"data"`
|
||||
Hash string `json:"hash,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// WasmFile represents WebAssembly data that needs to be loaded from a file.
|
||||
type WasmFile struct {
|
||||
Path string `json:"path"`
|
||||
Hash string `json:"hash,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// WasmUrl represents WebAssembly data that needs to be fetched from a URL.
|
||||
type WasmUrl struct {
|
||||
Url string `json:"url"`
|
||||
Hash string `json:"hash,omitempty"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
}
|
||||
|
||||
type concreteWasm struct {
|
||||
Data []byte `json:"data,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Url string `json:"url,omitempty"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Hash string `json:"hash,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
func (d WasmData) ToWasmData(ctx context.Context) (WasmData, error) {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (f WasmFile) ToWasmData(ctx context.Context) (WasmData, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return WasmData{}, ctx.Err()
|
||||
default:
|
||||
data, err := os.ReadFile(f.Path)
|
||||
if err != nil {
|
||||
return WasmData{}, err
|
||||
}
|
||||
|
||||
return WasmData{
|
||||
Data: data,
|
||||
Hash: f.Hash,
|
||||
Name: f.Name,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (u WasmUrl) ToWasmData(ctx context.Context) (WasmData, error) {
|
||||
client := http.DefaultClient
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, u.Method, u.Url, nil)
|
||||
if err != nil {
|
||||
return WasmData{}, err
|
||||
}
|
||||
|
||||
for key, value := range u.Headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return WasmData{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return WasmData{}, errors.New("failed to fetch Wasm data from URL")
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return WasmData{}, err
|
||||
}
|
||||
|
||||
return WasmData{
|
||||
Data: data,
|
||||
Hash: u.Hash,
|
||||
Name: u.Name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type ManifestMemory struct {
|
||||
MaxPages uint32 `json:"max_pages,omitempty"`
|
||||
MaxHttpResponseBytes int64 `json:"max_http_response_bytes,omitempty"`
|
||||
MaxVarBytes int64 `json:"max_var_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// Manifest represents the plugin's manifest, including Wasm modules and configuration.
|
||||
// See https://extism.org/docs/concepts/manifest for schema.
|
||||
type Manifest struct {
|
||||
Wasm []Wasm `json:"wasm"`
|
||||
Memory *ManifestMemory `json:"memory,omitempty"`
|
||||
Config map[string]string `json:"config,omitempty"`
|
||||
AllowedHosts []string `json:"allowed_hosts,omitempty"`
|
||||
AllowedPaths map[string]string `json:"allowed_paths,omitempty"`
|
||||
Timeout uint64 `json:"timeout_ms,omitempty"`
|
||||
}
|
||||
|
||||
type concreteManifest struct {
|
||||
Wasm []concreteWasm `json:"wasm"`
|
||||
Memory *struct {
|
||||
MaxPages uint32 `json:"max_pages,omitempty"`
|
||||
MaxHttpResponseBytes *int64 `json:"max_http_response_bytes,omitempty"`
|
||||
MaxVarBytes *int64 `json:"max_var_bytes,omitempty"`
|
||||
} `json:"memory,omitempty"`
|
||||
Config map[string]string `json:"config,omitempty"`
|
||||
AllowedHosts []string `json:"allowed_hosts,omitempty"`
|
||||
AllowedPaths map[string]string `json:"allowed_paths,omitempty"`
|
||||
Timeout uint64 `json:"timeout_ms,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Manifest) UnmarshalJSON(data []byte) error {
|
||||
tmp := concreteManifest{}
|
||||
err := json.Unmarshal(data, &tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.Memory = &ManifestMemory{}
|
||||
if tmp.Memory != nil {
|
||||
m.Memory.MaxPages = tmp.Memory.MaxPages
|
||||
if tmp.Memory.MaxHttpResponseBytes != nil {
|
||||
m.Memory.MaxHttpResponseBytes = *tmp.Memory.MaxHttpResponseBytes
|
||||
} else {
|
||||
m.Memory.MaxHttpResponseBytes = -1
|
||||
}
|
||||
|
||||
if tmp.Memory.MaxVarBytes != nil {
|
||||
m.Memory.MaxVarBytes = *tmp.Memory.MaxVarBytes
|
||||
} else {
|
||||
m.Memory.MaxVarBytes = -1
|
||||
}
|
||||
} else {
|
||||
m.Memory.MaxPages = 0
|
||||
m.Memory.MaxHttpResponseBytes = -1
|
||||
m.Memory.MaxVarBytes = -1
|
||||
}
|
||||
m.Config = tmp.Config
|
||||
m.AllowedHosts = tmp.AllowedHosts
|
||||
m.AllowedPaths = tmp.AllowedPaths
|
||||
m.Timeout = tmp.Timeout
|
||||
if m.Wasm == nil {
|
||||
m.Wasm = []Wasm{}
|
||||
}
|
||||
for _, w := range tmp.Wasm {
|
||||
if len(w.Data) > 0 {
|
||||
m.Wasm = append(m.Wasm, WasmData{Data: w.Data, Hash: w.Hash, Name: w.Name})
|
||||
} else if len(w.Path) > 0 {
|
||||
m.Wasm = append(m.Wasm, WasmFile{Path: w.Path, Hash: w.Hash, Name: w.Name})
|
||||
} else if len(w.Url) > 0 {
|
||||
m.Wasm = append(m.Wasm, WasmUrl{
|
||||
Url: w.Url,
|
||||
Headers: w.Headers,
|
||||
Method: w.Method,
|
||||
Hash: w.Hash,
|
||||
Name: w.Name,
|
||||
})
|
||||
} else {
|
||||
return errors.New("invalid Wasm entry")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the plugin by freeing the underlying resources.
|
||||
func (p *Plugin) Close(ctx context.Context) error {
|
||||
return p.CloseWithContext(ctx)
|
||||
}
|
||||
|
||||
// CloseWithContext closes the plugin by freeing the underlying resources.
|
||||
func (p *Plugin) CloseWithContext(ctx context.Context) error {
|
||||
for _, fn := range p.close {
|
||||
if err := fn(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// add an atomic global to store the plugin runtime-wide log level
|
||||
var pluginLogLevel = atomic.Int32{}
|
||||
|
||||
// SetPluginLogLevel sets the log level for the plugin
|
||||
func SetLogLevel(level LogLevel) {
|
||||
pluginLogLevel.Store(int32(level))
|
||||
}
|
||||
|
||||
// SetInput sets the input data for the plugin to be used in the next WebAssembly function call.
|
||||
func (p *Plugin) SetInput(data []byte) (uint64, error) {
|
||||
return p.SetInputWithContext(context.Background(), data)
|
||||
}
|
||||
|
||||
// SetInputWithContext sets the input data for the plugin to be used in the next WebAssembly function call.
|
||||
func (p *Plugin) SetInputWithContext(ctx context.Context, data []byte) (uint64, error) {
|
||||
_, err := p.extism.ExportedFunction("reset").Call(ctx)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return 0, errors.New("reset")
|
||||
}
|
||||
|
||||
ptr, err := p.extism.ExportedFunction("alloc").Call(ctx, uint64(len(data)))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
p.Memory().Write(uint32(ptr[0]), data)
|
||||
p.extism.ExportedFunction("input_set").Call(ctx, ptr[0], uint64(len(data)))
|
||||
return ptr[0], nil
|
||||
}
|
||||
|
||||
// GetOutput retrieves the output data from the last WebAssembly function call.
|
||||
func (p *Plugin) GetOutput() ([]byte, error) {
|
||||
return p.GetOutputWithContext(context.Background())
|
||||
}
|
||||
|
||||
// GetOutputWithContext retrieves the output data from the last WebAssembly function call.
|
||||
func (p *Plugin) GetOutputWithContext(ctx context.Context) ([]byte, error) {
|
||||
outputOffs, err := p.extism.ExportedFunction("output_offset").Call(ctx)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
outputLen, err := p.extism.ExportedFunction("output_length").Call(ctx)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
mem, _ := p.Memory().Read(uint32(outputOffs[0]), uint32(outputLen[0]))
|
||||
|
||||
// Make sure output is copied, because `Read` returns a write-through view
|
||||
buffer := make([]byte, len(mem))
|
||||
copy(buffer, mem)
|
||||
|
||||
return buffer, nil
|
||||
}
|
||||
|
||||
// Memory returns the plugin's WebAssembly memory interface.
|
||||
func (p *Plugin) Memory() api.Memory {
|
||||
return p.extism.ExportedMemory("memory")
|
||||
}
|
||||
|
||||
// GetError retrieves the error message from the last WebAssembly function call, if any.
|
||||
func (p *Plugin) GetError() string {
|
||||
return p.GetErrorWithContext(context.Background())
|
||||
}
|
||||
|
||||
// GetErrorWithContext retrieves the error message from the last WebAssembly function call.
|
||||
func (p *Plugin) GetErrorWithContext(ctx context.Context) string {
|
||||
errOffs, err := p.extism.ExportedFunction("error_get").Call(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
if errOffs[0] == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
errLen, err := p.extism.ExportedFunction("length").Call(ctx, errOffs[0])
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
mem, _ := p.Memory().Read(uint32(errOffs[0]), uint32(errLen[0]))
|
||||
return string(mem)
|
||||
}
|
||||
|
||||
// FunctionExists returns true when the named function is present in the plugin's main Module
|
||||
func (p *Plugin) FunctionExists(name string) bool {
|
||||
return p.mainModule.ExportedFunction(name) != nil
|
||||
}
|
||||
|
||||
// Call a function by name with the given input, returning the output
|
||||
func (p *Plugin) Call(name string, data []byte) (uint32, []byte, error) {
|
||||
return p.CallWithContext(context.Background(), name, data)
|
||||
}
|
||||
|
||||
// Call a function by name with the given input and context, returning the output
|
||||
func (p *Plugin) CallWithContext(ctx context.Context, name string, data []byte) (uint32, []byte, error) {
|
||||
if p.mainModule.IsClosed() {
|
||||
return 0, nil, fmt.Errorf("module is closed")
|
||||
}
|
||||
|
||||
ctx = context.WithValue(ctx, PluginCtxKey("extism"), p.extism)
|
||||
if p.Timeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, p.Timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
ctx = context.WithValue(ctx, PluginCtxKey("plugin"), p)
|
||||
|
||||
intputOffset, err := p.SetInput(data)
|
||||
if err != nil {
|
||||
return 1, []byte{}, err
|
||||
}
|
||||
|
||||
ctx = context.WithValue(ctx, InputOffsetKey("inputOffset"), intputOffset)
|
||||
|
||||
var f = p.mainModule.ExportedFunction(name)
|
||||
|
||||
if f == nil {
|
||||
return 1, []byte{}, fmt.Errorf("unknown function: %s", name)
|
||||
} else if n := len(f.Definition().ResultTypes()); n > 1 {
|
||||
return 1, []byte{}, fmt.Errorf("function %s has %v results, expected 0 or 1", name, n)
|
||||
}
|
||||
|
||||
var isStart = name == "_start" || name == "_initialize"
|
||||
if p.guestRuntime.init != nil && !isStart && !p.guestRuntime.initialized {
|
||||
err := p.guestRuntime.init(ctx)
|
||||
if err != nil {
|
||||
return 1, []byte{}, fmt.Errorf("failed to initialize runtime: %v", err)
|
||||
}
|
||||
p.guestRuntime.initialized = true
|
||||
}
|
||||
|
||||
p.Logf(LogLevelDebug, "Calling function : %v", name)
|
||||
|
||||
res, err := f.Call(ctx)
|
||||
|
||||
if p.traceCtx != nil {
|
||||
defer p.traceCtx.Finish()
|
||||
}
|
||||
|
||||
// Try to extact WASI exit code
|
||||
if exitErr, ok := err.(*sys.ExitError); ok {
|
||||
exitCode := exitErr.ExitCode()
|
||||
|
||||
if exitCode == 0 {
|
||||
err = nil
|
||||
}
|
||||
|
||||
if len(res) == 0 {
|
||||
res = []uint64{api.EncodeU32(exitCode)}
|
||||
}
|
||||
}
|
||||
|
||||
var rc uint32
|
||||
if len(res) == 0 {
|
||||
// As long as there is no error, we assume the call has succeeded
|
||||
if err == nil {
|
||||
rc = 0
|
||||
} else {
|
||||
rc = 1
|
||||
}
|
||||
} else {
|
||||
rc = api.DecodeU32(res[0])
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return rc, []byte{}, err
|
||||
}
|
||||
|
||||
var returnErr error = nil
|
||||
errMsg := p.GetErrorWithContext(ctx)
|
||||
if errMsg != "" {
|
||||
returnErr = errors.New(errMsg)
|
||||
}
|
||||
|
||||
output, err := p.GetOutputWithContext(ctx)
|
||||
if err != nil {
|
||||
e := fmt.Errorf("failed to get output: %v", err)
|
||||
if returnErr != nil {
|
||||
return rc, []byte{}, errors.Join(returnErr, e)
|
||||
} else {
|
||||
return rc, []byte{}, e
|
||||
}
|
||||
}
|
||||
|
||||
return rc, output, returnErr
|
||||
}
|
||||
|
||||
func calculateHash(data []byte) string {
|
||||
hasher := sha256.New()
|
||||
hasher.Write(data)
|
||||
return hex.EncodeToString(hasher.Sum(nil))
|
||||
}
|
||||
+694
@@ -0,0 +1,694 @@
|
||||
package extism
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"unsafe"
|
||||
|
||||
// TODO: is there a better package for this?
|
||||
"github.com/gobwas/glob"
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/api"
|
||||
)
|
||||
|
||||
type ValueType = byte
|
||||
|
||||
const (
|
||||
// ValueTypeI32 is a 32-bit integer.
|
||||
ValueTypeI32 = api.ValueTypeI32
|
||||
// ValueTypeI64 is a 64-bit integer.
|
||||
ValueTypeI64 = api.ValueTypeI64
|
||||
// ValueTypeF32 is a 32-bit floating point number.
|
||||
ValueTypeF32 = api.ValueTypeF32
|
||||
// ValueTypeF64 is a 64-bit floating point number.
|
||||
ValueTypeF64 = api.ValueTypeF64
|
||||
// ValueTypePTR represents a pointer to an Extism memory block. Alias for ValueTypeI64
|
||||
ValueTypePTR = ValueTypeI64
|
||||
)
|
||||
|
||||
// HostFunctionStackCallback is a Function implemented in Go instead of a wasm binary.
|
||||
// The plugin parameter is the calling plugin, used to access memory or
|
||||
// exported functions and logging.
|
||||
//
|
||||
// The stack is includes any parameters encoded according to their ValueType.
|
||||
// Its length is the max of parameter or result length. When there are results,
|
||||
// write them in order beginning at index zero. Do not use the stack after the
|
||||
// function returns.
|
||||
//
|
||||
// Here's a typical way to read three parameters and write back one.
|
||||
//
|
||||
// // read parameters in index order
|
||||
// argv, argvBuf := DecodeU32(inputs[0]), DecodeU32(inputs[1])
|
||||
//
|
||||
// // write results back to the stack in index order
|
||||
// stack[0] = EncodeU32(ErrnoSuccess)
|
||||
//
|
||||
// This function can be non-deterministic or cause side effects. It also
|
||||
// has special properties not defined in the WebAssembly Core specification.
|
||||
// Notably, this uses the caller's memory (via Module.Memory). See
|
||||
// https://www.w3.org/TR/wasm-core-1/#host-functions%E2%91%A0
|
||||
//
|
||||
// To safely decode/encode values from/to the uint64 inputs/ouputs, users are encouraged to use
|
||||
// Extism's EncodeXXX or DecodeXXX functions.
|
||||
type HostFunctionStackCallback func(ctx context.Context, p *CurrentPlugin, stack []uint64)
|
||||
|
||||
// HostFunction represents a custom function defined by the host.
|
||||
type HostFunction struct {
|
||||
stackCallback HostFunctionStackCallback
|
||||
Name string
|
||||
Namespace string
|
||||
Params []api.ValueType
|
||||
Returns []api.ValueType
|
||||
}
|
||||
|
||||
func (f *HostFunction) SetNamespace(namespace string) {
|
||||
f.Namespace = namespace
|
||||
}
|
||||
|
||||
// NewHostFunctionWithStack creates a new instance of a HostFunction, which is designed
|
||||
// to provide custom functionality in a given host environment.
|
||||
// Here's an example multiplication function that loads operands from memory:
|
||||
//
|
||||
// mult := NewHostFunctionWithStack(
|
||||
// "mult",
|
||||
// func(ctx context.Context, plugin *CurrentPlugin, stack []uint64) {
|
||||
// a := DecodeI32(stack[0])
|
||||
// b := DecodeI32(stack[1])
|
||||
//
|
||||
// stack[0] = EncodeI32(a * b)
|
||||
// },
|
||||
// []ValueType{ValueTypeI64, ValueTypeI64},
|
||||
// ValueTypeI64
|
||||
// )
|
||||
func NewHostFunctionWithStack(
|
||||
name string,
|
||||
callback HostFunctionStackCallback,
|
||||
params []ValueType,
|
||||
returnTypes []ValueType) HostFunction {
|
||||
|
||||
return HostFunction{
|
||||
stackCallback: callback,
|
||||
Name: name,
|
||||
Namespace: "extism:host/user",
|
||||
Params: params,
|
||||
Returns: returnTypes,
|
||||
}
|
||||
}
|
||||
|
||||
type CurrentPlugin struct {
|
||||
plugin *Plugin
|
||||
}
|
||||
|
||||
func (p *Plugin) currentPlugin() *CurrentPlugin {
|
||||
return &CurrentPlugin{p}
|
||||
}
|
||||
|
||||
func (p *CurrentPlugin) Log(level LogLevel, message string) {
|
||||
p.plugin.Log(level, message)
|
||||
}
|
||||
|
||||
func (p *CurrentPlugin) Logf(level LogLevel, format string, args ...any) {
|
||||
p.plugin.Logf(level, format, args...)
|
||||
}
|
||||
|
||||
// Memory returns the plugin's WebAssembly memory interface.
|
||||
func (p *CurrentPlugin) Memory() api.Memory {
|
||||
return p.plugin.Memory()
|
||||
}
|
||||
|
||||
// Alloc a new memory block of the given length, returning its offset
|
||||
func (p *CurrentPlugin) Alloc(n uint64) (uint64, error) {
|
||||
return p.AllocWithContext(context.Background(), n)
|
||||
}
|
||||
|
||||
// Alloc a new memory block of the given length, returning its offset
|
||||
func (p *CurrentPlugin) AllocWithContext(ctx context.Context, n uint64) (uint64, error) {
|
||||
out, err := p.plugin.extism.ExportedFunction("alloc").Call(ctx, uint64(n))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
} else if len(out) != 1 {
|
||||
return 0, fmt.Errorf("expected 1 return, go %v", len(out))
|
||||
}
|
||||
|
||||
return uint64(out[0]), nil
|
||||
}
|
||||
|
||||
// Free the memory block specified by the given offset
|
||||
func (p *CurrentPlugin) Free(offset uint64) error {
|
||||
return p.FreeWithContext(context.Background(), offset)
|
||||
}
|
||||
|
||||
// Free the memory block specified by the given offset
|
||||
func (p *CurrentPlugin) FreeWithContext(ctx context.Context, offset uint64) error {
|
||||
_, err := p.plugin.extism.ExportedFunction("free").Call(ctx, uint64(offset))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Length returns the number of bytes allocated at the specified offset
|
||||
func (p *CurrentPlugin) Length(offs uint64) (uint64, error) {
|
||||
return p.LengthWithContext(context.Background(), offs)
|
||||
}
|
||||
|
||||
// Length returns the number of bytes allocated at the specified offset
|
||||
func (p *CurrentPlugin) LengthWithContext(ctx context.Context, offs uint64) (uint64, error) {
|
||||
out, err := p.plugin.extism.ExportedFunction("length").Call(ctx, uint64(offs))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
} else if len(out) != 1 {
|
||||
return 0, fmt.Errorf("expected 1 return, go %v", len(out))
|
||||
}
|
||||
|
||||
return uint64(out[0]), nil
|
||||
}
|
||||
|
||||
// Write a string to wasm memory and return the offset
|
||||
func (p *CurrentPlugin) WriteString(s string) (uint64, error) {
|
||||
return p.WriteBytes([]byte(s))
|
||||
}
|
||||
|
||||
// WriteBytes writes a string to wasm memory and return the offset
|
||||
func (p *CurrentPlugin) WriteBytes(b []byte) (uint64, error) {
|
||||
ptr, err := p.Alloc(uint64(len(b)))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
ok := p.Memory().Write(uint32(ptr), b)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("failed to write to memory")
|
||||
}
|
||||
|
||||
return ptr, nil
|
||||
}
|
||||
|
||||
// ReadString reads a string from wasm memory
|
||||
func (p *CurrentPlugin) ReadString(offset uint64) (string, error) {
|
||||
buffer, err := p.ReadBytes(offset)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(buffer), nil
|
||||
}
|
||||
|
||||
// ReadBytes reads a byte array from memory
|
||||
func (p *CurrentPlugin) ReadBytes(offset uint64) ([]byte, error) {
|
||||
length, err := p.Length(offset)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
buffer, ok := p.Memory().Read(uint32(offset), uint32(length))
|
||||
if !ok {
|
||||
return []byte{}, fmt.Errorf("invalid memory block")
|
||||
}
|
||||
|
||||
cpy := make([]byte, len(buffer))
|
||||
copy(cpy, buffer)
|
||||
|
||||
return cpy, nil
|
||||
}
|
||||
|
||||
func buildHostModule(ctx context.Context, rt wazero.Runtime, name string, funcs []HostFunction) (api.Module, error) {
|
||||
builder := rt.NewHostModuleBuilder(name)
|
||||
|
||||
defineCustomHostFunctions(builder, funcs)
|
||||
|
||||
return builder.Instantiate(ctx)
|
||||
}
|
||||
|
||||
func defineCustomHostFunctions(builder wazero.HostModuleBuilder, funcs []HostFunction) {
|
||||
for _, f := range funcs {
|
||||
|
||||
// Go closures capture variables by reference, not by value.
|
||||
// This means that if you directly use f inside the closure without creating
|
||||
// a separate variable (closure) and assigning the value of f to it, you might run into unexpected behavior.
|
||||
// All the closures created in the loop would end up referencing the same f, which could lead to incorrect or unintended results.
|
||||
// See: https://github.com/extism/go-sdk/issues/5#issuecomment-1666774486
|
||||
closure := f.stackCallback
|
||||
|
||||
builder.NewFunctionBuilder().WithGoFunction(api.GoFunc(func(ctx context.Context, stack []uint64) {
|
||||
if plugin, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin); ok {
|
||||
closure(ctx, &CurrentPlugin{plugin}, stack)
|
||||
return
|
||||
}
|
||||
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
}), f.Params, f.Returns).Export(f.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func instantiateEnvModule(ctx context.Context, rt wazero.Runtime) (api.Module, error) {
|
||||
builder := rt.NewHostModuleBuilder("extism:host/env")
|
||||
|
||||
// A wrapper that creates allows calls from guest -> go host -> extism kernel wasm
|
||||
// See https://github.com/extism/proposals/blob/main/EIP-007-extism-runtime-kernel.md.
|
||||
extismFunc := func(name string, params []ValueType, results []ValueType) {
|
||||
builder.
|
||||
NewFunctionBuilder().
|
||||
WithGoModuleFunction(api.GoModuleFunc(func(ctx context.Context, m api.Module, stack []uint64) {
|
||||
extism, ok := ctx.Value(PluginCtxKey("extism")).(api.Module)
|
||||
if !ok {
|
||||
panic("Invalid context, `extism` key not found")
|
||||
}
|
||||
|
||||
f := extism.ExportedFunction(name)
|
||||
if f == nil {
|
||||
panic(fmt.Errorf("function %q not found in extism:host", name))
|
||||
}
|
||||
|
||||
err := f.CallWithStack(ctx, stack)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}), params, results).
|
||||
Export(name)
|
||||
}
|
||||
|
||||
extismFunc("alloc", []ValueType{ValueTypeI64}, []ValueType{ValueTypeI64})
|
||||
extismFunc("free", []ValueType{ValueTypeI64}, []ValueType{})
|
||||
extismFunc("load_u8", []ValueType{ValueTypeI64}, []ValueType{ValueTypeI32})
|
||||
extismFunc("input_load_u8", []ValueType{ValueTypeI64}, []ValueType{ValueTypeI32})
|
||||
extismFunc("store_u64", []ValueType{ValueTypeI64, ValueTypeI64}, []ValueType{})
|
||||
extismFunc("store_u8", []ValueType{ValueTypeI64, ValueTypeI32}, []ValueType{})
|
||||
extismFunc("input_set", []ValueType{ValueTypeI64, ValueTypeI64}, []ValueType{})
|
||||
extismFunc("output_set", []ValueType{ValueTypeI64, ValueTypeI64}, []ValueType{})
|
||||
extismFunc("input_length", []ValueType{}, []ValueType{ValueTypeI64})
|
||||
extismFunc("input_offset", []ValueType{}, []ValueType{ValueTypeI64})
|
||||
extismFunc("output_length", []ValueType{}, []ValueType{ValueTypeI64})
|
||||
extismFunc("output_offset", []ValueType{}, []ValueType{ValueTypeI64})
|
||||
extismFunc("length", []ValueType{ValueTypeI64}, []ValueType{ValueTypeI64})
|
||||
extismFunc("length_unsafe", []ValueType{ValueTypeI64}, []ValueType{ValueTypeI64})
|
||||
extismFunc("reset", []ValueType{}, []ValueType{})
|
||||
extismFunc("error_set", []ValueType{ValueTypeI64}, []ValueType{})
|
||||
extismFunc("error_get", []ValueType{}, []ValueType{ValueTypeI64})
|
||||
extismFunc("memory_bytes", []ValueType{}, []ValueType{ValueTypeI64})
|
||||
|
||||
builder.NewFunctionBuilder().
|
||||
WithGoModuleFunction(api.GoModuleFunc(api.GoModuleFunc(inputLoad_u64)), []ValueType{ValueTypeI64}, []ValueType{ValueTypeI64}).
|
||||
Export("input_load_u64")
|
||||
|
||||
builder.NewFunctionBuilder().
|
||||
WithGoModuleFunction(api.GoModuleFunc(load_u64), []ValueType{ValueTypeI64}, []ValueType{ValueTypeI64}).
|
||||
Export("load_u64")
|
||||
|
||||
builder.NewFunctionBuilder().
|
||||
WithGoModuleFunction(api.GoModuleFunc(store_u64), []ValueType{ValueTypeI64, ValueTypeI64}, []ValueType{}).
|
||||
Export("store_u64")
|
||||
|
||||
hostFunc := func(name string, f interface{}) {
|
||||
builder.NewFunctionBuilder().WithFunc(f).Export(name)
|
||||
}
|
||||
|
||||
hostFunc("config_get", configGet)
|
||||
hostFunc("var_get", varGet)
|
||||
hostFunc("var_set", varSet)
|
||||
hostFunc("http_request", httpRequest)
|
||||
hostFunc("http_status_code", httpStatusCode)
|
||||
hostFunc("http_headers", httpHeaders)
|
||||
hostFunc("get_log_level", getLogLevel)
|
||||
|
||||
logFunc := func(name string, level LogLevel) {
|
||||
hostFunc(name, func(ctx context.Context, m api.Module, offset uint64) {
|
||||
if plugin, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin); ok {
|
||||
if LogLevel(pluginLogLevel.Load()) > level {
|
||||
plugin.currentPlugin().Free(offset)
|
||||
return
|
||||
}
|
||||
|
||||
message, err := plugin.currentPlugin().ReadString(offset)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to read log message from memory: %v", err))
|
||||
}
|
||||
|
||||
plugin.Log(level, message)
|
||||
|
||||
plugin.currentPlugin().Free(offset)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
})
|
||||
}
|
||||
|
||||
logFunc("log_trace", LogLevelTrace)
|
||||
logFunc("log_debug", LogLevelDebug)
|
||||
logFunc("log_info", LogLevelInfo)
|
||||
logFunc("log_warn", LogLevelWarn)
|
||||
logFunc("log_error", LogLevelError)
|
||||
|
||||
return builder.Instantiate(ctx)
|
||||
}
|
||||
|
||||
func store_u64(ctx context.Context, mod api.Module, stack []uint64) {
|
||||
p, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin)
|
||||
if !ok {
|
||||
panic("Invalid context")
|
||||
}
|
||||
|
||||
offset := stack[0]
|
||||
value := stack[1]
|
||||
ok = p.Memory().WriteUint64Le(uint32(offset), value)
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("could not write value '%v' at offset: %v", value, offset))
|
||||
}
|
||||
}
|
||||
|
||||
func load_u64(ctx context.Context, mod api.Module, stack []uint64) {
|
||||
p, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin)
|
||||
if !ok {
|
||||
panic("Invalid context")
|
||||
}
|
||||
|
||||
stack[0], ok = p.Memory().ReadUint64Le(uint32(stack[0]))
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("could not read value at offset: %v", stack[0]))
|
||||
}
|
||||
}
|
||||
|
||||
func inputLoad_u64(ctx context.Context, mod api.Module, stack []uint64) {
|
||||
p, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin)
|
||||
if !ok {
|
||||
panic("Invalid context")
|
||||
}
|
||||
|
||||
offset, ok := ctx.Value(InputOffsetKey("inputOffset")).(uint64)
|
||||
if !ok {
|
||||
panic("Invalid context")
|
||||
}
|
||||
|
||||
stack[0], ok = p.Memory().ReadUint64Le(uint32(stack[0] + offset))
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("could not read value at offset: %v", stack[0]))
|
||||
}
|
||||
}
|
||||
|
||||
func configGet(ctx context.Context, m api.Module, offset uint64) uint64 {
|
||||
if plugin, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin); ok {
|
||||
cp := plugin.currentPlugin()
|
||||
|
||||
name, err := cp.ReadString(offset)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to read config name from memory: %v", err))
|
||||
}
|
||||
|
||||
value, ok := plugin.Config[name]
|
||||
if !ok {
|
||||
// Return 0 without an error if key is not found
|
||||
return 0
|
||||
}
|
||||
|
||||
offset, err = cp.WriteString(value)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to write config value to memory: %v", err))
|
||||
}
|
||||
|
||||
return offset
|
||||
}
|
||||
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
}
|
||||
|
||||
func varGet(ctx context.Context, m api.Module, offset uint64) uint64 {
|
||||
if plugin, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin); ok {
|
||||
cp := plugin.currentPlugin()
|
||||
|
||||
name, err := cp.ReadString(offset)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to read var name from memory: %v", err))
|
||||
}
|
||||
|
||||
cp.Free(offset)
|
||||
|
||||
value, ok := plugin.Var[name]
|
||||
if !ok {
|
||||
// Return 0 without an error if key is not found
|
||||
return 0
|
||||
}
|
||||
|
||||
offset, err = cp.WriteBytes(value)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to write var value to memory: %v", err))
|
||||
}
|
||||
|
||||
return offset
|
||||
}
|
||||
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
}
|
||||
|
||||
func varSet(ctx context.Context, m api.Module, nameOffset uint64, valueOffset uint64) {
|
||||
plugin, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin)
|
||||
if !ok {
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
}
|
||||
|
||||
if plugin.MaxVarBytes == 0 {
|
||||
panic("Vars are disabled by this host")
|
||||
}
|
||||
|
||||
cp := plugin.currentPlugin()
|
||||
|
||||
name, err := cp.ReadString(nameOffset)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to read var name from memory: %v", err))
|
||||
}
|
||||
|
||||
cp.Free(nameOffset)
|
||||
|
||||
// Remove if the value offset is 0
|
||||
if valueOffset == 0 {
|
||||
delete(plugin.Var, name)
|
||||
return
|
||||
}
|
||||
|
||||
value, err := cp.ReadBytes(valueOffset)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to read var value from memory: %v", err))
|
||||
}
|
||||
|
||||
cp.Free(valueOffset)
|
||||
|
||||
// Calculate size including current key/value
|
||||
size := int(unsafe.Sizeof([]byte{})+unsafe.Sizeof("")) + len(name) + len(value)
|
||||
for k, v := range plugin.Var {
|
||||
size += len(k)
|
||||
size += len(v)
|
||||
size += int(unsafe.Sizeof([]byte{}) + unsafe.Sizeof(""))
|
||||
}
|
||||
|
||||
if size >= int(plugin.MaxVarBytes) && valueOffset != 0 {
|
||||
panic("Variable store is full")
|
||||
}
|
||||
|
||||
plugin.Var[name] = value
|
||||
}
|
||||
|
||||
func httpRequest(ctx context.Context, m api.Module, requestOffset uint64, bodyOffset uint64) uint64 {
|
||||
if plugin, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin); ok {
|
||||
cp := plugin.currentPlugin()
|
||||
|
||||
if plugin.LastResponseHeaders != nil {
|
||||
for k := range plugin.LastResponseHeaders {
|
||||
delete(plugin.LastResponseHeaders, k)
|
||||
}
|
||||
}
|
||||
plugin.LastStatusCode = 0
|
||||
|
||||
requestJson, err := cp.ReadBytes(requestOffset)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("failed to read http request from memory: %v", err))
|
||||
}
|
||||
|
||||
var request HttpRequest
|
||||
err = json.Unmarshal(requestJson, &request)
|
||||
cp.Free(requestOffset)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("invalid http request: %v", err))
|
||||
}
|
||||
|
||||
// default method to GET and force to be upper
|
||||
if request.Method == "" {
|
||||
request.Method = "GET"
|
||||
}
|
||||
request.Method = strings.ToUpper(request.Method)
|
||||
|
||||
url, err := url.Parse(request.Url)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("invalid url: %v", err))
|
||||
}
|
||||
|
||||
// deny all requests by default
|
||||
hostMatches := false
|
||||
for _, allowedHost := range plugin.AllowedHosts {
|
||||
if allowedHost == url.Hostname() {
|
||||
hostMatches = true
|
||||
break
|
||||
}
|
||||
|
||||
pattern := glob.MustCompile(allowedHost)
|
||||
if pattern.Match(url.Hostname()) {
|
||||
hostMatches = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !hostMatches {
|
||||
panic(fmt.Errorf("HTTP request to '%v' is not allowed", request.Url))
|
||||
}
|
||||
|
||||
var bodyReader io.Reader = nil
|
||||
if bodyOffset != 0 {
|
||||
body, err := cp.ReadBytes(bodyOffset)
|
||||
if err != nil {
|
||||
panic("failed to read response body from memory")
|
||||
}
|
||||
|
||||
cp.Free(bodyOffset)
|
||||
|
||||
bodyReader = bytes.NewReader(body)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, request.Method, request.Url, bodyReader)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for key, value := range request.Headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
client := http.DefaultClient
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if plugin.LastResponseHeaders != nil {
|
||||
for k, v := range resp.Header {
|
||||
plugin.LastResponseHeaders[strings.ToLower(k)] = strings.Join(v, ",")
|
||||
}
|
||||
}
|
||||
|
||||
plugin.LastStatusCode = resp.StatusCode
|
||||
|
||||
limiter := http.MaxBytesReader(nil, resp.Body, int64(plugin.MaxHttpResponseBytes))
|
||||
body, err := io.ReadAll(limiter)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if len(body) == 0 {
|
||||
return 0
|
||||
} else {
|
||||
offset, err := cp.WriteBytes(body)
|
||||
if err != nil {
|
||||
panic("Failed to write resposne body to memory")
|
||||
}
|
||||
|
||||
return offset
|
||||
}
|
||||
}
|
||||
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
}
|
||||
|
||||
func httpStatusCode(ctx context.Context, m api.Module) int32 {
|
||||
if plugin, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin); ok {
|
||||
return int32(plugin.LastStatusCode)
|
||||
}
|
||||
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
}
|
||||
|
||||
func httpHeaders(ctx context.Context, _ api.Module) uint64 {
|
||||
if plugin, ok := ctx.Value(PluginCtxKey("plugin")).(*Plugin); ok {
|
||||
if plugin.LastResponseHeaders == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
data, err := json.Marshal(plugin.LastResponseHeaders)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
mem, err := plugin.currentPlugin().WriteBytes(data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return mem
|
||||
}
|
||||
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
}
|
||||
|
||||
func getLogLevel(ctx context.Context, m api.Module) int32 {
|
||||
// if _, ok := callCtx.Value(PluginCtxKey("plugin")).(*Plugin); ok {
|
||||
// panic("Invalid context, `plugin` key not found")
|
||||
// }
|
||||
return LogLevel(pluginLogLevel.Load()).ExtismCompat()
|
||||
}
|
||||
|
||||
// EncodeI32 encodes the input as a ValueTypeI32.
|
||||
func EncodeI32(input int32) uint64 {
|
||||
return api.EncodeI32(input)
|
||||
}
|
||||
|
||||
// DecodeI32 decodes the input as a ValueTypeI32.
|
||||
func DecodeI32(input uint64) int32 {
|
||||
return api.DecodeI32(input)
|
||||
}
|
||||
|
||||
// EncodeU32 encodes the input as a ValueTypeI32.
|
||||
func EncodeU32(input uint32) uint64 {
|
||||
return api.EncodeU32(input)
|
||||
}
|
||||
|
||||
// DecodeU32 decodes the input as a ValueTypeI32.
|
||||
func DecodeU32(input uint64) uint32 {
|
||||
return api.DecodeU32(input)
|
||||
}
|
||||
|
||||
// EncodeI64 encodes the input as a ValueTypeI64.
|
||||
func EncodeI64(input int64) uint64 {
|
||||
return api.EncodeI64(input)
|
||||
}
|
||||
|
||||
// EncodeF32 encodes the input as a ValueTypeF32.
|
||||
//
|
||||
// See DecodeF32
|
||||
func EncodeF32(input float32) uint64 {
|
||||
return api.EncodeF32(input)
|
||||
}
|
||||
|
||||
// DecodeF32 decodes the input as a ValueTypeF32.
|
||||
//
|
||||
// See EncodeF32
|
||||
func DecodeF32(input uint64) float32 {
|
||||
return api.DecodeF32(input)
|
||||
}
|
||||
|
||||
// EncodeF64 encodes the input as a ValueTypeF64.
|
||||
//
|
||||
// See EncodeF32
|
||||
func EncodeF64(input float64) uint64 {
|
||||
return api.EncodeF64(input)
|
||||
}
|
||||
|
||||
// DecodeF64 decodes the input as a ValueTypeF64.
|
||||
//
|
||||
// See EncodeF64
|
||||
func DecodeF64(input uint64) float64 {
|
||||
return api.DecodeF64(input)
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package extism
|
||||
|
||||
import "github.com/tetratelabs/wazero/api"
|
||||
|
||||
// Module is a wrapper around a wazero module. It allows us to provide
|
||||
// our own API and stability guarantees despite any changes that wazero
|
||||
// may choose to make.
|
||||
type Module struct {
|
||||
inner api.Module
|
||||
}
|
||||
|
||||
// ExportedFunctions returns a map of functions exported from the module
|
||||
// keyed by the function name.
|
||||
func (m *Module) ExportedFunctions() map[string]FunctionDefinition {
|
||||
v := make(map[string]FunctionDefinition)
|
||||
for name, def := range m.inner.ExportedFunctionDefinitions() {
|
||||
v[name] = FunctionDefinition{inner: def}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// FunctionDefinition represents a function defined in a module. It provides
|
||||
// a wrapper around the underlying wazero function definition.
|
||||
type FunctionDefinition struct {
|
||||
inner api.FunctionDefinition
|
||||
}
|
||||
|
||||
func (f *FunctionDefinition) Name() string {
|
||||
return f.inner.Name()
|
||||
}
|
||||
+421
@@ -0,0 +1,421 @@
|
||||
// new
|
||||
package extism
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
observe "github.com/dylibso/observe-sdk/go"
|
||||
"github.com/tetratelabs/wazero"
|
||||
"github.com/tetratelabs/wazero/api"
|
||||
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
|
||||
)
|
||||
|
||||
type CompiledPlugin struct {
|
||||
runtime wazero.Runtime
|
||||
main wazero.CompiledModule
|
||||
extism wazero.CompiledModule
|
||||
env api.Module
|
||||
modules map[string]wazero.CompiledModule
|
||||
|
||||
// when a module (main) is instantiated, it may have a module name that's added
|
||||
// to the data section of the wasm. If this is the case, we won't be able to
|
||||
// instantiate that module more than once. This counter acts as the module name
|
||||
// incrementing each time we instantiate the module.
|
||||
instanceCount atomic.Uint64
|
||||
|
||||
// this is the raw wasm bytes of the provided module, it is required when using a tracing observeAdapter.
|
||||
// If an adapter is not provided, this field will be nil.
|
||||
wasmBytes []byte
|
||||
hasWasi bool
|
||||
manifest Manifest
|
||||
observeAdapter *observe.AdapterBase
|
||||
observeOptions *observe.Options
|
||||
|
||||
maxHttp int64
|
||||
maxVar int64
|
||||
enableHttpResponseHeaders bool
|
||||
}
|
||||
|
||||
type PluginConfig struct {
|
||||
RuntimeConfig wazero.RuntimeConfig
|
||||
EnableWasi bool
|
||||
ObserveAdapter *observe.AdapterBase
|
||||
ObserveOptions *observe.Options
|
||||
EnableHttpResponseHeaders bool
|
||||
|
||||
// ModuleConfig is only used when a plugins are built using the NewPlugin
|
||||
// function. In this function, the plugin is both compiled, and an instance
|
||||
// of the plugin is instantiated, and the ModuleConfig is passed to the
|
||||
// instance.
|
||||
//
|
||||
// When plugins are built using NewCompiledPlugin, the ModuleConfig has no
|
||||
// effect because the instance is not created. Instead, the ModuleConfig is
|
||||
// passed directly in calls to the CompiledPlugin.Instance method.
|
||||
//
|
||||
// NOTE: Module name and start functions are ignored as they are overridden by Extism, also if Manifest contains
|
||||
// non-empty AllowedPaths, then FS is also ignored. If EXTISM_ENABLE_WASI_OUTPUT is set, then stdout and stderr are
|
||||
// set to os.Stdout and os.Stderr respectively (ignoring user defined module config).
|
||||
ModuleConfig wazero.ModuleConfig
|
||||
}
|
||||
|
||||
// NewPlugin creates compiles and instantiates a plugin that is ready
|
||||
// to be used. Plugins are not thread-safe. If you need to use a plugin
|
||||
// across multiple goroutines, use NewCompiledPlugin and create instances
|
||||
// of the plugin using the CompiledPlugin.Instance method.
|
||||
func NewPlugin(
|
||||
ctx context.Context,
|
||||
manifest Manifest,
|
||||
config PluginConfig,
|
||||
functions []HostFunction,
|
||||
) (*Plugin, error) {
|
||||
c, err := NewCompiledPlugin(ctx, manifest, config, functions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := c.Instance(ctx, PluginInstanceConfig{
|
||||
ModuleConfig: config.ModuleConfig,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.close = append(p.close, c.Close)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func calculateMaxHttp(manifest Manifest) int64 {
|
||||
// Default is 50MB
|
||||
maxHttp := int64(1024 * 1024 * 50)
|
||||
if manifest.Memory != nil && manifest.Memory.MaxHttpResponseBytes >= 0 {
|
||||
maxHttp = manifest.Memory.MaxHttpResponseBytes
|
||||
}
|
||||
return maxHttp
|
||||
}
|
||||
|
||||
func calculateMaxVar(manifest Manifest) int64 {
|
||||
// Default is 1MB
|
||||
maxVar := int64(1024 * 1024)
|
||||
if manifest.Memory != nil && manifest.Memory.MaxVarBytes >= 0 {
|
||||
maxVar = manifest.Memory.MaxVarBytes
|
||||
}
|
||||
return maxVar
|
||||
}
|
||||
|
||||
// NewCompiledPlugin creates a compiled plugin that is ready to be instantiated.
|
||||
// You can instantiate the plugin multiple times using the CompiledPlugin.Instance
|
||||
// method and run those instances concurrently.
|
||||
func NewCompiledPlugin(
|
||||
ctx context.Context,
|
||||
manifest Manifest,
|
||||
config PluginConfig,
|
||||
funcs []HostFunction,
|
||||
) (*CompiledPlugin, error) {
|
||||
count := len(manifest.Wasm)
|
||||
if count == 0 {
|
||||
return nil, fmt.Errorf("manifest can't be empty")
|
||||
}
|
||||
|
||||
runtimeConfig := config.RuntimeConfig
|
||||
if runtimeConfig == nil {
|
||||
runtimeConfig = wazero.NewRuntimeConfig()
|
||||
}
|
||||
|
||||
// Make sure function calls are cancelled if the context is cancelled
|
||||
if manifest.Timeout > 0 {
|
||||
runtimeConfig = runtimeConfig.WithCloseOnContextDone(true)
|
||||
}
|
||||
|
||||
if manifest.Memory != nil {
|
||||
if manifest.Memory.MaxPages > 0 {
|
||||
runtimeConfig = runtimeConfig.WithMemoryLimitPages(manifest.Memory.MaxPages)
|
||||
}
|
||||
}
|
||||
|
||||
p := CompiledPlugin{
|
||||
manifest: manifest,
|
||||
runtime: wazero.NewRuntimeWithConfig(ctx, runtimeConfig),
|
||||
observeAdapter: config.ObserveAdapter,
|
||||
observeOptions: config.ObserveOptions,
|
||||
enableHttpResponseHeaders: config.EnableHttpResponseHeaders,
|
||||
modules: make(map[string]wazero.CompiledModule),
|
||||
maxHttp: calculateMaxHttp(manifest),
|
||||
maxVar: calculateMaxVar(manifest),
|
||||
}
|
||||
|
||||
if config.EnableWasi {
|
||||
wasi_snapshot_preview1.MustInstantiate(ctx, p.runtime)
|
||||
p.hasWasi = true
|
||||
}
|
||||
|
||||
// Build host modules
|
||||
hostModules := make(map[string][]HostFunction)
|
||||
for _, f := range funcs {
|
||||
hostModules[f.Namespace] = append(hostModules[f.Namespace], f)
|
||||
}
|
||||
for name, funcs := range hostModules {
|
||||
_, err := buildHostModule(ctx, p.runtime, name, funcs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("building host module: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Compile the extism module
|
||||
var err error
|
||||
p.extism, err = p.runtime.CompileModule(ctx, extismRuntimeWasm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("instantiating extism module: %w", err)
|
||||
}
|
||||
|
||||
// Build and instantiate extism:host/env module
|
||||
p.env, err = instantiateEnvModule(ctx, p.runtime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Try to find the main module:
|
||||
// - There is always one main module
|
||||
// - If a Wasm value has the Name field set to "main" then use that module
|
||||
// - If there is only one module in the manifest then that is the main module by default
|
||||
// - Otherwise the last module listed is the main module
|
||||
|
||||
foundMain := false
|
||||
for i, wasm := range manifest.Wasm {
|
||||
data, err := wasm.ToWasmData(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if (data.Name == "" || i == len(manifest.Wasm)-1) && !foundMain {
|
||||
data.Name = "main"
|
||||
}
|
||||
|
||||
_, okm := p.modules[data.Name]
|
||||
|
||||
if data.Name == "extism:host/env" || okm {
|
||||
return nil, fmt.Errorf("module name collision: '%s'", data.Name)
|
||||
}
|
||||
|
||||
if data.Hash != "" {
|
||||
calculatedHash := calculateHash(data.Data)
|
||||
if data.Hash != calculatedHash {
|
||||
return nil, fmt.Errorf("hash mismatch for module '%s'", data.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if p.observeAdapter != nil {
|
||||
p.wasmBytes = data.Data
|
||||
}
|
||||
|
||||
compiledModule, err := p.runtime.CompileModule(ctx, data.Data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if data.Name == "main" {
|
||||
if foundMain {
|
||||
return nil, errors.New("can't have more than one main module")
|
||||
}
|
||||
p.main = compiledModule
|
||||
foundMain = true
|
||||
} else {
|
||||
// Store compiled module for instantiation
|
||||
p.modules[data.Name] = compiledModule
|
||||
// Create wrapper with original name that will forward calls to the actual module instance. See createModuleWrapper for more details.
|
||||
_, err = createModuleWrapper(ctx, p.runtime, data.Name, compiledModule)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create wrapper for %s: %w", data.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if p.main == nil {
|
||||
return nil, errors.New("no main module found")
|
||||
}
|
||||
|
||||
// We no longer need the wasm in the manifest so nil it
|
||||
// to make the slice eligible for garbage collection.
|
||||
p.manifest.Wasm = nil
|
||||
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
// createModuleWrapper creates a host module that acts as a proxy for module instances.
|
||||
// In Wazero, modules with the same name cannot be instantiated multiple times in the same runtime.
|
||||
// However, we need each Plugin instance to have its own copy of each module for isolation. To solve this, we:
|
||||
// 1. Create a host module wrapper that keeps the original module name (needed for imports to work)
|
||||
// 2. Instantiate actual module copies with unique names for each Plugin
|
||||
// 3. The wrapper forwards function calls to the correct module instance for each Plugin
|
||||
func createModuleWrapper(ctx context.Context, rt wazero.Runtime, name string, compiled wazero.CompiledModule) (api.Module, error) {
|
||||
builder := rt.NewHostModuleBuilder(name)
|
||||
|
||||
// Create proxy functions for each exported function from the original module.
|
||||
// These proxies will forward calls to the appropriate module instance.
|
||||
for _, export := range compiled.ExportedFunctions() {
|
||||
exportName := export.Name()
|
||||
|
||||
// Skip wrapping the _start function since it's automatically called by wazero during instantiation.
|
||||
// The wrapper functions require a Plugin instance in the context to work, but during wrapper
|
||||
// instantiation there is no Plugin instance yet.
|
||||
if exportName == "_start" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Create a proxy function that:
|
||||
// 1. Gets the calling Plugin instance from context
|
||||
// 2. Looks up that Plugin's copy of this module
|
||||
// 3. Forwards the call to the actual function
|
||||
wrapper := func(callCtx context.Context, mod api.Module, stack []uint64) {
|
||||
// Get the Plugin instance that's making this call
|
||||
plugin, ok := callCtx.Value(PluginCtxKey("plugin")).(*Plugin)
|
||||
if !ok {
|
||||
panic("Invalid context, `plugin` key not found")
|
||||
}
|
||||
|
||||
// Get this Plugin's instance of the module
|
||||
actualModule, ok := plugin.modules[name]
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("module %s not found in plugin", name))
|
||||
}
|
||||
|
||||
// Forward the call to the actual module instance
|
||||
fn := actualModule.ExportedFunction(exportName)
|
||||
if fn == nil {
|
||||
panic(fmt.Sprintf("function %s not found in module %s", exportName, name))
|
||||
}
|
||||
|
||||
err := fn.CallWithStack(callCtx, stack)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Export the proxy function with the same name and signature as the original
|
||||
builder.NewFunctionBuilder().
|
||||
WithGoModuleFunction(api.GoModuleFunc(wrapper), export.ParamTypes(), export.ResultTypes()).
|
||||
Export(exportName)
|
||||
}
|
||||
|
||||
return builder.Instantiate(ctx)
|
||||
}
|
||||
|
||||
func (p *CompiledPlugin) Close(ctx context.Context) error {
|
||||
return p.runtime.Close(ctx)
|
||||
}
|
||||
|
||||
func (p *CompiledPlugin) Instance(ctx context.Context, config PluginInstanceConfig) (*Plugin, error) {
|
||||
instanceNum := p.instanceCount.Add(1)
|
||||
|
||||
var closers []func(ctx context.Context) error
|
||||
|
||||
moduleConfig := config.ModuleConfig
|
||||
if moduleConfig == nil {
|
||||
moduleConfig = wazero.NewModuleConfig()
|
||||
}
|
||||
|
||||
// NOTE: we don't want wazero to call the start function, we will initialize
|
||||
// the guest runtime manually.
|
||||
// See: https://github.com/extism/go-sdk/pull/1#issuecomment-1650527495
|
||||
moduleConfig = moduleConfig.WithStartFunctions()
|
||||
|
||||
if len(p.manifest.AllowedPaths) > 0 {
|
||||
// NOTE: this is only necessary for guest modules because
|
||||
// host modules have the same access privileges as the host itself
|
||||
fs := wazero.NewFSConfig()
|
||||
for host, guest := range p.manifest.AllowedPaths {
|
||||
if strings.HasPrefix(host, "ro:") {
|
||||
trimmed := strings.TrimPrefix(host, "ro:")
|
||||
fs = fs.WithReadOnlyDirMount(trimmed, guest)
|
||||
} else {
|
||||
fs = fs.WithDirMount(host, guest)
|
||||
}
|
||||
}
|
||||
moduleConfig = moduleConfig.WithFSConfig(fs)
|
||||
}
|
||||
|
||||
_, wasiOutput := os.LookupEnv("EXTISM_ENABLE_WASI_OUTPUT")
|
||||
if p.hasWasi && wasiOutput {
|
||||
moduleConfig = moduleConfig.WithStderr(os.Stderr).WithStdout(os.Stdout)
|
||||
}
|
||||
|
||||
var trace *observe.TraceCtx
|
||||
var err error
|
||||
if p.observeAdapter != nil {
|
||||
trace, err = p.observeAdapter.NewTraceCtx(ctx, p.runtime, p.wasmBytes, p.observeOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize Observe Adapter: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Compile and instantiate the extism runtime. This runtime is stateful and needs to be
|
||||
// instantiated on a per-instance basis. We don't provide a name because the module needs
|
||||
// to be anonymous -- you cannot instantiate multiple modules with the same name into the
|
||||
// same runtime. It is okay that this is anonymous, because this module is only called
|
||||
// from Go host functions and not from the Wasm module itself.
|
||||
extism, err := p.runtime.InstantiateModule(ctx, p.extism, wazero.NewModuleConfig())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("instantiating extism module: %w", err)
|
||||
}
|
||||
|
||||
closers = append(closers, extism.Close)
|
||||
|
||||
// Instantiate all non-main modules first
|
||||
instancedModules := make(map[string]api.Module)
|
||||
for name, compiledModule := range p.modules {
|
||||
uniqueName := fmt.Sprintf("%s_%d", name, instanceNum)
|
||||
instance, err := p.runtime.InstantiateModule(ctx, compiledModule, moduleConfig.WithName(uniqueName))
|
||||
if err != nil {
|
||||
for _, closer := range closers {
|
||||
closer(ctx)
|
||||
}
|
||||
return nil, fmt.Errorf("instantiating module %s: %w", name, err)
|
||||
}
|
||||
instancedModules[name] = instance
|
||||
closers = append(closers, instance.Close)
|
||||
}
|
||||
|
||||
mainModuleName := fmt.Sprintf("main_%d", instanceNum)
|
||||
main, err := p.runtime.InstantiateModule(ctx, p.main, moduleConfig.WithName(mainModuleName))
|
||||
if err != nil {
|
||||
for _, closer := range closers {
|
||||
closer(ctx)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("instantiating module: %w", err)
|
||||
}
|
||||
|
||||
closers = append(closers, main.Close)
|
||||
|
||||
var headers map[string]string = nil
|
||||
if p.enableHttpResponseHeaders {
|
||||
headers = map[string]string{}
|
||||
}
|
||||
|
||||
instance := &Plugin{
|
||||
close: closers,
|
||||
extism: extism,
|
||||
hasWasi: p.hasWasi,
|
||||
mainModule: main,
|
||||
modules: instancedModules,
|
||||
Timeout: time.Duration(p.manifest.Timeout) * time.Millisecond,
|
||||
Config: p.manifest.Config,
|
||||
Var: make(map[string][]byte),
|
||||
AllowedHosts: p.manifest.AllowedHosts,
|
||||
AllowedPaths: p.manifest.AllowedPaths,
|
||||
LastStatusCode: 0,
|
||||
LastResponseHeaders: headers,
|
||||
MaxHttpResponseBytes: p.maxHttp,
|
||||
MaxVarBytes: p.maxVar,
|
||||
guestRuntime: guestRuntime{},
|
||||
Adapter: p.observeAdapter,
|
||||
log: logStd,
|
||||
traceCtx: trace,
|
||||
}
|
||||
instance.guestRuntime = detectGuestRuntime(instance)
|
||||
|
||||
return instance, nil
|
||||
}
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
package extism
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/tetratelabs/wazero/api"
|
||||
)
|
||||
|
||||
// TODO: test runtime initialization for WASI and Haskell
|
||||
|
||||
type runtimeType uint8
|
||||
|
||||
const (
|
||||
None runtimeType = iota
|
||||
Haskell
|
||||
Wasi
|
||||
)
|
||||
|
||||
type guestRuntime struct {
|
||||
mainRuntime moduleRuntime
|
||||
runtimes map[string]moduleRuntime
|
||||
init func(ctx context.Context) error
|
||||
initialized bool
|
||||
}
|
||||
|
||||
type moduleRuntime struct {
|
||||
runtimeType runtimeType
|
||||
init func(ctx context.Context) error
|
||||
initialized bool
|
||||
}
|
||||
|
||||
// detectGuestRuntime detects the runtime of the main module and all other modules
|
||||
// it returns a guest runtime with an initialization function specific that invokes
|
||||
// the initialization function of all the modules, with the main module last.
|
||||
func detectGuestRuntime(p *Plugin) guestRuntime {
|
||||
r := guestRuntime{runtimes: make(map[string]moduleRuntime)}
|
||||
|
||||
r.mainRuntime = detectModuleRuntime(p, p.mainModule)
|
||||
for k, m := range p.modules {
|
||||
r.runtimes[k] = detectModuleRuntime(p, m)
|
||||
}
|
||||
|
||||
r.init = func(ctx context.Context) error {
|
||||
|
||||
for k, v := range r.runtimes {
|
||||
p.Logf(LogLevelDebug, "Initializing runtime for module %v", k)
|
||||
err := v.init(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
v.initialized = true
|
||||
}
|
||||
|
||||
m := r.mainRuntime
|
||||
p.Logf(LogLevelDebug, "Initializing runtime for main module")
|
||||
err := m.init(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.initialized = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
// detectModuleRuntime detects the specific runtime of a given module
|
||||
// it returns a module runtime with an initialization function specific to that module
|
||||
func detectModuleRuntime(p *Plugin, m api.Module) moduleRuntime {
|
||||
runtime, ok := haskellRuntime(p, m)
|
||||
if ok {
|
||||
return runtime
|
||||
}
|
||||
|
||||
runtime, ok = wasiRuntime(p, m)
|
||||
if ok {
|
||||
return runtime
|
||||
}
|
||||
|
||||
p.Log(LogLevelTrace, "No runtime detected")
|
||||
return moduleRuntime{runtimeType: None, init: func(_ context.Context) error { return nil }, initialized: true}
|
||||
}
|
||||
|
||||
// Check for Haskell runtime initialization functions
|
||||
// Initialize Haskell runtime if `hs_init` and `hs_exit` are present,
|
||||
// by calling the `hs_init` export
|
||||
func haskellRuntime(p *Plugin, m api.Module) (moduleRuntime, bool) {
|
||||
initFunc := m.ExportedFunction("hs_init")
|
||||
if initFunc == nil {
|
||||
return moduleRuntime{}, false
|
||||
}
|
||||
|
||||
params := initFunc.Definition().ParamTypes()
|
||||
|
||||
if len(params) != 2 || params[0] != api.ValueTypeI32 || params[1] != api.ValueTypeI32 {
|
||||
p.Logf(LogLevelTrace, "hs_init function found with type %v", params)
|
||||
}
|
||||
|
||||
reactorInit := m.ExportedFunction("_initialize")
|
||||
|
||||
init := func(ctx context.Context) error {
|
||||
if reactorInit != nil {
|
||||
_, err := reactorInit.Call(ctx)
|
||||
if err != nil {
|
||||
p.Logf(LogLevelError, "Error running reactor _initialize: %s", err.Error())
|
||||
}
|
||||
}
|
||||
_, err := initFunc.Call(ctx, 0, 0)
|
||||
if err == nil {
|
||||
p.Log(LogLevelDebug, "Initialized Haskell language runtime.")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
p.Log(LogLevelTrace, "Haskell runtime detected")
|
||||
return moduleRuntime{runtimeType: Haskell, init: init}, true
|
||||
}
|
||||
|
||||
// Check for initialization functions defined by the WASI standard
|
||||
func wasiRuntime(p *Plugin, m api.Module) (moduleRuntime, bool) {
|
||||
if !p.hasWasi {
|
||||
return moduleRuntime{}, false
|
||||
}
|
||||
|
||||
// WASI supports two modules: Reactors and Commands
|
||||
// we prioritize Reactors over Commands
|
||||
// see: https://github.com/WebAssembly/WASI/blob/main/legacy/application-abi.md
|
||||
if r, ok := reactorModule(m, p); ok {
|
||||
return r, ok
|
||||
}
|
||||
|
||||
return commandModule(m, p)
|
||||
}
|
||||
|
||||
// Check for `_initialize` this is used by WASI to initialize certain interfaces.
|
||||
func reactorModule(m api.Module, p *Plugin) (moduleRuntime, bool) {
|
||||
init := findFunc(m, p, "_initialize")
|
||||
if init == nil {
|
||||
return moduleRuntime{}, false
|
||||
}
|
||||
|
||||
p.Logf(LogLevelTrace, "WASI runtime detected")
|
||||
p.Logf(LogLevelTrace, "Reactor module detected")
|
||||
|
||||
return moduleRuntime{runtimeType: Wasi, init: init}, true
|
||||
}
|
||||
|
||||
// Check for `__wasm__call_ctors`, this is used by WASI to
|
||||
// initialize certain interfaces.
|
||||
func commandModule(m api.Module, p *Plugin) (moduleRuntime, bool) {
|
||||
init := findFunc(m, p, "__wasm_call_ctors")
|
||||
if init == nil {
|
||||
return moduleRuntime{}, false
|
||||
}
|
||||
|
||||
p.Logf(LogLevelTrace, "WASI runtime detected")
|
||||
p.Logf(LogLevelTrace, "Command module detected")
|
||||
|
||||
return moduleRuntime{runtimeType: Wasi, init: init}, true
|
||||
}
|
||||
|
||||
func findFunc(m api.Module, p *Plugin, name string) func(context.Context) error {
|
||||
initFunc := m.ExportedFunction(name)
|
||||
if initFunc == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
params := initFunc.Definition().ParamTypes()
|
||||
if len(params) != 0 {
|
||||
p.Logf(LogLevelTrace, "%v function found with type %v", name, params)
|
||||
return nil
|
||||
}
|
||||
|
||||
return func(ctx context.Context) error {
|
||||
p.Logf(LogLevelDebug, "Calling %v", name)
|
||||
_, err := initFunc.Call(ctx)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func equal(actual []byte, expected []byte) bool {
|
||||
if len(actual) != len(expected) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i, k := range actual {
|
||||
if expected[i] != k {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
Reference in New Issue
Block a user