updated vendor

This commit is contained in:
2026-06-16 08:02:19 +02:00
parent 2f7f99d3f0
commit 77299d0c64
1283 changed files with 67302 additions and 208958 deletions
@@ -370,14 +370,14 @@ func (m *machine) resolveAddressingMode(arg0offset, ret0offset int64, i *instruc
// resolveRelativeAddresses resolves the relative addresses before encoding.
func (m *machine) resolveRelativeAddresses(ctx context.Context) {
for {
if len(m.unresolvedAddressModes) > 0 {
arg0offset, ret0offset := m.arg0OffsetFromSP(), m.ret0OffsetFromSP()
for _, i := range m.unresolvedAddressModes {
m.resolveAddressingMode(arg0offset, ret0offset, i)
}
if len(m.unresolvedAddressModes) > 0 {
arg0offset, ret0offset := m.arg0OffsetFromSP(), m.ret0OffsetFromSP()
for _, i := range m.unresolvedAddressModes {
m.resolveAddressingMode(arg0offset, ret0offset, i)
}
m.unresolvedAddressModes = m.unresolvedAddressModes[:0]
}
for {
// Reuse the slice to gather the unresolved conditional branches.
m.condBrRelocs = m.condBrRelocs[:0]
+188 -12
View File
@@ -43,6 +43,29 @@ type (
execCtxPtr uintptr
numberOfResults int
stackIteratorImpl stackIterator
// tryHandlers is the stack of active try_table exception handlers,
// used to match catch clauses when a throw exits to the dispatch loop.
tryHandlers []tryHandler
// pendingException holds the most recently caught exception, so handler
// code can read its params after re-entry.
pendingException *wasm.Exception
}
// tryHandler records the state at a try_table entry for exception handling.
// On match, we restore the stack to the checkpoint state and re-enter at returnAddress.
tryHandler struct {
// Cloned stack and state from the try_table entry checkpoint,
// using the same approach as experimental.Snapshot.
sp, fp, top uintptr
returnAddress *byte
savedRegisters [64][2]uint64
stack []byte // cloned stack
// catchClauses describes what exceptions this handler catches.
catchClauses []wazevoapi.CatchClauseInstance
// moduleInstance is the module that set up this try handler.
// Used for tag matching in doHandleException (the tag index in
// catch clauses is relative to this module's tag index space).
moduleInstance *wasm.ModuleInstance
}
// executionContext is the struct to be read/written by assembly functions.
@@ -90,6 +113,29 @@ type (
memoryWait64TrampolineAddress *byte
// memoryNotifyTrampolineAddress holds the address of the memory_notify trampoline function.
memoryNotifyTrampolineAddress *byte
// throwAllocTrampolineAddress holds the address of the throw-alloc trampoline:
// phase 1 of throw, which allocates the Exception heap object.
throwAllocTrampolineAddress *byte
// throwTrampolineAddress holds the address of the throw/throw_ref trampoline function.
throwTrampolineAddress *byte
// tryTableEnterTrampolineAddress holds the address of the try_table enter trampoline function.
tryTableEnterTrampolineAddress *byte
// tryTableLeaveTrampolineAddress holds the address of the try_table leave trampoline function.
tryTableLeaveTrampolineAddress *byte
// exceptionPtr holds the pointer to the Exception struct,
// used on the throw side (throwAlloc stores the new Exception)
// and on the catch side (catch_ref/catch_all_ref retrieve the exnref).
exceptionPtr uintptr
// exceptionParamsPtr points into exceptionPtr's Params slice
// backing array. On the throw side, throwAlloc sets it so compiled
// code can store params at [ptr + i*8]. On the catch side, compiled
// handler blocks load params from the same pointer.
exceptionParamsPtr uintptr
// caughtExceptionClauseIdx is set by the dispatch loop to -1 on
// TryTableEnter (normal path) or to the matched catch clause index
// when an exception is caught. Compiled code loads this from execCtx
// after the trampoline call to decide which handler to dispatch to.
caughtExceptionClauseIdx int64
}
)
@@ -217,6 +263,9 @@ func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint6
}
}
// Clear any stale try_table handlers from a previous call.
c.tryHandlers = c.tryHandlers[:0]
var paramResultPtr *uint64
if len(paramResultStack) > 0 {
paramResultPtr = &paramResultStack[0]
@@ -236,21 +285,25 @@ func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint6
var listeners []listenerForAbort
builder := wasmdebug.NewErrorBuilder()
def, lsn := c.addFrame(builder, uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress)))
if lsn != nil {
listeners = append(listeners, listenerForAbort{def, lsn})
}
returnAddrs := unwindStack(
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)),
c.execCtx.framePointerBeforeGoCall,
c.stackTop,
nil,
)
for _, retAddr := range returnAddrs[:len(returnAddrs)-1] { // the last return addr is the trampoline, so we skip it.
def, lsn = c.addFrame(builder, retAddr)
if c.execCtx.stackPointerBeforeGoCall != nil {
def, lsn := c.addFrame(builder, uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress)))
if lsn != nil {
listeners = append(listeners, listenerForAbort{def, lsn})
}
returnAddrs := unwindStack(
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)),
c.execCtx.framePointerBeforeGoCall,
c.stackTop,
nil,
)
if len(returnAddrs) > 1 {
for _, retAddr := range returnAddrs[:len(returnAddrs)-1] { // the last return addr is the trampoline, so we skip it.
def, lsn = c.addFrame(builder, retAddr)
if lsn != nil {
listeners = append(listeners, listenerForAbort{def, lsn})
}
}
}
}
err = builder.FromRecovered(r)
@@ -266,6 +319,7 @@ func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint6
if err != nil {
// Ensures that we can reuse this callEngine even after an error.
c.execCtx.exitCode = wazevoapi.ExitCodeOK
c.tryHandlers = c.tryHandlers[:0]
}
}()
@@ -505,12 +559,134 @@ func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint6
panic(wasmruntime.ErrRuntimeInvalidConversionToInteger)
case wazevoapi.ExitCodeUnalignedAtomic:
panic(wasmruntime.ErrRuntimeUnalignedAtomic)
case wazevoapi.ExitCodeThrowAlloc:
// Allocate the Exception heap object sized exactly to the tag's
// param count. Sets exceptionParamsPtr so compiled code can
// store params, and returns the exnref via the stack slot.
s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
tagIndex := int(s[0])
mod := c.callerModuleInstance()
tag := mod.Tags[tagIndex]
nParams := len(tag.Type.Params)
exn := &wasm.Exception{Tag: tag, Params: make([]uint64, nParams)}
c.pendingException = exn // GC root: keeps exn alive while compiled code writes params
if nParams > 0 {
c.execCtx.exceptionParamsPtr = uintptr(unsafe.Pointer(&exn.Params[0]))
}
// Return the exnref to compiled code via the stack slot.
s[0] = uint64(uintptr(unsafe.Pointer(exn)))
c.execCtx.exitCode = wazevoapi.ExitCodeOK
afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
case wazevoapi.ExitCodeThrow:
// Throw trampoline: (execCtx, exnref) → ().
// Reads the exnref from the stack, searches for a matching handler.
s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
// Read the Exception pointer directly from the uint64 value to avoid
// conversion from uintptr into unsafe.Pointer, which triggers checkptr.
exn := *(**wasm.Exception)(unsafe.Pointer(&s[0]))
if !c.doHandleException(exn) {
panic(wasmruntime.ErrRuntimeUncaughtException)
}
if len(exn.Params) > 0 {
c.execCtx.exceptionParamsPtr = uintptr(unsafe.Pointer(&exn.Params[0]))
}
c.execCtx.exceptionPtr = uintptr(unsafe.Pointer(exn))
c.execCtx.exitCode = wazevoapi.ExitCodeOK
afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
case wazevoapi.ExitCodeNullReference:
panic(wasmruntime.ErrRuntimeNullReference)
case wazevoapi.ExitCodeTryTableEnter:
// Save current state as a try handler checkpoint using stack cloning
// (same approach as experimental.Snapshot).
// The encoded exit code (with tryTableID in upper bits) is on the
// Go call stack as the second trampoline argument, not in execCtx.exitCode.
tryTableEnterStack := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
catchClauseTableIdx := wazevoapi.TryTableIDFromExitCode(wazevoapi.ExitCode(tryTableEnterStack[0]))
mod := c.callerModuleInstance()
me := mod.Engine.(*moduleEngine)
clauses := me.parent.catchClauseTable[catchClauseTableIdx]
returnAddress := c.execCtx.goCallReturnAddress
oldTop, oldSp := c.stackTop, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))
newSP, newFP, newTop, newStack := c.cloneStack(uintptr(len(c.stack)) + 16)
adjustClonedStack(oldSp, oldTop, newSP, newFP, newTop)
c.tryHandlers = append(c.tryHandlers, tryHandler{
sp: newSP,
fp: newFP,
top: newTop,
returnAddress: returnAddress,
savedRegisters: c.execCtx.savedRegisters,
stack: newStack,
catchClauses: clauses,
moduleInstance: mod,
})
// Set clauseIdx = -1 (no exception) in execCtx for the compiled code
// to read after the trampoline returns.
c.execCtx.caughtExceptionClauseIdx = -1
c.execCtx.exitCode = wazevoapi.ExitCodeOK
afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
case wazevoapi.ExitCodeTryTableLeave:
// Pop the most recent try handler.
if len(c.tryHandlers) > 0 {
c.tryHandlers = c.tryHandlers[:len(c.tryHandlers)-1]
}
c.execCtx.exitCode = wazevoapi.ExitCodeOK
afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
default:
panic("BUG")
}
}
}
// doHandleException tries to match the given exception against active try handlers.
// If a match is found, it restores the execution state to the handler's checkpoint
// (like snapshot.doRestore) and writes the matched clause index as the trampoline
// return value. Returns true if handled.
func (c *callEngine) doHandleException(exn *wasm.Exception) bool {
// Search try handlers from innermost (last) to outermost (first).
for i := len(c.tryHandlers) - 1; i >= 0; i-- {
h := &c.tryHandlers[i]
for clauseIdx, clause := range h.catchClauses {
// Use the module that set up the handler (not the one that threw)
// because clause.TagIndex is relative to that module's tag space.
mod := h.moduleInstance
matched := false
switch clause.Kind {
case wasm.CatchKindCatch, wasm.CatchKindCatchRef:
matched = mod.Tags[clause.TagIndex] == exn.Tag
case wasm.CatchKindCatchAll, wasm.CatchKindCatchAllRef:
matched = true
}
if matched {
// Pop all handlers at and above this one.
c.tryHandlers = c.tryHandlers[:i]
// Store the caught exception so handler code can read params.
c.pendingException = exn
// Restore the cloned stack (like snapshot.doRestore).
spp := *(**uint64)(unsafe.Pointer(&h.sp))
c.stack = h.stack
c.stackTop = h.top
ec := &c.execCtx
ec.stackBottomPtr = &c.stack[0]
ec.stackPointerBeforeGoCall = spp
ec.framePointerBeforeGoCall = h.fp
ec.goCallReturnAddress = h.returnAddress
ec.savedRegisters = h.savedRegisters
// Set the matched clause index in execCtx for compiled code to read.
ec.caughtExceptionClauseIdx = int64(clauseIdx)
return true
}
}
}
return false
}
func (c *callEngine) callerModuleInstance() *wasm.ModuleInstance {
return moduleInstanceFromOpaquePtr(c.execCtx.callerModuleContextPtr)
}
+67 -3
View File
@@ -67,7 +67,16 @@ type (
memoryWait64Address *byte
// memoryNotifyAddress is the address of memory.notify builtin function
memoryNotifyAddress *byte
listenerTrampolines listenerTrampolines
// throwAllocTrampolineAddress is the address of the throw-alloc trampoline:
// phase 1 of throw, which allocates the Exception heap object.
throwAllocTrampolineAddress *byte
// throwTrampolineAddress is the address of the throw/throw_ref trampoline function.
throwTrampolineAddress *byte
// tryTableEnterAddress is the address of try_table enter trampoline.
tryTableEnterAddress *byte
// tryTableLeaveAddress is the address of try_table leave trampoline.
tryTableLeaveAddress *byte
listenerTrampolines listenerTrampolines
}
listenerTrampolines = map[*wasm.FunctionType]struct {
@@ -93,6 +102,9 @@ type (
offsets wazevoapi.ModuleContextOffsetData
sharedFunctions *sharedFunctions
sourceMap sourceMap
// catchClauseTable stores catch clause info for each try_table,
// indexed by a try_table ID assigned during compilation.
catchClauseTable [][]wazevoapi.CatchClauseInstance
}
executables struct {
@@ -269,6 +281,7 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
relocator.appendFunction(fctx, module, cm, i, fidx, body, relsPerFunc, be.SourceOffsetInfo())
}
cm.catchClauseTable = fe.CatchClauseTable()
} else {
// Compile with N worker goroutines.
// Collect compiled functions across workers in a slice,
@@ -287,6 +300,10 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
ctx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
// Catch clause table IDs are baked into compiled machine code, so all
// workers must share a single table to ensure globally unique IDs.
sharedCCT := frontend.NewSharedCatchClauseTable()
var count atomic.Uint32
var wg sync.WaitGroup
wg.Add(workers)
@@ -299,7 +316,9 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
machine := newMachine()
ssaBuilder := ssa.NewBuilder()
be := backend.NewCompiler(ctx, machine, ssaBuilder)
fe := frontend.NewFrontendCompiler(module, ssaBuilder, &cm.offsets, ensureTermination, withListener, needSourceInfo)
fe := frontend.NewFrontendCompiler(
module, ssaBuilder, &cm.offsets, ensureTermination, withListener, needSourceInfo).
WithCatchClauseTable(sharedCCT)
for {
if err := ctx.Err(); err != nil {
@@ -346,6 +365,7 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
fn := &compiledFuncs[i]
relocator.appendFunction(fn.fctx, module, cm, fn.fnum, fn.fidx, fn.body, fn.relsPerFunc, fn.offsPerFunc)
}
cm.catchClauseTable = sharedCCT.Table()
}
// Allocate executable memory and then copy the generated machine code.
@@ -733,7 +753,7 @@ func (e *engine) NewModuleEngine(m *wasm.Module, mi *wasm.ModuleInstance) (wasm.
}
func (e *engine) compileSharedFunctions() {
var sizes [8]int
var sizes [12]int
var trampolines []byte
addTrampoline := func(i int, buf []byte) {
@@ -801,6 +821,38 @@ func (e *engine) compileSharedFunctions() {
Results: []ssa.Type{ssa.TypeI32},
}, false))
e.be.Init()
addTrampoline(8,
e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeThrowAlloc, &ssa.Signature{
// exec context, tag index → exnref
Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64},
Results: []ssa.Type{ssa.TypeI64},
}, false))
e.be.Init()
addTrampoline(9,
e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeThrow, &ssa.Signature{
// exec context, exnref
Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64},
Results: []ssa.Type{},
}, false))
e.be.Init()
addTrampoline(10,
e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeTryTableEnter, &ssa.Signature{
// exec context, catch clause info (encoded)
Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64},
Results: []ssa.Type{},
}, false))
e.be.Init()
addTrampoline(11,
e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeTryTableLeave, &ssa.Signature{
// exec context
Params: []ssa.Type{ssa.TypeI64},
Results: []ssa.Type{},
}, false))
fns := &sharedFunctions{
executable: mmapExecutable(trampolines),
listenerTrampolines: make(listenerTrampolines),
@@ -823,6 +875,14 @@ func (e *engine) compileSharedFunctions() {
fns.memoryWait64Address = &fns.executable[offset]
offset += sizes[6]
fns.memoryNotifyAddress = &fns.executable[offset]
offset += sizes[7]
fns.throwAllocTrampolineAddress = &fns.executable[offset]
offset += sizes[8]
fns.throwTrampolineAddress = &fns.executable[offset]
offset += sizes[9]
fns.tryTableEnterAddress = &fns.executable[offset]
offset += sizes[10]
fns.tryTableLeaveAddress = &fns.executable[offset]
if wazevoapi.PerfMapEnabled {
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.memoryGrowAddress)), uint64(sizes[0]), "memory_grow_trampoline")
@@ -833,6 +893,10 @@ func (e *engine) compileSharedFunctions() {
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.memoryWait32Address)), uint64(sizes[5]), "memory_wait32_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.memoryWait64Address)), uint64(sizes[6]), "memory_wait64_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.memoryNotifyAddress)), uint64(sizes[7]), "memory_notify_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.throwAllocTrampolineAddress)), uint64(sizes[8]), "throw_alloc_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.throwTrampolineAddress)), uint64(sizes[9]), "throw_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.tryTableEnterAddress)), uint64(sizes[10]), "try_table_enter_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.tryTableLeaveAddress)), uint64(sizes[11]), "try_table_leave_trampoline")
}
e.sharedFunctions = fns
@@ -182,6 +182,16 @@ func serializeCompiledModule(wazeroVersion string, cm *compiledModule) io.Reader
} else {
buf.WriteByte(0) // indicates that source map is not present.
}
// Catch clause table: number of try_tables (4 bytes), then for each:
// clause count (4 bytes), then for each clause: kind (1 byte) + tagIndex (4 bytes).
buf.Write(u32.LeBytes(uint32(len(cm.catchClauseTable))))
for _, clauses := range cm.catchClauseTable {
buf.Write(u32.LeBytes(uint32(len(clauses))))
for _, c := range clauses {
buf.WriteByte(c.Kind)
buf.Write(u32.LeBytes(c.TagIndex))
}
}
return bytes.NewReader(buf.Bytes())
}
@@ -291,6 +301,32 @@ func deserializeCompiledModule(wazeroVersion string, reader io.ReadCloser) (cm *
sm.executableOffsets = append(sm.executableOffsets, uintptr(executableRelativeOffset)+executableOffset)
}
}
// Catch clause table.
if _, err = io.ReadFull(reader, eightBytes[:4]); err != nil {
// Treat old cache entries without catch clause data as stale (trigger recompile).
return nil, true, nil
}
tableLen := binary.LittleEndian.Uint32(eightBytes[:4])
if tableLen > 0 {
cm.catchClauseTable = make([][]wazevoapi.CatchClauseInstance, tableLen)
for i := uint32(0); i < tableLen; i++ {
if _, err = io.ReadFull(reader, eightBytes[:4]); err != nil {
return nil, false, fmt.Errorf("compilationcache: error reading catch clause count for try_table[%d]: %v", i, err)
}
clauseCount := binary.LittleEndian.Uint32(eightBytes[:4])
clauses := make([]wazevoapi.CatchClauseInstance, clauseCount)
for j := uint32(0); j < clauseCount; j++ {
if _, err = io.ReadFull(reader, eightBytes[:5]); err != nil {
return nil, false, fmt.Errorf("compilationcache: error reading catch clause[%d][%d]: %v", i, j, err)
}
clauses[j] = wazevoapi.CatchClauseInstance{
Kind: eightBytes[0],
TagIndex: binary.LittleEndian.Uint32(eightBytes[1:5]),
}
}
cm.catchClauseTable[i] = clauses
}
}
return
}
@@ -4,6 +4,7 @@ package frontend
import (
"bytes"
"math"
"sync"
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
@@ -62,6 +63,20 @@ type Compiler struct {
execCtxPtrValue, moduleCtxPtrValue ssa.Value
// throwAllocSig is the signature for the throw-alloc trampoline:
// (execCtx, tagIndex) → (exnref). Allocates the Exception and returns
// its pointer so compiled code can pass it to the throw trampoline.
throwAllocSig ssa.Signature
// throwSig is the signature for the throw/throw_ref trampoline:
// (execCtx, exnref) → (). Searches for a matching handler and restores.
throwSig ssa.Signature
// tryTableEnterSig is the signature for the try_table enter trampoline.
tryTableEnterSig ssa.Signature
// tryTableLeaveSig is the signature for the try_table leave trampoline.
tryTableLeaveSig ssa.Signature
// catchClauseTable accumulates catch clause info for each try_table during compilation.
catchClauseTable catchClauseTable
// Following are reused for the known safe bounds analysis.
pointers []int
@@ -95,12 +110,75 @@ func NewFrontendCompiler(m *wasm.Module, ssaBuilder ssa.Builder, offset *wazevoa
offset: offset,
ensureTermination: ensureTermination,
needSourceOffsetInfo: sourceInfo,
catchClauseTable: &localCatchClauseTable{},
varLengthKnownSafeBoundWithIDPool: wazevoapi.NewVarLengthPool[knownSafeBoundWithID](),
}
c.declareSignatures(listenerOn)
return c
}
// catchClauseTable accumulates catch clause entries during compilation.
type catchClauseTable interface {
Append(clauses []wazevoapi.CatchClauseInstance) int
Table() [][]wazevoapi.CatchClauseInstance
}
// localCatchClauseTable is the single-threaded implementation.
type localCatchClauseTable struct {
table [][]wazevoapi.CatchClauseInstance
}
func (t *localCatchClauseTable) Append(clauses []wazevoapi.CatchClauseInstance) int {
id := len(t.table)
t.table = append(t.table, clauses)
return id
}
func (t *localCatchClauseTable) Table() [][]wazevoapi.CatchClauseInstance {
return t.table
}
// SharedCatchClauseTable is the thread-safe implementation for parallel compilation.
type SharedCatchClauseTable struct {
mu sync.Mutex
table [][]wazevoapi.CatchClauseInstance
finalized bool
}
// NewSharedCatchClauseTable creates a new SharedCatchClauseTable.
func NewSharedCatchClauseTable() *SharedCatchClauseTable {
return &SharedCatchClauseTable{}
}
func (s *SharedCatchClauseTable) Append(clauses []wazevoapi.CatchClauseInstance) int {
if s.finalized {
panic("already finalized")
}
s.mu.Lock()
id := len(s.table)
s.table = append(s.table, clauses)
s.mu.Unlock()
return id
}
func (s *SharedCatchClauseTable) Table() [][]wazevoapi.CatchClauseInstance {
s.finalized = true
return s.table
}
// WithCatchClauseTable replaces the catch clause table implementation.
// Used by the parallel compilation path to share a single mutex-protected
// table across workers.
func (c *Compiler) WithCatchClauseTable(t catchClauseTable) *Compiler {
c.catchClauseTable = t
return c
}
// CatchClauseTable returns the accumulated catch clause table.
func (c *Compiler) CatchClauseTable() [][]wazevoapi.CatchClauseInstance {
return c.catchClauseTable.Table()
}
func (c *Compiler) declareSignatures(listenerOn bool) {
m := c.m
c.signatures = make(map[*wasm.FunctionType]*ssa.Signature, len(m.TypeSection)+2)
@@ -194,6 +272,34 @@ func (c *Compiler) declareSignatures(listenerOn bool) {
Results: []ssa.Type{ssa.TypeI32},
}
c.ssaBuilder.DeclareSignature(&c.memoryNotifySig)
c.throwAllocSig = ssa.Signature{
ID: c.memoryNotifySig.ID + 1,
Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI64 /* tag index */},
Results: []ssa.Type{ssa.TypeI64 /* exnref */},
}
c.ssaBuilder.DeclareSignature(&c.throwAllocSig)
c.throwSig = ssa.Signature{
ID: c.throwAllocSig.ID + 1,
Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI64 /* exnref */},
Results: []ssa.Type{},
}
c.ssaBuilder.DeclareSignature(&c.throwSig)
c.tryTableEnterSig = ssa.Signature{
ID: c.throwSig.ID + 1,
Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI64 /* encoded exit code */},
Results: []ssa.Type{},
}
c.ssaBuilder.DeclareSignature(&c.tryTableEnterSig)
c.tryTableLeaveSig = ssa.Signature{
ID: c.tryTableEnterSig.ID + 1,
Params: []ssa.Type{ssa.TypeI64 /* exec context */},
Results: []ssa.Type{},
}
c.ssaBuilder.DeclareSignature(&c.tryTableLeaveSig)
}
// SignatureForWasmFunctionType returns the ssa.Signature for the given wasm.FunctionType.
@@ -234,7 +340,8 @@ func (c *Compiler) Init(idx, typIndex wasm.Index, typ *wasm.FunctionType, localT
// Note: this assumes 64-bit platform (I believe we won't have 32-bit backend ;)).
const executionContextPtrTyp, moduleContextPtrTyp = ssa.TypeI64, ssa.TypeI64
// LowerToSSA lowers the current function to SSA function which will be held by ssaBuilder.
// LowerToSSA lowers the current function to SSA IR which will be held by ssaBuilder.
//
// After calling this, the caller will be able to access the SSA info in *Compiler.ssaBuilder.
//
// Note that this only does the naive lowering, and do not do any optimization, instead the caller is expected to do so.
@@ -340,23 +447,7 @@ func (c *Compiler) declareNecessaryVariables() {
}
func (c *Compiler) declareWasmGlobal(typ wasm.ValueType, mutable bool) {
var st ssa.Type
switch typ {
case wasm.ValueTypeI32:
st = ssa.TypeI32
case wasm.ValueTypeI64,
// Both externref and funcref are represented as I64 since we only support 64-bit platforms.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
st = ssa.TypeI64
case wasm.ValueTypeF32:
st = ssa.TypeF32
case wasm.ValueTypeF64:
st = ssa.TypeF64
case wasm.ValueTypeV128:
st = ssa.TypeV128
default:
panic("TODO: " + wasm.ValueTypeName(typ))
}
st := WasmTypeToSSAType(typ)
v := c.ssaBuilder.DeclareVariable(st)
index := wasm.Index(len(c.globalVariables))
c.globalVariables = append(c.globalVariables, v)
@@ -372,8 +463,9 @@ func WasmTypeToSSAType(vt wasm.ValueType) ssa.Type {
case wasm.ValueTypeI32:
return ssa.TypeI32
case wasm.ValueTypeI64,
// Both externref and funcref are represented as I64 since we only support 64-bit platforms.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
// externref, funcref, and exnref are represented as I64 since we only support 64-bit platforms.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref,
wasm.ValueTypeExnref:
return ssa.TypeI64
case wasm.ValueTypeF32:
return ssa.TypeF32
@@ -382,6 +474,10 @@ func WasmTypeToSSAType(vt wasm.ValueType) ssa.Type {
case wasm.ValueTypeV128:
return ssa.TypeV128
default:
// Concrete ref types (ref $t) have variable bit patterns.
if vt.IsRef() {
return ssa.TypeI64
}
panic("TODO: " + wasm.ValueTypeName(vt))
}
}
@@ -65,6 +65,7 @@ const (
controlFrameKindIfWithElse
controlFrameKindIfWithoutElse
controlFrameKindBlock
controlFrameKindTryTable
)
// String implements fmt.Stringer for debugging.
@@ -80,6 +81,8 @@ func (k controlFrameKind) String() string {
return "if_without_else"
case controlFrameKindBlock:
return "block"
case controlFrameKindTryTable:
return "try_table"
default:
panic(k)
}
@@ -1443,6 +1446,11 @@ func (c *Compiler) lowerCurrentOpcode() {
unreachable := state.unreachable
if !unreachable {
// For try_table, emit the leave trampoline before the jump to the following block.
if ctrl.kind == controlFrameKindTryTable {
c.emitTryTableLeave()
}
// Top n-th args will be used as a result of the current control frame.
args := c.nPeekDup(len(ctrl.blockType.Results))
@@ -1477,6 +1485,7 @@ func (c *Compiler) lowerCurrentOpcode() {
break
}
c.emitTryTableLeaves(int(labelIndex))
targetBlk, argNum := state.brTargetArgNumFor(labelIndex)
args := c.nPeekDup(argNum)
c.insertJumpToBlock(args, targetBlk)
@@ -1494,6 +1503,21 @@ func (c *Compiler) lowerCurrentOpcode() {
targetBlk, argNum := state.brTargetArgNumFor(labelIndex)
args := c.nPeekDup(argNum)
var sealTargetBlk bool
// If the branch exits any try_table frames, emit TryTableLeave
// calls in a trampoline block that only runs on the taken path.
if c.branchExitsTryTable(int(labelIndex)) {
current := builder.CurrentBlock()
trampolineBlk := builder.AllocateBasicBlock()
builder.SetCurrentBlock(trampolineBlk)
c.emitTryTableLeaves(int(labelIndex))
c.insertJumpToBlock(args, targetBlk)
builder.SetCurrentBlock(current)
targetBlk = trampolineBlk
sealTargetBlk = true
args = ssa.ValuesNil
}
if c.needListener && targetBlk.ReturnBlock() { // In this case, we have to call the listener before returning.
// Save the currently active block.
current := builder.CurrentBlock()
@@ -1559,6 +1583,7 @@ func (c *Compiler) lowerCurrentOpcode() {
if state.unreachable {
break
}
c.emitTryTableLeaves(len(state.controlFrames))
if c.needListener {
c.callListenerAfter()
}
@@ -3355,7 +3380,12 @@ func (c *Compiler) lowerCurrentOpcode() {
state.push(refFuncRet)
case wasm.OpcodeRefNull:
c.loweringState.pc++ // skips the reference type as we treat both of them as i64(0).
switch reftype := c.wasmFunctionBody[c.loweringState.pc+1]; wasm.ValueType(reftype) {
case wasm.ValueTypeFuncref, wasm.ValueTypeExternref, wasm.ValueTypeExnref:
c.loweringState.pc++
default:
c.readI32u()
}
if state.unreachable {
break
}
@@ -3399,6 +3429,9 @@ func (c *Compiler) lowerCurrentOpcode() {
if state.unreachable {
break
}
// Per spec, return_call leaves the current frame, so all enclosing
// try_table handlers must be popped before the tail call.
c.emitTryTableLeaves(len(c.state().controlFrames))
_, _ = typeIndex, tableIndex
c.lowerTailCallReturnCallIndirect(typeIndex, tableIndex)
state.unreachable = true
@@ -3408,9 +3441,396 @@ func (c *Compiler) lowerCurrentOpcode() {
if state.unreachable {
break
}
// Per spec, return_call leaves the current frame, so all enclosing
// try_table handlers must be popped before the tail call.
c.emitTryTableLeaves(len(c.state().controlFrames))
c.lowerTailCallReturnCall(fnIndex)
state.unreachable = true
case wasm.OpcodeThrow:
tagIndex := c.readI32u()
if state.unreachable {
break
}
tagType := c.resolveTagType(tagIndex)
// Pop the tag's param values from the stack.
var throwParams []ssa.Value
if tagType != nil {
throwParams = make([]ssa.Value, len(tagType.Params))
for i := len(tagType.Params) - 1; i >= 0; i-- {
throwParams[i] = state.pop()
}
}
c.storeCallerModuleContext()
tagIdxVal := builder.AllocateInstruction().AsIconst64(uint64(tagIndex)).Insert(builder).Return()
// We need to store the throwParams in the exception and then throw it.
// However, each exception might have a variable number of parameters,
// so we let Go allocate the reference on the heap.
// The Go side allocates the Exception object (Params sized to nParams)
// and stores the pointer to the backing-array into execCtx.exceptionParamsPtr.
throwAllocPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetThrowAllocTrampolineAddress.U32(),
ssa.TypeI64,
).Insert(builder).Return()
throwAllocArgs := c.allocateVarLengthValues(2, c.execCtxPtrValue, tagIdxVal)
exnref := builder.AllocateInstruction().
AsCallIndirect(throwAllocPtr, &c.throwAllocSig, throwAllocArgs).
Insert(builder).Return()
// Reload memory pointers invalidated by the Go call.
c.reloadAfterCall()
// We can now store each param directly into Exception.Params using the pointer
// stored into execCtx.exceptionParamsPtr.
if len(throwParams) > 0 {
paramsPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetExceptionParamsPtr.U32(),
ssa.TypeI64,
).Insert(builder).Return()
for i, v := range throwParams {
switch v.Type() {
case ssa.TypeF32:
v = builder.AllocateInstruction().AsBitcast(v, ssa.TypeI32).Insert(builder).Return()
case ssa.TypeF64:
v = builder.AllocateInstruction().AsBitcast(v, ssa.TypeI64).Insert(builder).Return()
}
builder.AllocateInstruction().
AsStore(ssa.OpcodeStore, v, paramsPtr, uint32(i)*8).
Insert(builder)
}
}
// We return again control to Go to search and dispatch to a matching catch clause.
c.emitThrow(exnref)
state.unreachable = true
case wasm.OpcodeThrowRef:
if state.unreachable {
break
}
exnref := state.pop()
// Check for null exnref.
zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return()
isNull := builder.AllocateInstruction()
isNull.AsIcmp(exnref, zero, ssa.IntegerCmpCondEqual)
builder.InsertInstruction(isNull)
exitIfNull := builder.AllocateInstruction()
exitIfNull.AsExitIfTrueWithCode(c.execCtxPtrValue, isNull.Return(), wazevoapi.ExitCodeNullReference)
builder.InsertInstruction(exitIfNull)
c.storeCallerModuleContext()
c.emitThrow(exnref)
state.unreachable = true
case wasm.OpcodeTryTable:
bt := c.readBlockType()
if state.unreachable {
state.unreachableDepth++
// Still need to skip the catch clause bytes in the unreachable case.
c.skipTryTableCatchClauses()
break
}
// Parse catch clauses.
c.loweringState.pc++
catchCount, catchNum, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(catchNum) - 1
var catchClauses []catchClause
for i := uint32(0); i < catchCount; i++ {
c.loweringState.pc++
kind := c.wasmFunctionBody[c.loweringState.pc]
var tagIdx uint32
switch kind {
case wasm.CatchKindCatch, wasm.CatchKindCatchRef:
c.loweringState.pc++
var n uint64
tagIdx, n, _ = leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
case wasm.CatchKindCatchAll, wasm.CatchKindCatchAllRef:
// No tagIdx for catch_all variants.
}
c.loweringState.pc++
labelIdx, n, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
catchClauses = append(catchClauses, catchClause{kind: kind, tagIndex: tagIdx, labelIdx: labelIdx})
}
// Register catch clauses in the table and get the try_table ID.
var clauseInstances []wazevoapi.CatchClauseInstance
for _, cc := range catchClauses {
clauseInstances = append(clauseInstances, wazevoapi.CatchClauseInstance{
Kind: cc.kind,
TagIndex: cc.tagIndex,
})
}
tryTableID := c.catchClauseTable.Append(clauseInstances)
// Allocate the following block (after try_table end) and body block.
followingBlk := builder.AllocateBasicBlock()
c.addBlockParamsFromWasmTypes(bt.Results, followingBlk)
bodyBlk := builder.AllocateBasicBlock()
if len(catchClauses) > 0 {
// Store the caller module context so the dispatch loop can find the module.
c.storeCallerModuleContext()
// For each catch clause, create a handler block that loads exception
// params and jumps to the wasm target label.
// NOTE: catch clause label indices do NOT include the try_table itself
// (the try_table is pushed onto the control stack after the catch clauses
// are processed, per the spec). So we resolve labels BEFORE pushing.
varPool := builder.VarLengthPool()
targets := varPool.Allocate(len(catchClauses) + 1) // +1 for bodyBlk
currentBlk := builder.CurrentBlock()
for _, cc := range catchClauses {
handlerBlk := builder.AllocateBasicBlock()
builder.SetCurrentBlock(handlerBlk)
c.reloadAfterCall()
// Resolve the wasm target label.
targetBlk, _ := state.brTargetArgNumFor(cc.labelIdx)
// Load exception params and jump to wasm target.
var brArgs []ssa.Value
switch cc.kind {
case wasm.CatchKindCatch:
if tagType := c.resolveTagType(cc.tagIndex); tagType != nil {
brArgs = c.loadExceptionParams(tagType)
}
case wasm.CatchKindCatchRef:
if tagType := c.resolveTagType(cc.tagIndex); tagType != nil {
brArgs = c.loadExceptionParams(tagType)
}
brArgs = append(brArgs, c.loadExnRef())
case wasm.CatchKindCatchAll:
// No values.
case wasm.CatchKindCatchAllRef:
brArgs = append(brArgs, c.loadExnRef())
}
jmpArgs := c.allocateVarLengthValues(len(brArgs), brArgs...)
c.insertJumpToBlock(jmpArgs, targetBlk)
targets = targets.Append(varPool, ssa.Value(handlerBlk.ID()))
}
// Last target is the body block (default for clauseIdx == -1 / out of range).
targets = targets.Append(varPool, ssa.Value(bodyBlk.ID()))
// Back to the original block: call the try_table enter trampoline,
// then dispatch on the caught clause index.
builder.SetCurrentBlock(currentBlk)
encodedExitCode := uint64(wazevoapi.ExitCodeTryTableEnter | wazevoapi.ExitCode(tryTableID<<8))
// Load trampoline address from execCtx.
enterPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetTryTableEnterTrampolineAddress.U32(),
ssa.TypeI64,
).Insert(builder).Return()
// Call the trampoline: (execCtx, encodedExitCode) -> ().
exitCodeVal := builder.AllocateInstruction().AsIconst64(encodedExitCode).Insert(builder).Return()
args := c.allocateVarLengthValues(2, c.execCtxPtrValue, exitCodeVal)
builder.AllocateInstruction().
AsCallIndirect(enterPtr, &c.tryTableEnterSig, args).
Insert(builder)
// Load the caught clause index written by the dispatch loop.
clauseIdx := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetCaughtExceptionClauseIdx.U32(),
ssa.TypeI64,
).Insert(builder).Return()
// Dispatch to handler blocks or body block via br_table.
brTable := builder.AllocateInstruction()
brTable.AsBrTable(clauseIdx, targets)
builder.InsertInstruction(brTable)
// Seal handler blocks after BrTable is inserted (so predecessors are registered).
for _, targetID := range targets.View() {
blk := builder.BasicBlock(ssa.BasicBlockID(targetID))
if !blk.Sealed() {
builder.Seal(blk)
}
}
} else {
// No catch clauses — try_table acts as a plain block.
// Jump directly to body without entering exception handling.
c.insertJumpToBlock(ssa.ValuesNil, bodyBlk)
}
if !bodyBlk.Sealed() {
builder.Seal(bodyBlk)
}
builder.SetCurrentBlock(bodyBlk)
if len(catchClauses) > 0 {
// Body block is entered after the trampoline call, so we need to reload.
c.reloadAfterCall()
}
// Push the try_table control frame AFTER resolving catch labels.
state.ctrlPush(controlFrame{
kind: controlFrameKindTryTable,
originalStackLenWithoutParam: len(state.values) - len(bt.Params),
followingBlock: followingBlk,
blockType: bt,
})
case wasm.OpcodeRefAsNonNull:
if state.unreachable {
break
}
r := state.pop()
zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder)
checkNull := builder.AllocateInstruction().
AsIcmp(r, zero.Return(), ssa.IntegerCmpCondEqual).
Insert(builder).Return()
exitIfNull := builder.AllocateInstruction()
exitIfNull.AsExitIfTrueWithCode(c.execCtxPtrValue, checkNull, wazevoapi.ExitCodeNullReference)
builder.InsertInstruction(exitIfNull)
state.push(r)
case wasm.OpcodeBrOnNull:
labelIndex := c.readI32u()
if state.unreachable {
break
}
r := state.pop()
zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder)
isNull := builder.AllocateInstruction().
AsIcmp(r, zero.Return(), ssa.IntegerCmpCondEqual).
Insert(builder).Return()
targetBlk, argNum := state.brTargetArgNumFor(labelIndex)
args := c.nPeekDup(argNum)
var sealTargetBlk bool
if c.branchExitsTryTable(int(labelIndex)) {
current := builder.CurrentBlock()
trampolineBlk := builder.AllocateBasicBlock()
builder.SetCurrentBlock(trampolineBlk)
c.emitTryTableLeaves(int(labelIndex))
c.insertJumpToBlock(args, targetBlk)
builder.SetCurrentBlock(current)
targetBlk = trampolineBlk
sealTargetBlk = true
args = ssa.ValuesNil
}
if c.needListener && targetBlk.ReturnBlock() {
current := builder.CurrentBlock()
targetBlk = builder.AllocateBasicBlock()
builder.SetCurrentBlock(targetBlk)
sealTargetBlk = true
c.callListenerAfter()
instr := builder.AllocateInstruction()
instr.AsReturn(args)
builder.InsertInstruction(instr)
args = ssa.ValuesNil
builder.SetCurrentBlock(current)
}
brnz := builder.AllocateInstruction()
brnz.AsBrnz(isNull, args, targetBlk)
builder.InsertInstruction(brnz)
if sealTargetBlk {
builder.Seal(targetBlk)
}
// Fall-through: ref is non-null, push it back.
elseBlk := builder.AllocateBasicBlock()
c.insertJumpToBlock(ssa.ValuesNil, elseBlk)
builder.Seal(elseBlk)
builder.SetCurrentBlock(elseBlk)
state.push(r)
case wasm.OpcodeBrOnNonNull:
labelIndex := c.readI32u()
if state.unreachable {
break
}
r := state.pop()
zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder)
isNonNull := builder.AllocateInstruction().
AsIcmp(r, zero.Return(), ssa.IntegerCmpCondNotEqual).
Insert(builder).Return()
// When non-null, branch to label with args + the non-null ref.
targetBlk, argNum := state.brTargetArgNumFor(labelIndex)
// The branch delivers argNum-1 values from the stack plus the ref.
// The ref is the last value delivered to the label target.
args := c.nPeekDup(argNum - 1)
args = args.Append(builder.VarLengthPool(), r)
var sealTargetBlk bool
if c.branchExitsTryTable(int(labelIndex)) {
current := builder.CurrentBlock()
trampolineBlk := builder.AllocateBasicBlock()
builder.SetCurrentBlock(trampolineBlk)
c.emitTryTableLeaves(int(labelIndex))
c.insertJumpToBlock(args, targetBlk)
builder.SetCurrentBlock(current)
targetBlk = trampolineBlk
sealTargetBlk = true
args = ssa.ValuesNil
}
if c.needListener && targetBlk.ReturnBlock() {
current := builder.CurrentBlock()
targetBlk = builder.AllocateBasicBlock()
builder.SetCurrentBlock(targetBlk)
sealTargetBlk = true
c.callListenerAfter()
instr := builder.AllocateInstruction()
instr.AsReturn(args)
builder.InsertInstruction(instr)
args = ssa.ValuesNil
builder.SetCurrentBlock(current)
}
brnz := builder.AllocateInstruction()
brnz.AsBrnz(isNonNull, args, targetBlk)
builder.InsertInstruction(brnz)
if sealTargetBlk {
builder.Seal(targetBlk)
}
// Fall-through: ref is null, nothing extra pushed.
elseBlk := builder.AllocateBasicBlock()
c.insertJumpToBlock(ssa.ValuesNil, elseBlk)
builder.Seal(elseBlk)
builder.SetCurrentBlock(elseBlk)
case wasm.OpcodeCallRef:
typeIndex := c.readI32u()
if state.unreachable {
break
}
c.lowerCallRef(typeIndex)
case wasm.OpcodeReturnCallRef:
typeIndex := c.readI32u()
if state.unreachable {
break
}
c.emitTryTableLeaves(len(c.state().controlFrames))
c.lowerTailCallReturnCallRef(typeIndex)
state.unreachable = true
default:
panic("TODO: unsupported in wazevo yet: " + wasm.InstructionName(op))
}
@@ -3715,6 +4135,91 @@ func (c *Compiler) lowerTailCallReturnCallIndirect(typeIndex, tableIndex uint32)
c.lowerReturn(builder)
}
func (c *Compiler) prepareCallRef(typeIndex uint32) (ssa.Value, *wasm.FunctionType, ssa.Values) {
builder := c.ssaBuilder
state := c.state()
functionInstancePtr := state.pop()
// Check if it is not the null pointer.
zero := builder.AllocateInstruction()
zero.AsIconst64(0)
builder.InsertInstruction(zero)
checkNull := builder.AllocateInstruction()
checkNull.AsIcmp(functionInstancePtr, zero.Return(), ssa.IntegerCmpCondEqual)
builder.InsertInstruction(checkNull)
exitIfNull := builder.AllocateInstruction()
exitIfNull.AsExitIfTrueWithCode(c.execCtxPtrValue, checkNull.Return(), wazevoapi.ExitCodeNullReference)
builder.InsertInstruction(exitIfNull)
// Load the executable and moduleContextOpaquePtr from the function instance.
loadExecutablePtr := builder.AllocateInstruction()
loadExecutablePtr.AsLoad(functionInstancePtr, wazevoapi.FunctionInstanceExecutableOffset, ssa.TypeI64)
builder.InsertInstruction(loadExecutablePtr)
executablePtr := loadExecutablePtr.Return()
loadModuleContextOpaquePtr := builder.AllocateInstruction()
loadModuleContextOpaquePtr.AsLoad(functionInstancePtr, wazevoapi.FunctionInstanceModuleContextOpaquePtrOffset, ssa.TypeI64)
builder.InsertInstruction(loadModuleContextOpaquePtr)
moduleContextOpaquePtr := loadModuleContextOpaquePtr.Return()
typ := &c.m.TypeSection[typeIndex]
tail := len(state.values) - len(typ.Params)
vs := state.values[tail:]
state.values = state.values[:tail]
args := c.allocateVarLengthValues(2+len(vs), c.execCtxPtrValue, moduleContextOpaquePtr)
args = args.Append(builder.VarLengthPool(), vs...)
c.storeCallerModuleContext()
return executablePtr, typ, args
}
func (c *Compiler) lowerCallRef(typeIndex uint32) {
builder := c.ssaBuilder
state := c.state()
executablePtr, typ, args := c.prepareCallRef(typeIndex)
call := builder.AllocateInstruction()
call.AsCallIndirect(executablePtr, c.signatures[typ], args)
builder.InsertInstruction(call)
first, rest := call.Returns()
if first.Valid() {
state.push(first)
}
for _, v := range rest {
state.push(v)
}
c.reloadAfterCall()
}
func (c *Compiler) lowerTailCallReturnCallRef(typeIndex uint32) {
builder := c.ssaBuilder
state := c.state()
executablePtr, typ, args := c.prepareCallRef(typeIndex)
call := builder.AllocateInstruction()
call.AsTailCallReturnCallIndirect(executablePtr, c.signatures[typ], args)
builder.InsertInstruction(call)
// In a proper tail call, the following code is unreachable since execution
// transfers to the callee. However, sometimes the backend might need to fall back to
// a regular call, so we include return handling and let the backend delete it
// when redundant.
// For details, see internal/engine/RATIONALE.md
first, rest := call.Returns()
if first.Valid() {
state.push(first)
}
for _, v := range rest {
state.push(v)
}
c.reloadAfterCall()
c.lowerReturn(builder)
}
// memOpSetup inserts the bounds check and calculates the address of the memory operation (loads/stores).
func (c *Compiler) memOpSetup(baseAddr ssa.Value, constOffset, operationSizeInBytes uint64) (address ssa.Value) {
address = ssa.ValueInvalid
@@ -4026,6 +4531,185 @@ func (c *Compiler) storeCallerModuleContext() {
builder.InsertInstruction(store)
}
// resolveTagType returns the FunctionType for the tag at the given module-local index.
func (c *Compiler) resolveTagType(tagIndex uint32) *wasm.FunctionType {
if tagIndex < c.m.ImportTagCount {
cur := uint32(0)
for i := range c.m.ImportSection {
imp := &c.m.ImportSection[i]
if imp.Type != wasm.ExternTypeTag {
continue
}
if tagIndex == cur {
return &c.m.TypeSection[imp.DescTag]
}
cur++
}
} else {
tagSectionIdx := tagIndex - c.m.ImportTagCount
if tagSectionIdx < uint32(len(c.m.TagSection)) {
typeIdx := c.m.TagSection[tagSectionIdx].Type
return &c.m.TypeSection[typeIdx]
}
}
return nil
}
// emitThrow emits a call to the shared throw trampoline with the given exnref,
// followed by an unreachable exit (throw never returns).
func (c *Compiler) emitThrow(exnref ssa.Value) {
builder := c.ssaBuilder
throwPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetThrowTrampolineAddress.U32(),
ssa.TypeI64,
).Insert(builder).Return()
throwArgs := c.allocateVarLengthValues(2, c.execCtxPtrValue, exnref)
builder.AllocateInstruction().
AsCallIndirect(throwPtr, &c.throwSig, throwArgs).
Insert(builder)
exit := builder.AllocateInstruction()
exit.AsExitWithCode(c.execCtxPtrValue, wazevoapi.ExitCodeUnreachable)
builder.InsertInstruction(exit)
}
// emitTryTableLeave emits a trampoline call to pop the try handler in the dispatch loop.
func (c *Compiler) emitTryTableLeave() {
builder := c.ssaBuilder
c.storeCallerModuleContext()
leavePtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetTryTableLeaveTrampolineAddress.U32(),
ssa.TypeI64,
).Insert(builder).Return()
args := c.allocateVarLengthValues(1, c.execCtxPtrValue)
builder.AllocateInstruction().
AsCallIndirect(leavePtr, &c.tryTableLeaveSig, args).
Insert(builder)
}
// branchExitsTryTable returns true if a branch to the given depth would
// exit at least one try_table frame.
func (c *Compiler) branchExitsTryTable(depth int) bool {
state := c.state()
tail := len(state.controlFrames) - 1
for i := 0; i < depth; i++ {
if state.controlFrames[tail-i].kind == controlFrameKindTryTable {
return true
}
}
return false
}
// emitTryTableLeaves emits TryTableLeave calls for try_table frames
// that would be exited by a branch to the given depth.
func (c *Compiler) emitTryTableLeaves(depth int) {
state := c.state()
tail := len(state.controlFrames) - 1
for i := 0; i < depth; i++ {
if state.controlFrames[tail-i].kind == controlFrameKindTryTable {
c.emitTryTableLeave()
}
}
}
// catchClause holds a parsed catch clause from a try_table instruction.
type catchClause struct {
kind byte
tagIndex uint32
labelIdx uint32
}
// loadExceptionParams loads the exception params from the caught Exception's
// Params slice. The dispatch loop sets execCtx.exceptionParamsPtr to the
// slice's backing-array pointer after matching a handler. We load that pointer
// and then read each param from [ptr + i*8], mirroring the stores emitted by
// the throw lowering. Float params were bitcast to integers at the throw site,
// so we load as integer and bitcast back to the original type.
func (c *Compiler) loadExceptionParams(tagType *wasm.FunctionType) []ssa.Value {
if len(tagType.Params) == 0 {
return nil
}
builder := c.ssaBuilder
// Load the pointer to the caught Exception's Params backing array.
paramsPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetExceptionParamsPtr.U32(),
ssa.TypeI64,
).Insert(builder).Return()
var values []ssa.Value
for i, vt := range tagType.Params {
offset := uint32(i) * 8
ssaType := WasmTypeToSSAType(vt)
switch ssaType {
case ssa.TypeF32:
// Stored as i32 at throw site; bitcast back to f32.
raw := builder.AllocateInstruction().
AsLoad(paramsPtr, offset, ssa.TypeI32).
Insert(builder).Return()
val := builder.AllocateInstruction().AsBitcast(raw, ssa.TypeF32).Insert(builder).Return()
values = append(values, val)
case ssa.TypeF64:
// Stored as i64 at throw site; bitcast back to f64.
raw := builder.AllocateInstruction().
AsLoad(paramsPtr, offset, ssa.TypeI64).
Insert(builder).Return()
val := builder.AllocateInstruction().AsBitcast(raw, ssa.TypeF64).Insert(builder).Return()
values = append(values, val)
default:
val := builder.AllocateInstruction().
AsLoad(paramsPtr, offset, ssaType).
Insert(builder).Return()
values = append(values, val)
}
}
return values
}
// loadExnRef loads the exnref (pointer to Exception) from the executionContext.
// The dispatch loop writes it to exceptionPtr after matching a handler.
func (c *Compiler) loadExnRef() ssa.Value {
builder := c.ssaBuilder
return builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetExceptionPtr.U32(),
ssa.TypeI64,
).Insert(builder).Return()
}
// skipTryTableCatchClauses advances the bytecode PC past the catch clauses
// of a try_table instruction. This is used both in reachable and unreachable states.
func (c *Compiler) skipTryTableCatchClauses() {
c.loweringState.pc++
catchCount, catchNum, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(catchNum) - 1
for i := uint32(0); i < catchCount; i++ {
c.loweringState.pc++
kind := c.wasmFunctionBody[c.loweringState.pc]
switch kind {
case wasm.CatchKindCatch, wasm.CatchKindCatchRef:
// Read tag index.
c.loweringState.pc++
_, n, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
// Read label index.
c.loweringState.pc++
_, n, _ = leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
case wasm.CatchKindCatchAll, wasm.CatchKindCatchAllRef:
// Read label index.
c.loweringState.pc++
_, n, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
}
}
}
func (c *Compiler) readByte() byte {
v := c.wasmFunctionBody[c.loweringState.pc+1]
c.loweringState.pc++
@@ -4181,6 +4865,7 @@ func (c *Compiler) lowerBrTable(labels []uint32, index ssa.Value) {
targetBlk, _ := state.brTargetArgNumFor(l)
trampoline := builder.AllocateBasicBlock()
builder.SetCurrentBlock(trampoline)
c.emitTryTableLeaves(int(l))
c.insertJumpToBlock(args, targetBlk)
trampolineBlockIDs = trampolineBlockIDs.Append(builder.VarLengthPool(), ssa.Value(trampoline.ID()))
}
@@ -1,13 +1,17 @@
package wazevo
import (
"context"
"encoding/binary"
"fmt"
"unsafe"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
"github.com/tetratelabs/wazero/internal/internalapi"
"github.com/tetratelabs/wazero/internal/wasm"
"github.com/tetratelabs/wazero/internal/wasmdebug"
"github.com/tetratelabs/wazero/internal/wasmruntime"
)
@@ -132,6 +136,14 @@ func (m *moduleEngine) setupOpaque() {
}
}
if tagOffset := offsets.TagsBegin; tagOffset >= 0 {
for _, tag := range inst.Tags {
binary.LittleEndian.PutUint64(opaque[tagOffset:],
uint64(uintptr(unsafe.Pointer(tag))))
tagOffset += 8
}
}
if beforeListenerOffset := offsets.BeforeListenerTrampolines1stElement; beforeListenerOffset >= 0 {
binary.LittleEndian.PutUint64(opaque[beforeListenerOffset:], uint64(uintptr(unsafe.Pointer(&m.parent.listenerBeforeTrampolines[0]))))
}
@@ -160,6 +172,21 @@ func (m *moduleEngine) NewFunction(index wasm.Index) api.Function {
localIndex -= importedFnCount
}
if source := m.module.Source; source.IsHostModule {
// For host modules, we need to look up the GoFunction from the CodeSection.
def := source.FunctionDefinition(localIndex)
goF := source.CodeSection[localIndex].GoFunc
switch typed := goF.(type) {
case api.GoFunction:
// GoFunction doesn't need looked up module.
return &hostFunction{def: def, g: goFunctionAsGoModuleFunction(typed)}
case api.GoModuleFunction:
return &hostFunction{def: def, lookedUpModule: m.module, g: typed}
default:
panic(fmt.Sprintf("unexpected GoFunc type: %T", goF))
}
}
src := m.module.Source
typIndex := src.FunctionSection[localIndex]
typ := src.TypeSection[typIndex]
@@ -189,6 +216,10 @@ func (m *moduleEngine) NewFunction(index wasm.Index) api.Function {
ce.execCtx.memoryWait32TrampolineAddress = sharedFunctions.memoryWait32Address
ce.execCtx.memoryWait64TrampolineAddress = sharedFunctions.memoryWait64Address
ce.execCtx.memoryNotifyTrampolineAddress = sharedFunctions.memoryNotifyAddress
ce.execCtx.throwAllocTrampolineAddress = sharedFunctions.throwAllocTrampolineAddress
ce.execCtx.throwTrampolineAddress = sharedFunctions.throwTrampolineAddress
ce.execCtx.tryTableEnterTrampolineAddress = sharedFunctions.tryTableEnterAddress
ce.execCtx.tryTableLeaveTrampolineAddress = sharedFunctions.tryTableLeaveAddress
ce.execCtx.memmoveAddress = memmovPtr
ce.init()
return ce
@@ -242,15 +273,13 @@ func (m *moduleEngine) ResolveImportedFunction(index, descFunc, indexInImportedM
executableOffset, moduleCtxOffset, typeIDOffset := m.parent.offsets.ImportedFunctionOffset(index)
importedME := importedModuleEngine.(*moduleEngine)
if int(indexInImportedModule) >= len(importedME.importedFunctions) {
indexInImportedModule -= wasm.Index(len(importedME.importedFunctions))
} else {
if int(indexInImportedModule) < len(importedME.importedFunctions) {
imported := &importedME.importedFunctions[indexInImportedModule]
m.ResolveImportedFunction(index, descFunc, imported.indexInModule, imported.me)
return // Recursively resolve the imported function.
}
offset := importedME.parent.functionOffsets[indexInImportedModule]
offset := importedME.parent.functionOffsets[indexInImportedModule-wasm.Index(len(importedME.importedFunctions))]
typeID := m.module.TypeIDs[descFunc]
executable := &importedME.parent.executable[offset]
// Write functionInstance.
@@ -330,3 +359,45 @@ func (m *moduleEngine) LookupFunction(t *wasm.TableInstance, typeId wasm.Functio
func moduleInstanceFromOpaquePtr(ptr *byte) *wasm.ModuleInstance {
return *(**wasm.ModuleInstance)(unsafe.Pointer(ptr))
}
type hostFunction struct {
internalapi.WazeroOnly
def *wasm.FunctionDefinition
lookedUpModule *wasm.ModuleInstance
g api.GoModuleFunction
}
// goFunctionAsGoModuleFunction converts api.GoFunction to api.GoModuleFunction which ignores the api.Module argument.
func goFunctionAsGoModuleFunction(g api.GoFunction) api.GoModuleFunction {
return api.GoModuleFunc(func(ctx context.Context, _ api.Module, stack []uint64) {
g.Call(ctx, stack)
})
}
// Definition implements api.Function.
func (f *hostFunction) Definition() api.FunctionDefinition { return f.def }
// Call implements api.Function.
func (f *hostFunction) Call(ctx context.Context, params ...uint64) ([]uint64, error) {
typ := f.def.Functype
stackSize := typ.ParamNumInUint64
rn := typ.ResultNumInUint64
if rn > stackSize {
stackSize = rn
}
stack := make([]uint64, stackSize)
copy(stack, params)
return stack[:rn], f.CallWithStack(ctx, stack)
}
// CallWithStack implements api.Function.
func (f *hostFunction) CallWithStack(ctx context.Context, stack []uint64) (err error) {
defer func() {
if r := recover(); r != nil {
builder := wasmdebug.NewErrorBuilder()
err = builder.FromRecovered(r)
}
}()
f.g.Call(ctx, f.lookedUpModule, stack)
return nil
}
@@ -291,9 +291,9 @@ func (bb *basicBlock) addPred(blk BasicBlock, branch *Instruction) {
for i := range bb.preds {
existingPred := &bb.preds[i]
if existingPred.blk == pred && existingPred.branch != branch {
// If the target is already added, then this must come from the same BrTable,
// If the target is already added, then this must come from the same BrTable or TryTableDispatch,
// otherwise such redundant branch should be eliminated by the frontend. (which should be simpler).
panic(fmt.Sprintf("BUG: redundant non BrTable jumps in %s whose targes are the same", bb.Name()))
panic(fmt.Sprintf("BUG: redundant non BrTable/TryTableDispatch jumps in %s whose targets are the same", bb.Name()))
}
}
@@ -344,19 +344,21 @@ func (bb *basicBlock) validate(b *builder) {
}
}
var exp int
if bb.ReturnBlock() {
exp = len(b.currentSignature.Results)
} else {
exp = len(bb.params.View())
}
if pred.branch.opcode != OpcodeBrTable {
var exp int
if bb.ReturnBlock() {
exp = len(b.currentSignature.Results)
} else {
exp = len(bb.params.View())
}
if len(pred.branch.vs.View()) != exp {
panic(fmt.Sprintf(
"BUG: len(argument at %s) != len(params at %s): %d != %d: %s",
pred.blk.Name(), bb.Name(),
len(pred.branch.vs.View()), len(bb.params.View()), pred.branch.Format(b),
))
if len(pred.branch.vs.View()) != exp {
panic(fmt.Sprintf(
"BUG: len(argument at %s) != len(params at %s): %d != %d: %s",
pred.blk.Name(), bb.Name(),
len(pred.branch.vs.View()), len(bb.params.View()), pred.branch.Format(b),
))
}
}
}
@@ -30,6 +30,25 @@ const (
ExitCodeMemoryWait64
ExitCodeMemoryNotify
ExitCodeUnalignedAtomic
// ExitCodeThrowAlloc is the first phase of wasm throw: Go allocates the
// Exception heap object (with Params sized to the tag's param count) and
// writes its Params data pointer to execCtx.exceptionParamsPtr.
// Compiled code then stores params directly into the Exception.Params slice,
// followed by ExitCodeThrow to search for a matching handler.
ExitCodeThrowAlloc
// ExitCodeThrow is the shared throw/throw_ref exit code.
// The exnref is passed on the stack. The handler searches for a
// matching catch clause and restores the stack checkpoint.
ExitCodeThrow
// ExitCodeNullReference is an exit code for a null reference trap (throw_ref with null exnref).
ExitCodeNullReference
// ExitCodeTryTableEnter is an exit code for entering a try_table block.
// The catch clause info is encoded in the upper bits. The dispatch loop
// saves the current SP/FP/returnAddress as a try handler checkpoint.
ExitCodeTryTableEnter
// ExitCodeTryTableLeave is an exit code for leaving a try_table block.
// The dispatch loop pops the most recent try handler.
ExitCodeTryTableLeave
exitCodeMax
)
@@ -86,6 +105,16 @@ func (e ExitCode) String() string {
return "memory_wait64"
case ExitCodeMemoryNotify:
return "memory_notify"
case ExitCodeThrowAlloc:
return "throw_alloc"
case ExitCodeThrow:
return "throw"
case ExitCodeNullReference:
return "null_reference"
case ExitCodeTryTableEnter:
return "try_table_enter"
case ExitCodeTryTableLeave:
return "try_table_leave"
}
panic("TODO")
}
@@ -107,3 +136,15 @@ func ExitCodeCallGoFunctionWithIndex(index int, withListener bool) ExitCode {
func GoFunctionIndexFromExitCode(exitCode ExitCode) int {
return int(exitCode >> 8)
}
// TryTableIDFromExitCode extracts the try-table ID from an ExitCodeTryTableEnter
// exit code. Uses the same encoding as GoFunctionIndexFromExitCode (upper 24 bits).
func TryTableIDFromExitCode(exitCode ExitCode) int {
return GoFunctionIndexFromExitCode(exitCode)
}
// CatchClauseInstance is a runtime catch clause with resolved tag index.
type CatchClauseInstance struct {
Kind byte // wasm.CatchKindCatch, etc.
TagIndex uint32 // module-local tag index
}
@@ -53,6 +53,23 @@ const (
ExecutionContextOffsetMemoryWait32TrampolineAddress Offset = 1160
ExecutionContextOffsetMemoryWait64TrampolineAddress Offset = 1168
ExecutionContextOffsetMemoryNotifyTrampolineAddress Offset = 1176
// ExecutionContextOffsetThrowAllocTrampolineAddress is the address of the
// throw-alloc trampoline, which allocates the Exception heap object,
// sets exceptionParamsPtr, and returns the exnref.
ExecutionContextOffsetThrowAllocTrampolineAddress Offset = 1184
ExecutionContextOffsetThrowTrampolineAddress Offset = 1192
ExecutionContextOffsetTryTableEnterTrampolineAddress Offset = 1200
ExecutionContextOffsetTryTableLeaveTrampolineAddress Offset = 1208
// ExecutionContextOffsetExceptionPtr holds the pointer to the Exception struct,
// used on the throw side and by catch_ref/catch_all_ref handlers.
ExecutionContextOffsetExceptionPtr Offset = 1216
// ExecutionContextOffsetExceptionParamsPtr points into the Exception's
// Params slice backing array. Used by both throw (store params) and
// catch (load params) sides.
ExecutionContextOffsetExceptionParamsPtr Offset = 1224
// ExecutionContextOffsetCaughtExceptionClauseIdx is the matched catch clause index
// written by handleException and read by compiled handler dispatch code.
ExecutionContextOffsetCaughtExceptionClauseIdx Offset = 1232
)
// ModuleContextOffsetData allows the compilers to get the information about offsets to the fields of wazevo.moduleContextOpaque,
@@ -66,6 +83,7 @@ type ModuleContextOffsetData struct {
GlobalsBegin,
TypeIDs1stElement,
TablesBegin,
TagsBegin,
BeforeListenerTrampolines1stElement,
AfterListenerTrampolines1stElement,
DataInstances1stElement,
@@ -122,6 +140,11 @@ func (m *ModuleContextOffsetData) TableOffset(tableIndex int) Offset {
return m.TablesBegin + Offset(tableIndex)*8
}
// TagOffset returns an offset of the i-th tag instance pointer.
func (m *ModuleContextOffsetData) TagOffset(tagIndex int) Offset {
return m.TagsBegin + Offset(tagIndex)*8
}
// NewModuleContextOffsetData creates a ModuleContextOffsetData determining the structure of moduleContextOpaque for the given Module.
// The structure is described in the comment of wazevo.moduleContextOpaque.
func NewModuleContextOffsetData(m *wasm.Module, withListener bool) ModuleContextOffsetData {
@@ -185,6 +208,15 @@ func NewModuleContextOffsetData(m *wasm.Module, withListener bool) ModuleContext
ret.TablesBegin = -1
}
if tags := int(m.ImportTagCount) + len(m.TagSection); tags > 0 {
offset = align8(offset)
ret.TagsBegin = offset
// Pointers to *wasm.TagInstance.
offset += Offset(tags) * 8
} else {
ret.TagsBegin = -1
}
if withListener {
offset = align8(offset)
ret.BeforeListenerTrampolines1stElement = offset