updated vendor

This commit is contained in:
2026-06-16 08:02:19 +02:00
parent 2f7f99d3f0
commit 77299d0c64
1283 changed files with 67302 additions and 208958 deletions
+215 -6
View File
@@ -21,6 +21,7 @@ const (
controlFrameKindLoop
controlFrameKindIfWithElse
controlFrameKindIfWithoutElse
controlFrameKindTryTable
)
type (
@@ -57,7 +58,8 @@ func (c *controlFrame) asLabel() label {
case controlFrameKindFunction:
return newLabel(labelKindReturn, 0)
case controlFrameKindIfWithElse,
controlFrameKindIfWithoutElse:
controlFrameKindIfWithoutElse,
controlFrameKindTryTable:
return newLabel(labelKindContinuation, c.frameID)
}
panic(fmt.Sprintf("unreachable: a bug in interpreterir implementation: %v", c.kind))
@@ -187,6 +189,8 @@ type compiler struct {
funcs []uint32
// globals holds the global types for all declared globals in the module where the target function exists.
globals []wasm.GlobalType
// tags holds the type indexes for all declared tags in the module where the target function exists.
tags []uint32
// needSourceOffset is true if this module requires DWARF based stack trace.
needSourceOffset bool
@@ -254,6 +258,9 @@ type compilationResult struct {
LabelCallers map[label]uint32
// UsesMemory is true if this function might use memory.
UsesMemory bool
// PendingExceptionTable holds unresolved exception table entries, built during
// compilation. Labels are resolved to final PCs in lowerIR.
PendingExceptionTable []pendingExceptionTableEntry
// The following fields are per-module values, not per-function.
@@ -276,7 +283,7 @@ type compilationResult struct {
// newCompiler returns the new *compiler for the given parameters.
// Use compiler.Next function to get compilation result per function.
func newCompiler(enabledFeatures api.CoreFeatures, callFrameStackSizeInUint64 int, module *wasm.Module, ensureTermination bool) (*compiler, error) {
functions, globals, mem, tables, err := module.AllDeclarations()
functions, globals, mem, tables, tags, err := module.AllDeclarations()
if err != nil {
return nil, err
}
@@ -313,12 +320,14 @@ func newCompiler(enabledFeatures api.CoreFeatures, callFrameStackSizeInUint64 in
},
globals: globals,
funcs: functions,
tags: tags,
types: types,
ensureTermination: ensureTermination,
br: bytes.NewReader(nil),
funcTypeToSigs: funcTypeToIRSignatures{
indirectCalls: make([]*signature, len(types)),
directCalls: make([]*signature, len(types)),
callRefCalls: make([]*signature, len(types)),
wasmTypes: types,
},
needSourceOffset: module.DWARFLines != nil,
@@ -336,6 +345,7 @@ func (c *compiler) Next() (*compilationResult, error) {
c.result.Operations = c.result.Operations[:0]
c.result.IROperationSourceOffsetsInWasmBinary = c.result.IROperationSourceOffsetsInWasmBinary[:0]
c.result.UsesMemory = false
c.result.PendingExceptionTable = c.result.PendingExceptionTable[:0]
// Clears the existing entries in LabelCallers.
for frameID := uint32(0); frameID <= c.currentFrameID; frameID++ {
for k := labelKind(0); k < labelKindNum; k++ {
@@ -667,7 +677,8 @@ operatorSwitch:
// Initiate the continuation.
c.emit(newOperationLabel(continuationLabel))
case controlFrameKindBlockWithContinuationLabel,
controlFrameKindIfWithElse:
controlFrameKindIfWithElse,
controlFrameKindTryTable:
continuationLabel := newLabel(labelKindContinuation, frame.frameID)
c.result.LabelCallers[continuationLabel]++
c.emit(dropOp)
@@ -800,6 +811,95 @@ operatorSwitch:
// That means subsequent instructions in the current control frame are "unreachable"
// and can be safely removed.
c.markUnreachable()
case wasm.OpcodeThrow:
if c.unreachableState.on {
break operatorSwitch
}
// Pop the tag's param values from the stack.
if index < uint32(len(c.tags)) {
tagType := &c.types[c.tags[index]]
for i := len(tagType.Params) - 1; i >= 0; i-- {
c.stackPop()
}
}
c.emit(newOperationThrow(index))
c.markUnreachable()
case wasm.OpcodeThrowRef:
if c.unreachableState.on {
break operatorSwitch
}
// Pop the exnref from the stack.
c.stackPop()
c.emit(newOperationThrowRef())
c.markUnreachable()
case wasm.OpcodeTryTable:
c.br.Reset(c.body[c.pc+1:])
bt, num, err := wasm.DecodeBlockType(c.types, c.br, c.enabledFeatures)
if err != nil {
return fmt.Errorf("reading block type for try_table instruction: %w", err)
}
c.pc += num
if c.unreachableState.on {
c.unreachableState.depth++
// Still need to skip the catch clause bytes.
c.pc++
catchCount, catchNum, err := leb128.LoadUint32(c.body[c.pc:])
if err != nil {
return fmt.Errorf("reading catch count for try_table: %w", err)
}
c.pc += catchNum - 1
for i := uint32(0); i < catchCount; i++ {
if _, _, _, err := c.parseCatchClause(); err != nil {
return err
}
}
break operatorSwitch
}
// Read catch clause count.
c.pc++
catchCount, catchNum, err := leb128.LoadUint32(c.body[c.pc:])
if err != nil {
return fmt.Errorf("reading catch count for try_table: %w", err)
}
c.pc += catchNum - 1
// Parse catch clauses.
var pendingClauses []pendingCatchClause
for i := uint32(0); i < catchCount; i++ {
kind, tagIdx, labelIdx, err := c.parseCatchClause()
if err != nil {
return err
}
// Resolve the label from the control frame stack.
targetFrame := c.controlFrames.get(int(labelIdx))
targetFrame.ensureContinuation()
targetLabel := targetFrame.asLabel()
c.result.LabelCallers[targetLabel]++
pendingClauses = append(pendingClauses, pendingCatchClause{
kind: kind,
tagIndex: tagIdx,
targetLabel: targetLabel,
targetStackDepth: targetFrame.originalStackLenWithoutParamUint64,
})
}
// Create a control frame for the try_table block.
frameID := c.nextFrameID()
c.result.PendingExceptionTable = append(c.result.PendingExceptionTable, pendingExceptionTableEntry{
startOpIndex: len(c.result.Operations),
continuationFrameID: frameID,
clauses: pendingClauses,
})
frame := controlFrame{
frameID: frameID,
originalStackLenWithoutParam: len(c.stack) - len(bt.Params),
originalStackLenWithoutParamUint64: c.stackLenInUint64 - bt.ParamNumInUint64,
kind: controlFrameKindTryTable,
blockType: bt,
}
c.controlFrames.push(frame)
case wasm.OpcodeCall:
c.emit(
newOperationCall(index),
@@ -1616,7 +1716,18 @@ operatorSwitch:
newOperationRefFunc(index),
)
case wasm.OpcodeRefNull:
c.pc++ // Skip the type of reftype as every ref value is opaque pointer.
c.pc++
switch reftype := c.body[c.pc]; wasm.ValueType(reftype) {
case wasm.ValueTypeFuncref, wasm.ValueTypeExternref, wasm.ValueTypeExnref:
// Abstract ref types are a single byte; already skipped.
default:
// Concrete type index encoded as LEB128; skip it.
_, num, err := leb128.LoadUint32(c.body[c.pc:])
if err != nil {
return fmt.Errorf("failed to read type index for ref.null: %v", err)
}
c.pc += num - 1
}
c.emit(
newOperationConstI64(0),
)
@@ -3463,6 +3574,64 @@ operatorSwitch:
// and can be safely removed.
c.markUnreachable()
case wasm.OpcodeCallRef:
c.emit(newOperationCallRef(index))
case wasm.OpcodeReturnCallRef:
functionFrame := c.controlFrames.functionFrame()
dropRange := c.getFrameDropRange(functionFrame, false)
c.emit(newOperationReturnCallRef(index, dropRange, functionFrame.asLabel()))
c.markUnreachable()
case wasm.OpcodeRefAsNonNull:
c.emit(newOperationRefAsNonNull())
case wasm.OpcodeBrOnNull:
targetIndex, n, err := leb128.LoadUint32(c.body[c.pc+1:])
if err != nil {
return fmt.Errorf("read the target for br_on_null: %w", err)
}
c.pc += n
if c.unreachableState.on {
break operatorSwitch
}
targetFrame := c.controlFrames.get(int(targetIndex))
targetFrame.ensureContinuation()
drop := c.getFrameDropRange(targetFrame, false)
target := targetFrame.asLabel()
c.result.LabelCallers[target]++
continuationLabel := newLabel(labelKindHeader, c.nextFrameID())
c.result.LabelCallers[continuationLabel]++
c.emit(newOperationBrOnNull(target, continuationLabel, drop))
c.emit(newOperationLabel(continuationLabel))
// On fall-through (non-null), the ref is pushed back at runtime.
c.stackPush(unsignedTypeI64)
case wasm.OpcodeBrOnNonNull:
targetIndex, n, err := leb128.LoadUint32(c.body[c.pc+1:])
if err != nil {
return fmt.Errorf("read the target for br_on_non_null: %w", err)
}
c.pc += n
if c.unreachableState.on {
break operatorSwitch
}
targetFrame := c.controlFrames.get(int(targetIndex))
targetFrame.ensureContinuation()
drop := c.getFrameDropRange(targetFrame, false)
target := targetFrame.asLabel()
c.result.LabelCallers[target]++
continuationLabel := newLabel(labelKindHeader, c.nextFrameID())
c.result.LabelCallers[continuationLabel]++
c.emit(newOperationBrOnNonNull(target, continuationLabel, drop))
c.emit(newOperationLabel(continuationLabel))
default:
return fmt.Errorf("unsupported instruction in interpreterir: 0x%x", op)
}
@@ -3492,7 +3661,12 @@ func (c *compiler) applyToStack(opcode wasm.Opcode) (index uint32, err error) {
wasm.OpcodeGlobalSet,
// tail-call proposal
wasm.OpcodeTailCallReturnCall,
wasm.OpcodeTailCallReturnCallIndirect:
wasm.OpcodeTailCallReturnCallIndirect,
// exception handling - throw reads tag index
wasm.OpcodeThrow,
// typed function references
wasm.OpcodeCallRef,
wasm.OpcodeReturnCallRef:
// Assumes that we are at the opcode now so skip it before read immediates.
v, num, err := leb128.LoadUint32(c.body[c.pc+1:])
if err != nil {
@@ -3605,7 +3779,7 @@ func (c *compiler) emitDefaultValue(t wasm.ValueType) {
case wasm.ValueTypeI32:
c.stackPush(unsignedTypeI32)
c.emit(newOperationConstI32(0))
case wasm.ValueTypeI64, wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
case wasm.ValueTypeI64, wasm.ValueTypeExternref, wasm.ValueTypeFuncref, wasm.ValueTypeExnref:
c.stackPush(unsignedTypeI64)
c.emit(newOperationConstI64(0))
case wasm.ValueTypeF32:
@@ -3617,6 +3791,14 @@ func (c *compiler) emitDefaultValue(t wasm.ValueType) {
case wasm.ValueTypeV128:
c.stackPush(unsignedTypeV128)
c.emit(newOperationV128Const(0, 0))
default:
// Concrete ref types (ref $t) have variable bit patterns.
if t.IsRef() {
c.stackPush(unsignedTypeI64)
c.emit(newOperationConstI64(0))
} else {
panic(fmt.Sprintf("bug: unsupported value type for default value: 0x%x", t))
}
}
}
@@ -3673,3 +3855,30 @@ func (c *compiler) readMemoryArg(tag string) (memoryArg, error) {
c.pc += num
return memoryArg{Offset: offset, Alignment: alignment}, nil
}
// parseCatchClause parses a single catch clause from the bytecode at c.pc,
// advancing c.pc past the clause. Returns the kind, tag index (0 for catch_all
// variants), and label index.
func (c *compiler) parseCatchClause() (kind byte, tagIdx, labelIdx uint32, err error) {
var n uint64
c.pc++
kind = c.body[c.pc]
switch kind {
case wasm.CatchKindCatch, wasm.CatchKindCatchRef:
c.pc++
tagIdx, n, err = leb128.LoadUint32(c.body[c.pc:])
if err != nil {
err = fmt.Errorf("reading catch tag index: %w", err)
return
}
c.pc += n - 1
}
c.pc++
labelIdx, n, err = leb128.LoadUint32(c.body[c.pc:])
if err != nil {
err = fmt.Errorf("reading catch label index: %w", err)
return
}
c.pc += n - 1
return
}
@@ -130,6 +130,39 @@ func (e *moduleEngine) OwnsGlobals() bool { return false }
// MemoryGrown implements wasm.ModuleEngine.
func (e *moduleEngine) MemoryGrown() {}
// restorable is implemented by panic values that can restore callEngine state.
// Both *snapshot (snapshotter API) and *thrownException (exception handling)
// implement this interface.
type restorable interface {
// canRestore unwinds ce.frames to callerFrameCount and checks whether a
// handler exists at that depth. If no handler is found, the caller
// re-panics and the next outer callWithUnwind unwinds further.
canRestore(ce *callEngine, callerFrameCount int) bool
// doRestore restores the callEngine state to the given stack frame depth.
doRestore(ce *callEngine, callerFrameCount int)
}
// thrownException is the panic value for wasm exception propagation.
type thrownException struct {
exception *wasm.Exception
// Fields populated by canRestore for doRestore.
clause *exceptionTableCatchClause
values []uint64
}
func (t *thrownException) canRestore(ce *callEngine, callerFrameCount int) bool {
ce.frames = ce.frames[:callerFrameCount]
frame := ce.frames[callerFrameCount-1]
t.clause, t.values = searchExceptionTable(t.exception, frame)
return t.clause != nil
}
func (t *thrownException) doRestore(ce *callEngine, callerFrameCount int) {
frame := ce.frames[callerFrameCount-1]
ce.applyExceptionHandler(frame, t.clause, t.values)
t.clause, t.values = nil, nil
}
// callEngine holds context per moduleEngine.Call, and shared across all the
// function calls originating from the same moduleEngine.Call execution.
//
@@ -151,6 +184,94 @@ type callEngine struct {
stackIterator stackIterator
}
// matchCatchClause checks whether a single catch clause matches the given exception.
// Returns whether it matched and the values to push onto the stack.
func matchCatchClause(kind byte, clauseTag *wasm.TagInstance, exn *wasm.Exception) (matched bool, values []uint64) {
switch kind {
case wasm.CatchKindCatch:
if exn.Tag == clauseTag {
return true, slices.Clone(exn.Params)
}
case wasm.CatchKindCatchRef:
if exn.Tag == clauseTag {
values = slices.Clone(exn.Params)
values = append(values, uint64(uintptr(unsafe.Pointer(exn))))
return true, values
}
case wasm.CatchKindCatchAll:
return true, nil
case wasm.CatchKindCatchAllRef:
return true, []uint64{uint64(uintptr(unsafe.Pointer(exn)))}
}
return false, nil
}
// searchExceptionTable searches the compiled function's static exception table
// for a handler matching the given exception at the current PC. Returns the
// matched clause and catch values, or nil if no handler matches. Searches
// backwards so inner try_tables (which have higher indices) are checked first.
// This function is pure — it does not modify callEngine state.
func searchExceptionTable(exn *wasm.Exception, frame *callFrame) (*exceptionTableCatchClause, []uint64) {
table := frame.f.parent.exceptionTable
pc := frame.pc
for i := len(table) - 1; i >= 0; i-- {
entry := &table[i]
if pc < entry.startPC || pc >= entry.endPC {
continue
}
for j := range entry.clauses {
clause := &entry.clauses[j]
var clauseTag *wasm.TagInstance
if clause.kind == wasm.CatchKindCatch || clause.kind == wasm.CatchKindCatchRef {
clauseTag = frame.f.moduleInstance.Tags[clause.tagIndex]
}
matched, values := matchCatchClause(clause.kind, clauseTag, exn)
if matched {
return clause, values
}
}
}
return nil, nil
}
// applyExceptionHandler applies a matched exception table clause to the callEngine state.
func (ce *callEngine) applyExceptionHandler(frame *callFrame, clause *exceptionTableCatchClause, values []uint64) {
ce.stack = ce.stack[:frame.base-frame.f.funcType.ParamNumInUint64+clause.targetStackDepth]
ce.stack = append(ce.stack, values...)
frame.pc = clause.targetPC
}
// callWithUnwind calls the target function with support for stack unwinding
// (exception handling and snapshot restores). Returns true if the frame was
// unwound (caller should refresh frame/body/bodyLen and continue the loop).
// Returns false on normal return (caller should do frame.pc++).
func (ce *callEngine) callWithUnwind(ctx context.Context, m *wasm.ModuleInstance, tf *function) bool {
// Short-circuit: skip defer/recover overhead when neither exception
// handlers nor the snapshotter are active for the calling frame.
frame := ce.frames[len(ce.frames)-1]
if len(frame.f.parent.exceptionTable) == 0 && ctx.Value(expctxkeys.EnableSnapshotterKey{}) == nil {
ce.callFunction(ctx, m, tf)
return false
}
callerFrameCount := len(ce.frames)
caught := false
func() {
defer func() {
if r := recover(); r != nil {
if v, ok := r.(restorable); ok && v.canRestore(ce, callerFrameCount) {
v.doRestore(ce, callerFrameCount)
caught = true
return
}
panic(r)
}
}()
ce.callFunction(ctx, m, tf)
}()
return caught
}
func (e *moduleEngine) newCallEngine(compiled *function) *callEngine {
return &callEngine{f: compiled}
}
@@ -233,6 +354,7 @@ type callFrame struct {
type compiledFunction struct {
source *wasm.Module
body []unionOperation
exceptionTable []exceptionTableEntry
listener experimental.FunctionListener
offsetsInWasmBinary []uint64
hostFn interface{}
@@ -284,7 +406,11 @@ func (s *snapshot) Restore(ret []uint64) {
panic(s)
}
func (s *snapshot) doRestore() {
func (s *snapshot) canRestore(ce *callEngine, _ int) bool {
return s.ce == ce
}
func (s *snapshot) doRestore(_ *callEngine, _ int) {
ce := s.ce
ce.stack = s.stack
@@ -493,8 +619,45 @@ func (e *engine) lowerIR(ir *compilationResult, ret *compiledFunction) error {
}
case operationKindTailCallReturnCallIndirect:
e.setLabelAddress(&op.Us[1], label(op.Us[1]), labelAddressResolutions)
case operationKindBrOnNull:
e.setLabelAddress(&op.U1, label(op.U1), labelAddressResolutions)
e.setLabelAddress(&op.U2, label(op.U2), labelAddressResolutions)
case operationKindBrOnNonNull:
e.setLabelAddress(&op.U1, label(op.U1), labelAddressResolutions)
e.setLabelAddress(&op.U2, label(op.U2), labelAddressResolutions)
case operationKindReturnCallRef:
e.setLabelAddress(&op.Us[1], label(op.Us[1]), labelAddressResolutions)
}
}
// Resolve exception table entries (translate labels to PC).
if len(ir.PendingExceptionTable) > 0 {
ret.exceptionTable = make([]exceptionTableEntry, len(ir.PendingExceptionTable))
for i, pe := range ir.PendingExceptionTable {
contLabel := newLabel(labelKindContinuation, pe.continuationFrameID)
var endPC uint64
e.setLabelAddress(&endPC, contLabel, labelAddressResolutions)
clauses := make([]exceptionTableCatchClause, len(pe.clauses))
for j, clause := range pe.clauses {
var targetPC uint64
e.setLabelAddress(&targetPC, clause.targetLabel, labelAddressResolutions)
clauses[j] = exceptionTableCatchClause{
kind: clause.kind,
tagIndex: clause.tagIndex,
targetPC: targetPC,
targetStackDepth: clause.targetStackDepth,
}
}
ret.exceptionTable[i] = exceptionTableEntry{
startPC: uint64(pe.startOpIndex),
endPC: endPC,
clauses: clauses,
}
}
}
return nil
}
@@ -644,6 +807,11 @@ func (ce *callEngine) recoverOnCall(ctx context.Context, m *wasm.ModuleInstance,
panic(s)
}
// If an exception reached the top level without being caught, convert it to an uncaught exception error.
if _, ok := v.(*thrownException); ok {
v = wasmruntime.ErrRuntimeUncaughtException
}
builder := wasmdebug.NewErrorBuilder()
frameCount := len(ce.frames)
functionListeners := make([]functionListenerInvocation, 0, 16)
@@ -761,30 +929,26 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
ce.drop(op.Us[v+1])
frame.pc = op.Us[v]
case operationKindCall:
func() {
if ctx.Value(expctxkeys.EnableSnapshotterKey{}) != nil {
defer func() {
if r := recover(); r != nil {
if s, ok := r.(*snapshot); ok && s.ce == ce {
s.doRestore()
frame = ce.frames[len(ce.frames)-1]
body = frame.f.parent.body
bodyLen = uint64(len(body))
} else {
panic(r)
}
}
}()
}
ce.callFunction(ctx, f.moduleInstance, &functions[op.U1])
}()
frameUnwound := ce.callWithUnwind(ctx, f.moduleInstance, &functions[op.U1])
if frameUnwound {
frame = ce.frames[len(ce.frames)-1]
body = frame.f.parent.body
bodyLen = uint64(len(body))
continue
}
frame.pc++
case operationKindCallIndirect:
offset := ce.popValue()
table := tables[op.U2]
tf := ce.functionForOffset(table, offset, typeIDs[op.U1])
ce.callFunction(ctx, f.moduleInstance, tf)
frameUnwound := ce.callWithUnwind(ctx, f.moduleInstance, tf)
if frameUnwound {
frame = ce.frames[len(ce.frames)-1]
body = frame.f.parent.body
bodyLen = uint64(len(body))
continue
}
frame.pc++
case operationKindDrop:
ce.drop(op.U1)
@@ -973,7 +1137,10 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
case operationKindNe:
var b bool
switch unsignedType(op.B1) {
case unsignedTypeI32, unsignedTypeI64:
case unsignedTypeI32:
v2, v1 := ce.popValue(), ce.popValue()
b = uint32(v1) != uint32(v2)
case unsignedTypeI64:
v2, v1 := ce.popValue(), ce.popValue()
b = v1 != v2
case unsignedTypeF32:
@@ -990,7 +1157,11 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
}
frame.pc++
case operationKindEqz:
if ce.popValue() == 0 {
v := ce.popValue()
if unsignedInt(op.B1) == unsignedInt32 {
v = uint64(uint32(v))
}
if v == 0 {
ce.pushValue(1)
} else {
ce.pushValue(0)
@@ -1005,7 +1176,9 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
b = int32(v1) < int32(v2)
case signedTypeInt64:
b = int64(v1) < int64(v2)
case signedTypeUint32, signedTypeUint64:
case signedTypeUint32:
b = uint32(v1) < uint32(v2)
case signedTypeUint64:
b = v1 < v2
case signedTypeFloat32:
b = math.Float32frombits(uint32(v1)) < math.Float32frombits(uint32(v2))
@@ -1027,7 +1200,9 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
b = int32(v1) > int32(v2)
case signedTypeInt64:
b = int64(v1) > int64(v2)
case signedTypeUint32, signedTypeUint64:
case signedTypeUint32:
b = uint32(v1) > uint32(v2)
case signedTypeUint64:
b = v1 > v2
case signedTypeFloat32:
b = math.Float32frombits(uint32(v1)) > math.Float32frombits(uint32(v2))
@@ -1049,7 +1224,9 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
b = int32(v1) <= int32(v2)
case signedTypeInt64:
b = int64(v1) <= int64(v2)
case signedTypeUint32, signedTypeUint64:
case signedTypeUint32:
b = uint32(v1) <= uint32(v2)
case signedTypeUint64:
b = v1 <= v2
case signedTypeFloat32:
b = math.Float32frombits(uint32(v1)) <= math.Float32frombits(uint32(v2))
@@ -1071,7 +1248,9 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
b = int32(v1) >= int32(v2)
case signedTypeInt64:
b = int64(v1) >= int64(v2)
case signedTypeUint32, signedTypeUint64:
case signedTypeUint32:
b = uint32(v1) >= uint32(v2)
case signedTypeUint64:
b = v1 >= v2
case signedTypeFloat32:
b = math.Float32frombits(uint32(v1)) >= math.Float32frombits(uint32(v2))
@@ -1166,6 +1345,10 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
v2, v1 := ce.popValue(), ce.popValue()
switch t {
case signedTypeFloat32, signedTypeFloat64: // not integers
case signedTypeInt32, signedTypeUint32:
if uint32(v2) == 0 {
panic(wasmruntime.ErrRuntimeIntegerDivideByZero)
}
default:
if v2 == 0 {
panic(wasmruntime.ErrRuntimeIntegerDivideByZero)
@@ -1203,8 +1386,15 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
frame.pc++
case operationKindRem:
v2, v1 := ce.popValue(), ce.popValue()
if v2 == 0 {
panic(wasmruntime.ErrRuntimeIntegerDivideByZero)
switch signedInt(op.B1) {
case signedInt32, signedUint32:
if uint32(v2) == 0 {
panic(wasmruntime.ErrRuntimeIntegerDivideByZero)
}
default:
if v2 == 0 {
panic(wasmruntime.ErrRuntimeIntegerDivideByZero)
}
}
switch signedInt(op.B1) {
case signedInt32:
@@ -4346,6 +4536,35 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
memoryInst.Mux.Unlock()
ce.pushValue(uint64(old))
frame.pc++
case operationKindThrow:
tagIndex := uint32(op.U1)
tag := moduleInst.Tags[tagIndex]
paramCount := len(tag.Type.Params)
params := make([]uint64, paramCount)
for i := paramCount - 1; i >= 0; i-- {
params[i] = ce.popValue()
}
exn := &wasm.Exception{Tag: tag, Params: params}
if clause, values := searchExceptionTable(exn, frame); clause != nil {
ce.applyExceptionHandler(frame, clause, values)
continue
}
panic(&thrownException{exception: exn})
case operationKindThrowRef:
v := ce.popValue()
if v == 0 {
panic(wasmruntime.ErrRuntimeNullReference) // throw_ref on null exnref traps
}
// Read the Exception pointer directly from the uint64 value to avoid
// conversion from uintptr into unsafe.Pointer, which triggers checkptr.
exn := *(**wasm.Exception)(unsafe.Pointer(&v))
if clause, values := searchExceptionTable(exn, frame); clause != nil {
ce.applyExceptionHandler(frame, clause, values)
continue
}
panic(&thrownException{exception: exn})
case operationKindTailCallReturnCall:
f := &functions[op.U1]
ce.dropForTailCall(frame, f)
@@ -4361,7 +4580,13 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
// For details, see internal/engine/RATIONALE.md
if tf.moduleInstance != f.moduleInstance {
// Revert to a normal call.
ce.callFunction(ctx, f.moduleInstance, tf)
frameUnwound := ce.callWithUnwind(ctx, f.moduleInstance, tf)
if frameUnwound {
frame = ce.frames[len(ce.frames)-1]
body = frame.f.parent.body
bodyLen = uint64(len(body))
continue
}
// Return
ce.drop(op.Us[0])
// Jump to the function frame (return)
@@ -4372,6 +4597,73 @@ func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance
ce.dropForTailCall(frame, tf)
body, bodyLen = ce.resetPc(frame, tf)
case operationKindCallRef:
ref := ce.popValue()
if ref == 0 {
panic(wasmruntime.ErrRuntimeNullReference)
}
tf := functionFromUintptr(uintptr(ref))
frameUnwound := ce.callWithUnwind(ctx, f.moduleInstance, tf)
if frameUnwound {
frame = ce.frames[len(ce.frames)-1]
body = frame.f.parent.body
bodyLen = uint64(len(body))
continue
}
frame.pc++
case operationKindReturnCallRef:
ref := ce.popValue()
if ref == 0 {
panic(wasmruntime.ErrRuntimeNullReference)
}
tf := functionFromUintptr(uintptr(ref))
if tf.moduleInstance != f.moduleInstance {
frameUnwound := ce.callWithUnwind(ctx, f.moduleInstance, tf)
if frameUnwound {
frame = ce.frames[len(ce.frames)-1]
body = frame.f.parent.body
bodyLen = uint64(len(body))
continue
}
ce.drop(op.Us[0])
frame.pc = op.Us[1]
continue
}
ce.dropForTailCall(frame, tf)
body, bodyLen = ce.resetPc(frame, tf)
case operationKindRefAsNonNull:
ref := ce.popValue()
if ref == 0 {
panic(wasmruntime.ErrRuntimeNullReference)
}
ce.pushValue(ref)
frame.pc++
case operationKindBrOnNull:
ref := ce.popValue()
if ref == 0 {
ce.drop(op.U3)
frame.pc = op.U1
} else {
ce.pushValue(ref)
frame.pc = op.U2
}
case operationKindBrOnNonNull:
ref := ce.popValue()
if ref != 0 {
ce.drop(op.U3)
ce.pushValue(ref)
frame.pc = op.U1
} else {
frame.pc = op.U2
}
default:
frame.pc++
}
@@ -4623,7 +4915,9 @@ func (ce *callEngine) callNativeFuncWithListener(ctx context.Context, m *wasm.Mo
// popMemoryOffset takes a memory offset off the stack for use in load and store instructions.
// As the top of stack value is 64-bit, this ensures it is in range before returning it.
func (ce *callEngine) popMemoryOffset(op *unionOperation) uint32 {
offset := op.U2 + ce.popValue()
// Memory addresses are i32; mask to 32 bits to ignore any
// garbage in the upper bits of the uint64 stack slot.
offset := op.U2 + uint64(uint32(ce.popValue()))
if offset > math.MaxUint32 {
panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess)
}
@@ -449,6 +449,20 @@ func (o operationKind) String() (ret string) {
ret = "operationKindTailCallReturnCall"
case operationKindTailCallReturnCallIndirect:
ret = "operationKindTailCallReturnCallIndirect"
case operationKindThrow:
ret = "operationKindThrow"
case operationKindThrowRef:
ret = "operationKindThrowRef"
case operationKindCallRef:
ret = "operationKindCallRef"
case operationKindReturnCallRef:
ret = "operationKindReturnCallRef"
case operationKindRefAsNonNull:
ret = "operationKindRefAsNonNull"
case operationKindBrOnNull:
ret = "operationKindBrOnNull"
case operationKindBrOnNonNull:
ret = "operationKindBrOnNonNull"
default:
panic(fmt.Errorf("unknown operation %d", o))
}
@@ -777,6 +791,22 @@ const (
// operationKindTailCallReturnCallIndirect is the Kind for newOperationKindTailCallReturnCallIndirect.
operationKindTailCallReturnCallIndirect
// operationKindThrow is the Kind for throw instruction.
operationKindThrow
// operationKindThrowRef is the Kind for throw_ref instruction.
operationKindThrowRef
// operationKindCallRef is the Kind for call_ref instruction.
operationKindCallRef
// operationKindReturnCallRef is the Kind for return_call_ref instruction.
operationKindReturnCallRef
// operationKindRefAsNonNull is the Kind for ref.as_non_null instruction.
operationKindRefAsNonNull
// operationKindBrOnNull is the Kind for br_on_null instruction.
operationKindBrOnNull
// operationKindBrOnNonNull is the Kind for br_on_non_null instruction.
operationKindBrOnNonNull
// operationKindEnd is always placed at the bottom of this iota definition to be used in the test.
operationKindEnd
)
@@ -1112,6 +1142,27 @@ func (o unionOperation) String() string {
case operationKindTailCallReturnCallIndirect:
return fmt.Sprintf("%s %d %d", o.Kind, o.U1, o.U2)
case operationKindThrow:
return fmt.Sprintf("%s %d", o.Kind, o.U1)
case operationKindThrowRef:
return o.Kind.String()
case operationKindCallRef:
return fmt.Sprintf("%s %d", o.Kind, o.U1)
case operationKindReturnCallRef:
return fmt.Sprintf("%s %d", o.Kind, o.U1)
case operationKindRefAsNonNull:
return o.Kind.String()
case operationKindBrOnNull:
return fmt.Sprintf("%s %s %s", o.Kind, label(o.U1).String(), label(o.U2).String())
case operationKindBrOnNonNull:
return fmt.Sprintf("%s %s %s", o.Kind, label(o.U1).String(), label(o.U2).String())
default:
panic(fmt.Sprintf("TODO: %v", o.Kind))
}
@@ -2843,3 +2894,85 @@ func newOperationTailCallReturnCall(functionIndex uint32) unionOperation {
func newOperationTailCallReturnCallIndirect(typeIndex, tableIndex uint32, dropDepth inclusiveRange, l label) unionOperation {
return unionOperation{Kind: operationKindTailCallReturnCallIndirect, U1: uint64(typeIndex), U2: uint64(tableIndex), Us: []uint64{dropDepth.AsU64(), uint64(l)}}
}
// newOperationThrow is a constructor for unionOperation with operationKindThrow.
// U1 stores the tag index.
func newOperationThrow(tagIndex uint32) unionOperation {
return unionOperation{Kind: operationKindThrow, U1: uint64(tagIndex)}
}
// newOperationThrowRef is a constructor for unionOperation with operationKindThrowRef.
func newOperationThrowRef() unionOperation {
return unionOperation{Kind: operationKindThrowRef}
}
// newOperationCallRef is a constructor for operationKindCallRef.
// U1 = type index.
func newOperationCallRef(typeIndex uint32) unionOperation {
return unionOperation{Kind: operationKindCallRef, U1: uint64(typeIndex)}
}
// newOperationReturnCallRef is a constructor for operationKindReturnCallRef.
// U1 = type index, U2 = table index (unused), Us = [dropDepth, label].
func newOperationReturnCallRef(typeIndex uint32, dropDepth inclusiveRange, l label) unionOperation {
return unionOperation{Kind: operationKindReturnCallRef, U1: uint64(typeIndex), Us: []uint64{dropDepth.AsU64(), uint64(l)}}
}
// newOperationRefAsNonNull is a constructor for operationKindRefAsNonNull.
func newOperationRefAsNonNull() unionOperation {
return unionOperation{Kind: operationKindRefAsNonNull}
}
// newOperationBrOnNull is a constructor for operationKindBrOnNull.
// If ref is null, branch to U1 (thenTarget) with drop U3; otherwise continue at U2 (elseTarget).
func newOperationBrOnNull(thenTarget, elseTarget label, thenDrop inclusiveRange) unionOperation {
return unionOperation{
Kind: operationKindBrOnNull,
U1: uint64(thenTarget),
U2: uint64(elseTarget),
U3: thenDrop.AsU64(),
}
}
// newOperationBrOnNonNull is a constructor for operationKindBrOnNonNull.
// If ref is non-null, branch to U1 (thenTarget) with drop U3; otherwise continue at U2 (elseTarget).
func newOperationBrOnNonNull(thenTarget, elseTarget label, thenDrop inclusiveRange) unionOperation {
return unionOperation{
Kind: operationKindBrOnNonNull,
U1: uint64(thenTarget),
U2: uint64(elseTarget),
U3: thenDrop.AsU64(),
}
}
// exceptionTableEntry represents one try_table's exception handling scope.
// Built at compile time and stored per compiledFunction.
type exceptionTableEntry struct {
startPC uint64 // first PC inside the try_table body
endPC uint64 // PC of continuation label (exclusive)
clauses []exceptionTableCatchClause
}
// exceptionTableCatchClause is a single catch clause within an exception table entry.
type exceptionTableCatchClause struct {
kind byte // CatchKindCatch, CatchKindCatchRef, CatchKindCatchAll, CatchKindCatchAllRef
tagIndex uint32 // tag index for catch/catch_ref
targetPC uint64 // resolved PC to jump to on match
targetStackDepth int // = targetFrame.originalStackLenWithoutParamUint64
}
// pendingExceptionTableEntry is an unresolved exception table entry built during compilation.
// Labels are resolved to final PCs in lowerIR.
type pendingExceptionTableEntry struct {
startOpIndex int // index in Operations[] of the first instruction inside the try_table body
continuationFrameID uint32
clauses []pendingCatchClause
}
// pendingCatchClause is an unresolved catch clause within a pending exception table entry.
type pendingCatchClause struct {
kind byte
tagIndex uint32
targetLabel label // unresolved label, resolved in lowerIR
targetStackDepth int // = targetFrame.originalStackLenWithoutParamUint64
}
@@ -268,6 +268,9 @@ func (c *compiler) wasmOpcodeSignature(op wasm.Opcode, index uint32) (*signature
return signature_I32_None, nil
case wasm.OpcodeElse, wasm.OpcodeEnd, wasm.OpcodeBr:
return signature_None_None, nil
case wasm.OpcodeThrow, wasm.OpcodeThrowRef, wasm.OpcodeTryTable:
// Stack manipulation handled dynamically by the compiler.
return signature_None_None, nil
case wasm.OpcodeBrIf, wasm.OpcodeBrTable:
return signature_I32_None, nil
case wasm.OpcodeReturn:
@@ -276,6 +279,17 @@ func (c *compiler) wasmOpcodeSignature(op wasm.Opcode, index uint32) (*signature
return c.funcTypeToSigs.get(c.funcs[index], false /* direct */), nil
case wasm.OpcodeCallIndirect, wasm.OpcodeTailCallReturnCallIndirect:
return c.funcTypeToSigs.get(index, true /* call_indirect */), nil
case wasm.OpcodeCallRef, wasm.OpcodeReturnCallRef:
return c.funcTypeToSigs.getCallRef(index), nil
case wasm.OpcodeRefAsNonNull:
// Pop a ref (i64), push it back (i64). Traps if null at runtime.
return signature_I64_I64, nil
case wasm.OpcodeBrOnNull:
// Pop a ref (i64). If null, branch; if non-null, push ref back.
return signature_I64_None, nil
case wasm.OpcodeBrOnNonNull:
// Pop a ref (i64). If non-null, push and branch; if null, fall through.
return signature_I64_None, nil
case wasm.OpcodeDrop:
return signature_Unknown_None, nil
case wasm.OpcodeSelect, wasm.OpcodeTypedSelect:
@@ -650,6 +664,7 @@ func (c *compiler) wasmOpcodeSignature(op wasm.Opcode, index uint32) (*signature
type funcTypeToIRSignatures struct {
directCalls []*signature
indirectCalls []*signature
callRefCalls []*signature
wasmTypes []wasm.FunctionType
}
@@ -694,13 +709,36 @@ func (f *funcTypeToIRSignatures) get(typeIndex wasm.Index, indirect bool) *signa
return sig
}
// getCallRef returns the *signature for call_ref, which is like a direct call
// but with an extra i64 input for the funcref operand.
func (f *funcTypeToIRSignatures) getCallRef(typeIndex wasm.Index) *signature {
if sig := f.callRefCalls[typeIndex]; sig != nil {
return sig
}
tp := &f.wasmTypes[typeIndex]
sig := &signature{
in: make([]unsignedType, 0, len(tp.Params)+1),
out: make([]unsignedType, 0, len(tp.Results)),
}
for _, vt := range tp.Params {
sig.in = append(sig.in, wasmValueTypeTounsignedType(vt))
}
sig.in = append(sig.in, unsignedTypeI64) // funcref operand
for _, vt := range tp.Results {
sig.out = append(sig.out, wasmValueTypeTounsignedType(vt))
}
f.callRefCalls[typeIndex] = sig
return sig
}
func wasmValueTypeTounsignedType(vt wasm.ValueType) unsignedType {
switch vt {
case wasm.ValueTypeI32:
return unsignedTypeI32
case wasm.ValueTypeI64,
// From interpreterir layer, ref type values are opaque 64-bit pointers.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
wasm.ValueTypeExternref, wasm.ValueTypeFuncref,
wasm.ValueTypeExnref:
return unsignedTypeI64
case wasm.ValueTypeF32:
return unsignedTypeF32
@@ -708,6 +746,11 @@ func wasmValueTypeTounsignedType(vt wasm.ValueType) unsignedType {
return unsignedTypeF64
case wasm.ValueTypeV128:
return unsignedTypeV128
default:
// Concrete ref types (ref $t) have variable bit patterns.
if vt.IsRef() {
return unsignedTypeI64
}
}
panic("unreachable")
}
@@ -718,7 +761,8 @@ func wasmValueTypeToUnsignedOutSignature(vt wasm.ValueType) *signature {
return signature_None_I32
case wasm.ValueTypeI64,
// From interpreterir layer, ref type values are opaque 64-bit pointers.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
wasm.ValueTypeExternref, wasm.ValueTypeFuncref,
wasm.ValueTypeExnref:
return signature_None_I64
case wasm.ValueTypeF32:
return signature_None_F32
@@ -726,6 +770,10 @@ func wasmValueTypeToUnsignedOutSignature(vt wasm.ValueType) *signature {
return signature_None_F64
case wasm.ValueTypeV128:
return signature_None_V128
default:
if vt.IsRef() {
return signature_None_I64
}
}
panic("unreachable")
}
@@ -736,7 +784,8 @@ func wasmValueTypeToUnsignedInSignature(vt wasm.ValueType) *signature {
return signature_I32_None
case wasm.ValueTypeI64,
// From interpreterir layer, ref type values are opaque 64-bit pointers.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
wasm.ValueTypeExternref, wasm.ValueTypeFuncref,
wasm.ValueTypeExnref:
return signature_I64_None
case wasm.ValueTypeF32:
return signature_F32_None
@@ -744,6 +793,10 @@ func wasmValueTypeToUnsignedInSignature(vt wasm.ValueType) *signature {
return signature_F64_None
case wasm.ValueTypeV128:
return signature_V128_None
default:
if vt.IsRef() {
return signature_I64_None
}
}
panic("unreachable")
}
@@ -754,7 +807,8 @@ func wasmValueTypeToUnsignedInOutSignature(vt wasm.ValueType) *signature {
return signature_I32_I32
case wasm.ValueTypeI64,
// At interpreterir layer, ref type values are opaque 64-bit pointers.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
wasm.ValueTypeExternref, wasm.ValueTypeFuncref,
wasm.ValueTypeExnref:
return signature_I64_I64
case wasm.ValueTypeF32:
return signature_F32_F32
@@ -762,6 +816,10 @@ func wasmValueTypeToUnsignedInOutSignature(vt wasm.ValueType) *signature {
return signature_F64_F64
case wasm.ValueTypeV128:
return signature_V128_V128
default:
if vt.IsRef() {
return signature_I64_I64
}
}
panic("unreachable")
}
@@ -370,14 +370,14 @@ func (m *machine) resolveAddressingMode(arg0offset, ret0offset int64, i *instruc
// resolveRelativeAddresses resolves the relative addresses before encoding.
func (m *machine) resolveRelativeAddresses(ctx context.Context) {
for {
if len(m.unresolvedAddressModes) > 0 {
arg0offset, ret0offset := m.arg0OffsetFromSP(), m.ret0OffsetFromSP()
for _, i := range m.unresolvedAddressModes {
m.resolveAddressingMode(arg0offset, ret0offset, i)
}
if len(m.unresolvedAddressModes) > 0 {
arg0offset, ret0offset := m.arg0OffsetFromSP(), m.ret0OffsetFromSP()
for _, i := range m.unresolvedAddressModes {
m.resolveAddressingMode(arg0offset, ret0offset, i)
}
m.unresolvedAddressModes = m.unresolvedAddressModes[:0]
}
for {
// Reuse the slice to gather the unresolved conditional branches.
m.condBrRelocs = m.condBrRelocs[:0]
+188 -12
View File
@@ -43,6 +43,29 @@ type (
execCtxPtr uintptr
numberOfResults int
stackIteratorImpl stackIterator
// tryHandlers is the stack of active try_table exception handlers,
// used to match catch clauses when a throw exits to the dispatch loop.
tryHandlers []tryHandler
// pendingException holds the most recently caught exception, so handler
// code can read its params after re-entry.
pendingException *wasm.Exception
}
// tryHandler records the state at a try_table entry for exception handling.
// On match, we restore the stack to the checkpoint state and re-enter at returnAddress.
tryHandler struct {
// Cloned stack and state from the try_table entry checkpoint,
// using the same approach as experimental.Snapshot.
sp, fp, top uintptr
returnAddress *byte
savedRegisters [64][2]uint64
stack []byte // cloned stack
// catchClauses describes what exceptions this handler catches.
catchClauses []wazevoapi.CatchClauseInstance
// moduleInstance is the module that set up this try handler.
// Used for tag matching in doHandleException (the tag index in
// catch clauses is relative to this module's tag index space).
moduleInstance *wasm.ModuleInstance
}
// executionContext is the struct to be read/written by assembly functions.
@@ -90,6 +113,29 @@ type (
memoryWait64TrampolineAddress *byte
// memoryNotifyTrampolineAddress holds the address of the memory_notify trampoline function.
memoryNotifyTrampolineAddress *byte
// throwAllocTrampolineAddress holds the address of the throw-alloc trampoline:
// phase 1 of throw, which allocates the Exception heap object.
throwAllocTrampolineAddress *byte
// throwTrampolineAddress holds the address of the throw/throw_ref trampoline function.
throwTrampolineAddress *byte
// tryTableEnterTrampolineAddress holds the address of the try_table enter trampoline function.
tryTableEnterTrampolineAddress *byte
// tryTableLeaveTrampolineAddress holds the address of the try_table leave trampoline function.
tryTableLeaveTrampolineAddress *byte
// exceptionPtr holds the pointer to the Exception struct,
// used on the throw side (throwAlloc stores the new Exception)
// and on the catch side (catch_ref/catch_all_ref retrieve the exnref).
exceptionPtr uintptr
// exceptionParamsPtr points into exceptionPtr's Params slice
// backing array. On the throw side, throwAlloc sets it so compiled
// code can store params at [ptr + i*8]. On the catch side, compiled
// handler blocks load params from the same pointer.
exceptionParamsPtr uintptr
// caughtExceptionClauseIdx is set by the dispatch loop to -1 on
// TryTableEnter (normal path) or to the matched catch clause index
// when an exception is caught. Compiled code loads this from execCtx
// after the trampoline call to decide which handler to dispatch to.
caughtExceptionClauseIdx int64
}
)
@@ -217,6 +263,9 @@ func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint6
}
}
// Clear any stale try_table handlers from a previous call.
c.tryHandlers = c.tryHandlers[:0]
var paramResultPtr *uint64
if len(paramResultStack) > 0 {
paramResultPtr = &paramResultStack[0]
@@ -236,21 +285,25 @@ func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint6
var listeners []listenerForAbort
builder := wasmdebug.NewErrorBuilder()
def, lsn := c.addFrame(builder, uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress)))
if lsn != nil {
listeners = append(listeners, listenerForAbort{def, lsn})
}
returnAddrs := unwindStack(
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)),
c.execCtx.framePointerBeforeGoCall,
c.stackTop,
nil,
)
for _, retAddr := range returnAddrs[:len(returnAddrs)-1] { // the last return addr is the trampoline, so we skip it.
def, lsn = c.addFrame(builder, retAddr)
if c.execCtx.stackPointerBeforeGoCall != nil {
def, lsn := c.addFrame(builder, uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress)))
if lsn != nil {
listeners = append(listeners, listenerForAbort{def, lsn})
}
returnAddrs := unwindStack(
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)),
c.execCtx.framePointerBeforeGoCall,
c.stackTop,
nil,
)
if len(returnAddrs) > 1 {
for _, retAddr := range returnAddrs[:len(returnAddrs)-1] { // the last return addr is the trampoline, so we skip it.
def, lsn = c.addFrame(builder, retAddr)
if lsn != nil {
listeners = append(listeners, listenerForAbort{def, lsn})
}
}
}
}
err = builder.FromRecovered(r)
@@ -266,6 +319,7 @@ func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint6
if err != nil {
// Ensures that we can reuse this callEngine even after an error.
c.execCtx.exitCode = wazevoapi.ExitCodeOK
c.tryHandlers = c.tryHandlers[:0]
}
}()
@@ -505,12 +559,134 @@ func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint6
panic(wasmruntime.ErrRuntimeInvalidConversionToInteger)
case wazevoapi.ExitCodeUnalignedAtomic:
panic(wasmruntime.ErrRuntimeUnalignedAtomic)
case wazevoapi.ExitCodeThrowAlloc:
// Allocate the Exception heap object sized exactly to the tag's
// param count. Sets exceptionParamsPtr so compiled code can
// store params, and returns the exnref via the stack slot.
s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
tagIndex := int(s[0])
mod := c.callerModuleInstance()
tag := mod.Tags[tagIndex]
nParams := len(tag.Type.Params)
exn := &wasm.Exception{Tag: tag, Params: make([]uint64, nParams)}
c.pendingException = exn // GC root: keeps exn alive while compiled code writes params
if nParams > 0 {
c.execCtx.exceptionParamsPtr = uintptr(unsafe.Pointer(&exn.Params[0]))
}
// Return the exnref to compiled code via the stack slot.
s[0] = uint64(uintptr(unsafe.Pointer(exn)))
c.execCtx.exitCode = wazevoapi.ExitCodeOK
afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
case wazevoapi.ExitCodeThrow:
// Throw trampoline: (execCtx, exnref) → ().
// Reads the exnref from the stack, searches for a matching handler.
s := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
// Read the Exception pointer directly from the uint64 value to avoid
// conversion from uintptr into unsafe.Pointer, which triggers checkptr.
exn := *(**wasm.Exception)(unsafe.Pointer(&s[0]))
if !c.doHandleException(exn) {
panic(wasmruntime.ErrRuntimeUncaughtException)
}
if len(exn.Params) > 0 {
c.execCtx.exceptionParamsPtr = uintptr(unsafe.Pointer(&exn.Params[0]))
}
c.execCtx.exceptionPtr = uintptr(unsafe.Pointer(exn))
c.execCtx.exitCode = wazevoapi.ExitCodeOK
afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
case wazevoapi.ExitCodeNullReference:
panic(wasmruntime.ErrRuntimeNullReference)
case wazevoapi.ExitCodeTryTableEnter:
// Save current state as a try handler checkpoint using stack cloning
// (same approach as experimental.Snapshot).
// The encoded exit code (with tryTableID in upper bits) is on the
// Go call stack as the second trampoline argument, not in execCtx.exitCode.
tryTableEnterStack := goCallStackView(c.execCtx.stackPointerBeforeGoCall)
catchClauseTableIdx := wazevoapi.TryTableIDFromExitCode(wazevoapi.ExitCode(tryTableEnterStack[0]))
mod := c.callerModuleInstance()
me := mod.Engine.(*moduleEngine)
clauses := me.parent.catchClauseTable[catchClauseTableIdx]
returnAddress := c.execCtx.goCallReturnAddress
oldTop, oldSp := c.stackTop, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall))
newSP, newFP, newTop, newStack := c.cloneStack(uintptr(len(c.stack)) + 16)
adjustClonedStack(oldSp, oldTop, newSP, newFP, newTop)
c.tryHandlers = append(c.tryHandlers, tryHandler{
sp: newSP,
fp: newFP,
top: newTop,
returnAddress: returnAddress,
savedRegisters: c.execCtx.savedRegisters,
stack: newStack,
catchClauses: clauses,
moduleInstance: mod,
})
// Set clauseIdx = -1 (no exception) in execCtx for the compiled code
// to read after the trampoline returns.
c.execCtx.caughtExceptionClauseIdx = -1
c.execCtx.exitCode = wazevoapi.ExitCodeOK
afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
case wazevoapi.ExitCodeTryTableLeave:
// Pop the most recent try handler.
if len(c.tryHandlers) > 0 {
c.tryHandlers = c.tryHandlers[:len(c.tryHandlers)-1]
}
c.execCtx.exitCode = wazevoapi.ExitCodeOK
afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr,
uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall)
default:
panic("BUG")
}
}
}
// doHandleException tries to match the given exception against active try handlers.
// If a match is found, it restores the execution state to the handler's checkpoint
// (like snapshot.doRestore) and writes the matched clause index as the trampoline
// return value. Returns true if handled.
func (c *callEngine) doHandleException(exn *wasm.Exception) bool {
// Search try handlers from innermost (last) to outermost (first).
for i := len(c.tryHandlers) - 1; i >= 0; i-- {
h := &c.tryHandlers[i]
for clauseIdx, clause := range h.catchClauses {
// Use the module that set up the handler (not the one that threw)
// because clause.TagIndex is relative to that module's tag space.
mod := h.moduleInstance
matched := false
switch clause.Kind {
case wasm.CatchKindCatch, wasm.CatchKindCatchRef:
matched = mod.Tags[clause.TagIndex] == exn.Tag
case wasm.CatchKindCatchAll, wasm.CatchKindCatchAllRef:
matched = true
}
if matched {
// Pop all handlers at and above this one.
c.tryHandlers = c.tryHandlers[:i]
// Store the caught exception so handler code can read params.
c.pendingException = exn
// Restore the cloned stack (like snapshot.doRestore).
spp := *(**uint64)(unsafe.Pointer(&h.sp))
c.stack = h.stack
c.stackTop = h.top
ec := &c.execCtx
ec.stackBottomPtr = &c.stack[0]
ec.stackPointerBeforeGoCall = spp
ec.framePointerBeforeGoCall = h.fp
ec.goCallReturnAddress = h.returnAddress
ec.savedRegisters = h.savedRegisters
// Set the matched clause index in execCtx for compiled code to read.
ec.caughtExceptionClauseIdx = int64(clauseIdx)
return true
}
}
}
return false
}
func (c *callEngine) callerModuleInstance() *wasm.ModuleInstance {
return moduleInstanceFromOpaquePtr(c.execCtx.callerModuleContextPtr)
}
+67 -3
View File
@@ -67,7 +67,16 @@ type (
memoryWait64Address *byte
// memoryNotifyAddress is the address of memory.notify builtin function
memoryNotifyAddress *byte
listenerTrampolines listenerTrampolines
// throwAllocTrampolineAddress is the address of the throw-alloc trampoline:
// phase 1 of throw, which allocates the Exception heap object.
throwAllocTrampolineAddress *byte
// throwTrampolineAddress is the address of the throw/throw_ref trampoline function.
throwTrampolineAddress *byte
// tryTableEnterAddress is the address of try_table enter trampoline.
tryTableEnterAddress *byte
// tryTableLeaveAddress is the address of try_table leave trampoline.
tryTableLeaveAddress *byte
listenerTrampolines listenerTrampolines
}
listenerTrampolines = map[*wasm.FunctionType]struct {
@@ -93,6 +102,9 @@ type (
offsets wazevoapi.ModuleContextOffsetData
sharedFunctions *sharedFunctions
sourceMap sourceMap
// catchClauseTable stores catch clause info for each try_table,
// indexed by a try_table ID assigned during compilation.
catchClauseTable [][]wazevoapi.CatchClauseInstance
}
executables struct {
@@ -269,6 +281,7 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
relocator.appendFunction(fctx, module, cm, i, fidx, body, relsPerFunc, be.SourceOffsetInfo())
}
cm.catchClauseTable = fe.CatchClauseTable()
} else {
// Compile with N worker goroutines.
// Collect compiled functions across workers in a slice,
@@ -287,6 +300,10 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
ctx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
// Catch clause table IDs are baked into compiled machine code, so all
// workers must share a single table to ensure globally unique IDs.
sharedCCT := frontend.NewSharedCatchClauseTable()
var count atomic.Uint32
var wg sync.WaitGroup
wg.Add(workers)
@@ -299,7 +316,9 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
machine := newMachine()
ssaBuilder := ssa.NewBuilder()
be := backend.NewCompiler(ctx, machine, ssaBuilder)
fe := frontend.NewFrontendCompiler(module, ssaBuilder, &cm.offsets, ensureTermination, withListener, needSourceInfo)
fe := frontend.NewFrontendCompiler(
module, ssaBuilder, &cm.offsets, ensureTermination, withListener, needSourceInfo).
WithCatchClauseTable(sharedCCT)
for {
if err := ctx.Err(); err != nil {
@@ -346,6 +365,7 @@ func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listene
fn := &compiledFuncs[i]
relocator.appendFunction(fn.fctx, module, cm, fn.fnum, fn.fidx, fn.body, fn.relsPerFunc, fn.offsPerFunc)
}
cm.catchClauseTable = sharedCCT.Table()
}
// Allocate executable memory and then copy the generated machine code.
@@ -733,7 +753,7 @@ func (e *engine) NewModuleEngine(m *wasm.Module, mi *wasm.ModuleInstance) (wasm.
}
func (e *engine) compileSharedFunctions() {
var sizes [8]int
var sizes [12]int
var trampolines []byte
addTrampoline := func(i int, buf []byte) {
@@ -801,6 +821,38 @@ func (e *engine) compileSharedFunctions() {
Results: []ssa.Type{ssa.TypeI32},
}, false))
e.be.Init()
addTrampoline(8,
e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeThrowAlloc, &ssa.Signature{
// exec context, tag index → exnref
Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64},
Results: []ssa.Type{ssa.TypeI64},
}, false))
e.be.Init()
addTrampoline(9,
e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeThrow, &ssa.Signature{
// exec context, exnref
Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64},
Results: []ssa.Type{},
}, false))
e.be.Init()
addTrampoline(10,
e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeTryTableEnter, &ssa.Signature{
// exec context, catch clause info (encoded)
Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64},
Results: []ssa.Type{},
}, false))
e.be.Init()
addTrampoline(11,
e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeTryTableLeave, &ssa.Signature{
// exec context
Params: []ssa.Type{ssa.TypeI64},
Results: []ssa.Type{},
}, false))
fns := &sharedFunctions{
executable: mmapExecutable(trampolines),
listenerTrampolines: make(listenerTrampolines),
@@ -823,6 +875,14 @@ func (e *engine) compileSharedFunctions() {
fns.memoryWait64Address = &fns.executable[offset]
offset += sizes[6]
fns.memoryNotifyAddress = &fns.executable[offset]
offset += sizes[7]
fns.throwAllocTrampolineAddress = &fns.executable[offset]
offset += sizes[8]
fns.throwTrampolineAddress = &fns.executable[offset]
offset += sizes[9]
fns.tryTableEnterAddress = &fns.executable[offset]
offset += sizes[10]
fns.tryTableLeaveAddress = &fns.executable[offset]
if wazevoapi.PerfMapEnabled {
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.memoryGrowAddress)), uint64(sizes[0]), "memory_grow_trampoline")
@@ -833,6 +893,10 @@ func (e *engine) compileSharedFunctions() {
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.memoryWait32Address)), uint64(sizes[5]), "memory_wait32_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.memoryWait64Address)), uint64(sizes[6]), "memory_wait64_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.memoryNotifyAddress)), uint64(sizes[7]), "memory_notify_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.throwAllocTrampolineAddress)), uint64(sizes[8]), "throw_alloc_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.throwTrampolineAddress)), uint64(sizes[9]), "throw_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.tryTableEnterAddress)), uint64(sizes[10]), "try_table_enter_trampoline")
wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(fns.tryTableLeaveAddress)), uint64(sizes[11]), "try_table_leave_trampoline")
}
e.sharedFunctions = fns
@@ -182,6 +182,16 @@ func serializeCompiledModule(wazeroVersion string, cm *compiledModule) io.Reader
} else {
buf.WriteByte(0) // indicates that source map is not present.
}
// Catch clause table: number of try_tables (4 bytes), then for each:
// clause count (4 bytes), then for each clause: kind (1 byte) + tagIndex (4 bytes).
buf.Write(u32.LeBytes(uint32(len(cm.catchClauseTable))))
for _, clauses := range cm.catchClauseTable {
buf.Write(u32.LeBytes(uint32(len(clauses))))
for _, c := range clauses {
buf.WriteByte(c.Kind)
buf.Write(u32.LeBytes(c.TagIndex))
}
}
return bytes.NewReader(buf.Bytes())
}
@@ -291,6 +301,32 @@ func deserializeCompiledModule(wazeroVersion string, reader io.ReadCloser) (cm *
sm.executableOffsets = append(sm.executableOffsets, uintptr(executableRelativeOffset)+executableOffset)
}
}
// Catch clause table.
if _, err = io.ReadFull(reader, eightBytes[:4]); err != nil {
// Treat old cache entries without catch clause data as stale (trigger recompile).
return nil, true, nil
}
tableLen := binary.LittleEndian.Uint32(eightBytes[:4])
if tableLen > 0 {
cm.catchClauseTable = make([][]wazevoapi.CatchClauseInstance, tableLen)
for i := uint32(0); i < tableLen; i++ {
if _, err = io.ReadFull(reader, eightBytes[:4]); err != nil {
return nil, false, fmt.Errorf("compilationcache: error reading catch clause count for try_table[%d]: %v", i, err)
}
clauseCount := binary.LittleEndian.Uint32(eightBytes[:4])
clauses := make([]wazevoapi.CatchClauseInstance, clauseCount)
for j := uint32(0); j < clauseCount; j++ {
if _, err = io.ReadFull(reader, eightBytes[:5]); err != nil {
return nil, false, fmt.Errorf("compilationcache: error reading catch clause[%d][%d]: %v", i, j, err)
}
clauses[j] = wazevoapi.CatchClauseInstance{
Kind: eightBytes[0],
TagIndex: binary.LittleEndian.Uint32(eightBytes[1:5]),
}
}
cm.catchClauseTable[i] = clauses
}
}
return
}
@@ -4,6 +4,7 @@ package frontend
import (
"bytes"
"math"
"sync"
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
@@ -62,6 +63,20 @@ type Compiler struct {
execCtxPtrValue, moduleCtxPtrValue ssa.Value
// throwAllocSig is the signature for the throw-alloc trampoline:
// (execCtx, tagIndex) → (exnref). Allocates the Exception and returns
// its pointer so compiled code can pass it to the throw trampoline.
throwAllocSig ssa.Signature
// throwSig is the signature for the throw/throw_ref trampoline:
// (execCtx, exnref) → (). Searches for a matching handler and restores.
throwSig ssa.Signature
// tryTableEnterSig is the signature for the try_table enter trampoline.
tryTableEnterSig ssa.Signature
// tryTableLeaveSig is the signature for the try_table leave trampoline.
tryTableLeaveSig ssa.Signature
// catchClauseTable accumulates catch clause info for each try_table during compilation.
catchClauseTable catchClauseTable
// Following are reused for the known safe bounds analysis.
pointers []int
@@ -95,12 +110,75 @@ func NewFrontendCompiler(m *wasm.Module, ssaBuilder ssa.Builder, offset *wazevoa
offset: offset,
ensureTermination: ensureTermination,
needSourceOffsetInfo: sourceInfo,
catchClauseTable: &localCatchClauseTable{},
varLengthKnownSafeBoundWithIDPool: wazevoapi.NewVarLengthPool[knownSafeBoundWithID](),
}
c.declareSignatures(listenerOn)
return c
}
// catchClauseTable accumulates catch clause entries during compilation.
type catchClauseTable interface {
Append(clauses []wazevoapi.CatchClauseInstance) int
Table() [][]wazevoapi.CatchClauseInstance
}
// localCatchClauseTable is the single-threaded implementation.
type localCatchClauseTable struct {
table [][]wazevoapi.CatchClauseInstance
}
func (t *localCatchClauseTable) Append(clauses []wazevoapi.CatchClauseInstance) int {
id := len(t.table)
t.table = append(t.table, clauses)
return id
}
func (t *localCatchClauseTable) Table() [][]wazevoapi.CatchClauseInstance {
return t.table
}
// SharedCatchClauseTable is the thread-safe implementation for parallel compilation.
type SharedCatchClauseTable struct {
mu sync.Mutex
table [][]wazevoapi.CatchClauseInstance
finalized bool
}
// NewSharedCatchClauseTable creates a new SharedCatchClauseTable.
func NewSharedCatchClauseTable() *SharedCatchClauseTable {
return &SharedCatchClauseTable{}
}
func (s *SharedCatchClauseTable) Append(clauses []wazevoapi.CatchClauseInstance) int {
if s.finalized {
panic("already finalized")
}
s.mu.Lock()
id := len(s.table)
s.table = append(s.table, clauses)
s.mu.Unlock()
return id
}
func (s *SharedCatchClauseTable) Table() [][]wazevoapi.CatchClauseInstance {
s.finalized = true
return s.table
}
// WithCatchClauseTable replaces the catch clause table implementation.
// Used by the parallel compilation path to share a single mutex-protected
// table across workers.
func (c *Compiler) WithCatchClauseTable(t catchClauseTable) *Compiler {
c.catchClauseTable = t
return c
}
// CatchClauseTable returns the accumulated catch clause table.
func (c *Compiler) CatchClauseTable() [][]wazevoapi.CatchClauseInstance {
return c.catchClauseTable.Table()
}
func (c *Compiler) declareSignatures(listenerOn bool) {
m := c.m
c.signatures = make(map[*wasm.FunctionType]*ssa.Signature, len(m.TypeSection)+2)
@@ -194,6 +272,34 @@ func (c *Compiler) declareSignatures(listenerOn bool) {
Results: []ssa.Type{ssa.TypeI32},
}
c.ssaBuilder.DeclareSignature(&c.memoryNotifySig)
c.throwAllocSig = ssa.Signature{
ID: c.memoryNotifySig.ID + 1,
Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI64 /* tag index */},
Results: []ssa.Type{ssa.TypeI64 /* exnref */},
}
c.ssaBuilder.DeclareSignature(&c.throwAllocSig)
c.throwSig = ssa.Signature{
ID: c.throwAllocSig.ID + 1,
Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI64 /* exnref */},
Results: []ssa.Type{},
}
c.ssaBuilder.DeclareSignature(&c.throwSig)
c.tryTableEnterSig = ssa.Signature{
ID: c.throwSig.ID + 1,
Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI64 /* encoded exit code */},
Results: []ssa.Type{},
}
c.ssaBuilder.DeclareSignature(&c.tryTableEnterSig)
c.tryTableLeaveSig = ssa.Signature{
ID: c.tryTableEnterSig.ID + 1,
Params: []ssa.Type{ssa.TypeI64 /* exec context */},
Results: []ssa.Type{},
}
c.ssaBuilder.DeclareSignature(&c.tryTableLeaveSig)
}
// SignatureForWasmFunctionType returns the ssa.Signature for the given wasm.FunctionType.
@@ -234,7 +340,8 @@ func (c *Compiler) Init(idx, typIndex wasm.Index, typ *wasm.FunctionType, localT
// Note: this assumes 64-bit platform (I believe we won't have 32-bit backend ;)).
const executionContextPtrTyp, moduleContextPtrTyp = ssa.TypeI64, ssa.TypeI64
// LowerToSSA lowers the current function to SSA function which will be held by ssaBuilder.
// LowerToSSA lowers the current function to SSA IR which will be held by ssaBuilder.
//
// After calling this, the caller will be able to access the SSA info in *Compiler.ssaBuilder.
//
// Note that this only does the naive lowering, and do not do any optimization, instead the caller is expected to do so.
@@ -340,23 +447,7 @@ func (c *Compiler) declareNecessaryVariables() {
}
func (c *Compiler) declareWasmGlobal(typ wasm.ValueType, mutable bool) {
var st ssa.Type
switch typ {
case wasm.ValueTypeI32:
st = ssa.TypeI32
case wasm.ValueTypeI64,
// Both externref and funcref are represented as I64 since we only support 64-bit platforms.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
st = ssa.TypeI64
case wasm.ValueTypeF32:
st = ssa.TypeF32
case wasm.ValueTypeF64:
st = ssa.TypeF64
case wasm.ValueTypeV128:
st = ssa.TypeV128
default:
panic("TODO: " + wasm.ValueTypeName(typ))
}
st := WasmTypeToSSAType(typ)
v := c.ssaBuilder.DeclareVariable(st)
index := wasm.Index(len(c.globalVariables))
c.globalVariables = append(c.globalVariables, v)
@@ -372,8 +463,9 @@ func WasmTypeToSSAType(vt wasm.ValueType) ssa.Type {
case wasm.ValueTypeI32:
return ssa.TypeI32
case wasm.ValueTypeI64,
// Both externref and funcref are represented as I64 since we only support 64-bit platforms.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref:
// externref, funcref, and exnref are represented as I64 since we only support 64-bit platforms.
wasm.ValueTypeExternref, wasm.ValueTypeFuncref,
wasm.ValueTypeExnref:
return ssa.TypeI64
case wasm.ValueTypeF32:
return ssa.TypeF32
@@ -382,6 +474,10 @@ func WasmTypeToSSAType(vt wasm.ValueType) ssa.Type {
case wasm.ValueTypeV128:
return ssa.TypeV128
default:
// Concrete ref types (ref $t) have variable bit patterns.
if vt.IsRef() {
return ssa.TypeI64
}
panic("TODO: " + wasm.ValueTypeName(vt))
}
}
@@ -65,6 +65,7 @@ const (
controlFrameKindIfWithElse
controlFrameKindIfWithoutElse
controlFrameKindBlock
controlFrameKindTryTable
)
// String implements fmt.Stringer for debugging.
@@ -80,6 +81,8 @@ func (k controlFrameKind) String() string {
return "if_without_else"
case controlFrameKindBlock:
return "block"
case controlFrameKindTryTable:
return "try_table"
default:
panic(k)
}
@@ -1443,6 +1446,11 @@ func (c *Compiler) lowerCurrentOpcode() {
unreachable := state.unreachable
if !unreachable {
// For try_table, emit the leave trampoline before the jump to the following block.
if ctrl.kind == controlFrameKindTryTable {
c.emitTryTableLeave()
}
// Top n-th args will be used as a result of the current control frame.
args := c.nPeekDup(len(ctrl.blockType.Results))
@@ -1477,6 +1485,7 @@ func (c *Compiler) lowerCurrentOpcode() {
break
}
c.emitTryTableLeaves(int(labelIndex))
targetBlk, argNum := state.brTargetArgNumFor(labelIndex)
args := c.nPeekDup(argNum)
c.insertJumpToBlock(args, targetBlk)
@@ -1494,6 +1503,21 @@ func (c *Compiler) lowerCurrentOpcode() {
targetBlk, argNum := state.brTargetArgNumFor(labelIndex)
args := c.nPeekDup(argNum)
var sealTargetBlk bool
// If the branch exits any try_table frames, emit TryTableLeave
// calls in a trampoline block that only runs on the taken path.
if c.branchExitsTryTable(int(labelIndex)) {
current := builder.CurrentBlock()
trampolineBlk := builder.AllocateBasicBlock()
builder.SetCurrentBlock(trampolineBlk)
c.emitTryTableLeaves(int(labelIndex))
c.insertJumpToBlock(args, targetBlk)
builder.SetCurrentBlock(current)
targetBlk = trampolineBlk
sealTargetBlk = true
args = ssa.ValuesNil
}
if c.needListener && targetBlk.ReturnBlock() { // In this case, we have to call the listener before returning.
// Save the currently active block.
current := builder.CurrentBlock()
@@ -1559,6 +1583,7 @@ func (c *Compiler) lowerCurrentOpcode() {
if state.unreachable {
break
}
c.emitTryTableLeaves(len(state.controlFrames))
if c.needListener {
c.callListenerAfter()
}
@@ -3355,7 +3380,12 @@ func (c *Compiler) lowerCurrentOpcode() {
state.push(refFuncRet)
case wasm.OpcodeRefNull:
c.loweringState.pc++ // skips the reference type as we treat both of them as i64(0).
switch reftype := c.wasmFunctionBody[c.loweringState.pc+1]; wasm.ValueType(reftype) {
case wasm.ValueTypeFuncref, wasm.ValueTypeExternref, wasm.ValueTypeExnref:
c.loweringState.pc++
default:
c.readI32u()
}
if state.unreachable {
break
}
@@ -3399,6 +3429,9 @@ func (c *Compiler) lowerCurrentOpcode() {
if state.unreachable {
break
}
// Per spec, return_call leaves the current frame, so all enclosing
// try_table handlers must be popped before the tail call.
c.emitTryTableLeaves(len(c.state().controlFrames))
_, _ = typeIndex, tableIndex
c.lowerTailCallReturnCallIndirect(typeIndex, tableIndex)
state.unreachable = true
@@ -3408,9 +3441,396 @@ func (c *Compiler) lowerCurrentOpcode() {
if state.unreachable {
break
}
// Per spec, return_call leaves the current frame, so all enclosing
// try_table handlers must be popped before the tail call.
c.emitTryTableLeaves(len(c.state().controlFrames))
c.lowerTailCallReturnCall(fnIndex)
state.unreachable = true
case wasm.OpcodeThrow:
tagIndex := c.readI32u()
if state.unreachable {
break
}
tagType := c.resolveTagType(tagIndex)
// Pop the tag's param values from the stack.
var throwParams []ssa.Value
if tagType != nil {
throwParams = make([]ssa.Value, len(tagType.Params))
for i := len(tagType.Params) - 1; i >= 0; i-- {
throwParams[i] = state.pop()
}
}
c.storeCallerModuleContext()
tagIdxVal := builder.AllocateInstruction().AsIconst64(uint64(tagIndex)).Insert(builder).Return()
// We need to store the throwParams in the exception and then throw it.
// However, each exception might have a variable number of parameters,
// so we let Go allocate the reference on the heap.
// The Go side allocates the Exception object (Params sized to nParams)
// and stores the pointer to the backing-array into execCtx.exceptionParamsPtr.
throwAllocPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetThrowAllocTrampolineAddress.U32(),
ssa.TypeI64,
).Insert(builder).Return()
throwAllocArgs := c.allocateVarLengthValues(2, c.execCtxPtrValue, tagIdxVal)
exnref := builder.AllocateInstruction().
AsCallIndirect(throwAllocPtr, &c.throwAllocSig, throwAllocArgs).
Insert(builder).Return()
// Reload memory pointers invalidated by the Go call.
c.reloadAfterCall()
// We can now store each param directly into Exception.Params using the pointer
// stored into execCtx.exceptionParamsPtr.
if len(throwParams) > 0 {
paramsPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetExceptionParamsPtr.U32(),
ssa.TypeI64,
).Insert(builder).Return()
for i, v := range throwParams {
switch v.Type() {
case ssa.TypeF32:
v = builder.AllocateInstruction().AsBitcast(v, ssa.TypeI32).Insert(builder).Return()
case ssa.TypeF64:
v = builder.AllocateInstruction().AsBitcast(v, ssa.TypeI64).Insert(builder).Return()
}
builder.AllocateInstruction().
AsStore(ssa.OpcodeStore, v, paramsPtr, uint32(i)*8).
Insert(builder)
}
}
// We return again control to Go to search and dispatch to a matching catch clause.
c.emitThrow(exnref)
state.unreachable = true
case wasm.OpcodeThrowRef:
if state.unreachable {
break
}
exnref := state.pop()
// Check for null exnref.
zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return()
isNull := builder.AllocateInstruction()
isNull.AsIcmp(exnref, zero, ssa.IntegerCmpCondEqual)
builder.InsertInstruction(isNull)
exitIfNull := builder.AllocateInstruction()
exitIfNull.AsExitIfTrueWithCode(c.execCtxPtrValue, isNull.Return(), wazevoapi.ExitCodeNullReference)
builder.InsertInstruction(exitIfNull)
c.storeCallerModuleContext()
c.emitThrow(exnref)
state.unreachable = true
case wasm.OpcodeTryTable:
bt := c.readBlockType()
if state.unreachable {
state.unreachableDepth++
// Still need to skip the catch clause bytes in the unreachable case.
c.skipTryTableCatchClauses()
break
}
// Parse catch clauses.
c.loweringState.pc++
catchCount, catchNum, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(catchNum) - 1
var catchClauses []catchClause
for i := uint32(0); i < catchCount; i++ {
c.loweringState.pc++
kind := c.wasmFunctionBody[c.loweringState.pc]
var tagIdx uint32
switch kind {
case wasm.CatchKindCatch, wasm.CatchKindCatchRef:
c.loweringState.pc++
var n uint64
tagIdx, n, _ = leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
case wasm.CatchKindCatchAll, wasm.CatchKindCatchAllRef:
// No tagIdx for catch_all variants.
}
c.loweringState.pc++
labelIdx, n, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
catchClauses = append(catchClauses, catchClause{kind: kind, tagIndex: tagIdx, labelIdx: labelIdx})
}
// Register catch clauses in the table and get the try_table ID.
var clauseInstances []wazevoapi.CatchClauseInstance
for _, cc := range catchClauses {
clauseInstances = append(clauseInstances, wazevoapi.CatchClauseInstance{
Kind: cc.kind,
TagIndex: cc.tagIndex,
})
}
tryTableID := c.catchClauseTable.Append(clauseInstances)
// Allocate the following block (after try_table end) and body block.
followingBlk := builder.AllocateBasicBlock()
c.addBlockParamsFromWasmTypes(bt.Results, followingBlk)
bodyBlk := builder.AllocateBasicBlock()
if len(catchClauses) > 0 {
// Store the caller module context so the dispatch loop can find the module.
c.storeCallerModuleContext()
// For each catch clause, create a handler block that loads exception
// params and jumps to the wasm target label.
// NOTE: catch clause label indices do NOT include the try_table itself
// (the try_table is pushed onto the control stack after the catch clauses
// are processed, per the spec). So we resolve labels BEFORE pushing.
varPool := builder.VarLengthPool()
targets := varPool.Allocate(len(catchClauses) + 1) // +1 for bodyBlk
currentBlk := builder.CurrentBlock()
for _, cc := range catchClauses {
handlerBlk := builder.AllocateBasicBlock()
builder.SetCurrentBlock(handlerBlk)
c.reloadAfterCall()
// Resolve the wasm target label.
targetBlk, _ := state.brTargetArgNumFor(cc.labelIdx)
// Load exception params and jump to wasm target.
var brArgs []ssa.Value
switch cc.kind {
case wasm.CatchKindCatch:
if tagType := c.resolveTagType(cc.tagIndex); tagType != nil {
brArgs = c.loadExceptionParams(tagType)
}
case wasm.CatchKindCatchRef:
if tagType := c.resolveTagType(cc.tagIndex); tagType != nil {
brArgs = c.loadExceptionParams(tagType)
}
brArgs = append(brArgs, c.loadExnRef())
case wasm.CatchKindCatchAll:
// No values.
case wasm.CatchKindCatchAllRef:
brArgs = append(brArgs, c.loadExnRef())
}
jmpArgs := c.allocateVarLengthValues(len(brArgs), brArgs...)
c.insertJumpToBlock(jmpArgs, targetBlk)
targets = targets.Append(varPool, ssa.Value(handlerBlk.ID()))
}
// Last target is the body block (default for clauseIdx == -1 / out of range).
targets = targets.Append(varPool, ssa.Value(bodyBlk.ID()))
// Back to the original block: call the try_table enter trampoline,
// then dispatch on the caught clause index.
builder.SetCurrentBlock(currentBlk)
encodedExitCode := uint64(wazevoapi.ExitCodeTryTableEnter | wazevoapi.ExitCode(tryTableID<<8))
// Load trampoline address from execCtx.
enterPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetTryTableEnterTrampolineAddress.U32(),
ssa.TypeI64,
).Insert(builder).Return()
// Call the trampoline: (execCtx, encodedExitCode) -> ().
exitCodeVal := builder.AllocateInstruction().AsIconst64(encodedExitCode).Insert(builder).Return()
args := c.allocateVarLengthValues(2, c.execCtxPtrValue, exitCodeVal)
builder.AllocateInstruction().
AsCallIndirect(enterPtr, &c.tryTableEnterSig, args).
Insert(builder)
// Load the caught clause index written by the dispatch loop.
clauseIdx := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetCaughtExceptionClauseIdx.U32(),
ssa.TypeI64,
).Insert(builder).Return()
// Dispatch to handler blocks or body block via br_table.
brTable := builder.AllocateInstruction()
brTable.AsBrTable(clauseIdx, targets)
builder.InsertInstruction(brTable)
// Seal handler blocks after BrTable is inserted (so predecessors are registered).
for _, targetID := range targets.View() {
blk := builder.BasicBlock(ssa.BasicBlockID(targetID))
if !blk.Sealed() {
builder.Seal(blk)
}
}
} else {
// No catch clauses — try_table acts as a plain block.
// Jump directly to body without entering exception handling.
c.insertJumpToBlock(ssa.ValuesNil, bodyBlk)
}
if !bodyBlk.Sealed() {
builder.Seal(bodyBlk)
}
builder.SetCurrentBlock(bodyBlk)
if len(catchClauses) > 0 {
// Body block is entered after the trampoline call, so we need to reload.
c.reloadAfterCall()
}
// Push the try_table control frame AFTER resolving catch labels.
state.ctrlPush(controlFrame{
kind: controlFrameKindTryTable,
originalStackLenWithoutParam: len(state.values) - len(bt.Params),
followingBlock: followingBlk,
blockType: bt,
})
case wasm.OpcodeRefAsNonNull:
if state.unreachable {
break
}
r := state.pop()
zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder)
checkNull := builder.AllocateInstruction().
AsIcmp(r, zero.Return(), ssa.IntegerCmpCondEqual).
Insert(builder).Return()
exitIfNull := builder.AllocateInstruction()
exitIfNull.AsExitIfTrueWithCode(c.execCtxPtrValue, checkNull, wazevoapi.ExitCodeNullReference)
builder.InsertInstruction(exitIfNull)
state.push(r)
case wasm.OpcodeBrOnNull:
labelIndex := c.readI32u()
if state.unreachable {
break
}
r := state.pop()
zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder)
isNull := builder.AllocateInstruction().
AsIcmp(r, zero.Return(), ssa.IntegerCmpCondEqual).
Insert(builder).Return()
targetBlk, argNum := state.brTargetArgNumFor(labelIndex)
args := c.nPeekDup(argNum)
var sealTargetBlk bool
if c.branchExitsTryTable(int(labelIndex)) {
current := builder.CurrentBlock()
trampolineBlk := builder.AllocateBasicBlock()
builder.SetCurrentBlock(trampolineBlk)
c.emitTryTableLeaves(int(labelIndex))
c.insertJumpToBlock(args, targetBlk)
builder.SetCurrentBlock(current)
targetBlk = trampolineBlk
sealTargetBlk = true
args = ssa.ValuesNil
}
if c.needListener && targetBlk.ReturnBlock() {
current := builder.CurrentBlock()
targetBlk = builder.AllocateBasicBlock()
builder.SetCurrentBlock(targetBlk)
sealTargetBlk = true
c.callListenerAfter()
instr := builder.AllocateInstruction()
instr.AsReturn(args)
builder.InsertInstruction(instr)
args = ssa.ValuesNil
builder.SetCurrentBlock(current)
}
brnz := builder.AllocateInstruction()
brnz.AsBrnz(isNull, args, targetBlk)
builder.InsertInstruction(brnz)
if sealTargetBlk {
builder.Seal(targetBlk)
}
// Fall-through: ref is non-null, push it back.
elseBlk := builder.AllocateBasicBlock()
c.insertJumpToBlock(ssa.ValuesNil, elseBlk)
builder.Seal(elseBlk)
builder.SetCurrentBlock(elseBlk)
state.push(r)
case wasm.OpcodeBrOnNonNull:
labelIndex := c.readI32u()
if state.unreachable {
break
}
r := state.pop()
zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder)
isNonNull := builder.AllocateInstruction().
AsIcmp(r, zero.Return(), ssa.IntegerCmpCondNotEqual).
Insert(builder).Return()
// When non-null, branch to label with args + the non-null ref.
targetBlk, argNum := state.brTargetArgNumFor(labelIndex)
// The branch delivers argNum-1 values from the stack plus the ref.
// The ref is the last value delivered to the label target.
args := c.nPeekDup(argNum - 1)
args = args.Append(builder.VarLengthPool(), r)
var sealTargetBlk bool
if c.branchExitsTryTable(int(labelIndex)) {
current := builder.CurrentBlock()
trampolineBlk := builder.AllocateBasicBlock()
builder.SetCurrentBlock(trampolineBlk)
c.emitTryTableLeaves(int(labelIndex))
c.insertJumpToBlock(args, targetBlk)
builder.SetCurrentBlock(current)
targetBlk = trampolineBlk
sealTargetBlk = true
args = ssa.ValuesNil
}
if c.needListener && targetBlk.ReturnBlock() {
current := builder.CurrentBlock()
targetBlk = builder.AllocateBasicBlock()
builder.SetCurrentBlock(targetBlk)
sealTargetBlk = true
c.callListenerAfter()
instr := builder.AllocateInstruction()
instr.AsReturn(args)
builder.InsertInstruction(instr)
args = ssa.ValuesNil
builder.SetCurrentBlock(current)
}
brnz := builder.AllocateInstruction()
brnz.AsBrnz(isNonNull, args, targetBlk)
builder.InsertInstruction(brnz)
if sealTargetBlk {
builder.Seal(targetBlk)
}
// Fall-through: ref is null, nothing extra pushed.
elseBlk := builder.AllocateBasicBlock()
c.insertJumpToBlock(ssa.ValuesNil, elseBlk)
builder.Seal(elseBlk)
builder.SetCurrentBlock(elseBlk)
case wasm.OpcodeCallRef:
typeIndex := c.readI32u()
if state.unreachable {
break
}
c.lowerCallRef(typeIndex)
case wasm.OpcodeReturnCallRef:
typeIndex := c.readI32u()
if state.unreachable {
break
}
c.emitTryTableLeaves(len(c.state().controlFrames))
c.lowerTailCallReturnCallRef(typeIndex)
state.unreachable = true
default:
panic("TODO: unsupported in wazevo yet: " + wasm.InstructionName(op))
}
@@ -3715,6 +4135,91 @@ func (c *Compiler) lowerTailCallReturnCallIndirect(typeIndex, tableIndex uint32)
c.lowerReturn(builder)
}
func (c *Compiler) prepareCallRef(typeIndex uint32) (ssa.Value, *wasm.FunctionType, ssa.Values) {
builder := c.ssaBuilder
state := c.state()
functionInstancePtr := state.pop()
// Check if it is not the null pointer.
zero := builder.AllocateInstruction()
zero.AsIconst64(0)
builder.InsertInstruction(zero)
checkNull := builder.AllocateInstruction()
checkNull.AsIcmp(functionInstancePtr, zero.Return(), ssa.IntegerCmpCondEqual)
builder.InsertInstruction(checkNull)
exitIfNull := builder.AllocateInstruction()
exitIfNull.AsExitIfTrueWithCode(c.execCtxPtrValue, checkNull.Return(), wazevoapi.ExitCodeNullReference)
builder.InsertInstruction(exitIfNull)
// Load the executable and moduleContextOpaquePtr from the function instance.
loadExecutablePtr := builder.AllocateInstruction()
loadExecutablePtr.AsLoad(functionInstancePtr, wazevoapi.FunctionInstanceExecutableOffset, ssa.TypeI64)
builder.InsertInstruction(loadExecutablePtr)
executablePtr := loadExecutablePtr.Return()
loadModuleContextOpaquePtr := builder.AllocateInstruction()
loadModuleContextOpaquePtr.AsLoad(functionInstancePtr, wazevoapi.FunctionInstanceModuleContextOpaquePtrOffset, ssa.TypeI64)
builder.InsertInstruction(loadModuleContextOpaquePtr)
moduleContextOpaquePtr := loadModuleContextOpaquePtr.Return()
typ := &c.m.TypeSection[typeIndex]
tail := len(state.values) - len(typ.Params)
vs := state.values[tail:]
state.values = state.values[:tail]
args := c.allocateVarLengthValues(2+len(vs), c.execCtxPtrValue, moduleContextOpaquePtr)
args = args.Append(builder.VarLengthPool(), vs...)
c.storeCallerModuleContext()
return executablePtr, typ, args
}
func (c *Compiler) lowerCallRef(typeIndex uint32) {
builder := c.ssaBuilder
state := c.state()
executablePtr, typ, args := c.prepareCallRef(typeIndex)
call := builder.AllocateInstruction()
call.AsCallIndirect(executablePtr, c.signatures[typ], args)
builder.InsertInstruction(call)
first, rest := call.Returns()
if first.Valid() {
state.push(first)
}
for _, v := range rest {
state.push(v)
}
c.reloadAfterCall()
}
func (c *Compiler) lowerTailCallReturnCallRef(typeIndex uint32) {
builder := c.ssaBuilder
state := c.state()
executablePtr, typ, args := c.prepareCallRef(typeIndex)
call := builder.AllocateInstruction()
call.AsTailCallReturnCallIndirect(executablePtr, c.signatures[typ], args)
builder.InsertInstruction(call)
// In a proper tail call, the following code is unreachable since execution
// transfers to the callee. However, sometimes the backend might need to fall back to
// a regular call, so we include return handling and let the backend delete it
// when redundant.
// For details, see internal/engine/RATIONALE.md
first, rest := call.Returns()
if first.Valid() {
state.push(first)
}
for _, v := range rest {
state.push(v)
}
c.reloadAfterCall()
c.lowerReturn(builder)
}
// memOpSetup inserts the bounds check and calculates the address of the memory operation (loads/stores).
func (c *Compiler) memOpSetup(baseAddr ssa.Value, constOffset, operationSizeInBytes uint64) (address ssa.Value) {
address = ssa.ValueInvalid
@@ -4026,6 +4531,185 @@ func (c *Compiler) storeCallerModuleContext() {
builder.InsertInstruction(store)
}
// resolveTagType returns the FunctionType for the tag at the given module-local index.
func (c *Compiler) resolveTagType(tagIndex uint32) *wasm.FunctionType {
if tagIndex < c.m.ImportTagCount {
cur := uint32(0)
for i := range c.m.ImportSection {
imp := &c.m.ImportSection[i]
if imp.Type != wasm.ExternTypeTag {
continue
}
if tagIndex == cur {
return &c.m.TypeSection[imp.DescTag]
}
cur++
}
} else {
tagSectionIdx := tagIndex - c.m.ImportTagCount
if tagSectionIdx < uint32(len(c.m.TagSection)) {
typeIdx := c.m.TagSection[tagSectionIdx].Type
return &c.m.TypeSection[typeIdx]
}
}
return nil
}
// emitThrow emits a call to the shared throw trampoline with the given exnref,
// followed by an unreachable exit (throw never returns).
func (c *Compiler) emitThrow(exnref ssa.Value) {
builder := c.ssaBuilder
throwPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetThrowTrampolineAddress.U32(),
ssa.TypeI64,
).Insert(builder).Return()
throwArgs := c.allocateVarLengthValues(2, c.execCtxPtrValue, exnref)
builder.AllocateInstruction().
AsCallIndirect(throwPtr, &c.throwSig, throwArgs).
Insert(builder)
exit := builder.AllocateInstruction()
exit.AsExitWithCode(c.execCtxPtrValue, wazevoapi.ExitCodeUnreachable)
builder.InsertInstruction(exit)
}
// emitTryTableLeave emits a trampoline call to pop the try handler in the dispatch loop.
func (c *Compiler) emitTryTableLeave() {
builder := c.ssaBuilder
c.storeCallerModuleContext()
leavePtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetTryTableLeaveTrampolineAddress.U32(),
ssa.TypeI64,
).Insert(builder).Return()
args := c.allocateVarLengthValues(1, c.execCtxPtrValue)
builder.AllocateInstruction().
AsCallIndirect(leavePtr, &c.tryTableLeaveSig, args).
Insert(builder)
}
// branchExitsTryTable returns true if a branch to the given depth would
// exit at least one try_table frame.
func (c *Compiler) branchExitsTryTable(depth int) bool {
state := c.state()
tail := len(state.controlFrames) - 1
for i := 0; i < depth; i++ {
if state.controlFrames[tail-i].kind == controlFrameKindTryTable {
return true
}
}
return false
}
// emitTryTableLeaves emits TryTableLeave calls for try_table frames
// that would be exited by a branch to the given depth.
func (c *Compiler) emitTryTableLeaves(depth int) {
state := c.state()
tail := len(state.controlFrames) - 1
for i := 0; i < depth; i++ {
if state.controlFrames[tail-i].kind == controlFrameKindTryTable {
c.emitTryTableLeave()
}
}
}
// catchClause holds a parsed catch clause from a try_table instruction.
type catchClause struct {
kind byte
tagIndex uint32
labelIdx uint32
}
// loadExceptionParams loads the exception params from the caught Exception's
// Params slice. The dispatch loop sets execCtx.exceptionParamsPtr to the
// slice's backing-array pointer after matching a handler. We load that pointer
// and then read each param from [ptr + i*8], mirroring the stores emitted by
// the throw lowering. Float params were bitcast to integers at the throw site,
// so we load as integer and bitcast back to the original type.
func (c *Compiler) loadExceptionParams(tagType *wasm.FunctionType) []ssa.Value {
if len(tagType.Params) == 0 {
return nil
}
builder := c.ssaBuilder
// Load the pointer to the caught Exception's Params backing array.
paramsPtr := builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetExceptionParamsPtr.U32(),
ssa.TypeI64,
).Insert(builder).Return()
var values []ssa.Value
for i, vt := range tagType.Params {
offset := uint32(i) * 8
ssaType := WasmTypeToSSAType(vt)
switch ssaType {
case ssa.TypeF32:
// Stored as i32 at throw site; bitcast back to f32.
raw := builder.AllocateInstruction().
AsLoad(paramsPtr, offset, ssa.TypeI32).
Insert(builder).Return()
val := builder.AllocateInstruction().AsBitcast(raw, ssa.TypeF32).Insert(builder).Return()
values = append(values, val)
case ssa.TypeF64:
// Stored as i64 at throw site; bitcast back to f64.
raw := builder.AllocateInstruction().
AsLoad(paramsPtr, offset, ssa.TypeI64).
Insert(builder).Return()
val := builder.AllocateInstruction().AsBitcast(raw, ssa.TypeF64).Insert(builder).Return()
values = append(values, val)
default:
val := builder.AllocateInstruction().
AsLoad(paramsPtr, offset, ssaType).
Insert(builder).Return()
values = append(values, val)
}
}
return values
}
// loadExnRef loads the exnref (pointer to Exception) from the executionContext.
// The dispatch loop writes it to exceptionPtr after matching a handler.
func (c *Compiler) loadExnRef() ssa.Value {
builder := c.ssaBuilder
return builder.AllocateInstruction().
AsLoad(c.execCtxPtrValue,
wazevoapi.ExecutionContextOffsetExceptionPtr.U32(),
ssa.TypeI64,
).Insert(builder).Return()
}
// skipTryTableCatchClauses advances the bytecode PC past the catch clauses
// of a try_table instruction. This is used both in reachable and unreachable states.
func (c *Compiler) skipTryTableCatchClauses() {
c.loweringState.pc++
catchCount, catchNum, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(catchNum) - 1
for i := uint32(0); i < catchCount; i++ {
c.loweringState.pc++
kind := c.wasmFunctionBody[c.loweringState.pc]
switch kind {
case wasm.CatchKindCatch, wasm.CatchKindCatchRef:
// Read tag index.
c.loweringState.pc++
_, n, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
// Read label index.
c.loweringState.pc++
_, n, _ = leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
case wasm.CatchKindCatchAll, wasm.CatchKindCatchAllRef:
// Read label index.
c.loweringState.pc++
_, n, _ := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc:])
c.loweringState.pc += int(n) - 1
}
}
}
func (c *Compiler) readByte() byte {
v := c.wasmFunctionBody[c.loweringState.pc+1]
c.loweringState.pc++
@@ -4181,6 +4865,7 @@ func (c *Compiler) lowerBrTable(labels []uint32, index ssa.Value) {
targetBlk, _ := state.brTargetArgNumFor(l)
trampoline := builder.AllocateBasicBlock()
builder.SetCurrentBlock(trampoline)
c.emitTryTableLeaves(int(l))
c.insertJumpToBlock(args, targetBlk)
trampolineBlockIDs = trampolineBlockIDs.Append(builder.VarLengthPool(), ssa.Value(trampoline.ID()))
}
@@ -1,13 +1,17 @@
package wazevo
import (
"context"
"encoding/binary"
"fmt"
"unsafe"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/experimental"
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
"github.com/tetratelabs/wazero/internal/internalapi"
"github.com/tetratelabs/wazero/internal/wasm"
"github.com/tetratelabs/wazero/internal/wasmdebug"
"github.com/tetratelabs/wazero/internal/wasmruntime"
)
@@ -132,6 +136,14 @@ func (m *moduleEngine) setupOpaque() {
}
}
if tagOffset := offsets.TagsBegin; tagOffset >= 0 {
for _, tag := range inst.Tags {
binary.LittleEndian.PutUint64(opaque[tagOffset:],
uint64(uintptr(unsafe.Pointer(tag))))
tagOffset += 8
}
}
if beforeListenerOffset := offsets.BeforeListenerTrampolines1stElement; beforeListenerOffset >= 0 {
binary.LittleEndian.PutUint64(opaque[beforeListenerOffset:], uint64(uintptr(unsafe.Pointer(&m.parent.listenerBeforeTrampolines[0]))))
}
@@ -160,6 +172,21 @@ func (m *moduleEngine) NewFunction(index wasm.Index) api.Function {
localIndex -= importedFnCount
}
if source := m.module.Source; source.IsHostModule {
// For host modules, we need to look up the GoFunction from the CodeSection.
def := source.FunctionDefinition(localIndex)
goF := source.CodeSection[localIndex].GoFunc
switch typed := goF.(type) {
case api.GoFunction:
// GoFunction doesn't need looked up module.
return &hostFunction{def: def, g: goFunctionAsGoModuleFunction(typed)}
case api.GoModuleFunction:
return &hostFunction{def: def, lookedUpModule: m.module, g: typed}
default:
panic(fmt.Sprintf("unexpected GoFunc type: %T", goF))
}
}
src := m.module.Source
typIndex := src.FunctionSection[localIndex]
typ := src.TypeSection[typIndex]
@@ -189,6 +216,10 @@ func (m *moduleEngine) NewFunction(index wasm.Index) api.Function {
ce.execCtx.memoryWait32TrampolineAddress = sharedFunctions.memoryWait32Address
ce.execCtx.memoryWait64TrampolineAddress = sharedFunctions.memoryWait64Address
ce.execCtx.memoryNotifyTrampolineAddress = sharedFunctions.memoryNotifyAddress
ce.execCtx.throwAllocTrampolineAddress = sharedFunctions.throwAllocTrampolineAddress
ce.execCtx.throwTrampolineAddress = sharedFunctions.throwTrampolineAddress
ce.execCtx.tryTableEnterTrampolineAddress = sharedFunctions.tryTableEnterAddress
ce.execCtx.tryTableLeaveTrampolineAddress = sharedFunctions.tryTableLeaveAddress
ce.execCtx.memmoveAddress = memmovPtr
ce.init()
return ce
@@ -242,15 +273,13 @@ func (m *moduleEngine) ResolveImportedFunction(index, descFunc, indexInImportedM
executableOffset, moduleCtxOffset, typeIDOffset := m.parent.offsets.ImportedFunctionOffset(index)
importedME := importedModuleEngine.(*moduleEngine)
if int(indexInImportedModule) >= len(importedME.importedFunctions) {
indexInImportedModule -= wasm.Index(len(importedME.importedFunctions))
} else {
if int(indexInImportedModule) < len(importedME.importedFunctions) {
imported := &importedME.importedFunctions[indexInImportedModule]
m.ResolveImportedFunction(index, descFunc, imported.indexInModule, imported.me)
return // Recursively resolve the imported function.
}
offset := importedME.parent.functionOffsets[indexInImportedModule]
offset := importedME.parent.functionOffsets[indexInImportedModule-wasm.Index(len(importedME.importedFunctions))]
typeID := m.module.TypeIDs[descFunc]
executable := &importedME.parent.executable[offset]
// Write functionInstance.
@@ -330,3 +359,45 @@ func (m *moduleEngine) LookupFunction(t *wasm.TableInstance, typeId wasm.Functio
func moduleInstanceFromOpaquePtr(ptr *byte) *wasm.ModuleInstance {
return *(**wasm.ModuleInstance)(unsafe.Pointer(ptr))
}
type hostFunction struct {
internalapi.WazeroOnly
def *wasm.FunctionDefinition
lookedUpModule *wasm.ModuleInstance
g api.GoModuleFunction
}
// goFunctionAsGoModuleFunction converts api.GoFunction to api.GoModuleFunction which ignores the api.Module argument.
func goFunctionAsGoModuleFunction(g api.GoFunction) api.GoModuleFunction {
return api.GoModuleFunc(func(ctx context.Context, _ api.Module, stack []uint64) {
g.Call(ctx, stack)
})
}
// Definition implements api.Function.
func (f *hostFunction) Definition() api.FunctionDefinition { return f.def }
// Call implements api.Function.
func (f *hostFunction) Call(ctx context.Context, params ...uint64) ([]uint64, error) {
typ := f.def.Functype
stackSize := typ.ParamNumInUint64
rn := typ.ResultNumInUint64
if rn > stackSize {
stackSize = rn
}
stack := make([]uint64, stackSize)
copy(stack, params)
return stack[:rn], f.CallWithStack(ctx, stack)
}
// CallWithStack implements api.Function.
func (f *hostFunction) CallWithStack(ctx context.Context, stack []uint64) (err error) {
defer func() {
if r := recover(); r != nil {
builder := wasmdebug.NewErrorBuilder()
err = builder.FromRecovered(r)
}
}()
f.g.Call(ctx, f.lookedUpModule, stack)
return nil
}
@@ -291,9 +291,9 @@ func (bb *basicBlock) addPred(blk BasicBlock, branch *Instruction) {
for i := range bb.preds {
existingPred := &bb.preds[i]
if existingPred.blk == pred && existingPred.branch != branch {
// If the target is already added, then this must come from the same BrTable,
// If the target is already added, then this must come from the same BrTable or TryTableDispatch,
// otherwise such redundant branch should be eliminated by the frontend. (which should be simpler).
panic(fmt.Sprintf("BUG: redundant non BrTable jumps in %s whose targes are the same", bb.Name()))
panic(fmt.Sprintf("BUG: redundant non BrTable/TryTableDispatch jumps in %s whose targets are the same", bb.Name()))
}
}
@@ -344,19 +344,21 @@ func (bb *basicBlock) validate(b *builder) {
}
}
var exp int
if bb.ReturnBlock() {
exp = len(b.currentSignature.Results)
} else {
exp = len(bb.params.View())
}
if pred.branch.opcode != OpcodeBrTable {
var exp int
if bb.ReturnBlock() {
exp = len(b.currentSignature.Results)
} else {
exp = len(bb.params.View())
}
if len(pred.branch.vs.View()) != exp {
panic(fmt.Sprintf(
"BUG: len(argument at %s) != len(params at %s): %d != %d: %s",
pred.blk.Name(), bb.Name(),
len(pred.branch.vs.View()), len(bb.params.View()), pred.branch.Format(b),
))
if len(pred.branch.vs.View()) != exp {
panic(fmt.Sprintf(
"BUG: len(argument at %s) != len(params at %s): %d != %d: %s",
pred.blk.Name(), bb.Name(),
len(pred.branch.vs.View()), len(bb.params.View()), pred.branch.Format(b),
))
}
}
}
@@ -30,6 +30,25 @@ const (
ExitCodeMemoryWait64
ExitCodeMemoryNotify
ExitCodeUnalignedAtomic
// ExitCodeThrowAlloc is the first phase of wasm throw: Go allocates the
// Exception heap object (with Params sized to the tag's param count) and
// writes its Params data pointer to execCtx.exceptionParamsPtr.
// Compiled code then stores params directly into the Exception.Params slice,
// followed by ExitCodeThrow to search for a matching handler.
ExitCodeThrowAlloc
// ExitCodeThrow is the shared throw/throw_ref exit code.
// The exnref is passed on the stack. The handler searches for a
// matching catch clause and restores the stack checkpoint.
ExitCodeThrow
// ExitCodeNullReference is an exit code for a null reference trap (throw_ref with null exnref).
ExitCodeNullReference
// ExitCodeTryTableEnter is an exit code for entering a try_table block.
// The catch clause info is encoded in the upper bits. The dispatch loop
// saves the current SP/FP/returnAddress as a try handler checkpoint.
ExitCodeTryTableEnter
// ExitCodeTryTableLeave is an exit code for leaving a try_table block.
// The dispatch loop pops the most recent try handler.
ExitCodeTryTableLeave
exitCodeMax
)
@@ -86,6 +105,16 @@ func (e ExitCode) String() string {
return "memory_wait64"
case ExitCodeMemoryNotify:
return "memory_notify"
case ExitCodeThrowAlloc:
return "throw_alloc"
case ExitCodeThrow:
return "throw"
case ExitCodeNullReference:
return "null_reference"
case ExitCodeTryTableEnter:
return "try_table_enter"
case ExitCodeTryTableLeave:
return "try_table_leave"
}
panic("TODO")
}
@@ -107,3 +136,15 @@ func ExitCodeCallGoFunctionWithIndex(index int, withListener bool) ExitCode {
func GoFunctionIndexFromExitCode(exitCode ExitCode) int {
return int(exitCode >> 8)
}
// TryTableIDFromExitCode extracts the try-table ID from an ExitCodeTryTableEnter
// exit code. Uses the same encoding as GoFunctionIndexFromExitCode (upper 24 bits).
func TryTableIDFromExitCode(exitCode ExitCode) int {
return GoFunctionIndexFromExitCode(exitCode)
}
// CatchClauseInstance is a runtime catch clause with resolved tag index.
type CatchClauseInstance struct {
Kind byte // wasm.CatchKindCatch, etc.
TagIndex uint32 // module-local tag index
}
@@ -53,6 +53,23 @@ const (
ExecutionContextOffsetMemoryWait32TrampolineAddress Offset = 1160
ExecutionContextOffsetMemoryWait64TrampolineAddress Offset = 1168
ExecutionContextOffsetMemoryNotifyTrampolineAddress Offset = 1176
// ExecutionContextOffsetThrowAllocTrampolineAddress is the address of the
// throw-alloc trampoline, which allocates the Exception heap object,
// sets exceptionParamsPtr, and returns the exnref.
ExecutionContextOffsetThrowAllocTrampolineAddress Offset = 1184
ExecutionContextOffsetThrowTrampolineAddress Offset = 1192
ExecutionContextOffsetTryTableEnterTrampolineAddress Offset = 1200
ExecutionContextOffsetTryTableLeaveTrampolineAddress Offset = 1208
// ExecutionContextOffsetExceptionPtr holds the pointer to the Exception struct,
// used on the throw side and by catch_ref/catch_all_ref handlers.
ExecutionContextOffsetExceptionPtr Offset = 1216
// ExecutionContextOffsetExceptionParamsPtr points into the Exception's
// Params slice backing array. Used by both throw (store params) and
// catch (load params) sides.
ExecutionContextOffsetExceptionParamsPtr Offset = 1224
// ExecutionContextOffsetCaughtExceptionClauseIdx is the matched catch clause index
// written by handleException and read by compiled handler dispatch code.
ExecutionContextOffsetCaughtExceptionClauseIdx Offset = 1232
)
// ModuleContextOffsetData allows the compilers to get the information about offsets to the fields of wazevo.moduleContextOpaque,
@@ -66,6 +83,7 @@ type ModuleContextOffsetData struct {
GlobalsBegin,
TypeIDs1stElement,
TablesBegin,
TagsBegin,
BeforeListenerTrampolines1stElement,
AfterListenerTrampolines1stElement,
DataInstances1stElement,
@@ -122,6 +140,11 @@ func (m *ModuleContextOffsetData) TableOffset(tableIndex int) Offset {
return m.TablesBegin + Offset(tableIndex)*8
}
// TagOffset returns an offset of the i-th tag instance pointer.
func (m *ModuleContextOffsetData) TagOffset(tagIndex int) Offset {
return m.TagsBegin + Offset(tagIndex)*8
}
// NewModuleContextOffsetData creates a ModuleContextOffsetData determining the structure of moduleContextOpaque for the given Module.
// The structure is described in the comment of wazevo.moduleContextOpaque.
func NewModuleContextOffsetData(m *wasm.Module, withListener bool) ModuleContextOffsetData {
@@ -185,6 +208,15 @@ func NewModuleContextOffsetData(m *wasm.Module, withListener bool) ModuleContext
ret.TablesBegin = -1
}
if tags := int(m.ImportTagCount) + len(m.TagSection); tags > 0 {
offset = align8(offset)
ret.TagsBegin = offset
// Pointers to *wasm.TagInstance.
offset += Offset(tags) * 8
} else {
ret.TagsBegin = -1
}
if withListener {
offset = align8(offset)
ret.BeforeListenerTrampolines1stElement = offset