working commit

2026-03-13 19:02:42 +02:00
parent bebbf79c7a
commit 5c1da77f4c
1329 changed files with 314708 additions and 39 deletions
@@ -0,0 +1,26 @@
+package platform
+
+// CpuFeatureFlags exposes methods for querying CPU capabilities
+type CpuFeatureFlags uint64
+
+const (
+	// CpuFeatureAmd64SSE4_1 is the flag to query CpuFeatureFlags.Has for SSEv4.1 capabilities on amd64
+	CpuFeatureAmd64SSE4_1 = 1 << iota
+	// CpuFeatureAmd64BMI1 is the flag to query CpuFeatureFlags.Has for Bit Manipulation Instruction Set 1 (e.g. TZCNT) on amd64
+	CpuFeatureAmd64BMI1
+	// CpuExtraFeatureABM is the flag to query CpuFeatureFlags.Has for Advanced Bit Manipulation capabilities (e.g. LZCNT) on amd64
+	CpuFeatureAmd64ABM
+)
+
+const (
+	// CpuFeatureArm64Atomic is the flag to query CpuFeatureFlags.Has for Large System Extensions capabilities on arm64
+	CpuFeatureArm64Atomic CpuFeatureFlags = 1 << iota
+)
+
+func (c CpuFeatureFlags) Has(f CpuFeatureFlags) bool {
+	return c&f != 0
+}
+
+func (c CpuFeatureFlags) Raw() uint64 {
+	return uint64(c)
+}
@@ -0,0 +1,22 @@
+package platform
+
+import "golang.org/x/sys/cpu"
+
+// CpuFeatures exposes the capabilities for this CPU, queried via the Has method.
+var CpuFeatures = loadCpuFeatureFlags()
+
+func loadCpuFeatureFlags() (flags CpuFeatureFlags) {
+	if cpu.X86.HasSSE41 {
+		flags |= CpuFeatureAmd64SSE4_1
+	}
+	if cpu.X86.HasBMI1 {
+		flags |= CpuFeatureAmd64BMI1
+	}
+	// x/sys/cpu does not track the ABM explicitly.
+	// LZCNT combined with BMI1 and BMI2 completes the expanded ABM instruction set.
+	// Intel includes LZCNT in BMI1, and all AMD CPUs with POPCNT also have LZCNT.
+	if cpu.X86.HasBMI1 && cpu.X86.HasBMI2 && cpu.X86.HasPOPCNT {
+		flags |= CpuFeatureAmd64ABM
+	}
+	return
+}
@@ -0,0 +1,28 @@
+package platform
+
+import (
+	"runtime"
+
+	"golang.org/x/sys/cpu"
+)
+
+// CpuFeatures exposes the capabilities for this CPU, queried via the Has method.
+var CpuFeatures = loadCpuFeatureFlags()
+
+func loadCpuFeatureFlags() (flags CpuFeatureFlags) {
+	switch runtime.GOOS {
+	case "darwin", "windows":
+		// These OSes do not allow userland to read the instruction set attribute registers,
+		// but basically require atomic instructions:
+		// - "darwin" is the desktop version (mobile version is "ios"),
+		//   and the M1 is a ARMv8.4.
+		// - "windows" requires them from Windows 11, see page 12
+		//   https://download.microsoft.com/download/7/8/8/788bf5ab-0751-4928-a22c-dffdc23c27f2/Minimum%20Hardware%20Requirements%20for%20Windows%2011.pdf
+		flags |= CpuFeatureArm64Atomic
+	default:
+		if cpu.ARM64.HasATOMICS {
+			flags |= CpuFeatureArm64Atomic
+		}
+	}
+	return
+}
@@ -0,0 +1,5 @@
+//go:build !(amd64 || arm64)
+
+package platform
+
+const CpuFeatures CpuFeatureFlags = 0
@@ -0,0 +1,17 @@
+package platform
+
+import (
+	"io"
+	"math/rand"
+)
+
+// seed is a fixed seed value for NewFakeRandSource.
+//
+// Trivia: While arbitrary, 42 was chosen as it is the "Ultimate Answer" in
+// the Douglas Adams novel "The Hitchhiker's Guide to the Galaxy."
+const seed = int64(42)
+
+// NewFakeRandSource returns a deterministic source of random values.
+func NewFakeRandSource() io.Reader {
+	return rand.New(rand.NewSource(seed))
+}
@@ -0,0 +1,72 @@
+package platform
+
+import (
+	"math/bits"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sys/unix"
+)
+
+var hugePagesConfigs []hugePagesConfig
+
+type hugePagesConfig struct {
+	size int
+	flag int
+}
+
+func (hpc *hugePagesConfig) match(size int) bool {
+	return (size & (hpc.size - 1)) == 0
+}
+
+func init() {
+	dirents, err := os.ReadDir("/sys/kernel/mm/hugepages/")
+	if err != nil {
+		return
+	}
+
+	for _, dirent := range dirents {
+		name := dirent.Name()
+		if !strings.HasPrefix(name, "hugepages-") {
+			continue
+		}
+		if !strings.HasSuffix(name, "kB") {
+			continue
+		}
+		n, err := strconv.ParseUint(name[10:len(name)-2], 10, 64)
+		if err != nil {
+			continue
+		}
+		if bits.OnesCount64(n) != 1 {
+			continue
+		}
+		n *= 1024
+		hugePagesConfigs = append(hugePagesConfigs, hugePagesConfig{
+			size: int(n),
+			flag: int(bits.TrailingZeros64(n)<<unix.MAP_HUGE_SHIFT) | unix.MAP_HUGETLB,
+		})
+	}
+
+	sort.Slice(hugePagesConfigs, func(i, j int) bool {
+		return hugePagesConfigs[i].size > hugePagesConfigs[j].size
+	})
+}
+
+func mmapCodeSegment(size int) ([]byte, error) {
+	flag := unix.MAP_ANON | unix.MAP_PRIVATE
+	prot := unix.PROT_READ | unix.PROT_WRITE
+
+	for _, hugePagesConfig := range hugePagesConfigs {
+		if hugePagesConfig.match(size) {
+			b, err := unix.Mmap(-1, 0, size, prot, flag|hugePagesConfig.flag)
+			if err != nil {
+				continue
+			}
+			return b, nil
+		}
+	}
+
+	return unix.Mmap(-1, 0, size, prot, flag)
+}
@@ -0,0 +1,19 @@
+// Separated from linux which has support for huge pages.
+
+//go:build unix && !linux
+
+package platform
+
+import "golang.org/x/sys/unix"
+
+func mmapCodeSegment(size int) ([]byte, error) {
+	return unix.Mmap(
+		-1,
+		0,
+		size,
+		unix.PROT_READ|unix.PROT_WRITE,
+		// Anonymous as this is not an actual file, but a memory,
+		// Private as this is in-process memory region.
+		unix.MAP_ANON|unix.MAP_PRIVATE,
+	)
+}
@@ -0,0 +1,14 @@
+//go:build unix
+
+package platform
+
+import "golang.org/x/sys/unix"
+
+func munmapCodeSegment(code []byte) error {
+	return unix.Munmap(code)
+}
+
+// MprotectCodeSegment is like unix.Mprotect with RX permission.
+func MprotectCodeSegment(b []byte) (err error) {
+	return unix.Mprotect(b, unix.PROT_READ|unix.PROT_EXEC)
+}
@@ -0,0 +1,22 @@
+//go:build !(unix || windows)
+
+package platform
+
+import (
+	"fmt"
+	"runtime"
+)
+
+var errUnsupported = fmt.Errorf("mmap unsupported on GOOS=%s. Use interpreter instead.", runtime.GOOS)
+
+func munmapCodeSegment(code []byte) error {
+	panic(errUnsupported)
+}
+
+func mmapCodeSegment(size int) ([]byte, error) {
+	panic(errUnsupported)
+}
+
+func MprotectCodeSegment(b []byte) (err error) {
+	panic(errUnsupported)
+}
@@ -0,0 +1,31 @@
+package platform
+
+import (
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+func munmapCodeSegment(code []byte) error {
+	address := unsafe.Pointer(&code[0])
+	size := uintptr(0) // size must be 0 because we're using MEM_RELEASE.
+	return windows.VirtualFree(uintptr(address), size, windows.MEM_RELEASE)
+}
+
+func mmapCodeSegment(size int) ([]byte, error) {
+	address := uintptr(0) // system determines where to allocate the region.
+	p, err := windows.VirtualAlloc(address, uintptr(size),
+		windows.MEM_COMMIT, windows.PAGE_READWRITE)
+	if err != nil {
+		return nil, err
+	}
+
+	return unsafe.Slice((*byte)(unsafe.Pointer(p)), size), nil
+}
+
+var old = uint32(windows.PAGE_READWRITE)
+
+func MprotectCodeSegment(b []byte) (err error) {
+	address := unsafe.Pointer(&b[0])
+	return windows.VirtualProtect(uintptr(address), uintptr(len(b)), windows.PAGE_EXECUTE_READ, &old)
+}
@@ -0,0 +1,6 @@
+//go:build !windows
+
+package platform
+
+// ToPosixPath returns the input, as only windows might return backslashes.
+func ToPosixPath(in string) string { return in }
@@ -0,0 +1,17 @@
+package platform
+
+import "strings"
+
+// ToPosixPath returns the input, converting any backslashes to forward ones.
+func ToPosixPath(in string) string {
+	// strings.Map only allocates on change, which is good enough especially as
+	// path.Join uses forward slash even on windows.
+	return strings.Map(windowsToPosixSeparator, in)
+}
+
+func windowsToPosixSeparator(r rune) rune {
+	if r == '\\' {
+		return '/'
+	}
+	return r
+}
@@ -0,0 +1,49 @@
+// Package platform includes runtime-specific code needed for the compiler or otherwise.
+package platform
+
+import (
+	"runtime"
+
+	"github.com/tetratelabs/wazero/api"
+	"github.com/tetratelabs/wazero/experimental"
+)
+
+// CompilerSupported includes constraints here and also the assembler.
+func CompilerSupported() bool {
+	return CompilerSupports(api.CoreFeaturesV2)
+}
+
+func CompilerSupports(features api.CoreFeatures) bool {
+	switch runtime.GOOS {
+	case "linux", "darwin", "freebsd", "netbsd", "windows":
+		if runtime.GOARCH == "arm64" {
+			if features.IsEnabled(experimental.CoreFeaturesThreads) {
+				return CpuFeatures.Has(CpuFeatureArm64Atomic)
+			}
+			return true
+		}
+		fallthrough
+	case "dragonfly", "solaris", "illumos":
+		return runtime.GOARCH == "amd64" && CpuFeatures.Has(CpuFeatureAmd64SSE4_1)
+	default:
+		return false
+	}
+}
+
+// MmapCodeSegment copies the code into the executable region and returns the byte slice of the region.
+//
+// See https://man7.org/linux/man-pages/man2/mmap.2.html for mmap API and flags.
+func MmapCodeSegment(size int) ([]byte, error) {
+	if size == 0 {
+		panic("BUG: MmapCodeSegment with zero length")
+	}
+	return mmapCodeSegment(size)
+}
+
+// MunmapCodeSegment unmaps the given memory region.
+func MunmapCodeSegment(code []byte) error {
+	if len(code) == 0 {
+		panic("BUG: MunmapCodeSegment with zero length")
+	}
+	return munmapCodeSegment(code)
+}
@@ -0,0 +1,76 @@
+package platform
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/tetratelabs/wazero/sys"
+)
+
+const (
+	ms = int64(time.Millisecond)
+	// FakeEpochNanos is midnight UTC 2022-01-01 and exposed for testing
+	FakeEpochNanos = 1640995200000 * ms
+)
+
+// NewFakeWalltime implements sys.Walltime with FakeEpochNanos that increases by 1ms each reading.
+// See /RATIONALE.md
+func NewFakeWalltime() sys.Walltime {
+	// AddInt64 returns the new value. Adjust so the first reading will be FakeEpochNanos
+	t := FakeEpochNanos - ms
+	return func() (sec int64, nsec int32) {
+		wt := atomic.AddInt64(&t, ms)
+		return wt / 1e9, int32(wt % 1e9)
+	}
+}
+
+// NewFakeNanotime implements sys.Nanotime that increases by 1ms each reading.
+// See /RATIONALE.md
+func NewFakeNanotime() sys.Nanotime {
+	// AddInt64 returns the new value. Adjust so the first reading will be zero.
+	t := int64(0) - ms
+	return func() int64 {
+		return atomic.AddInt64(&t, ms)
+	}
+}
+
+// FakeNanosleep implements sys.Nanosleep by returning without sleeping.
+var FakeNanosleep = sys.Nanosleep(func(int64) {})
+
+// FakeOsyield implements sys.Osyield by returning without yielding.
+var FakeOsyield = sys.Osyield(func() {})
+
+// Walltime implements sys.Walltime with time.Now.
+//
+// Note: This is only notably less efficient than it could be is reading
+// runtime.walltime(). time.Now defensively reads nanotime also, just in case
+// time.Since is used. This doubles the performance impact. However, wall time
+// is likely to be read less frequently than Nanotime. Also, doubling the cost
+// matters less on fast platforms that can return both in <=100ns.
+func Walltime() (sec int64, nsec int32) {
+	t := time.Now()
+	return t.Unix(), int32(t.Nanosecond())
+}
+
+// nanoBase uses time.Now to ensure a monotonic clock reading on all platforms
+// via time.Since.
+var nanoBase = time.Now()
+
+// nanotimePortable implements sys.Nanotime with time.Since.
+//
+// Note: This is less efficient than it could be is reading runtime.nanotime(),
+// Just to do that requires CGO.
+func nanotimePortable() int64 {
+	return time.Since(nanoBase).Nanoseconds()
+}
+
+// Nanotime implements sys.Nanotime with runtime.nanotime() if CGO is available
+// and time.Since if not.
+func Nanotime() int64 {
+	return nanotime()
+}
+
+// Nanosleep implements sys.Nanosleep with time.Sleep.
+func Nanosleep(ns int64) {
+	time.Sleep(time.Duration(ns))
+}
@@ -0,0 +1,11 @@
+//go:build !windows
+
+package platform
+
+import _ "unsafe" // for go:linkname
+
+// nanotime uses runtime.nanotime as it is available on all platforms and
+// benchmarks faster than using time.Since.
+//
+//go:linkname nanotime runtime.nanotime
+func nanotime() int64
@@ -0,0 +1,40 @@
+//go:build windows
+
+package platform
+
+import (
+	"math/bits"
+	"syscall"
+	"time"
+	"unsafe"
+)
+
+var (
+	kernel32                   = syscall.NewLazyDLL("kernel32.dll")
+	_QueryPerformanceCounter   = kernel32.NewProc("QueryPerformanceCounter")
+	_QueryPerformanceFrequency = kernel32.NewProc("QueryPerformanceFrequency")
+)
+
+var qpcfreq uint64
+
+func init() {
+	_, _, _ = _QueryPerformanceFrequency.Call(uintptr(unsafe.Pointer(&qpcfreq)))
+}
+
+// On Windows, time.Time handled in time package cannot have the nanosecond precision.
+// The reason is that by default, it doesn't use QueryPerformanceCounter[1], but instead, use "interrupt time"
+// which doesn't support nanoseconds precision (though it is a monotonic) [2, 3].
+//
+// [1] https://learn.microsoft.com/en-us/windows/win32/api/profileapi/nf-profileapi-queryperformancecounter
+// [2] https://github.com/golang/go/blob/go1.24.0/src/runtime/sys_windows_amd64.s#L279-L284
+// [3] https://github.com/golang/go/blob/go1.24.0/src/runtime/time_windows.h#L7-L13
+//
+// Therefore, on Windows, we directly invoke the syscall for QPC instead of time.Now or runtime.nanotime.
+// See https://github.com/golang/go/issues/31160 for example.
+func nanotime() int64 {
+	var counter uint64
+	_, _, _ = _QueryPerformanceCounter.Call(uintptr(unsafe.Pointer(&counter)))
+	hi, lo := bits.Mul64(counter, uint64(time.Second))
+	nanos, _ := bits.Div64(hi, lo, qpcfreq)
+	return int64(nanos)
+}