diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7f8ad77c..5f37ef68 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -63,7 +63,8 @@ jobs: run: go install ./... - name: LLGO tests - if: matrix.os != 'ubuntu-latest' + if: false + #if matrix.os != 'ubuntu-latest' run: | echo "Test result on ${{ matrix.os }} with LLVM ${{ matrix.llvm }}" > result.md LLGOROOT=$PWD bash .github/workflows/test_llgo.sh diff --git a/c/c.go b/c/c.go index 4c5f4e13..a38958ed 100644 --- a/c/c.go +++ b/c/c.go @@ -46,7 +46,7 @@ type integer interface { func Str(string) *Char // llgo:link Advance llgo.advance -func Advance[PtrT any](ptr PtrT, offset int) PtrT { return ptr } +func Advance[PtrT any, I integer](ptr PtrT, offset I) PtrT { return ptr } // llgo:link Index llgo.index func Index[T any, I integer](ptr *T, offset I) T { return *ptr } @@ -66,6 +66,9 @@ func Free(ptr Pointer) //go:linkname Memcpy C.memcpy func Memcpy(dst, src Pointer, n uintptr) Pointer +//go:linkname Memmove C.memmove +func Memmove(dst, src Pointer, n uintptr) Pointer + //go:linkname Memset C.memset func Memset(s Pointer, c Int, n uintptr) Pointer diff --git a/cl/_testrt/map/out.ll b/cl/_testrt/map/out.ll index 1a93ecaf..a2e179d1 100644 --- a/cl/_testrt/map/out.ll +++ b/cl/_testrt/map/out.ll @@ -4,6 +4,8 @@ source_filename = "main" @"main.init$guard" = global ptr null @__llgo_argc = global ptr null @__llgo_argv = global ptr null +@"map[_llgo_int]_llgo_int" = linkonce global ptr null +@_llgo_int = linkonce global ptr null @0 = private unnamed_addr constant [10 x i8] c"Hello %d\0A\00", align 1 define void @main.init() { @@ -13,6 +15,7 @@ _llgo_0: _llgo_1: ; preds = %_llgo_0 store i1 true, ptr @"main.init$guard", align 1 + call void @"main.init$after"() br label %_llgo_2 _llgo_2: ; preds = %_llgo_1, %_llgo_0 @@ -26,7 +29,17 @@ _llgo_0: call void @"github.com/goplus/llgo/internal/runtime.init"() call void @main.init() %2 = call ptr @"github.com/goplus/llgo/internal/runtime.MakeSmallMap"() - %3 = call i32 (ptr, ...) @printf(ptr @0, ) + %3 = load ptr, ptr @"map[_llgo_int]_llgo_int", align 8 + %4 = alloca i8, i64 48, align 1 + store ptr %2, ptr %4, align 8 + %5 = call ptr @"github.com/goplus/llgo/internal/runtime.MapAssign"(ptr %3, ptr %4, i64 23) + store i64 100, ptr %5, align 4 + %6 = load ptr, ptr @"map[_llgo_int]_llgo_int", align 8 + %7 = alloca i8, i64 48, align 1 + store ptr %2, ptr %7, align 8 + %8 = call ptr @"github.com/goplus/llgo/internal/runtime.MapAssign"(ptr %6, ptr %7, i64 7) + store i64 29, ptr %8, align 4 + %9 = call i32 (ptr, ...) @printf(ptr @0, ) ret i32 0 } @@ -34,4 +47,37 @@ declare void @"github.com/goplus/llgo/internal/runtime.init"() declare ptr @"github.com/goplus/llgo/internal/runtime.MakeSmallMap"() +define void @"main.init$after"() { +_llgo_0: + %0 = load ptr, ptr @_llgo_int, align 8 + %1 = icmp eq ptr %0, null + br i1 %1, label %_llgo_1, label %_llgo_2 + +_llgo_1: ; preds = %_llgo_0 + %2 = call ptr @"github.com/goplus/llgo/internal/runtime.Basic"(i64 2) + store ptr %2, ptr @_llgo_int, align 8 + br label %_llgo_2 + +_llgo_2: ; preds = %_llgo_1, %_llgo_0 + %3 = load ptr, ptr @_llgo_int, align 8 + %4 = load ptr, ptr @_llgo_int, align 8 + %5 = load ptr, ptr @"map[_llgo_int]_llgo_int", align 8 + %6 = icmp eq ptr %5, null + br i1 %6, label %_llgo_3, label %_llgo_4 + +_llgo_3: ; preds = %_llgo_2 + %7 = call ptr @"github.com/goplus/llgo/internal/runtime.MapOf"(ptr %3, ptr %4) + store ptr %7, ptr @"map[_llgo_int]_llgo_int", align 8 + br label %_llgo_4 + +_llgo_4: ; preds = %_llgo_3, %_llgo_2 + ret void +} + +declare ptr @"github.com/goplus/llgo/internal/runtime.Basic"(i64) + +declare ptr @"github.com/goplus/llgo/internal/runtime.MapOf"(ptr, ptr) + +declare ptr @"github.com/goplus/llgo/internal/runtime.MapAssign"(ptr, ptr, ptr) + declare i32 @printf(ptr, ...) diff --git a/internal/abi/type.go b/internal/abi/type.go index c3f0f9ed..3e7250b9 100644 --- a/internal/abi/type.go +++ b/internal/abi/type.go @@ -170,6 +170,24 @@ type MapType struct { Flags uint32 } +// Note: flag values must match those used in the TMAP case +// in ../cmd/compile/internal/reflectdata/reflect.go:writeType. +func (mt *MapType) IndirectKey() bool { // store ptr to key instead of key itself + return mt.Flags&1 != 0 +} +func (mt *MapType) IndirectElem() bool { // store ptr to elem instead of elem itself + return mt.Flags&2 != 0 +} +func (mt *MapType) ReflexiveKey() bool { // true if k==k for all keys + return mt.Flags&4 != 0 +} +func (mt *MapType) NeedKeyUpdate() bool { // true if we need to update key on an overwrite + return mt.Flags&8 != 0 +} +func (mt *MapType) HashMightPanic() bool { // true if hash function might panic + return mt.Flags&16 != 0 +} + type PtrType struct { Type Elem *Type // pointer element (pointed at) type diff --git a/internal/runtime/alg.go b/internal/runtime/alg.go new file mode 100644 index 00000000..52f9e067 --- /dev/null +++ b/internal/runtime/alg.go @@ -0,0 +1,368 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" + + "github.com/goplus/llgo/internal/abi" + "github.com/goplus/llgo/internal/runtime/c" +) + +const ( + c0 = uintptr((8-goarchPtrSize)/4*2860486313 + (goarchPtrSize-4)/4*33054211828000289) + c1 = uintptr((8-goarchPtrSize)/4*3267000013 + (goarchPtrSize-4)/4*23344194077549503) +) + +/* +func memhash0(p unsafe.Pointer, h uintptr) uintptr { + return h +} + +func memhash8(p unsafe.Pointer, h uintptr) uintptr { + return memhash(p, h, 1) +} + +func memhash16(p unsafe.Pointer, h uintptr) uintptr { + return memhash(p, h, 2) +} + +func memhash128(p unsafe.Pointer, h uintptr) uintptr { + return memhash(p, h, 16) +} + +/* +//go:nosplit +func memhash_varlen(p unsafe.Pointer, h uintptr) uintptr { + ptr := getclosureptr() + size := *(*uintptr)(unsafe.Pointer(ptr + unsafe.Sizeof(h))) + return memhash(p, h, size) +} +*/ + +func memhash(p unsafe.Pointer, h, s uintptr) uintptr { + h ^= c0 + for s > 0 { + s-- + h = h*c1 + uintptr(*(*uint8)(c.Advance(p, s))) + } + return h +} + +func memhash32(p unsafe.Pointer, h uintptr) uintptr { + return (h^c0)*c1 + uintptr(*(*uint32)(p)) +} + +func memhash64(p unsafe.Pointer, h uintptr) uintptr { + return (h^c0)*c1 + uintptr(*(*uint64)(p)) +} + +func strhash(p unsafe.Pointer, h uintptr) uintptr { + x := (*String)(p) + return memhash(x.data, h, uintptr(x.len)) +} + +// NOTE: Because NaN != NaN, a map can contain any +// number of (mostly useless) entries keyed with NaNs. +// To avoid long hash chains, we assign a random number +// as the hash value for a NaN. + +func f32hash(p unsafe.Pointer, h uintptr) uintptr { + f := *(*float32)(p) + switch { + case f == 0: + return c1 * (c0 ^ h) // +0, -0 + case f != f: + return c1 * (c0 ^ h ^ uintptr(fastrand())) // any kind of NaN + default: + return memhash(p, h, 4) + } +} + +func f64hash(p unsafe.Pointer, h uintptr) uintptr { + f := *(*float64)(p) + switch { + case f == 0: + return c1 * (c0 ^ h) // +0, -0 + case f != f: + return c1 * (c0 ^ h ^ uintptr(fastrand())) // any kind of NaN + default: + return memhash(p, h, 8) + } +} + +func c64hash(p unsafe.Pointer, h uintptr) uintptr { + x := (*[2]float32)(p) + return f32hash(unsafe.Pointer(&x[1]), f32hash(unsafe.Pointer(&x[0]), h)) +} + +func c128hash(p unsafe.Pointer, h uintptr) uintptr { + x := (*[2]float64)(p) + return f64hash(unsafe.Pointer(&x[1]), f64hash(unsafe.Pointer(&x[0]), h)) +} + +func interhash(p unsafe.Pointer, h uintptr) uintptr { + a := (*iface)(p) + tab := a.tab + if tab == nil { + return h + } + t := tab._type + if t.Equal == nil { + // Check hashability here. We could do this check inside + // typehash, but we want to report the topmost type in + // the error text (e.g. in a struct with a field of slice type + // we want to report the struct, not the slice). + panic(errorString("hash of unhashable type " + t.Name())) + } + if isDirectIface(t) { + return c1 * typehash(t, unsafe.Pointer(&a.data), h^c0) + } else { + return c1 * typehash(t, a.data, h^c0) + } +} + +func nilinterhash(p unsafe.Pointer, h uintptr) uintptr { + a := (*eface)(p) + t := a._type + if t == nil { + return h + } + if t.Equal == nil { + // See comment in interhash above. + panic(errorString("hash of unhashable type " + t.Name())) + } + if isDirectIface(t) { + return c1 * typehash(t, unsafe.Pointer(&a.data), h^c0) + } else { + return c1 * typehash(t, a.data, h^c0) + } +} + +// typehash computes the hash of the object of type t at address p. +// h is the seed. +// This function is seldom used. Most maps use for hashing either +// fixed functions (e.g. f32hash) or compiler-generated functions +// (e.g. for a type like struct { x, y string }). This implementation +// is slower but more general and is used for hashing interface types +// (called from interhash or nilinterhash, above) or for hashing in +// maps generated by reflect.MapOf (reflect_typehash, below). +// Note: this function must match the compiler generated +// functions exactly. See issue 37716. +func typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr { + if t.TFlag&abi.TFlagRegularMemory != 0 { + // Handle ptr sizes specially, see issue 37086. + switch t.Size_ { + case 4: + return memhash32(p, h) + case 8: + return memhash64(p, h) + default: + return memhash(p, h, t.Size_) + } + } + switch t.Kind() { + case abi.Float32: + return f32hash(p, h) + case abi.Float64: + return f64hash(p, h) + case abi.Complex64: + return c64hash(p, h) + case abi.Complex128: + return c128hash(p, h) + case abi.String: + return strhash(p, h) + case abi.Interface: + i := (*interfacetype)(unsafe.Pointer(t)) + if len(i.Methods) == 0 { + return nilinterhash(p, h) + } + return interhash(p, h) + case abi.Array: + a := (*abi.ArrayType)(unsafe.Pointer(t)) + for i := uintptr(0); i < a.Len; i++ { + h = typehash(a.Elem, add(p, i*a.Elem.Size_), h) + } + return h + case abi.Struct: + s := (*abi.StructType)(unsafe.Pointer(t)) + for _, f := range s.Fields { + /* TODO(xsw): skip blank field + if f.Name.IsBlank() { + continue + } + */ + h = typehash(f.Typ, add(p, f.Offset), h) + } + return h + default: + // Should never happen, as typehash should only be called + // with comparable types. + panic(errorString("hash of unhashable type " + t.Name())) + } +} + +/* +//go:linkname reflect_typehash reflect.typehash +func reflect_typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr { + return typehash(t, p, h) +} + +func memequal0(p, q unsafe.Pointer) bool { + return true +} +func memequal8(p, q unsafe.Pointer) bool { + return *(*int8)(p) == *(*int8)(q) +} +func memequal16(p, q unsafe.Pointer) bool { + return *(*int16)(p) == *(*int16)(q) +} +func memequal32(p, q unsafe.Pointer) bool { + return *(*int32)(p) == *(*int32)(q) +} +func memequal64(p, q unsafe.Pointer) bool { + return *(*int64)(p) == *(*int64)(q) +} +func memequal128(p, q unsafe.Pointer) bool { + return *(*[2]int64)(p) == *(*[2]int64)(q) +} +func f32equal(p, q unsafe.Pointer) bool { + return *(*float32)(p) == *(*float32)(q) +} +func f64equal(p, q unsafe.Pointer) bool { + return *(*float64)(p) == *(*float64)(q) +} +func c64equal(p, q unsafe.Pointer) bool { + return *(*complex64)(p) == *(*complex64)(q) +} +func c128equal(p, q unsafe.Pointer) bool { + return *(*complex128)(p) == *(*complex128)(q) +} +func strequal(p, q unsafe.Pointer) bool { + return *(*string)(p) == *(*string)(q) +} +func interequal(p, q unsafe.Pointer) bool { + x := *(*iface)(p) + y := *(*iface)(q) + return x.tab == y.tab && ifaceeq(x.tab, x.data, y.data) +} +func nilinterequal(p, q unsafe.Pointer) bool { + x := *(*eface)(p) + y := *(*eface)(q) + return x._type == y._type && efaceeq(x._type, x.data, y.data) +} +func efaceeq(t *_type, x, y unsafe.Pointer) bool { + if t == nil { + return true + } + eq := t.Equal + if eq == nil { + panic(errorString("comparing uncomparable type " + t.Name())) + } + if isDirectIface(t) { + // Direct interface types are ptr, chan, map, func, and single-element structs/arrays thereof. + // Maps and funcs are not comparable, so they can't reach here. + // Ptrs, chans, and single-element items can be compared directly using ==. + return x == y + } + return eq(x, y) +} +func ifaceeq(tab *itab, x, y unsafe.Pointer) bool { + if tab == nil { + return true + } + t := tab._type + eq := t.Equal + if eq == nil { + panic(errorString("comparing uncomparable type " + t.Name())) + } + if isDirectIface(t) { + // See comment in efaceeq. + return x == y + } + return eq(x, y) +} + +// Testing adapters for hash quality tests (see hash_test.go) +func stringHash(s string, seed uintptr) uintptr { + return strhash(unsafe.Pointer(&s), seed) +} + +func bytesHash(b []byte, seed uintptr) uintptr { + s := (*slice)(unsafe.Pointer(&b)) + return memhash(s.data, seed, uintptr(s.len)) +} + +func int32Hash(i uint32, seed uintptr) uintptr { + return memhash32(unsafe.Pointer(&i), seed) +} + +func int64Hash(i uint64, seed uintptr) uintptr { + return memhash64(unsafe.Pointer(&i), seed) +} + +func efaceHash(i any, seed uintptr) uintptr { + return nilinterhash(unsafe.Pointer(&i), seed) +} + +func ifaceHash(i interface { + F() +}, seed uintptr) uintptr { + return interhash(unsafe.Pointer(&i), seed) +} + +/* +const hashRandomBytes = goarch.PtrSize / 4 * 64 + +// used in asm_{386,amd64,arm64}.s to seed the hash function +var aeskeysched [hashRandomBytes]byte + +// used in hash{32,64}.go to seed the hash function +var hashkey [4]uintptr + +func alginit() { + // Install AES hash algorithms if the instructions needed are present. + if (GOARCH == "386" || GOARCH == "amd64") && + cpu.X86.HasAES && // AESENC + cpu.X86.HasSSSE3 && // PSHUFB + cpu.X86.HasSSE41 { // PINSR{D,Q} + initAlgAES() + return + } + if GOARCH == "arm64" && cpu.ARM64.HasAES { + initAlgAES() + return + } + getRandomData((*[len(hashkey) * goarch.PtrSize]byte)(unsafe.Pointer(&hashkey))[:]) + hashkey[0] |= 1 // make sure these numbers are odd + hashkey[1] |= 1 + hashkey[2] |= 1 + hashkey[3] |= 1 +} + +func initAlgAES() { + useAeshash = true + // Initialize with random data so hash collisions will be hard to engineer. + getRandomData(aeskeysched[:]) +} + +// Note: These routines perform the read with a native endianness. +func readUnaligned32(p unsafe.Pointer) uint32 { + q := (*[4]byte)(p) + if goarch.BigEndian { + return uint32(q[3]) | uint32(q[2])<<8 | uint32(q[1])<<16 | uint32(q[0])<<24 + } + return uint32(q[0]) | uint32(q[1])<<8 | uint32(q[2])<<16 | uint32(q[3])<<24 +} + +func readUnaligned64(p unsafe.Pointer) uint64 { + q := (*[8]byte)(p) + if goarch.BigEndian { + return uint64(q[7]) | uint64(q[6])<<8 | uint64(q[5])<<16 | uint64(q[4])<<24 | + uint64(q[3])<<32 | uint64(q[2])<<40 | uint64(q[1])<<48 | uint64(q[0])<<56 + } + return uint64(q[0]) | uint64(q[1])<<8 | uint64(q[2])<<16 | uint64(q[3])<<24 | uint64(q[4])<<32 | uint64(q[5])<<40 | uint64(q[6])<<48 | uint64(q[7])<<56 +} +*/ diff --git a/internal/runtime/c/c.go b/internal/runtime/c/c.go index cea55d8b..91a6fc1e 100644 --- a/internal/runtime/c/c.go +++ b/internal/runtime/c/c.go @@ -30,11 +30,15 @@ type ( FilePtr = unsafe.Pointer ) +type integer interface { + ~int | ~uint | ~uintptr | ~int32 | ~uint32 | ~int64 | ~uint64 +} + //go:linkname Str llgo.cstr func Str(string) *Char // llgo:link Advance llgo.advance -func Advance[PtrT any](ptr PtrT, offset int) PtrT { return ptr } +func Advance[PtrT any, I integer](ptr PtrT, offset I) PtrT { return ptr } //go:linkname Alloca llgo.alloca func Alloca(size uintptr) Pointer diff --git a/internal/runtime/error.go b/internal/runtime/error.go new file mode 100644 index 00000000..a4205aa9 --- /dev/null +++ b/internal/runtime/error.go @@ -0,0 +1,334 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +/* +import "internal/bytealg" + +// The Error interface identifies a run time error. +type Error interface { + error + + // RuntimeError is a no-op function but + // serves to distinguish types that are run time + // errors from ordinary errors: a type is a + // run time error if it has a RuntimeError method. + RuntimeError() +} + +// A TypeAssertionError explains a failed type assertion. +type TypeAssertionError struct { + _interface *_type + concrete *_type + asserted *_type + missingMethod string // one method needed by Interface, missing from Concrete +} + +func (*TypeAssertionError) RuntimeError() {} + +func (e *TypeAssertionError) Error() string { + inter := "interface" + if e._interface != nil { + inter = toRType(e._interface).string() + } + as := toRType(e.asserted).string() + if e.concrete == nil { + return "interface conversion: " + inter + " is nil, not " + as + } + cs := toRType(e.concrete).string() + if e.missingMethod == "" { + msg := "interface conversion: " + inter + " is " + cs + ", not " + as + if cs == as { + // provide slightly clearer error message + if toRType(e.concrete).pkgpath() != toRType(e.asserted).pkgpath() { + msg += " (types from different packages)" + } else { + msg += " (types from different scopes)" + } + } + return msg + } + return "interface conversion: " + cs + " is not " + as + + ": missing method " + e.missingMethod +} + +// itoa converts val to a decimal representation. The result is +// written somewhere within buf and the location of the result is returned. +// buf must be at least 20 bytes. +// +//go:nosplit +func itoa(buf []byte, val uint64) []byte { + i := len(buf) - 1 + for val >= 10 { + buf[i] = byte(val%10 + '0') + i-- + val /= 10 + } + buf[i] = byte(val + '0') + return buf[i:] +} + +// An errorString represents a runtime error described by a single string. +type errorString string + +func (e errorString) RuntimeError() {} + +func (e errorString) Error() string { + return "runtime error: " + string(e) +} + +type errorAddressString struct { + msg string // error message + addr uintptr // memory address where the error occurred +} + +func (e errorAddressString) RuntimeError() {} + +func (e errorAddressString) Error() string { + return "runtime error: " + e.msg +} + +// Addr returns the memory address where a fault occurred. +// The address provided is best-effort. +// The veracity of the result may depend on the platform. +// Errors providing this method will only be returned as +// a result of using runtime/debug.SetPanicOnFault. +func (e errorAddressString) Addr() uintptr { + return e.addr +} +*/ + +// plainError represents a runtime error described a string without +// the prefix "runtime error: " after invoking errorString.Error(). +// See Issue #14965. +type plainError string + +func (e plainError) RuntimeError() {} + +func (e plainError) Error() string { + return string(e) +} + +/* +// A boundsError represents an indexing or slicing operation gone wrong. +type boundsError struct { + x int64 + y int + // Values in an index or slice expression can be signed or unsigned. + // That means we'd need 65 bits to encode all possible indexes, from -2^63 to 2^64-1. + // Instead, we keep track of whether x should be interpreted as signed or unsigned. + // y is known to be nonnegative and to fit in an int. + signed bool + code boundsErrorCode +} + +type boundsErrorCode uint8 + +const ( + boundsIndex boundsErrorCode = iota // s[x], 0 <= x < len(s) failed + + boundsSliceAlen // s[?:x], 0 <= x <= len(s) failed + boundsSliceAcap // s[?:x], 0 <= x <= cap(s) failed + boundsSliceB // s[x:y], 0 <= x <= y failed (but boundsSliceA didn't happen) + + boundsSlice3Alen // s[?:?:x], 0 <= x <= len(s) failed + boundsSlice3Acap // s[?:?:x], 0 <= x <= cap(s) failed + boundsSlice3B // s[?:x:y], 0 <= x <= y failed (but boundsSlice3A didn't happen) + boundsSlice3C // s[x:y:?], 0 <= x <= y failed (but boundsSlice3A/B didn't happen) + + boundsConvert // (*[x]T)(s), 0 <= x <= len(s) failed + // Note: in the above, len(s) and cap(s) are stored in y +) + +// boundsErrorFmts provide error text for various out-of-bounds panics. +// Note: if you change these strings, you should adjust the size of the buffer +// in boundsError.Error below as well. +var boundsErrorFmts = [...]string{ + boundsIndex: "index out of range [%x] with length %y", + boundsSliceAlen: "slice bounds out of range [:%x] with length %y", + boundsSliceAcap: "slice bounds out of range [:%x] with capacity %y", + boundsSliceB: "slice bounds out of range [%x:%y]", + boundsSlice3Alen: "slice bounds out of range [::%x] with length %y", + boundsSlice3Acap: "slice bounds out of range [::%x] with capacity %y", + boundsSlice3B: "slice bounds out of range [:%x:%y]", + boundsSlice3C: "slice bounds out of range [%x:%y:]", + boundsConvert: "cannot convert slice with length %y to array or pointer to array with length %x", +} + +// boundsNegErrorFmts are overriding formats if x is negative. In this case there's no need to report y. +var boundsNegErrorFmts = [...]string{ + boundsIndex: "index out of range [%x]", + boundsSliceAlen: "slice bounds out of range [:%x]", + boundsSliceAcap: "slice bounds out of range [:%x]", + boundsSliceB: "slice bounds out of range [%x:]", + boundsSlice3Alen: "slice bounds out of range [::%x]", + boundsSlice3Acap: "slice bounds out of range [::%x]", + boundsSlice3B: "slice bounds out of range [:%x:]", + boundsSlice3C: "slice bounds out of range [%x::]", +} + +func (e boundsError) RuntimeError() {} + +func appendIntStr(b []byte, v int64, signed bool) []byte { + if signed && v < 0 { + b = append(b, '-') + v = -v + } + var buf [20]byte + b = append(b, itoa(buf[:], uint64(v))...) + return b +} + +func (e boundsError) Error() string { + fmt := boundsErrorFmts[e.code] + if e.signed && e.x < 0 { + fmt = boundsNegErrorFmts[e.code] + } + // max message length is 99: "runtime error: slice bounds out of range [::%x] with capacity %y" + // x can be at most 20 characters. y can be at most 19. + b := make([]byte, 0, 100) + b = append(b, "runtime error: "...) + for i := 0; i < len(fmt); i++ { + c := fmt[i] + if c != '%' { + b = append(b, c) + continue + } + i++ + switch fmt[i] { + case 'x': + b = appendIntStr(b, e.x, e.signed) + case 'y': + b = appendIntStr(b, int64(e.y), true) + } + } + return string(b) +} + +type stringer interface { + String() string +} + +// printany prints an argument passed to panic. +// If panic is called with a value that has a String or Error method, +// it has already been converted into a string by preprintpanics. +func printany(i any) { + switch v := i.(type) { + case nil: + print("nil") + case bool: + print(v) + case int: + print(v) + case int8: + print(v) + case int16: + print(v) + case int32: + print(v) + case int64: + print(v) + case uint: + print(v) + case uint8: + print(v) + case uint16: + print(v) + case uint32: + print(v) + case uint64: + print(v) + case uintptr: + print(v) + case float32: + print(v) + case float64: + print(v) + case complex64: + print(v) + case complex128: + print(v) + case string: + print(v) + default: + printanycustomtype(i) + } +} + +func printanycustomtype(i any) { + eface := efaceOf(&i) + typestring := toRType(eface._type).string() + + switch eface._type.Kind_ { + case kindString: + print(typestring, `("`, *(*string)(eface.data), `")`) + case kindBool: + print(typestring, "(", *(*bool)(eface.data), ")") + case kindInt: + print(typestring, "(", *(*int)(eface.data), ")") + case kindInt8: + print(typestring, "(", *(*int8)(eface.data), ")") + case kindInt16: + print(typestring, "(", *(*int16)(eface.data), ")") + case kindInt32: + print(typestring, "(", *(*int32)(eface.data), ")") + case kindInt64: + print(typestring, "(", *(*int64)(eface.data), ")") + case kindUint: + print(typestring, "(", *(*uint)(eface.data), ")") + case kindUint8: + print(typestring, "(", *(*uint8)(eface.data), ")") + case kindUint16: + print(typestring, "(", *(*uint16)(eface.data), ")") + case kindUint32: + print(typestring, "(", *(*uint32)(eface.data), ")") + case kindUint64: + print(typestring, "(", *(*uint64)(eface.data), ")") + case kindUintptr: + print(typestring, "(", *(*uintptr)(eface.data), ")") + case kindFloat32: + print(typestring, "(", *(*float32)(eface.data), ")") + case kindFloat64: + print(typestring, "(", *(*float64)(eface.data), ")") + case kindComplex64: + print(typestring, *(*complex64)(eface.data)) + case kindComplex128: + print(typestring, *(*complex128)(eface.data)) + default: + print("(", typestring, ") ", eface.data) + } +} + +// panicwrap generates a panic for a call to a wrapped value method +// with a nil pointer receiver. +// +// It is called from the generated wrapper code. +func panicwrap() { + pc := getcallerpc() + name := funcNameForPrint(funcname(findfunc(pc))) + // name is something like "main.(*T).F". + // We want to extract pkg ("main"), typ ("T"), and meth ("F"). + // Do it by finding the parens. + i := bytealg.IndexByteString(name, '(') + if i < 0 { + throw("panicwrap: no ( in " + name) + } + pkg := name[:i-1] + if i+2 >= len(name) || name[i-1:i+2] != ".(*" { + throw("panicwrap: unexpected string after package name: " + name) + } + name = name[i+2:] + i = bytealg.IndexByteString(name, ')') + if i < 0 { + throw("panicwrap: no ) in " + name) + } + if i+2 >= len(name) || name[i:i+2] != ")." { + throw("panicwrap: unexpected string after type name: " + name) + } + typ := name[:i] + meth := name[i+2:] + panic(plainError("value method " + pkg + "." + typ + "." + meth + " called using nil *" + typ + " pointer")) +} +*/ diff --git a/internal/runtime/malloc.go b/internal/runtime/malloc.go new file mode 100644 index 00000000..bcfd5216 --- /dev/null +++ b/internal/runtime/malloc.go @@ -0,0 +1,343 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" +) + +const ( + bigAlloc = 1 << (goarchPtrSize*8 - 6) +) + +// implementation of new builtin +// compiler (both frontend and SSA backend) knows the signature +// of this function. +func newobject(typ *_type) unsafe.Pointer { + return AllocZ(typ.Size_) +} + +/* +//go:linkname reflect_unsafe_New reflect.unsafe_New +func reflect_unsafe_New(typ *_type) unsafe.Pointer { + return mallocgc(typ.Size_, typ, true) +} + +//go:linkname reflectlite_unsafe_New internal/reflectlite.unsafe_New +func reflectlite_unsafe_New(typ *_type) unsafe.Pointer { + return mallocgc(typ.Size_, typ, true) +} +*/ + +const mathMaxUintptr = ^uintptr(0) + +// mathMulUintptr returns a * b and whether the multiplication overflowed. +// On supported platforms this is an intrinsic lowered by the compiler. +func mathMulUintptr(a, b uintptr) (uintptr, bool) { + if a|b < 1<<(4*goarchPtrSize) || a == 0 { + return a * b, false + } + overflow := b > mathMaxUintptr/a + return a * b, overflow +} + +// newarray allocates an array of n elements of type typ. +func newarray(typ *_type, n int) unsafe.Pointer { + if n == 1 { + return AllocZ(typ.Size_) + } + mem, overflow := mathMulUintptr(typ.Size_, uintptr(n)) + if overflow || n < 0 { + panic(plainError("runtime: allocation size out of range")) + } + return AllocZ(mem) +} + +/* +//go:linkname reflect_unsafe_NewArray reflect.unsafe_NewArray +func reflect_unsafe_NewArray(typ *_type, n int) unsafe.Pointer { + return newarray(typ, n) +} + +func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { + c := getMCache(mp) + if c == nil { + throw("profilealloc called without a P or outside bootstrapping") + } + c.nextSample = nextSample() + mProf_Malloc(x, size) +} + +// nextSample returns the next sampling point for heap profiling. The goal is +// to sample allocations on average every MemProfileRate bytes, but with a +// completely random distribution over the allocation timeline; this +// corresponds to a Poisson process with parameter MemProfileRate. In Poisson +// processes, the distance between two samples follows the exponential +// distribution (exp(MemProfileRate)), so the best return value is a random +// number taken from an exponential distribution whose mean is MemProfileRate. +func nextSample() uintptr { + if MemProfileRate == 1 { + // Callers assign our return value to + // mcache.next_sample, but next_sample is not used + // when the rate is 1. So avoid the math below and + // just return something. + return 0 + } + if GOOS == "plan9" { + // Plan 9 doesn't support floating point in note handler. + if gp := getg(); gp == gp.m.gsignal { + return nextSampleNoFP() + } + } + + return uintptr(fastexprand(MemProfileRate)) +} + +// fastexprand returns a random number from an exponential distribution with +// the specified mean. +func fastexprand(mean int) int32 { + // Avoid overflow. Maximum possible step is + // -ln(1/(1< 0x7000000: + mean = 0x7000000 + case mean == 0: + return 0 + } + + // Take a random sample of the exponential distribution exp(-mean*x). + // The probability distribution function is mean*exp(-mean*x), so the CDF is + // p = 1 - exp(-mean*x), so + // q = 1 - p == exp(-mean*x) + // log_e(q) = -mean*x + // -log_e(q)/mean = x + // x = -log_e(q) * mean + // x = log_2(q) * (-log_e(2)) * mean ; Using log_2 for efficiency + const randomBitCount = 26 + q := fastrandn(1< 0 { + qlog = 0 + } + const minusLog2 = -0.6931471805599453 // -ln(2) + return int32(qlog*(minusLog2*float64(mean))) + 1 +} + +// nextSampleNoFP is similar to nextSample, but uses older, +// simpler code to avoid floating point. +func nextSampleNoFP() uintptr { + // Set first allocation sample size. + rate := MemProfileRate + if rate > 0x3fffffff { // make 2*rate not overflow + rate = 0x3fffffff + } + if rate != 0 { + return uintptr(fastrandn(uint32(2 * rate))) + } + return 0 +} + +type persistentAlloc struct { + base *notInHeap + off uintptr +} + +var globalAlloc struct { + mutex + persistentAlloc +} + +// persistentChunkSize is the number of bytes we allocate when we grow +// a persistentAlloc. +const persistentChunkSize = 256 << 10 + +// persistentChunks is a list of all the persistent chunks we have +// allocated. The list is maintained through the first word in the +// persistent chunk. This is updated atomically. +var persistentChunks *notInHeap + +// Wrapper around sysAlloc that can allocate small chunks. +// There is no associated free operation. +// Intended for things like function/type/debug-related persistent data. +// If align is 0, uses default align (currently 8). +// The returned memory will be zeroed. +// sysStat must be non-nil. +// +// Consider marking persistentalloc'd types not in heap by embedding +// runtime/internal/sys.NotInHeap. +func persistentalloc(size, align uintptr, sysStat *sysMemStat) unsafe.Pointer { + var p *notInHeap + systemstack(func() { + p = persistentalloc1(size, align, sysStat) + }) + return unsafe.Pointer(p) +} + +// Must run on system stack because stack growth can (re)invoke it. +// See issue 9174. +// +//go:systemstack +func persistentalloc1(size, align uintptr, sysStat *sysMemStat) *notInHeap { + const ( + maxBlock = 64 << 10 // VM reservation granularity is 64K on windows + ) + + if size == 0 { + throw("persistentalloc: size == 0") + } + if align != 0 { + if align&(align-1) != 0 { + throw("persistentalloc: align is not a power of 2") + } + if align > _PageSize { + throw("persistentalloc: align is too large") + } + } else { + align = 8 + } + + if size >= maxBlock { + return (*notInHeap)(sysAlloc(size, sysStat)) + } + + mp := acquirem() + var persistent *persistentAlloc + if mp != nil && mp.p != 0 { + persistent = &mp.p.ptr().palloc + } else { + lock(&globalAlloc.mutex) + persistent = &globalAlloc.persistentAlloc + } + persistent.off = alignUp(persistent.off, align) + if persistent.off+size > persistentChunkSize || persistent.base == nil { + persistent.base = (*notInHeap)(sysAlloc(persistentChunkSize, &memstats.other_sys)) + if persistent.base == nil { + if persistent == &globalAlloc.persistentAlloc { + unlock(&globalAlloc.mutex) + } + throw("runtime: cannot allocate memory") + } + + // Add the new chunk to the persistentChunks list. + for { + chunks := uintptr(unsafe.Pointer(persistentChunks)) + *(*uintptr)(unsafe.Pointer(persistent.base)) = chunks + if atomic.Casuintptr((*uintptr)(unsafe.Pointer(&persistentChunks)), chunks, uintptr(unsafe.Pointer(persistent.base))) { + break + } + } + persistent.off = alignUp(goarch.PtrSize, align) + } + p := persistent.base.add(persistent.off) + persistent.off += size + releasem(mp) + if persistent == &globalAlloc.persistentAlloc { + unlock(&globalAlloc.mutex) + } + + if sysStat != &memstats.other_sys { + sysStat.add(int64(size)) + memstats.other_sys.add(-int64(size)) + } + return p +} + +// inPersistentAlloc reports whether p points to memory allocated by +// persistentalloc. This must be nosplit because it is called by the +// cgo checker code, which is called by the write barrier code. +// +//go:nosplit +func inPersistentAlloc(p uintptr) bool { + chunk := atomic.Loaduintptr((*uintptr)(unsafe.Pointer(&persistentChunks))) + for chunk != 0 { + if p >= chunk && p < chunk+persistentChunkSize { + return true + } + chunk = *(*uintptr)(unsafe.Pointer(chunk)) + } + return false +} + +// linearAlloc is a simple linear allocator that pre-reserves a region +// of memory and then optionally maps that region into the Ready state +// as needed. +// +// The caller is responsible for locking. +type linearAlloc struct { + next uintptr // next free byte + mapped uintptr // one byte past end of mapped space + end uintptr // end of reserved space + + mapMemory bool // transition memory from Reserved to Ready if true +} + +func (l *linearAlloc) init(base, size uintptr, mapMemory bool) { + if base+size < base { + // Chop off the last byte. The runtime isn't prepared + // to deal with situations where the bounds could overflow. + // Leave that memory reserved, though, so we don't map it + // later. + size -= 1 + } + l.next, l.mapped = base, base + l.end = base + size + l.mapMemory = mapMemory +} + +func (l *linearAlloc) alloc(size, align uintptr, sysStat *sysMemStat) unsafe.Pointer { + p := alignUp(l.next, align) + if p+size > l.end { + return nil + } + l.next = p + size + if pEnd := alignUp(l.next-1, physPageSize); pEnd > l.mapped { + if l.mapMemory { + // Transition from Reserved to Prepared to Ready. + n := pEnd - l.mapped + sysMap(unsafe.Pointer(l.mapped), n, sysStat) + sysUsed(unsafe.Pointer(l.mapped), n, n) + } + l.mapped = pEnd + } + return unsafe.Pointer(p) +} + +// notInHeap is off-heap memory allocated by a lower-level allocator +// like sysAlloc or persistentAlloc. +// +// In general, it's better to use real types which embed +// runtime/internal/sys.NotInHeap, but this serves as a generic type +// for situations where that isn't possible (like in the allocators). +// +// TODO: Use this as the return type of sysAlloc, persistentAlloc, etc? +type notInHeap struct{ _ sys.NotInHeap } + +func (p *notInHeap) add(bytes uintptr) *notInHeap { + return (*notInHeap)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + bytes)) +} + +// computeRZlog computes the size of the redzone. +// Refer to the implementation of the compiler-rt. +func computeRZlog(userSize uintptr) uintptr { + switch { + case userSize <= (64 - 16): + return 16 << 0 + case userSize <= (128 - 32): + return 16 << 1 + case userSize <= (512 - 64): + return 16 << 2 + case userSize <= (4096 - 128): + return 16 << 3 + case userSize <= (1<<14)-256: + return 16 << 4 + case userSize <= (1<<15)-512: + return 16 << 5 + case userSize <= (1<<16)-1024: + return 16 << 6 + default: + return 16 << 7 + } +} +*/ diff --git a/internal/runtime/map.go b/internal/runtime/map.go index 4c83a751..23fad661 100644 --- a/internal/runtime/map.go +++ b/internal/runtime/map.go @@ -59,6 +59,12 @@ import ( "github.com/goplus/llgo/internal/abi" ) +type maptype = abi.MapType + +const ( + goarchPtrSize = unsafe.Sizeof(uintptr(0)) +) + const ( // Maximum number of key/elem pairs a bucket can hold. bucketCntBits = abi.MapBucketCountBits @@ -74,8 +80,9 @@ const ( // Must fit in a uint8. // Fast versions cannot handle big elems - the cutoff size for // fast versions in cmd/compile/internal/gc/walk.go must be at most this elem. - maxKeySize = abi.MapMaxKeyBytes - maxElemSize = abi.MapMaxElemBytes + // + // maxKeySize = abi.MapMaxKeyBytes + // maxElemSize = abi.MapMaxElemBytes // data offset should be the size of the bmap struct, but needs to be // aligned correctly. For amd64p32 this means 64-bit alignment @@ -179,11 +186,12 @@ type hiter struct { bucket uintptr checkBucket uintptr } +*/ // bucketShift returns 1<> (goarch.PtrSize*8 - 8)) + top := uint8(hash >> (goarchPtrSize*8 - 8)) if top < minTopHash { top += minTopHash } @@ -206,16 +214,18 @@ func evacuated(b *bmap) bool { } func (b *bmap) overflow(t *maptype) *bmap { - return *(**bmap)(add(unsafe.Pointer(b), uintptr(t.BucketSize)-goarch.PtrSize)) + return *(**bmap)(add(unsafe.Pointer(b), uintptr(t.BucketSize)-goarchPtrSize)) } func (b *bmap) setoverflow(t *maptype, ovf *bmap) { - *(**bmap)(add(unsafe.Pointer(b), uintptr(t.BucketSize)-goarch.PtrSize)) = ovf + *(**bmap)(add(unsafe.Pointer(b), uintptr(t.BucketSize)-goarchPtrSize)) = ovf } +/* func (b *bmap) keys() unsafe.Pointer { return add(unsafe.Pointer(b), dataOffset) } +*/ // incrnoverflow increments h.noverflow. // noverflow counts the number of overflow buckets. @@ -280,6 +290,7 @@ func (h *hmap) createOverflow() { } } +/* func makemap64(t *maptype, hint int64, h *hmap) *hmap { if int64(int(hint)) != hint { hint = 0 @@ -297,15 +308,14 @@ func makemap_small() *hmap { return h } -/* // makemap implements Go map creation for make(map[k]v, hint). // If the compiler has determined that the map or the first bucket // can be created on the stack, h and/or bucket may be non-nil. // If h != nil, the map can be created directly in h. // If h.buckets != nil, bucket pointed to can be used as the first bucket. func makemap(t *maptype, hint int, h *hmap) *hmap { - mem, overflow := math.MulUintptr(uintptr(hint), t.Bucket.Size_) - if overflow || mem > maxAlloc { + mem, overflow := mathMulUintptr(uintptr(hint), t.Bucket.Size_) + if overflow || mem > bigAlloc { hint = 0 } @@ -389,6 +399,7 @@ func makeBucketArray(t *maptype, b uint8, dirtyalloc unsafe.Pointer) (buckets un return buckets, nextOverflow } +/* // mapaccess1 returns a pointer to h[key]. Never returns nil, instead // it will return a reference to the zero object for the elem type if // the key is not in the map. @@ -575,24 +586,13 @@ func mapaccess2_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) (unsafe.Point } return e, true } +*/ // Like mapaccess, but allocates a slot for the key if it is not present in the map. func mapassign(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer { if h == nil { panic(plainError("assignment to entry in nil map")) } - if raceenabled { - callerpc := getcallerpc() - pc := abi.FuncPCABIInternal(mapassign) - racewritepc(unsafe.Pointer(h), callerpc, pc) - raceReadObjectPC(t.Key, key, callerpc, pc) - } - if msanenabled { - msanread(key, t.Key.Size_) - } - if asanenabled { - asanread(key, t.Key.Size_) - } if h.flags&hashWriting != 0 { fatal("concurrent map writes") } @@ -694,6 +694,7 @@ done: return elem } +/* func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) { if raceenabled && h != nil { callerpc := getcallerpc() @@ -1055,6 +1056,7 @@ func mapclear(t *maptype, h *hmap) { } h.flags &^= hashWriting } +*/ func hashGrow(t *maptype, h *hmap) { // If we've hit the load factor, get bigger. @@ -1305,6 +1307,7 @@ func advanceEvacuationMark(h *hmap, t *maptype, newbit uintptr) { } } +/* // Reflect stubs. Called from ../reflect/asm_*.s //go:linkname reflect_makemap reflect.makemap diff --git a/internal/runtime/mbarrier.go b/internal/runtime/mbarrier.go new file mode 100644 index 00000000..01edc942 --- /dev/null +++ b/internal/runtime/mbarrier.go @@ -0,0 +1,349 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Garbage collector: write barriers. +// +// For the concurrent garbage collector, the Go compiler implements +// updates to pointer-valued fields that may be in heap objects by +// emitting calls to write barriers. The main write barrier for +// individual pointer writes is gcWriteBarrier and is implemented in +// assembly. This file contains write barrier entry points for bulk +// operations. See also mwbbuf.go. + +package runtime + +import ( + "unsafe" + + "github.com/goplus/llgo/internal/abi" +) + +/* +import ( + "internal/abi" + "internal/goarch" + "internal/goexperiment" + "unsafe" +) + +// Go uses a hybrid barrier that combines a Yuasa-style deletion +// barrier—which shades the object whose reference is being +// overwritten—with Dijkstra insertion barrier—which shades the object +// whose reference is being written. The insertion part of the barrier +// is necessary while the calling goroutine's stack is grey. In +// pseudocode, the barrier is: +// +// writePointer(slot, ptr): +// shade(*slot) +// if current stack is grey: +// shade(ptr) +// *slot = ptr +// +// slot is the destination in Go code. +// ptr is the value that goes into the slot in Go code. +// +// Shade indicates that it has seen a white pointer by adding the referent +// to wbuf as well as marking it. +// +// The two shades and the condition work together to prevent a mutator +// from hiding an object from the garbage collector: +// +// 1. shade(*slot) prevents a mutator from hiding an object by moving +// the sole pointer to it from the heap to its stack. If it attempts +// to unlink an object from the heap, this will shade it. +// +// 2. shade(ptr) prevents a mutator from hiding an object by moving +// the sole pointer to it from its stack into a black object in the +// heap. If it attempts to install the pointer into a black object, +// this will shade it. +// +// 3. Once a goroutine's stack is black, the shade(ptr) becomes +// unnecessary. shade(ptr) prevents hiding an object by moving it from +// the stack to the heap, but this requires first having a pointer +// hidden on the stack. Immediately after a stack is scanned, it only +// points to shaded objects, so it's not hiding anything, and the +// shade(*slot) prevents it from hiding any other pointers on its +// stack. +// +// For a detailed description of this barrier and proof of +// correctness, see https://github.com/golang/proposal/blob/master/design/17503-eliminate-rescan.md +// +// +// +// Dealing with memory ordering: +// +// Both the Yuasa and Dijkstra barriers can be made conditional on the +// color of the object containing the slot. We chose not to make these +// conditional because the cost of ensuring that the object holding +// the slot doesn't concurrently change color without the mutator +// noticing seems prohibitive. +// +// Consider the following example where the mutator writes into +// a slot and then loads the slot's mark bit while the GC thread +// writes to the slot's mark bit and then as part of scanning reads +// the slot. +// +// Initially both [slot] and [slotmark] are 0 (nil) +// Mutator thread GC thread +// st [slot], ptr st [slotmark], 1 +// +// ld r1, [slotmark] ld r2, [slot] +// +// Without an expensive memory barrier between the st and the ld, the final +// result on most HW (including 386/amd64) can be r1==r2==0. This is a classic +// example of what can happen when loads are allowed to be reordered with older +// stores (avoiding such reorderings lies at the heart of the classic +// Peterson/Dekker algorithms for mutual exclusion). Rather than require memory +// barriers, which will slow down both the mutator and the GC, we always grey +// the ptr object regardless of the slot's color. +// +// Another place where we intentionally omit memory barriers is when +// accessing mheap_.arena_used to check if a pointer points into the +// heap. On relaxed memory machines, it's possible for a mutator to +// extend the size of the heap by updating arena_used, allocate an +// object from this new region, and publish a pointer to that object, +// but for tracing running on another processor to observe the pointer +// but use the old value of arena_used. In this case, tracing will not +// mark the object, even though it's reachable. However, the mutator +// is guaranteed to execute a write barrier when it publishes the +// pointer, so it will take care of marking the object. A general +// consequence of this is that the garbage collector may cache the +// value of mheap_.arena_used. (See issue #9984.) +// +// +// Stack writes: +// +// The compiler omits write barriers for writes to the current frame, +// but if a stack pointer has been passed down the call stack, the +// compiler will generate a write barrier for writes through that +// pointer (because it doesn't know it's not a heap pointer). +// +// +// Global writes: +// +// The Go garbage collector requires write barriers when heap pointers +// are stored in globals. Many garbage collectors ignore writes to +// globals and instead pick up global -> heap pointers during +// termination. This increases pause time, so we instead rely on write +// barriers for writes to globals so that we don't have to rescan +// global during mark termination. +// +// +// Publication ordering: +// +// The write barrier is *pre-publication*, meaning that the write +// barrier happens prior to the *slot = ptr write that may make ptr +// reachable by some goroutine that currently cannot reach it. +// +// +// Signal handler pointer writes: +// +// In general, the signal handler cannot safely invoke the write +// barrier because it may run without a P or even during the write +// barrier. +// +// There is exactly one exception: profbuf.go omits a barrier during +// signal handler profile logging. That's safe only because of the +// deletion barrier. See profbuf.go for a detailed argument. If we +// remove the deletion barrier, we'll have to work out a new way to +// handle the profile logging. +*/ + +// typedmemmove copies a value of type typ to dst from src. +// Must be nosplit, see #16026. +// +// TODO: Perfect for go:nosplitrec since we can't have a safe point +// anywhere in the bulk barrier or memmove. +// +//go:nosplit +func typedmemmove(typ *abi.Type, dst, src unsafe.Pointer) { + if dst == src { + return + } + // There's a race here: if some other goroutine can write to + // src, it may change some pointer in src after we've + // performed the write barrier but before we perform the + // memory copy. This safe because the write performed by that + // other goroutine must also be accompanied by a write + // barrier, so at worst we've unnecessarily greyed the old + // pointer that was in src. + memmove(dst, src, typ.Size_) +} + +/* +// wbZero performs the write barrier operations necessary before +// zeroing a region of memory at address dst of type typ. +// Does not actually do the zeroing. +// +//go:nowritebarrierrec +//go:nosplit +func wbZero(typ *_type, dst unsafe.Pointer) { + bulkBarrierPreWrite(uintptr(dst), 0, typ.PtrBytes) +} + +// wbMove performs the write barrier operations necessary before +// copying a region of memory from src to dst of type typ. +// Does not actually do the copying. +// +//go:nowritebarrierrec +//go:nosplit +func wbMove(typ *_type, dst, src unsafe.Pointer) { + bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.PtrBytes) +} + +//go:linkname reflect_typedmemmove reflect.typedmemmove +func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) { + if raceenabled { + raceWriteObjectPC(typ, dst, getcallerpc(), abi.FuncPCABIInternal(reflect_typedmemmove)) + raceReadObjectPC(typ, src, getcallerpc(), abi.FuncPCABIInternal(reflect_typedmemmove)) + } + if msanenabled { + msanwrite(dst, typ.Size_) + msanread(src, typ.Size_) + } + if asanenabled { + asanwrite(dst, typ.Size_) + asanread(src, typ.Size_) + } + typedmemmove(typ, dst, src) +} + +//go:linkname reflectlite_typedmemmove internal/reflectlite.typedmemmove +func reflectlite_typedmemmove(typ *_type, dst, src unsafe.Pointer) { + reflect_typedmemmove(typ, dst, src) +} + +// reflectcallmove is invoked by reflectcall to copy the return values +// out of the stack and into the heap, invoking the necessary write +// barriers. dst, src, and size describe the return value area to +// copy. typ describes the entire frame (not just the return values). +// typ may be nil, which indicates write barriers are not needed. +// +// It must be nosplit and must only call nosplit functions because the +// stack map of reflectcall is wrong. +// +//go:nosplit +func reflectcallmove(typ *_type, dst, src unsafe.Pointer, size uintptr, regs *abi.RegArgs) { + if writeBarrier.needed && typ != nil && typ.PtrBytes != 0 && size >= goarch.PtrSize { + bulkBarrierPreWrite(uintptr(dst), uintptr(src), size) + } + memmove(dst, src, size) + + // Move pointers returned in registers to a place where the GC can see them. + for i := range regs.Ints { + if regs.ReturnIsPtr.Get(i) { + regs.Ptrs[i] = unsafe.Pointer(regs.Ints[i]) + } + } +} + +//go:nosplit +func typedslicecopy(typ *_type, dstPtr unsafe.Pointer, dstLen int, srcPtr unsafe.Pointer, srcLen int) int { + n := dstLen + if n > srcLen { + n = srcLen + } + if n == 0 { + return 0 + } + + // The compiler emits calls to typedslicecopy before + // instrumentation runs, so unlike the other copying and + // assignment operations, it's not instrumented in the calling + // code and needs its own instrumentation. + if raceenabled { + callerpc := getcallerpc() + pc := abi.FuncPCABIInternal(slicecopy) + racewriterangepc(dstPtr, uintptr(n)*typ.Size_, callerpc, pc) + racereadrangepc(srcPtr, uintptr(n)*typ.Size_, callerpc, pc) + } + if msanenabled { + msanwrite(dstPtr, uintptr(n)*typ.Size_) + msanread(srcPtr, uintptr(n)*typ.Size_) + } + if asanenabled { + asanwrite(dstPtr, uintptr(n)*typ.Size_) + asanread(srcPtr, uintptr(n)*typ.Size_) + } + + if goexperiment.CgoCheck2 { + cgoCheckSliceCopy(typ, dstPtr, srcPtr, n) + } + + if dstPtr == srcPtr { + return n + } + + // Note: No point in checking typ.PtrBytes here: + // compiler only emits calls to typedslicecopy for types with pointers, + // and growslice and reflect_typedslicecopy check for pointers + // before calling typedslicecopy. + size := uintptr(n) * typ.Size_ + if writeBarrier.needed { + pwsize := size - typ.Size_ + typ.PtrBytes + bulkBarrierPreWrite(uintptr(dstPtr), uintptr(srcPtr), pwsize) + } + // See typedmemmove for a discussion of the race between the + // barrier and memmove. + memmove(dstPtr, srcPtr, size) + return n +} + +//go:linkname reflect_typedslicecopy reflect.typedslicecopy +func reflect_typedslicecopy(elemType *_type, dst, src slice) int { + if elemType.PtrBytes == 0 { + return slicecopy(dst.array, dst.len, src.array, src.len, elemType.Size_) + } + return typedslicecopy(elemType, dst.array, dst.len, src.array, src.len) +} + +// typedmemclr clears the typed memory at ptr with type typ. The +// memory at ptr must already be initialized (and hence in type-safe +// state). If the memory is being initialized for the first time, see +// memclrNoHeapPointers. +// +// If the caller knows that typ has pointers, it can alternatively +// call memclrHasPointers. +// +// TODO: A "go:nosplitrec" annotation would be perfect for this. +// +//go:nosplit +func typedmemclr(typ *_type, ptr unsafe.Pointer) { + if writeBarrier.needed && typ.PtrBytes != 0 { + bulkBarrierPreWrite(uintptr(ptr), 0, typ.PtrBytes) + } + memclrNoHeapPointers(ptr, typ.Size_) +} + +//go:linkname reflect_typedmemclr reflect.typedmemclr +func reflect_typedmemclr(typ *_type, ptr unsafe.Pointer) { + typedmemclr(typ, ptr) +} + +//go:linkname reflect_typedmemclrpartial reflect.typedmemclrpartial +func reflect_typedmemclrpartial(typ *_type, ptr unsafe.Pointer, off, size uintptr) { + if writeBarrier.needed && typ.PtrBytes != 0 { + bulkBarrierPreWrite(uintptr(ptr), 0, size) + } + memclrNoHeapPointers(ptr, size) +} + +//go:linkname reflect_typedarrayclear reflect.typedarrayclear +func reflect_typedarrayclear(typ *_type, ptr unsafe.Pointer, len int) { + size := typ.Size_ * uintptr(len) + if writeBarrier.needed && typ.PtrBytes != 0 { + bulkBarrierPreWrite(uintptr(ptr), 0, size) + } + memclrNoHeapPointers(ptr, size) +} +*/ + +// memclrHasPointers clears n bytes of typed memory starting at ptr. +// The caller must ensure that the type of the object at ptr has +// pointers, usually by checking typ.PtrBytes. However, ptr +// does not have to point to the start of the allocation. +func memclrHasPointers(ptr unsafe.Pointer, n uintptr) { + bulkBarrierPreWrite(uintptr(ptr), 0, n) + memclrNoHeapPointers(ptr, n) +} diff --git a/internal/runtime/mbitmap.go b/internal/runtime/mbitmap.go new file mode 100644 index 00000000..e28c4f42 --- /dev/null +++ b/internal/runtime/mbitmap.go @@ -0,0 +1,1446 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Garbage collector: type and heap bitmaps. +// +// Stack, data, and bss bitmaps +// +// Stack frames and global variables in the data and bss sections are +// described by bitmaps with 1 bit per pointer-sized word. A "1" bit +// means the word is a live pointer to be visited by the GC (referred to +// as "pointer"). A "0" bit means the word should be ignored by GC +// (referred to as "scalar", though it could be a dead pointer value). +// +// Heap bitmap +// +// The heap bitmap comprises 1 bit for each pointer-sized word in the heap, +// recording whether a pointer is stored in that word or not. This bitmap +// is stored in the heapArena metadata backing each heap arena. +// That is, if ha is the heapArena for the arena starting at "start", +// then ha.bitmap[0] holds the 64 bits for the 64 words "start" +// through start+63*ptrSize, ha.bitmap[1] holds the entries for +// start+64*ptrSize through start+127*ptrSize, and so on. +// Bits correspond to words in little-endian order. ha.bitmap[0]&1 represents +// the word at "start", ha.bitmap[0]>>1&1 represents the word at start+8, etc. +// (For 32-bit platforms, s/64/32/.) +// +// We also keep a noMorePtrs bitmap which allows us to stop scanning +// the heap bitmap early in certain situations. If ha.noMorePtrs[i]>>j&1 +// is 1, then the object containing the last word described by ha.bitmap[8*i+j] +// has no more pointers beyond those described by ha.bitmap[8*i+j]. +// If ha.noMorePtrs[i]>>j&1 is set, the entries in ha.bitmap[8*i+j+1] and +// beyond must all be zero until the start of the next object. +// +// The bitmap for noscan spans is set to all zero at span allocation time. +// +// The bitmap for unallocated objects in scannable spans is not maintained +// (can be junk). + +package runtime + +/* +import ( + "internal/goarch" + "runtime/internal/atomic" + "runtime/internal/sys" + "unsafe" +) + +// addb returns the byte pointer p+n. +// +//go:nowritebarrier +//go:nosplit +func addb(p *byte, n uintptr) *byte { + // Note: wrote out full expression instead of calling add(p, n) + // to reduce the number of temporaries generated by the + // compiler for this trivial expression during inlining. + return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + n)) +} + +// subtractb returns the byte pointer p-n. +// +//go:nowritebarrier +//go:nosplit +func subtractb(p *byte, n uintptr) *byte { + // Note: wrote out full expression instead of calling add(p, -n) + // to reduce the number of temporaries generated by the + // compiler for this trivial expression during inlining. + return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) - n)) +} + +// add1 returns the byte pointer p+1. +// +//go:nowritebarrier +//go:nosplit +func add1(p *byte) *byte { + // Note: wrote out full expression instead of calling addb(p, 1) + // to reduce the number of temporaries generated by the + // compiler for this trivial expression during inlining. + return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) + 1)) +} + +// subtract1 returns the byte pointer p-1. +// +// nosplit because it is used during write barriers and must not be preempted. +// +//go:nowritebarrier +//go:nosplit +func subtract1(p *byte) *byte { + // Note: wrote out full expression instead of calling subtractb(p, 1) + // to reduce the number of temporaries generated by the + // compiler for this trivial expression during inlining. + return (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(p)) - 1)) +} + +// markBits provides access to the mark bit for an object in the heap. +// bytep points to the byte holding the mark bit. +// mask is a byte with a single bit set that can be &ed with *bytep +// to see if the bit has been set. +// *m.byte&m.mask != 0 indicates the mark bit is set. +// index can be used along with span information to generate +// the address of the object in the heap. +// We maintain one set of mark bits for allocation and one for +// marking purposes. +type markBits struct { + bytep *uint8 + mask uint8 + index uintptr +} + +//go:nosplit +func (s *mspan) allocBitsForIndex(allocBitIndex uintptr) markBits { + bytep, mask := s.allocBits.bitp(allocBitIndex) + return markBits{bytep, mask, allocBitIndex} +} + +// refillAllocCache takes 8 bytes s.allocBits starting at whichByte +// and negates them so that ctz (count trailing zeros) instructions +// can be used. It then places these 8 bytes into the cached 64 bit +// s.allocCache. +func (s *mspan) refillAllocCache(whichByte uintptr) { + bytes := (*[8]uint8)(unsafe.Pointer(s.allocBits.bytep(whichByte))) + aCache := uint64(0) + aCache |= uint64(bytes[0]) + aCache |= uint64(bytes[1]) << (1 * 8) + aCache |= uint64(bytes[2]) << (2 * 8) + aCache |= uint64(bytes[3]) << (3 * 8) + aCache |= uint64(bytes[4]) << (4 * 8) + aCache |= uint64(bytes[5]) << (5 * 8) + aCache |= uint64(bytes[6]) << (6 * 8) + aCache |= uint64(bytes[7]) << (7 * 8) + s.allocCache = ^aCache +} + +// nextFreeIndex returns the index of the next free object in s at +// or after s.freeindex. +// There are hardware instructions that can be used to make this +// faster if profiling warrants it. +func (s *mspan) nextFreeIndex() uintptr { + sfreeindex := s.freeindex + snelems := s.nelems + if sfreeindex == snelems { + return sfreeindex + } + if sfreeindex > snelems { + throw("s.freeindex > s.nelems") + } + + aCache := s.allocCache + + bitIndex := sys.TrailingZeros64(aCache) + for bitIndex == 64 { + // Move index to start of next cached bits. + sfreeindex = (sfreeindex + 64) &^ (64 - 1) + if sfreeindex >= snelems { + s.freeindex = snelems + return snelems + } + whichByte := sfreeindex / 8 + // Refill s.allocCache with the next 64 alloc bits. + s.refillAllocCache(whichByte) + aCache = s.allocCache + bitIndex = sys.TrailingZeros64(aCache) + // nothing available in cached bits + // grab the next 8 bytes and try again. + } + result := sfreeindex + uintptr(bitIndex) + if result >= snelems { + s.freeindex = snelems + return snelems + } + + s.allocCache >>= uint(bitIndex + 1) + sfreeindex = result + 1 + + if sfreeindex%64 == 0 && sfreeindex != snelems { + // We just incremented s.freeindex so it isn't 0. + // As each 1 in s.allocCache was encountered and used for allocation + // it was shifted away. At this point s.allocCache contains all 0s. + // Refill s.allocCache so that it corresponds + // to the bits at s.allocBits starting at s.freeindex. + whichByte := sfreeindex / 8 + s.refillAllocCache(whichByte) + } + s.freeindex = sfreeindex + return result +} + +// isFree reports whether the index'th object in s is unallocated. +// +// The caller must ensure s.state is mSpanInUse, and there must have +// been no preemption points since ensuring this (which could allow a +// GC transition, which would allow the state to change). +func (s *mspan) isFree(index uintptr) bool { + if index < s.freeIndexForScan { + return false + } + bytep, mask := s.allocBits.bitp(index) + return *bytep&mask == 0 +} + +// divideByElemSize returns n/s.elemsize. +// n must be within [0, s.npages*_PageSize), +// or may be exactly s.npages*_PageSize +// if s.elemsize is from sizeclasses.go. +// +// nosplit, because it is called by objIndex, which is nosplit +// +//go:nosplit +func (s *mspan) divideByElemSize(n uintptr) uintptr { + const doubleCheck = false + + // See explanation in mksizeclasses.go's computeDivMagic. + q := uintptr((uint64(n) * uint64(s.divMul)) >> 32) + + if doubleCheck && q != n/s.elemsize { + println(n, "/", s.elemsize, "should be", n/s.elemsize, "but got", q) + throw("bad magic division") + } + return q +} + +// nosplit, because it is called by other nosplit code like findObject +// +//go:nosplit +func (s *mspan) objIndex(p uintptr) uintptr { + return s.divideByElemSize(p - s.base()) +} + +func markBitsForAddr(p uintptr) markBits { + s := spanOf(p) + objIndex := s.objIndex(p) + return s.markBitsForIndex(objIndex) +} + +func (s *mspan) markBitsForIndex(objIndex uintptr) markBits { + bytep, mask := s.gcmarkBits.bitp(objIndex) + return markBits{bytep, mask, objIndex} +} + +func (s *mspan) markBitsForBase() markBits { + return markBits{&s.gcmarkBits.x, uint8(1), 0} +} + +// isMarked reports whether mark bit m is set. +func (m markBits) isMarked() bool { + return *m.bytep&m.mask != 0 +} + +// setMarked sets the marked bit in the markbits, atomically. +func (m markBits) setMarked() { + // Might be racing with other updates, so use atomic update always. + // We used to be clever here and use a non-atomic update in certain + // cases, but it's not worth the risk. + atomic.Or8(m.bytep, m.mask) +} + +// setMarkedNonAtomic sets the marked bit in the markbits, non-atomically. +func (m markBits) setMarkedNonAtomic() { + *m.bytep |= m.mask +} + +// clearMarked clears the marked bit in the markbits, atomically. +func (m markBits) clearMarked() { + // Might be racing with other updates, so use atomic update always. + // We used to be clever here and use a non-atomic update in certain + // cases, but it's not worth the risk. + atomic.And8(m.bytep, ^m.mask) +} + +// markBitsForSpan returns the markBits for the span base address base. +func markBitsForSpan(base uintptr) (mbits markBits) { + mbits = markBitsForAddr(base) + if mbits.mask != 1 { + throw("markBitsForSpan: unaligned start") + } + return mbits +} + +// advance advances the markBits to the next object in the span. +func (m *markBits) advance() { + if m.mask == 1<<7 { + m.bytep = (*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(m.bytep)) + 1)) + m.mask = 1 + } else { + m.mask = m.mask << 1 + } + m.index++ +} + +// clobberdeadPtr is a special value that is used by the compiler to +// clobber dead stack slots, when -clobberdead flag is set. +const clobberdeadPtr = uintptr(0xdeaddead | 0xdeaddead<<((^uintptr(0)>>63)*32)) + +// badPointer throws bad pointer in heap panic. +func badPointer(s *mspan, p, refBase, refOff uintptr) { + // Typically this indicates an incorrect use + // of unsafe or cgo to store a bad pointer in + // the Go heap. It may also indicate a runtime + // bug. + // + // TODO(austin): We could be more aggressive + // and detect pointers to unallocated objects + // in allocated spans. + printlock() + print("runtime: pointer ", hex(p)) + if s != nil { + state := s.state.get() + if state != mSpanInUse { + print(" to unallocated span") + } else { + print(" to unused region of span") + } + print(" span.base()=", hex(s.base()), " span.limit=", hex(s.limit), " span.state=", state) + } + print("\n") + if refBase != 0 { + print("runtime: found in object at *(", hex(refBase), "+", hex(refOff), ")\n") + gcDumpObject("object", refBase, refOff) + } + getg().m.traceback = 2 + throw("found bad pointer in Go heap (incorrect use of unsafe or cgo?)") +} + +// findObject returns the base address for the heap object containing +// the address p, the object's span, and the index of the object in s. +// If p does not point into a heap object, it returns base == 0. +// +// If p points is an invalid heap pointer and debug.invalidptr != 0, +// findObject panics. +// +// refBase and refOff optionally give the base address of the object +// in which the pointer p was found and the byte offset at which it +// was found. These are used for error reporting. +// +// It is nosplit so it is safe for p to be a pointer to the current goroutine's stack. +// Since p is a uintptr, it would not be adjusted if the stack were to move. +// +//go:nosplit +func findObject(p, refBase, refOff uintptr) (base uintptr, s *mspan, objIndex uintptr) { + s = spanOf(p) + // If s is nil, the virtual address has never been part of the heap. + // This pointer may be to some mmap'd region, so we allow it. + if s == nil { + if (GOARCH == "amd64" || GOARCH == "arm64") && p == clobberdeadPtr && debug.invalidptr != 0 { + // Crash if clobberdeadPtr is seen. Only on AMD64 and ARM64 for now, + // as they are the only platform where compiler's clobberdead mode is + // implemented. On these platforms clobberdeadPtr cannot be a valid address. + badPointer(s, p, refBase, refOff) + } + return + } + // If p is a bad pointer, it may not be in s's bounds. + // + // Check s.state to synchronize with span initialization + // before checking other fields. See also spanOfHeap. + if state := s.state.get(); state != mSpanInUse || p < s.base() || p >= s.limit { + // Pointers into stacks are also ok, the runtime manages these explicitly. + if state == mSpanManual { + return + } + // The following ensures that we are rigorous about what data + // structures hold valid pointers. + if debug.invalidptr != 0 { + badPointer(s, p, refBase, refOff) + } + return + } + + objIndex = s.objIndex(p) + base = s.base() + objIndex*s.elemsize + return +} + +// reflect_verifyNotInHeapPtr reports whether converting the not-in-heap pointer into a unsafe.Pointer is ok. +// +//go:linkname reflect_verifyNotInHeapPtr reflect.verifyNotInHeapPtr +func reflect_verifyNotInHeapPtr(p uintptr) bool { + // Conversion to a pointer is ok as long as findObject above does not call badPointer. + // Since we're already promised that p doesn't point into the heap, just disallow heap + // pointers and the special clobbered pointer. + return spanOf(p) == nil && p != clobberdeadPtr +} + +const ptrBits = 8 * goarch.PtrSize + +// heapBits provides access to the bitmap bits for a single heap word. +// The methods on heapBits take value receivers so that the compiler +// can more easily inline calls to those methods and registerize the +// struct fields independently. +type heapBits struct { + // heapBits will report on pointers in the range [addr,addr+size). + // The low bit of mask contains the pointerness of the word at addr + // (assuming valid>0). + addr, size uintptr + + // The next few pointer bits representing words starting at addr. + // Those bits already returned by next() are zeroed. + mask uintptr + // Number of bits in mask that are valid. mask is always less than 1<> off + valid := ptrBits - off + + // Process depending on where the object ends. + nptr := size / goarch.PtrSize + if nptr < valid { + // Bits for this object end before the end of this bitmap word. + // Squash bits for the following objects. + mask &= 1<<(nptr&(ptrBits-1)) - 1 + valid = nptr + } else if nptr == valid { + // Bits for this object end at exactly the end of this bitmap word. + // All good. + } else { + // Bits for this object extend into the next bitmap word. See if there + // may be any pointers recorded there. + if uintptr(ha.noMorePtrs[idx/8])>>(idx%8)&1 != 0 { + // No more pointers in this object after this bitmap word. + // Update size so we know not to look there. + size = valid * goarch.PtrSize + } + } + + return heapBits{addr: addr, size: size, mask: mask, valid: valid} +} + +// Returns the (absolute) address of the next known pointer and +// a heapBits iterator representing any remaining pointers. +// If there are no more pointers, returns address 0. +// Note that next does not modify h. The caller must record the result. +// +// nosplit because it is used during write barriers and must not be preempted. +// +//go:nosplit +func (h heapBits) next() (heapBits, uintptr) { + for { + if h.mask != 0 { + var i int + if goarch.PtrSize == 8 { + i = sys.TrailingZeros64(uint64(h.mask)) + } else { + i = sys.TrailingZeros32(uint32(h.mask)) + } + h.mask ^= uintptr(1) << (i & (ptrBits - 1)) + return h, h.addr + uintptr(i)*goarch.PtrSize + } + + // Skip words that we've already processed. + h.addr += h.valid * goarch.PtrSize + h.size -= h.valid * goarch.PtrSize + if h.size == 0 { + return h, 0 // no more pointers + } + + // Grab more bits and try again. + h = heapBitsForAddr(h.addr, h.size) + } +} + +// nextFast is like next, but can return 0 even when there are more pointers +// to be found. Callers should call next if nextFast returns 0 as its second +// return value. +// +// if addr, h = h.nextFast(); addr == 0 { +// if addr, h = h.next(); addr == 0 { +// ... no more pointers ... +// } +// } +// ... process pointer at addr ... +// +// nextFast is designed to be inlineable. +// +//go:nosplit +func (h heapBits) nextFast() (heapBits, uintptr) { + // TESTQ/JEQ + if h.mask == 0 { + return h, 0 + } + // BSFQ + var i int + if goarch.PtrSize == 8 { + i = sys.TrailingZeros64(uint64(h.mask)) + } else { + i = sys.TrailingZeros32(uint32(h.mask)) + } + // BTCQ + h.mask ^= uintptr(1) << (i & (ptrBits - 1)) + // LEAQ (XX)(XX*8) + return h, h.addr + uintptr(i)*goarch.PtrSize +} +*/ + +// bulkBarrierPreWrite executes a write barrier +// for every pointer slot in the memory range [src, src+size), +// using pointer/scalar information from [dst, dst+size). +// This executes the write barriers necessary before a memmove. +// src, dst, and size must be pointer-aligned. +// The range [dst, dst+size) must lie within a single object. +// It does not perform the actual writes. +// +// As a special case, src == 0 indicates that this is being used for a +// memclr. bulkBarrierPreWrite will pass 0 for the src of each write +// barrier. +// +// Callers should call bulkBarrierPreWrite immediately before +// calling memmove(dst, src, size). This function is marked nosplit +// to avoid being preempted; the GC must not stop the goroutine +// between the memmove and the execution of the barriers. +// The caller is also responsible for cgo pointer checks if this +// may be writing Go pointers into non-Go memory. +// +// The pointer bitmap is not maintained for allocations containing +// no pointers at all; any caller of bulkBarrierPreWrite must first +// make sure the underlying allocation contains pointers, usually +// by checking typ.PtrBytes. +// +// Callers must perform cgo checks if goexperiment.CgoCheck2. +func bulkBarrierPreWrite(dst, src, size uintptr) { +} + +/* +// bulkBarrierPreWriteSrcOnly is like bulkBarrierPreWrite but +// does not execute write barriers for [dst, dst+size). +// +// In addition to the requirements of bulkBarrierPreWrite +// callers need to ensure [dst, dst+size) is zeroed. +// +// This is used for special cases where e.g. dst was just +// created and zeroed with malloc. +// +//go:nosplit +func bulkBarrierPreWriteSrcOnly(dst, src, size uintptr) { + if (dst|src|size)&(goarch.PtrSize-1) != 0 { + throw("bulkBarrierPreWrite: unaligned arguments") + } + if !writeBarrier.needed { + return + } + buf := &getg().m.p.ptr().wbBuf + h := heapBitsForAddr(dst, size) + for { + var addr uintptr + if h, addr = h.next(); addr == 0 { + break + } + srcx := (*uintptr)(unsafe.Pointer(addr - dst + src)) + p := buf.get1() + p[0] = *srcx + } +} + +// bulkBarrierBitmap executes write barriers for copying from [src, +// src+size) to [dst, dst+size) using a 1-bit pointer bitmap. src is +// assumed to start maskOffset bytes into the data covered by the +// bitmap in bits (which may not be a multiple of 8). +// +// This is used by bulkBarrierPreWrite for writes to data and BSS. +// +//go:nosplit +func bulkBarrierBitmap(dst, src, size, maskOffset uintptr, bits *uint8) { + word := maskOffset / goarch.PtrSize + bits = addb(bits, word/8) + mask := uint8(1) << (word % 8) + + buf := &getg().m.p.ptr().wbBuf + for i := uintptr(0); i < size; i += goarch.PtrSize { + if mask == 0 { + bits = addb(bits, 1) + if *bits == 0 { + // Skip 8 words. + i += 7 * goarch.PtrSize + continue + } + mask = 1 + } + if *bits&mask != 0 { + dstx := (*uintptr)(unsafe.Pointer(dst + i)) + if src == 0 { + p := buf.get1() + p[0] = *dstx + } else { + srcx := (*uintptr)(unsafe.Pointer(src + i)) + p := buf.get2() + p[0] = *dstx + p[1] = *srcx + } + } + mask <<= 1 + } +} + +// typeBitsBulkBarrier executes a write barrier for every +// pointer that would be copied from [src, src+size) to [dst, +// dst+size) by a memmove using the type bitmap to locate those +// pointer slots. +// +// The type typ must correspond exactly to [src, src+size) and [dst, dst+size). +// dst, src, and size must be pointer-aligned. +// The type typ must have a plain bitmap, not a GC program. +// The only use of this function is in channel sends, and the +// 64 kB channel element limit takes care of this for us. +// +// Must not be preempted because it typically runs right before memmove, +// and the GC must observe them as an atomic action. +// +// Callers must perform cgo checks if goexperiment.CgoCheck2. +// +//go:nosplit +func typeBitsBulkBarrier(typ *_type, dst, src, size uintptr) { + if typ == nil { + throw("runtime: typeBitsBulkBarrier without type") + } + if typ.Size_ != size { + println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " of size ", typ.Size_, " but memory size", size) + throw("runtime: invalid typeBitsBulkBarrier") + } + if typ.Kind_&kindGCProg != 0 { + println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " with GC prog") + throw("runtime: invalid typeBitsBulkBarrier") + } + if !writeBarrier.needed { + return + } + ptrmask := typ.GCData + buf := &getg().m.p.ptr().wbBuf + var bits uint32 + for i := uintptr(0); i < typ.PtrBytes; i += goarch.PtrSize { + if i&(goarch.PtrSize*8-1) == 0 { + bits = uint32(*ptrmask) + ptrmask = addb(ptrmask, 1) + } else { + bits = bits >> 1 + } + if bits&1 != 0 { + dstx := (*uintptr)(unsafe.Pointer(dst + i)) + srcx := (*uintptr)(unsafe.Pointer(src + i)) + p := buf.get2() + p[0] = *dstx + p[1] = *srcx + } + } +} + +// initHeapBits initializes the heap bitmap for a span. +// If this is a span of single pointer allocations, it initializes all +// words to pointer. If force is true, clears all bits. +func (s *mspan) initHeapBits(forceClear bool) { + if forceClear || s.spanclass.noscan() { + // Set all the pointer bits to zero. We do this once + // when the span is allocated so we don't have to do it + // for each object allocation. + base := s.base() + size := s.npages * pageSize + h := writeHeapBitsForAddr(base) + h.flush(base, size) + return + } + isPtrs := goarch.PtrSize == 8 && s.elemsize == goarch.PtrSize + if !isPtrs { + return // nothing to do + } + h := writeHeapBitsForAddr(s.base()) + size := s.npages * pageSize + nptrs := size / goarch.PtrSize + for i := uintptr(0); i < nptrs; i += ptrBits { + h = h.write(^uintptr(0), ptrBits) + } + h.flush(s.base(), size) +} + +// countAlloc returns the number of objects allocated in span s by +// scanning the allocation bitmap. +func (s *mspan) countAlloc() int { + count := 0 + bytes := divRoundUp(s.nelems, 8) + // Iterate over each 8-byte chunk and count allocations + // with an intrinsic. Note that newMarkBits guarantees that + // gcmarkBits will be 8-byte aligned, so we don't have to + // worry about edge cases, irrelevant bits will simply be zero. + for i := uintptr(0); i < bytes; i += 8 { + // Extract 64 bits from the byte pointer and get a OnesCount. + // Note that the unsafe cast here doesn't preserve endianness, + // but that's OK. We only care about how many bits are 1, not + // about the order we discover them in. + mrkBits := *(*uint64)(unsafe.Pointer(s.gcmarkBits.bytep(i))) + count += sys.OnesCount64(mrkBits) + } + return count +} + +type writeHeapBits struct { + addr uintptr // address that the low bit of mask represents the pointer state of. + mask uintptr // some pointer bits starting at the address addr. + valid uintptr // number of bits in buf that are valid (including low) + low uintptr // number of low-order bits to not overwrite +} + +func writeHeapBitsForAddr(addr uintptr) (h writeHeapBits) { + // We start writing bits maybe in the middle of a heap bitmap word. + // Remember how many bits into the word we started, so we can be sure + // not to overwrite the previous bits. + h.low = addr / goarch.PtrSize % ptrBits + + // round down to heap word that starts the bitmap word. + h.addr = addr - h.low*goarch.PtrSize + + // We don't have any bits yet. + h.mask = 0 + h.valid = h.low + + return +} + +// write appends the pointerness of the next valid pointer slots +// using the low valid bits of bits. 1=pointer, 0=scalar. +func (h writeHeapBits) write(bits, valid uintptr) writeHeapBits { + if h.valid+valid <= ptrBits { + // Fast path - just accumulate the bits. + h.mask |= bits << h.valid + h.valid += valid + return h + } + // Too many bits to fit in this word. Write the current word + // out and move on to the next word. + + data := h.mask | bits<> (ptrBits - h.valid) // leftover for next word + h.valid += valid - ptrBits // have h.valid+valid bits, writing ptrBits of them + + // Flush mask to the memory bitmap. + // TODO: figure out how to cache arena lookup. + ai := arenaIndex(h.addr) + ha := mheap_.arenas[ai.l1()][ai.l2()] + idx := h.addr / (ptrBits * goarch.PtrSize) % heapArenaBitmapWords + m := uintptr(1)< ptrBits { + h = h.write(0, ptrBits) + words -= ptrBits + } + return h.write(0, words) +} + +// Flush the bits that have been written, and add zeros as needed +// to cover the full object [addr, addr+size). +func (h writeHeapBits) flush(addr, size uintptr) { + // zeros counts the number of bits needed to represent the object minus the + // number of bits we've already written. This is the number of 0 bits + // that need to be added. + zeros := (addr+size-h.addr)/goarch.PtrSize - h.valid + + // Add zero bits up to the bitmap word boundary + if zeros > 0 { + z := ptrBits - h.valid + if z > zeros { + z = zeros + } + h.valid += z + zeros -= z + } + + // Find word in bitmap that we're going to write. + ai := arenaIndex(h.addr) + ha := mheap_.arenas[ai.l1()][ai.l2()] + idx := h.addr / (ptrBits * goarch.PtrSize) % heapArenaBitmapWords + + // Write remaining bits. + if h.valid != h.low { + m := uintptr(1)< 8 { + h = h.write(uintptr(*p), 8) + p = add1(p) + j -= 8 + } + h = h.write(uintptr(*p), j) + + if i+typ.Size_ == dataSize { + break // no padding after last element + } + + // Pad with zeros to the start of the next element. + h = h.pad(typ.Size_ - n*goarch.PtrSize) + } + + h.flush(x, size) + + // Erase the expanded GC program. + memclrNoHeapPointers(unsafe.Pointer(obj), (n+7)/8) + return + } + + // Note about sizes: + // + // typ.Size is the number of words in the object, + // and typ.PtrBytes is the number of words in the prefix + // of the object that contains pointers. That is, the final + // typ.Size - typ.PtrBytes words contain no pointers. + // This allows optimization of a common pattern where + // an object has a small header followed by a large scalar + // buffer. If we know the pointers are over, we don't have + // to scan the buffer's heap bitmap at all. + // The 1-bit ptrmasks are sized to contain only bits for + // the typ.PtrBytes prefix, zero padded out to a full byte + // of bitmap. If there is more room in the allocated object, + // that space is pointerless. The noMorePtrs bitmap will prevent + // scanning large pointerless tails of an object. + // + // Replicated copies are not as nice: if there is an array of + // objects with scalar tails, all but the last tail does have to + // be initialized, because there is no way to say "skip forward". + + ptrs := typ.PtrBytes / goarch.PtrSize + if typ.Size_ == dataSize { // Single element + if ptrs <= ptrBits { // Single small element + m := readUintptr(typ.GCData) + h = h.write(m, ptrs) + } else { // Single large element + p := typ.GCData + for { + h = h.write(readUintptr(p), ptrBits) + p = addb(p, ptrBits/8) + ptrs -= ptrBits + if ptrs <= ptrBits { + break + } + } + m := readUintptr(p) + h = h.write(m, ptrs) + } + } else { // Repeated element + words := typ.Size_ / goarch.PtrSize // total words, including scalar tail + if words <= ptrBits { // Repeated small element + n := dataSize / typ.Size_ + m := readUintptr(typ.GCData) + // Make larger unit to repeat + for words <= ptrBits/2 { + if n&1 != 0 { + h = h.write(m, words) + } + n /= 2 + m |= m << words + ptrs += words + words *= 2 + if n == 1 { + break + } + } + for n > 1 { + h = h.write(m, words) + n-- + } + h = h.write(m, ptrs) + } else { // Repeated large element + for i := uintptr(0); true; i += typ.Size_ { + p := typ.GCData + j := ptrs + for j > ptrBits { + h = h.write(readUintptr(p), ptrBits) + p = addb(p, ptrBits/8) + j -= ptrBits + } + m := readUintptr(p) + h = h.write(m, j) + if i+typ.Size_ == dataSize { + break // don't need the trailing nonptr bits on the last element. + } + // Pad with zeros to the start of the next element. + h = h.pad(typ.Size_ - typ.PtrBytes) + } + } + } + h.flush(x, size) + + if doubleCheck { + h := heapBitsForAddr(x, size) + for i := uintptr(0); i < size; i += goarch.PtrSize { + // Compute the pointer bit we want at offset i. + want := false + if i < dataSize { + off := i % typ.Size_ + if off < typ.PtrBytes { + j := off / goarch.PtrSize + want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0 + } + } + if want { + var addr uintptr + h, addr = h.next() + if addr != x+i { + throw("heapBitsSetType: pointer entry not correct") + } + } + } + if _, addr := h.next(); addr != 0 { + throw("heapBitsSetType: extra pointer") + } + } +} + +var debugPtrmask struct { + lock mutex + data *byte +} + +// progToPointerMask returns the 1-bit pointer mask output by the GC program prog. +// size the size of the region described by prog, in bytes. +// The resulting bitvector will have no more than size/goarch.PtrSize bits. +func progToPointerMask(prog *byte, size uintptr) bitvector { + n := (size/goarch.PtrSize + 7) / 8 + x := (*[1 << 30]byte)(persistentalloc(n+1, 1, &memstats.buckhash_sys))[:n+1] + x[len(x)-1] = 0xa1 // overflow check sentinel + n = runGCProg(prog, &x[0]) + if x[len(x)-1] != 0xa1 { + throw("progToPointerMask: overflow") + } + return bitvector{int32(n), &x[0]} +} + +// Packed GC pointer bitmaps, aka GC programs. +// +// For large types containing arrays, the type information has a +// natural repetition that can be encoded to save space in the +// binary and in the memory representation of the type information. +// +// The encoding is a simple Lempel-Ziv style bytecode machine +// with the following instructions: +// +// 00000000: stop +// 0nnnnnnn: emit n bits copied from the next (n+7)/8 bytes +// 10000000 n c: repeat the previous n bits c times; n, c are varints +// 1nnnnnnn c: repeat the previous n bits c times; c is a varint + +// runGCProg returns the number of 1-bit entries written to memory. +func runGCProg(prog, dst *byte) uintptr { + dstStart := dst + + // Bits waiting to be written to memory. + var bits uintptr + var nbits uintptr + + p := prog +Run: + for { + // Flush accumulated full bytes. + // The rest of the loop assumes that nbits <= 7. + for ; nbits >= 8; nbits -= 8 { + *dst = uint8(bits) + dst = add1(dst) + bits >>= 8 + } + + // Process one instruction. + inst := uintptr(*p) + p = add1(p) + n := inst & 0x7F + if inst&0x80 == 0 { + // Literal bits; n == 0 means end of program. + if n == 0 { + // Program is over. + break Run + } + nbyte := n / 8 + for i := uintptr(0); i < nbyte; i++ { + bits |= uintptr(*p) << nbits + p = add1(p) + *dst = uint8(bits) + dst = add1(dst) + bits >>= 8 + } + if n %= 8; n > 0 { + bits |= uintptr(*p) << nbits + p = add1(p) + nbits += n + } + continue Run + } + + // Repeat. If n == 0, it is encoded in a varint in the next bytes. + if n == 0 { + for off := uint(0); ; off += 7 { + x := uintptr(*p) + p = add1(p) + n |= (x & 0x7F) << off + if x&0x80 == 0 { + break + } + } + } + + // Count is encoded in a varint in the next bytes. + c := uintptr(0) + for off := uint(0); ; off += 7 { + x := uintptr(*p) + p = add1(p) + c |= (x & 0x7F) << off + if x&0x80 == 0 { + break + } + } + c *= n // now total number of bits to copy + + // If the number of bits being repeated is small, load them + // into a register and use that register for the entire loop + // instead of repeatedly reading from memory. + // Handling fewer than 8 bits here makes the general loop simpler. + // The cutoff is goarch.PtrSize*8 - 7 to guarantee that when we add + // the pattern to a bit buffer holding at most 7 bits (a partial byte) + // it will not overflow. + src := dst + const maxBits = goarch.PtrSize*8 - 7 + if n <= maxBits { + // Start with bits in output buffer. + pattern := bits + npattern := nbits + + // If we need more bits, fetch them from memory. + src = subtract1(src) + for npattern < n { + pattern <<= 8 + pattern |= uintptr(*src) + src = subtract1(src) + npattern += 8 + } + + // We started with the whole bit output buffer, + // and then we loaded bits from whole bytes. + // Either way, we might now have too many instead of too few. + // Discard the extra. + if npattern > n { + pattern >>= npattern - n + npattern = n + } + + // Replicate pattern to at most maxBits. + if npattern == 1 { + // One bit being repeated. + // If the bit is 1, make the pattern all 1s. + // If the bit is 0, the pattern is already all 0s, + // but we can claim that the number of bits + // in the word is equal to the number we need (c), + // because right shift of bits will zero fill. + if pattern == 1 { + pattern = 1<8 bits, there will be full bytes to flush + // on each iteration. + for ; c >= npattern; c -= npattern { + bits |= pattern << nbits + nbits += npattern + for nbits >= 8 { + *dst = uint8(bits) + dst = add1(dst) + bits >>= 8 + nbits -= 8 + } + } + + // Add final fragment to bit buffer. + if c > 0 { + pattern &= 1< nbits because n > maxBits and nbits <= 7 + // Leading src fragment. + src = subtractb(src, (off+7)/8) + if frag := off & 7; frag != 0 { + bits |= uintptr(*src) >> (8 - frag) << nbits + src = add1(src) + nbits += frag + c -= frag + } + // Main loop: load one byte, write another. + // The bits are rotating through the bit buffer. + for i := c / 8; i > 0; i-- { + bits |= uintptr(*src) << nbits + src = add1(src) + *dst = uint8(bits) + dst = add1(dst) + bits >>= 8 + } + // Final src fragment. + if c %= 8; c > 0 { + bits |= (uintptr(*src) & (1< 0; nbits -= 8 { + *dst = uint8(bits) + dst = add1(dst) + bits >>= 8 + } + return totalBits +} + +// materializeGCProg allocates space for the (1-bit) pointer bitmask +// for an object of size ptrdata. Then it fills that space with the +// pointer bitmask specified by the program prog. +// The bitmask starts at s.startAddr. +// The result must be deallocated with dematerializeGCProg. +func materializeGCProg(ptrdata uintptr, prog *byte) *mspan { + // Each word of ptrdata needs one bit in the bitmap. + bitmapBytes := divRoundUp(ptrdata, 8*goarch.PtrSize) + // Compute the number of pages needed for bitmapBytes. + pages := divRoundUp(bitmapBytes, pageSize) + s := mheap_.allocManual(pages, spanAllocPtrScalarBits) + runGCProg(addb(prog, 4), (*byte)(unsafe.Pointer(s.startAddr))) + return s +} +func dematerializeGCProg(s *mspan) { + mheap_.freeManual(s, spanAllocPtrScalarBits) +} + +func dumpGCProg(p *byte) { + nptr := 0 + for { + x := *p + p = add1(p) + if x == 0 { + print("\t", nptr, " end\n") + break + } + if x&0x80 == 0 { + print("\t", nptr, " lit ", x, ":") + n := int(x+7) / 8 + for i := 0; i < n; i++ { + print(" ", hex(*p)) + p = add1(p) + } + print("\n") + nptr += int(x) + } else { + nbit := int(x &^ 0x80) + if nbit == 0 { + for nb := uint(0); ; nb += 7 { + x := *p + p = add1(p) + nbit |= int(x&0x7f) << nb + if x&0x80 == 0 { + break + } + } + } + count := 0 + for nb := uint(0); ; nb += 7 { + x := *p + p = add1(p) + count |= int(x&0x7f) << nb + if x&0x80 == 0 { + break + } + } + print("\t", nptr, " repeat ", nbit, " × ", count, "\n") + nptr += nbit * count + } + } +} + +// Testing. + +// reflect_gcbits returns the GC type info for x, for testing. +// The result is the bitmap entries (0 or 1), one entry per byte. +// +//go:linkname reflect_gcbits reflect.gcbits +func reflect_gcbits(x any) []byte { + return getgcmask(x) +} + +// Returns GC type info for the pointer stored in ep for testing. +// If ep points to the stack, only static live information will be returned +// (i.e. not for objects which are only dynamically live stack objects). +func getgcmask(ep any) (mask []byte) { + e := *efaceOf(&ep) + p := e.data + t := e._type + // data or bss + for _, datap := range activeModules() { + // data + if datap.data <= uintptr(p) && uintptr(p) < datap.edata { + bitmap := datap.gcdatamask.bytedata + n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_ + mask = make([]byte, n/goarch.PtrSize) + for i := uintptr(0); i < n; i += goarch.PtrSize { + off := (uintptr(p) + i - datap.data) / goarch.PtrSize + mask[i/goarch.PtrSize] = (*addb(bitmap, off/8) >> (off % 8)) & 1 + } + return + } + + // bss + if datap.bss <= uintptr(p) && uintptr(p) < datap.ebss { + bitmap := datap.gcbssmask.bytedata + n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_ + mask = make([]byte, n/goarch.PtrSize) + for i := uintptr(0); i < n; i += goarch.PtrSize { + off := (uintptr(p) + i - datap.bss) / goarch.PtrSize + mask[i/goarch.PtrSize] = (*addb(bitmap, off/8) >> (off % 8)) & 1 + } + return + } + } + + // heap + if base, s, _ := findObject(uintptr(p), 0, 0); base != 0 { + if s.spanclass.noscan() { + return nil + } + n := s.elemsize + hbits := heapBitsForAddr(base, n) + mask = make([]byte, n/goarch.PtrSize) + for { + var addr uintptr + if hbits, addr = hbits.next(); addr == 0 { + break + } + mask[(addr-base)/goarch.PtrSize] = 1 + } + // Callers expect this mask to end at the last pointer. + for len(mask) > 0 && mask[len(mask)-1] == 0 { + mask = mask[:len(mask)-1] + } + return + } + + // stack + if gp := getg(); gp.m.curg.stack.lo <= uintptr(p) && uintptr(p) < gp.m.curg.stack.hi { + found := false + var u unwinder + for u.initAt(gp.m.curg.sched.pc, gp.m.curg.sched.sp, 0, gp.m.curg, 0); u.valid(); u.next() { + if u.frame.sp <= uintptr(p) && uintptr(p) < u.frame.varp { + found = true + break + } + } + if found { + locals, _, _ := u.frame.getStackMap(nil, false) + if locals.n == 0 { + return + } + size := uintptr(locals.n) * goarch.PtrSize + n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_ + mask = make([]byte, n/goarch.PtrSize) + for i := uintptr(0); i < n; i += goarch.PtrSize { + off := (uintptr(p) + i - u.frame.varp + size) / goarch.PtrSize + mask[i/goarch.PtrSize] = locals.ptrbit(off) + } + } + return + } + + // otherwise, not something the GC knows about. + // possibly read-only data, like malloc(0). + // must not have pointers + return +} +*/ diff --git a/internal/runtime/msize.go b/internal/runtime/msize.go new file mode 100644 index 00000000..c74a9754 --- /dev/null +++ b/internal/runtime/msize.go @@ -0,0 +1,29 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Malloc small size classes. +// +// See malloc.go for overview. +// See also mksizeclasses.go for how we decide what size classes to use. + +package runtime + +// Returns size of the memory block that mallocgc will allocate if you ask for the size. +func roundupsize(size uintptr) uintptr { + return size +} + +/* if size < _MaxSmallSize { + if size <= smallSizeMax-8 { + return uintptr(class_to_size[size_to_class8[divRoundUp(size, smallSizeDiv)]]) + } else { + return uintptr(class_to_size[size_to_class128[divRoundUp(size-smallSizeMax, largeSizeDiv)]]) + } + } + if size+_PageSize < size { + return size + } + return alignUp(size, _PageSize) +} +*/ diff --git a/internal/runtime/panic.go b/internal/runtime/panic.go index 1d90e032..c6f4cdd6 100644 --- a/internal/runtime/panic.go +++ b/internal/runtime/panic.go @@ -1052,21 +1052,14 @@ func sync_throw(s string) { func sync_fatal(s string) { fatal(s) } +*/ // throw triggers a fatal error that dumps a stack trace and exits. // // throw should be used for runtime-internal fatal errors where Go itself, // rather than user code, may be at fault for the failure. -// -//go:nosplit func throw(s string) { - // Everything throw does should be recursively nosplit so it - // can be called even when it's unsafe to grow the stack. - systemstack(func() { - print("fatal error: ", s, "\n") - }) - - fatalthrow(throwTypeRuntime) + fatal(s) } // fatal triggers a fatal error that dumps a stack trace and exits. @@ -1076,10 +1069,11 @@ func throw(s string) { // // fatal does not include runtime frames, system goroutines, or frame metadata // (fp, sp, pc) in the stack trace unless GOTRACEBACK=system or higher. -// -//go:nosplit func fatal(s string) { - // Everything fatal does should be recursively nosplit so it + panic("fatal error: " + s) +} + +/* // Everything throw does should be recursively nosplit so it // can be called even when it's unsafe to grow the stack. systemstack(func() { print("fatal error: ", s, "\n") @@ -1088,6 +1082,7 @@ func fatal(s string) { fatalthrow(throwTypeUser) } +/* // runningPanicDefers is non-zero while running deferred functions for panic. // This is used to try hard to get a panic stack trace out when exiting. var runningPanicDefers atomic.Uint32 @@ -1139,17 +1134,7 @@ func recovery(gp *g) { // fatalthrow implements an unrecoverable runtime throw. It freezes the // system, prints stack traces starting from its caller, and terminates the // process. -// -//go:nosplit func fatalthrow(t throwType) { - pc := getcallerpc() - sp := getcallersp() - gp := getg() - - if gp.m.throwing == throwTypeNone { - gp.m.throwing = t - } - // Switch to the system stack to avoid any stack growth, which may make // things worse if the runtime is in a bad state. systemstack(func() { @@ -1172,6 +1157,7 @@ func fatalthrow(t throwType) { *(*int)(nil) = 0 // not reached } +/* // fatalpanic implements an unrecoverable panic. It is like fatalthrow, except // that if msgs != nil, fatalpanic also prints panic messages and decrements // runningPanicDefers once main is blocked from exiting. diff --git a/internal/runtime/stubs.go b/internal/runtime/stubs.go index ed8b9a8d..349aecc4 100644 --- a/internal/runtime/stubs.go +++ b/internal/runtime/stubs.go @@ -4,27 +4,148 @@ package runtime -import _ "unsafe" +import ( + "unsafe" + + "github.com/goplus/llgo/c" +) + +// Should be a built-in for unsafe.Pointer? +// +//go:linkname add llgo.advance +func add(p unsafe.Pointer, x uintptr) unsafe.Pointer + +/* +// mcall switches from the g to the g0 stack and invokes fn(g), +// where g is the goroutine that made the call. +// mcall saves g's current PC/SP in g->sched so that it can be restored later. +// It is up to fn to arrange for that later execution, typically by recording +// g in a data structure, causing something to call ready(g) later. +// mcall returns to the original goroutine g later, when g has been rescheduled. +// fn must not return at all; typically it ends by calling schedule, to let the m +// run other goroutines. +// +// mcall can only be called from g stacks (not g0, not gsignal). +// +// This must NOT be go:noescape: if fn is a stack-allocated closure, +// fn puts g on a run queue, and g executes before fn returns, the +// closure will be invalidated while it is still executing. +func mcall(fn func(*g)) + +// systemstack runs fn on a system stack. +// If systemstack is called from the per-OS-thread (g0) stack, or +// if systemstack is called from the signal handling (gsignal) stack, +// systemstack calls fn directly and returns. +// Otherwise, systemstack is being called from the limited stack +// of an ordinary goroutine. In this case, systemstack switches +// to the per-OS-thread stack, calls fn, and switches back. +// It is common to use a func literal as the argument, in order +// to share inputs and outputs with the code around the call +// to system stack: +// +// ... set up y ... +// systemstack(func() { +// x = bigcall(y) +// }) +// ... use x ... +// +//go:noescape +func systemstack(fn func()) + +//go:nosplit +//go:nowritebarrierrec +func badsystemstack() { + writeErrStr("fatal: systemstack called from unexpected goroutine") +} +*/ + +// memclrNoHeapPointers clears n bytes starting at ptr. +// +// Usually you should use typedmemclr. memclrNoHeapPointers should be +// used only when the caller knows that *ptr contains no heap pointers +// because either: +// +// *ptr is initialized memory and its type is pointer-free, or +// +// *ptr is uninitialized memory (e.g., memory that's being reused +// for a new allocation) and hence contains only "junk". +// +// memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n +// is a multiple of the pointer size, then any pointer-aligned, +// pointer-sized portion is cleared atomically. Despite the function +// name, this is necessary because this function is the underlying +// implementation of typedmemclr and memclrHasPointers. See the doc of +// memmove for more details. +// +// The (CPU-specific) implementations of this function are in memclr_*.s. +func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { + c.Memset(ptr, 0, n) +} + +// Zeroinit initializes memory to zero. +func Zeroinit(p unsafe.Pointer, size uintptr) unsafe.Pointer { + return c.Memset(p, 0, size) +} + +/* +//go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers +func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { + memclrNoHeapPointers(ptr, n) +} +*/ + +// memmove copies n bytes from "from" to "to". +// +// memmove ensures that any pointer in "from" is written to "to" with +// an indivisible write, so that racy reads cannot observe a +// half-written pointer. This is necessary to prevent the garbage +// collector from observing invalid pointers, and differs from memmove +// in unmanaged languages. However, memmove is only required to do +// this if "from" and "to" may contain pointers, which can only be the +// case if "from", "to", and "n" are all be word-aligned. +// +//go:linkname memmove C.memmove +func memmove(to, from unsafe.Pointer, n uintptr) + +/* +// Outside assembly calls memmove. Make sure it has ABI wrappers. +// +//go:linkname memmove + +//go:linkname reflect_memmove reflect.memmove +func reflect_memmove(to, from unsafe.Pointer, n uintptr) { + memmove(to, from, n) +} + +// exported value for testing +const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) +*/ //go:linkname fastrand C.rand func fastrand() uint32 -/* TODO(xsw): -func fastrand() uint32 { +/* +//go:nosplit +func fastrandn(n uint32) uint32 { + // This is similar to fastrand() % n, but faster. + // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32(uint64(fastrand()) * uint64(n) >> 32) +} + +func fastrand64() uint64 { mp := getg().m // Implement wyrand: https://github.com/wangyi-fudan/wyhash // Only the platform that math.Mul64 can be lowered // by the compiler should be in this list. if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64| goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le| - goarch.IsS390x|goarch.IsRiscv64|goarch.IsLoong64 == 1 { + goarch.IsS390x|goarch.IsRiscv64 == 1 { mp.fastrand += 0xa0761d6478bd642f hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db) - return uint32(hi ^ lo) + return hi ^ lo } // Implement xorshift64+: 2 32-bit xorshift sequences added together. - // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf // This generator passes the SmallCrush suite, part of TestU01 framework: // http://simul.iro.umontreal.ca/testu01/tu01.html @@ -32,7 +153,324 @@ func fastrand() uint32 { s1, s0 := t[0], t[1] s1 ^= s1 << 17 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 + r := uint64(s0 + s1) + + s0, s1 = s1, s0 + s1 ^= s1 << 17 + s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 + r += uint64(s0+s1) << 32 + t[0], t[1] = s0, s1 - return s0 + s1 + return r } + +func fastrandu() uint { + if goarch.PtrSize == 4 { + return uint(fastrand()) + } + return uint(fastrand64()) +} + +//go:linkname rand_fastrand64 math/rand.fastrand64 +func rand_fastrand64() uint64 { return fastrand64() } + +//go:linkname sync_fastrandn sync.fastrandn +func sync_fastrandn(n uint32) uint32 { return fastrandn(n) } + +//go:linkname net_fastrandu net.fastrandu +func net_fastrandu() uint { return fastrandu() } + +//go:linkname os_fastrand os.fastrand +func os_fastrand() uint32 { return fastrand() } + +// in internal/bytealg/equal_*.s +// +//go:noescape +func memequal(a, b unsafe.Pointer, size uintptr) bool + +// noescape hides a pointer from escape analysis. noescape is +// the identity function but escape analysis doesn't think the +// output depends on the input. noescape is inlined and currently +// compiles down to zero instructions. +// USE CAREFULLY! +func noescape(p unsafe.Pointer) unsafe.Pointer { + x := uintptr(p) + return unsafe.Pointer(x ^ 0) +} + +// noEscapePtr hides a pointer from escape analysis. See noescape. +// USE CAREFULLY! +// +//go:nosplit +func noEscapePtr[T any](p *T) *T { + x := uintptr(unsafe.Pointer(p)) + return (*T)(unsafe.Pointer(x ^ 0)) +} + +// Not all cgocallback frames are actually cgocallback, +// so not all have these arguments. Mark them uintptr so that the GC +// does not misinterpret memory when the arguments are not present. +// cgocallback is not called from Go, only from crosscall2. +// This in turn calls cgocallbackg, which is where we'll find +// pointer-declared arguments. +// +// When fn is nil (frame is saved g), call dropm instead, +// this is used when the C thread is exiting. +func cgocallback(fn, frame, ctxt uintptr) + +func gogo(buf *gobuf) + +func asminit() +func setg(gg *g) +func breakpoint() + +// reflectcall calls fn with arguments described by stackArgs, stackArgsSize, +// frameSize, and regArgs. +// +// Arguments passed on the stack and space for return values passed on the stack +// must be laid out at the space pointed to by stackArgs (with total length +// stackArgsSize) according to the ABI. +// +// stackRetOffset must be some value <= stackArgsSize that indicates the +// offset within stackArgs where the return value space begins. +// +// frameSize is the total size of the argument frame at stackArgs and must +// therefore be >= stackArgsSize. It must include additional space for spilling +// register arguments for stack growth and preemption. +// +// TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, +// since frameSize will be redundant with stackArgsSize. +// +// Arguments passed in registers must be laid out in regArgs according to the ABI. +// regArgs will hold any return values passed in registers after the call. +// +// reflectcall copies stack arguments from stackArgs to the goroutine stack, and +// then copies back stackArgsSize-stackRetOffset bytes back to the return space +// in stackArgs once fn has completed. It also "unspills" argument registers from +// regArgs before calling fn, and spills them back into regArgs immediately +// following the call to fn. If there are results being returned on the stack, +// the caller should pass the argument frame type as stackArgsType so that +// reflectcall can execute appropriate write barriers during the copy. +// +// reflectcall expects regArgs.ReturnIsPtr to be populated indicating which +// registers on the return path will contain Go pointers. It will then store +// these pointers in regArgs.Ptrs such that they are visible to the GC. +// +// Package reflect passes a frame type. In package runtime, there is only +// one call that copies results back, in callbackWrap in syscall_windows.go, and it +// does NOT pass a frame type, meaning there are no write barriers invoked. See that +// call site for justification. +// +// Package reflect accesses this symbol through a linkname. +// +// Arguments passed through to reflectcall do not escape. The type is used +// only in a very limited callee of reflectcall, the stackArgs are copied, and +// regArgs is only used in the reflectcall frame. +// +//go:noescape +func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) + +func procyield(cycles uint32) + +type neverCallThisFunction struct{} + +// goexit is the return stub at the top of every goroutine call stack. +// Each goroutine stack is constructed as if goexit called the +// goroutine's entry point function, so that when the entry point +// function returns, it will return to goexit, which will call goexit1 +// to perform the actual exit. +// +// This function must never be called directly. Call goexit1 instead. +// gentraceback assumes that goexit terminates the stack. A direct +// call on the stack will cause gentraceback to stop walking the stack +// prematurely and if there is leftover state it may panic. +func goexit(neverCallThisFunction) + +// publicationBarrier performs a store/store barrier (a "publication" +// or "export" barrier). Some form of synchronization is required +// between initializing an object and making that object accessible to +// another processor. Without synchronization, the initialization +// writes and the "publication" write may be reordered, allowing the +// other processor to follow the pointer and observe an uninitialized +// object. In general, higher-level synchronization should be used, +// such as locking or an atomic pointer write. publicationBarrier is +// for when those aren't an option, such as in the implementation of +// the memory manager. +// +// There's no corresponding barrier for the read side because the read +// side naturally has a data dependency order. All architectures that +// Go supports or seems likely to ever support automatically enforce +// data dependency ordering. +func publicationBarrier() + +// getcallerpc returns the program counter (PC) of its caller's caller. +// getcallersp returns the stack pointer (SP) of its caller's caller. +// The implementation may be a compiler intrinsic; there is not +// necessarily code implementing this on every platform. +// +// For example: +// +// func f(arg1, arg2, arg3 int) { +// pc := getcallerpc() +// sp := getcallersp() +// } +// +// These two lines find the PC and SP immediately following +// the call to f (where f will return). +// +// The call to getcallerpc and getcallersp must be done in the +// frame being asked about. +// +// The result of getcallersp is correct at the time of the return, +// but it may be invalidated by any subsequent call to a function +// that might relocate the stack in order to grow or shrink it. +// A general rule is that the result of getcallersp should be used +// immediately and can only be passed to nosplit functions. + +//go:noescape +func getcallerpc() uintptr + +//go:noescape +func getcallersp() uintptr // implemented as an intrinsic on all platforms + +// getclosureptr returns the pointer to the current closure. +// getclosureptr can only be used in an assignment statement +// at the entry of a function. Moreover, go:nosplit directive +// must be specified at the declaration of caller function, +// so that the function prolog does not clobber the closure register. +// for example: +// +// //go:nosplit +// func f(arg1, arg2, arg3 int) { +// dx := getclosureptr() +// } +// +// The compiler rewrites calls to this function into instructions that fetch the +// pointer from a well-known register (DX on x86 architecture, etc.) directly. +func getclosureptr() uintptr + +//go:noescape +func asmcgocall(fn, arg unsafe.Pointer) int32 + +func morestack() +func morestack_noctxt() +func rt0_go() + +// return0 is a stub used to return 0 from deferproc. +// It is called at the very end of deferproc to signal +// the calling Go function that it should not jump +// to deferreturn. +// in asm_*.s +func return0() + +// in asm_*.s +// not called directly; definitions here supply type information for traceback. +// These must have the same signature (arg pointer map) as reflectcall. +func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) + +func systemstack_switch() + +// alignUp rounds n up to a multiple of a. a must be a power of 2. +func alignUp(n, a uintptr) uintptr { + return (n + a - 1) &^ (a - 1) +} + +// alignDown rounds n down to a multiple of a. a must be a power of 2. +func alignDown(n, a uintptr) uintptr { + return n &^ (a - 1) +} + +// divRoundUp returns ceil(n / a). +func divRoundUp(n, a uintptr) uintptr { + // a is generally a power of two. This will get inlined and + // the compiler will optimize the division. + return (n + a - 1) / a +} + +// checkASM reports whether assembly runtime checks have passed. +func checkASM() bool + +func memequal_varlen(a, b unsafe.Pointer) bool + +// bool2int returns 0 if x is false or 1 if x is true. +func bool2int(x bool) int { + // Avoid branches. In the SSA compiler, this compiles to + // exactly what you would want it to. + return int(uint8(*(*uint8)(unsafe.Pointer(&x)))) +} + +// abort crashes the runtime in situations where even throw might not +// work. In general it should do something a debugger will recognize +// (e.g., an INT3 on x86). A crash in abort is recognized by the +// signal handler, which will attempt to tear down the runtime +// immediately. +func abort() + +// Called from compiled code; declared for vet; do NOT call from Go. +func gcWriteBarrier1() +func gcWriteBarrier2() +func gcWriteBarrier3() +func gcWriteBarrier4() +func gcWriteBarrier5() +func gcWriteBarrier6() +func gcWriteBarrier7() +func gcWriteBarrier8() +func duffzero() +func duffcopy() + +// Called from linker-generated .initarray; declared for go vet; do NOT call from Go. +func addmoduledata() + +// Injected by the signal handler for panicking signals. +// Initializes any registers that have fixed meaning at calls but +// are scratch in bodies and calls sigpanic. +// On many platforms it just jumps to sigpanic. +func sigpanic0() + +// intArgRegs is used by the various register assignment +// algorithm implementations in the runtime. These include:. +// - Finalizers (mfinal.go) +// - Windows callbacks (syscall_windows.go) +// +// Both are stripped-down versions of the algorithm since they +// only have to deal with a subset of cases (finalizers only +// take a pointer or interface argument, Go Windows callbacks +// don't support floating point). +// +// It should be modified with care and are generally only +// modified when testing this package. +// +// It should never be set higher than its internal/abi +// constant counterparts, because the system relies on a +// structure that is at least large enough to hold the +// registers the system supports. +// +// Protected by finlock. +var intArgRegs = abi.IntArgRegs */ diff --git a/internal/runtime/type.go b/internal/runtime/type.go deleted file mode 100644 index 8e8b4332..00000000 --- a/internal/runtime/type.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Runtime type representation. - -package runtime - -import ( - "github.com/goplus/llgo/internal/abi" -) - -type _type = abi.Type - -/* -type maptype = abi.MapType - -type arraytype = abi.ArrayType - -type chantype = abi.ChanType - -type slicetype = abi.SliceType - -type functype = abi.FuncType - -type ptrtype = abi.PtrType - -type name = abi.Name - -type structtype = abi.StructType -*/ diff --git a/internal/runtime/z_face.go b/internal/runtime/z_face.go index 2da98913..6574369f 100644 --- a/internal/runtime/z_face.go +++ b/internal/runtime/z_face.go @@ -23,6 +23,13 @@ import ( "github.com/goplus/llgo/internal/runtime/c" ) +type _type = abi.Type + +// isDirectIface reports whether t is stored directly in an interface value. +func isDirectIface(t *_type) bool { + return t.Kind_&abi.KindDirectIface != 0 +} + type eface struct { _type *_type data unsafe.Pointer diff --git a/internal/runtime/z_map.go b/internal/runtime/z_map.go index 8d618f8e..500fa1f1 100644 --- a/internal/runtime/z_map.go +++ b/internal/runtime/z_map.go @@ -16,10 +16,113 @@ package runtime +import ( + "unsafe" + + "github.com/goplus/llgo/internal/abi" +) + // Map represents a Go map. type Map = hmap +type MapType = abi.MapType // MakeSmallMap creates a new small map. func MakeSmallMap() *Map { return makemap_small() } + +// MakeMap creates a new map. +func MakeMap(t *MapType, hint int, at *Map) *Map { + return makemap(t, hint, at) +} + +// MapAssign finds a key in map m and returns the elem address to assign. +func MapAssign(t *MapType, m *Map, key unsafe.Pointer) unsafe.Pointer { + return mapassign(t, m, key) +} + +func isReflexive(key *Type) bool { + return true // TODO(xsw): false for float/complex type +} + +func hashOf(t *Type) func(key unsafe.Pointer, hash0 uintptr) uintptr { + if t.TFlag&abi.TFlagRegularMemory != 0 { + switch t.Size_ { + case 4: + return memhash32 + case 8: + return memhash64 + } + return func(key unsafe.Pointer, hash0 uintptr) uintptr { + return memhash(key, hash0, t.Size_) + } + } + switch t.Kind() { + case abi.Float32: + return f32hash + case abi.Float64: + return f64hash + case abi.Complex64: + return c64hash + case abi.Complex128: + return c128hash + case abi.String: + return strhash + case abi.Interface: + i := (*interfacetype)(unsafe.Pointer(t)) + if len(i.Methods) == 0 { + return nilinterhash + } + return interhash + } + return func(key unsafe.Pointer, hash0 uintptr) uintptr { + return typehash(t, key, hash0) + } +} + +// MapOf creates a new map type. +func MapOf(key, elem *Type) *MapType { + var flags uint32 + + keySlot, elemSlot := key, elem + ptrTy := Basic(abi.UnsafePointer) + if keySlot.Size_ > 128 { + keySlot = ptrTy + flags |= 1 + } + if elemSlot.Size_ > 128 { + elemSlot = ptrTy + flags |= 2 + } + if isReflexive(key) { + flags |= 4 + } + + tophashTy := ArrayOf(bucketCnt, Basic(abi.Uint8)) + keysTy := ArrayOf(bucketCnt, keySlot) + elemsTy := ArrayOf(bucketCnt, elemSlot) + + tophash := StructField("tophash", tophashTy, 0, "", false) + keys := StructField("keys", keysTy, tophashTy.Size_, "", false) + elems := StructField("elems", elemsTy, keys.Offset+keysTy.Size_, "", false) + overflow := StructField("overflow", ptrTy, elems.Offset+elemsTy.Size_, "", false) + + bucket := Struct("", overflow.Offset+ptrTy.Size_, tophash, keys, elems, overflow) + + ret := &abi.MapType{ + Type: abi.Type{ + Size_: unsafe.Sizeof(uintptr(0)), + Hash: uint32(abi.Map), + Kind_: uint8(abi.Map), + }, + Key: key, + Elem: elem, + Bucket: bucket, + Hasher: hashOf(key), + KeySize: uint8(keySlot.Size_), // size of key slot + ValueSize: uint8(elemSlot.Size_), // size of elem slot + BucketSize: uint16(bucket.Size_), // size of bucket + Flags: flags, + } + return ret +} diff --git a/internal/runtime/z_rt.go b/internal/runtime/z_rt.go index 222c6e5c..a592c11b 100644 --- a/internal/runtime/z_rt.go +++ b/internal/runtime/z_rt.go @@ -93,10 +93,3 @@ func stringTracef(fp c.FilePtr, format *c.Char, s String) { } // ----------------------------------------------------------------------------- - -// Zeroinit initializes memory to zero. -func Zeroinit(p unsafe.Pointer, size uintptr) unsafe.Pointer { - return c.Memset(p, 0, size) -} - -// ----------------------------------------------------------------------------- diff --git a/internal/runtime/z_slice.go b/internal/runtime/z_slice.go index f4711e47..3657b22c 100644 --- a/internal/runtime/z_slice.go +++ b/internal/runtime/z_slice.go @@ -24,6 +24,8 @@ import ( // ----------------------------------------------------------------------------- +// type Slice = slice + // Slice is the runtime representation of a slice. type Slice struct { data unsafe.Pointer diff --git a/py/README.md b/py/README.md index 16d3bbc3..32479857 100644 --- a/py/README.md +++ b/py/README.md @@ -19,10 +19,10 @@ To run the demos in directory `_demo`, you need to set the `LLGO_LIB_PYTHON` env export LLGO_LIB_PYTHON=/foo/bar/python3.12 ``` -For example, `/opt/homebrew/Frameworks/Python.framework/Versions/3.12/libpython3.12.dylib` is a typical python lib location under macOS. So we should set it like this: +For example, `/opt/homebrew/Frameworks/Python.framework/Versions/3.12/lib/libpython3.12.dylib` is a typical python lib location under macOS. So we should set it like this: ```sh -export LLGO_LIB_PYTHON=/opt/homebrew/Frameworks/Python.framework/Versions/3.12/python3.12 +export LLGO_LIB_PYTHON=/opt/homebrew/Frameworks/Python.framework/Versions/3.12/lib/python3.12 ``` Then you can run the demos in directory `_demo`: diff --git a/ssa/abi/abi.go b/ssa/abi/abi.go index 4841b895..28e065b3 100644 --- a/ssa/abi/abi.go +++ b/ssa/abi/abi.go @@ -159,6 +159,10 @@ func (b *Builder) TypeName(t types.Type) (ret string, pub bool) { return "_llgo_any", true } return b.InterfaceName(t) + case *types.Map: + key, pub1 := b.TypeName(t.Key()) + elem, pub2 := b.TypeName(t.Elem()) + return fmt.Sprintf("map[%s]%s", key, elem), pub1 && pub2 } log.Panicf("todo: %T\n", t) return diff --git a/ssa/abitype.go b/ssa/abitype.go index 36c14489..da29c957 100644 --- a/ssa/abitype.go +++ b/ssa/abitype.go @@ -66,6 +66,8 @@ func (b Builder) abiTypeOf(t types.Type) func() Expr { return b.abiFuncOf(t) case *types.Slice: return b.abiSliceOf(t) + case *types.Map: + return b.abiMapOf(t) case *types.Array: return b.abiArrayOf(t) } @@ -244,6 +246,14 @@ func (b Builder) abiPointerOf(t *types.Pointer) func() Expr { } } +func (b Builder) abiMapOf(t *types.Map) func() Expr { + key := b.abiType(t.Key()) + elem := b.abiType(t.Elem()) + return func() Expr { + return b.Call(b.Pkg.rtFunc("MapOf"), key, elem) + } +} + func (b Builder) abiSliceOf(t *types.Slice) func() Expr { elem := b.abiType(t.Elem()) return func() Expr { diff --git a/ssa/datastruct.go b/ssa/datastruct.go index aa09b8c0..bac55714 100644 --- a/ssa/datastruct.go +++ b/ssa/datastruct.go @@ -360,8 +360,63 @@ func (b Builder) MapUpdate(m, k, v Expr) { if debugInstr { log.Printf("MapUpdate %v[%v] = %v\n", m.impl, k.impl, v.impl) } - // TODO(xsw) - // panic("todo") + t := m.Type + if t.kind != vkMap { + panic("TODO: not a map") + } + tabi := b.abiType(t.raw.Type) + prog := b.Prog + mptr := b.dupAlloca(m) + ptrimpl := b.InlineCall(b.Pkg.rtFunc("MapAssign"), tabi, mptr, k).impl + ptr := Expr{ptrimpl, prog.Pointer(v.Type)} + b.Store(ptr, v) // TODO(xsw): indirect store +} + +// ----------------------------------------------------------------------------- + +// The Range instruction yields an iterator over the domain and range +// of X, which must be a string or map. +// +// Elements are accessed via Next. +// +// Type() returns an opaque and degenerate "rangeIter" type. +// +// Pos() returns the ast.RangeStmt.For. +// +// Example printed form: +// +// t0 = range "hello":string +func (b Builder) Range(x Expr) Expr { + switch x.kind { + case vkString: + return b.InlineCall(b.Pkg.rtFunc("NewStringIter"), x) + } + panic("todo") +} + +// The Next instruction reads and advances the (map or string) +// iterator Iter and returns a 3-tuple value (ok, k, v). If the +// iterator is not exhausted, ok is true and k and v are the next +// elements of the domain and range, respectively. Otherwise ok is +// false and k and v are undefined. +// +// Components of the tuple are accessed using Extract. +// +// The IsString field distinguishes iterators over strings from those +// over maps, as the Type() alone is insufficient: consider +// map[int]rune. +// +// Type() returns a *types.Tuple for the triple (ok, k, v). +// The types of k and/or v may be types.Invalid. +// +// Example printed form: +// +// t1 = next t0 +func (b Builder) Next(iter Expr, isString bool) (ret Expr) { + if isString { + return b.InlineCall(b.Pkg.rtFunc("StringIterNext"), iter) + } + panic("todo") } // ----------------------------------------------------------------------------- diff --git a/ssa/expr.go b/ssa/expr.go index f41ff644..d9573891 100644 --- a/ssa/expr.go +++ b/ssa/expr.go @@ -669,53 +669,6 @@ func castPtr(b llvm.Builder, x llvm.Value, t llvm.Type) llvm.Value { // ----------------------------------------------------------------------------- -// The Range instruction yields an iterator over the domain and range -// of X, which must be a string or map. -// -// Elements are accessed via Next. -// -// Type() returns an opaque and degenerate "rangeIter" type. -// -// Pos() returns the ast.RangeStmt.For. -// -// Example printed form: -// -// t0 = range "hello":string -func (b Builder) Range(x Expr) Expr { - switch x.kind { - case vkString: - return b.InlineCall(b.Pkg.rtFunc("NewStringIter"), x) - } - panic("todo") -} - -// The Next instruction reads and advances the (map or string) -// iterator Iter and returns a 3-tuple value (ok, k, v). If the -// iterator is not exhausted, ok is true and k and v are the next -// elements of the domain and range, respectively. Otherwise ok is -// false and k and v are undefined. -// -// Components of the tuple are accessed using Extract. -// -// The IsString field distinguishes iterators over strings from those -// over maps, as the Type() alone is insufficient: consider -// map[int]rune. -// -// Type() returns a *types.Tuple for the triple (ok, k, v). -// The types of k and/or v may be types.Invalid. -// -// Example printed form: -// -// t1 = next t0 -func (b Builder) Next(iter Expr, isString bool) (ret Expr) { - if isString { - return b.InlineCall(b.Pkg.rtFunc("StringIterNext"), iter) - } - panic("todo") -} - -// ----------------------------------------------------------------------------- - // The MakeClosure instruction yields a closure value whose code is // Fn and whose free variables' values are supplied by Bindings. // diff --git a/ssa/memory.go b/ssa/memory.go index d63df457..8ece634c 100644 --- a/ssa/memory.go +++ b/ssa/memory.go @@ -107,16 +107,15 @@ func aggregateInit(b llvm.Builder, ptr llvm.Value, tll llvm.Type, flds ...llvm.V } } -/* -func (b Builder) dupMalloc(v Expr) Expr { +func (b Builder) dupAlloca(v Expr) Expr { prog := b.Prog n := prog.SizeOf(v.Type) tptr := prog.Pointer(v.Type) - ptr := b.malloc(prog.Val(uintptr(n))).impl - b.Store(Expr{ptr, tptr}, v) - return Expr{ptr, tptr} + ptr := b.Alloca(prog.Val(uintptr(n))).impl + ret := Expr{ptr, tptr} + b.Store(ret, v) + return ret } -*/ // ----------------------------------------------------------------------------- diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index fb14d9c6..5231a7df 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -27,6 +27,17 @@ import ( "github.com/goplus/llvm" ) +func TestMapUpdate(t *testing.T) { + var b Builder + var m = Expr{Type: &aType{}} + defer func() { + if e := recover(); e == nil { + t.Log("MapUpdate: no error?") + } + }() + b.MapUpdate(m, m, m) +} + func TestEndDefer(t *testing.T) { prog := NewProgram(nil) pkg := prog.NewPackage("foo", "foo")