Initial commit: Go 1.23 release state

This commit is contained in:
Vorapol Rinsatitnon
2024-09-21 23:49:08 +10:00
commit 17cd57a668
13231 changed files with 3114330 additions and 0 deletions

View File

@@ -0,0 +1,21 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build (darwin || linux || windows || freebsd) && (amd64 || arm64)
package fuzz
import (
"unsafe"
)
// coverage returns a []byte containing unique 8-bit counters for each edge of
// the instrumented source code. This coverage data will only be generated if
// `-d=libfuzzer` is set at build time. This can be used to understand the code
// coverage of a test execution.
func coverage() []byte {
addr := unsafe.Pointer(&_counters)
size := uintptr(unsafe.Pointer(&_ecounters)) - uintptr(addr)
return unsafe.Slice((*byte)(addr), int(size))
}

View File

@@ -0,0 +1,24 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// TODO: expand the set of supported platforms, with testing. Nothing about
// the instrumentation is OS specific, but only amd64 and arm64 are
// supported in the runtime. See src/runtime/libfuzzer*.
//
// If you update this constraint, also update internal/platform.FuzzInstrumented.
//
//go:build !((darwin || linux || windows || freebsd) && (amd64 || arm64))
package fuzz
// TODO(#48504): re-enable on platforms where instrumentation works.
// In theory, we shouldn't need this file at all: if the binary was built
// without coverage, then _counters and _ecounters should have the same address.
// However, this caused an init failure on aix/ppc64, so it's disabled here.
// coverage returns a []byte containing unique 8-bit counters for each edge of
// the instrumented source code. This coverage data will only be generated if
// `-d=libfuzzer` is set at build time. This can be used to understand the code
// coverage of a test execution.
func coverage() []byte { return nil }

View File

@@ -0,0 +1,105 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"fmt"
"math/bits"
)
// ResetCoverage sets all of the counters for each edge of the instrumented
// source code to 0.
func ResetCoverage() {
cov := coverage()
clear(cov)
}
// SnapshotCoverage copies the current counter values into coverageSnapshot,
// preserving them for later inspection. SnapshotCoverage also rounds each
// counter down to the nearest power of two. This lets the coordinator store
// multiple values for each counter by OR'ing them together.
func SnapshotCoverage() {
cov := coverage()
for i, b := range cov {
b |= b >> 1
b |= b >> 2
b |= b >> 4
b -= b >> 1
coverageSnapshot[i] = b
}
}
// diffCoverage returns a set of bits set in snapshot but not in base.
// If there are no new bits set, diffCoverage returns nil.
func diffCoverage(base, snapshot []byte) []byte {
if len(base) != len(snapshot) {
panic(fmt.Sprintf("the number of coverage bits changed: before=%d, after=%d", len(base), len(snapshot)))
}
found := false
for i := range snapshot {
if snapshot[i]&^base[i] != 0 {
found = true
break
}
}
if !found {
return nil
}
diff := make([]byte, len(snapshot))
for i := range diff {
diff[i] = snapshot[i] &^ base[i]
}
return diff
}
// countNewCoverageBits returns the number of bits set in snapshot that are not
// set in base.
func countNewCoverageBits(base, snapshot []byte) int {
n := 0
for i := range snapshot {
n += bits.OnesCount8(snapshot[i] &^ base[i])
}
return n
}
// isCoverageSubset returns true if all the base coverage bits are set in
// snapshot.
func isCoverageSubset(base, snapshot []byte) bool {
for i, v := range base {
if v&snapshot[i] != v {
return false
}
}
return true
}
// hasCoverageBit returns true if snapshot has at least one bit set that is
// also set in base.
func hasCoverageBit(base, snapshot []byte) bool {
for i := range snapshot {
if snapshot[i]&base[i] != 0 {
return true
}
}
return false
}
func countBits(cov []byte) int {
n := 0
for _, c := range cov {
n += bits.OnesCount8(c)
}
return n
}
var (
coverageEnabled = len(coverage()) > 0
coverageSnapshot = make([]byte, len(coverage()))
// _counters and _ecounters mark the start and end, respectively, of where
// the 8-bit coverage counters reside in memory. They're known to cmd/link,
// which specially assigns their addresses for this purpose.
_counters, _ecounters [0]byte
)

View File

@@ -0,0 +1,361 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"bytes"
"fmt"
"go/ast"
"go/parser"
"go/token"
"math"
"strconv"
"strings"
"unicode/utf8"
)
// encVersion1 will be the first line of a file with version 1 encoding.
var encVersion1 = "go test fuzz v1"
// marshalCorpusFile encodes an arbitrary number of arguments into the file format for the
// corpus.
func marshalCorpusFile(vals ...any) []byte {
if len(vals) == 0 {
panic("must have at least one value to marshal")
}
b := bytes.NewBuffer([]byte(encVersion1 + "\n"))
// TODO(katiehockman): keep uint8 and int32 encoding where applicable,
// instead of changing to byte and rune respectively.
for _, val := range vals {
switch t := val.(type) {
case int, int8, int16, int64, uint, uint16, uint32, uint64, bool:
fmt.Fprintf(b, "%T(%v)\n", t, t)
case float32:
if math.IsNaN(float64(t)) && math.Float32bits(t) != math.Float32bits(float32(math.NaN())) {
// We encode unusual NaNs as hex values, because that is how users are
// likely to encounter them in literature about floating-point encoding.
// This allows us to reproduce fuzz failures that depend on the specific
// NaN representation (for float32 there are about 2^24 possibilities!),
// not just the fact that the value is *a* NaN.
//
// Note that the specific value of float32(math.NaN()) can vary based on
// whether the architecture represents signaling NaNs using a low bit
// (as is common) or a high bit (as commonly implemented on MIPS
// hardware before around 2012). We believe that the increase in clarity
// from identifying "NaN" with math.NaN() is worth the slight ambiguity
// from a platform-dependent value.
fmt.Fprintf(b, "math.Float32frombits(0x%x)\n", math.Float32bits(t))
} else {
// We encode all other values — including the NaN value that is
// bitwise-identical to float32(math.Nan()) — using the default
// formatting, which is equivalent to strconv.FormatFloat with format
// 'g' and can be parsed by strconv.ParseFloat.
//
// For an ordinary floating-point number this format includes
// sufficiently many digits to reconstruct the exact value. For positive
// or negative infinity it is the string "+Inf" or "-Inf". For positive
// or negative zero it is "0" or "-0". For NaN, it is the string "NaN".
fmt.Fprintf(b, "%T(%v)\n", t, t)
}
case float64:
if math.IsNaN(t) && math.Float64bits(t) != math.Float64bits(math.NaN()) {
fmt.Fprintf(b, "math.Float64frombits(0x%x)\n", math.Float64bits(t))
} else {
fmt.Fprintf(b, "%T(%v)\n", t, t)
}
case string:
fmt.Fprintf(b, "string(%q)\n", t)
case rune: // int32
// Although rune and int32 are represented by the same type, only a subset
// of valid int32 values can be expressed as rune literals. Notably,
// negative numbers, surrogate halves, and values above unicode.MaxRune
// have no quoted representation.
//
// fmt with "%q" (and the corresponding functions in the strconv package)
// would quote out-of-range values to the Unicode replacement character
// instead of the original value (see https://go.dev/issue/51526), so
// they must be treated as int32 instead.
//
// We arbitrarily draw the line at UTF-8 validity, which biases toward the
// "rune" interpretation. (However, we accept either format as input.)
if utf8.ValidRune(t) {
fmt.Fprintf(b, "rune(%q)\n", t)
} else {
fmt.Fprintf(b, "int32(%v)\n", t)
}
case byte: // uint8
// For bytes, we arbitrarily prefer the character interpretation.
// (Every byte has a valid character encoding.)
fmt.Fprintf(b, "byte(%q)\n", t)
case []byte: // []uint8
fmt.Fprintf(b, "[]byte(%q)\n", t)
default:
panic(fmt.Sprintf("unsupported type: %T", t))
}
}
return b.Bytes()
}
// unmarshalCorpusFile decodes corpus bytes into their respective values.
func unmarshalCorpusFile(b []byte) ([]any, error) {
if len(b) == 0 {
return nil, fmt.Errorf("cannot unmarshal empty string")
}
lines := bytes.Split(b, []byte("\n"))
if len(lines) < 2 {
return nil, fmt.Errorf("must include version and at least one value")
}
version := strings.TrimSuffix(string(lines[0]), "\r")
if version != encVersion1 {
return nil, fmt.Errorf("unknown encoding version: %s", version)
}
var vals []any
for _, line := range lines[1:] {
line = bytes.TrimSpace(line)
if len(line) == 0 {
continue
}
v, err := parseCorpusValue(line)
if err != nil {
return nil, fmt.Errorf("malformed line %q: %v", line, err)
}
vals = append(vals, v)
}
return vals, nil
}
func parseCorpusValue(line []byte) (any, error) {
fs := token.NewFileSet()
expr, err := parser.ParseExprFrom(fs, "(test)", line, 0)
if err != nil {
return nil, err
}
call, ok := expr.(*ast.CallExpr)
if !ok {
return nil, fmt.Errorf("expected call expression")
}
if len(call.Args) != 1 {
return nil, fmt.Errorf("expected call expression with 1 argument; got %d", len(call.Args))
}
arg := call.Args[0]
if arrayType, ok := call.Fun.(*ast.ArrayType); ok {
if arrayType.Len != nil {
return nil, fmt.Errorf("expected []byte or primitive type")
}
elt, ok := arrayType.Elt.(*ast.Ident)
if !ok || elt.Name != "byte" {
return nil, fmt.Errorf("expected []byte")
}
lit, ok := arg.(*ast.BasicLit)
if !ok || lit.Kind != token.STRING {
return nil, fmt.Errorf("string literal required for type []byte")
}
s, err := strconv.Unquote(lit.Value)
if err != nil {
return nil, err
}
return []byte(s), nil
}
var idType *ast.Ident
if selector, ok := call.Fun.(*ast.SelectorExpr); ok {
xIdent, ok := selector.X.(*ast.Ident)
if !ok || xIdent.Name != "math" {
return nil, fmt.Errorf("invalid selector type")
}
switch selector.Sel.Name {
case "Float64frombits":
idType = &ast.Ident{Name: "float64-bits"}
case "Float32frombits":
idType = &ast.Ident{Name: "float32-bits"}
default:
return nil, fmt.Errorf("invalid selector type")
}
} else {
idType, ok = call.Fun.(*ast.Ident)
if !ok {
return nil, fmt.Errorf("expected []byte or primitive type")
}
if idType.Name == "bool" {
id, ok := arg.(*ast.Ident)
if !ok {
return nil, fmt.Errorf("malformed bool")
}
if id.Name == "true" {
return true, nil
} else if id.Name == "false" {
return false, nil
} else {
return nil, fmt.Errorf("true or false required for type bool")
}
}
}
var (
val string
kind token.Token
)
if op, ok := arg.(*ast.UnaryExpr); ok {
switch lit := op.X.(type) {
case *ast.BasicLit:
if op.Op != token.SUB {
return nil, fmt.Errorf("unsupported operation on int/float: %v", op.Op)
}
// Special case for negative numbers.
val = op.Op.String() + lit.Value // e.g. "-" + "124"
kind = lit.Kind
case *ast.Ident:
if lit.Name != "Inf" {
return nil, fmt.Errorf("expected operation on int or float type")
}
if op.Op == token.SUB {
val = "-Inf"
} else {
val = "+Inf"
}
kind = token.FLOAT
default:
return nil, fmt.Errorf("expected operation on int or float type")
}
} else {
switch lit := arg.(type) {
case *ast.BasicLit:
val, kind = lit.Value, lit.Kind
case *ast.Ident:
if lit.Name != "NaN" {
return nil, fmt.Errorf("literal value required for primitive type")
}
val, kind = "NaN", token.FLOAT
default:
return nil, fmt.Errorf("literal value required for primitive type")
}
}
switch typ := idType.Name; typ {
case "string":
if kind != token.STRING {
return nil, fmt.Errorf("string literal value required for type string")
}
return strconv.Unquote(val)
case "byte", "rune":
if kind == token.INT {
switch typ {
case "rune":
return parseInt(val, typ)
case "byte":
return parseUint(val, typ)
}
}
if kind != token.CHAR {
return nil, fmt.Errorf("character literal required for byte/rune types")
}
n := len(val)
if n < 2 {
return nil, fmt.Errorf("malformed character literal, missing single quotes")
}
code, _, _, err := strconv.UnquoteChar(val[1:n-1], '\'')
if err != nil {
return nil, err
}
if typ == "rune" {
return code, nil
}
if code >= 256 {
return nil, fmt.Errorf("can only encode single byte to a byte type")
}
return byte(code), nil
case "int", "int8", "int16", "int32", "int64":
if kind != token.INT {
return nil, fmt.Errorf("integer literal required for int types")
}
return parseInt(val, typ)
case "uint", "uint8", "uint16", "uint32", "uint64":
if kind != token.INT {
return nil, fmt.Errorf("integer literal required for uint types")
}
return parseUint(val, typ)
case "float32":
if kind != token.FLOAT && kind != token.INT {
return nil, fmt.Errorf("float or integer literal required for float32 type")
}
v, err := strconv.ParseFloat(val, 32)
return float32(v), err
case "float64":
if kind != token.FLOAT && kind != token.INT {
return nil, fmt.Errorf("float or integer literal required for float64 type")
}
return strconv.ParseFloat(val, 64)
case "float32-bits":
if kind != token.INT {
return nil, fmt.Errorf("integer literal required for math.Float32frombits type")
}
bits, err := parseUint(val, "uint32")
if err != nil {
return nil, err
}
return math.Float32frombits(bits.(uint32)), nil
case "float64-bits":
if kind != token.FLOAT && kind != token.INT {
return nil, fmt.Errorf("integer literal required for math.Float64frombits type")
}
bits, err := parseUint(val, "uint64")
if err != nil {
return nil, err
}
return math.Float64frombits(bits.(uint64)), nil
default:
return nil, fmt.Errorf("expected []byte or primitive type")
}
}
// parseInt returns an integer of value val and type typ.
func parseInt(val, typ string) (any, error) {
switch typ {
case "int":
// The int type may be either 32 or 64 bits. If 32, the fuzz tests in the
// corpus may include 64-bit values produced by fuzzing runs on 64-bit
// architectures. When running those tests, we implicitly wrap the values to
// fit in a regular int. (The test case is still “interesting”, even if the
// specific values of its inputs are platform-dependent.)
i, err := strconv.ParseInt(val, 0, 64)
return int(i), err
case "int8":
i, err := strconv.ParseInt(val, 0, 8)
return int8(i), err
case "int16":
i, err := strconv.ParseInt(val, 0, 16)
return int16(i), err
case "int32", "rune":
i, err := strconv.ParseInt(val, 0, 32)
return int32(i), err
case "int64":
return strconv.ParseInt(val, 0, 64)
default:
panic("unreachable")
}
}
// parseUint returns an unsigned integer of value val and type typ.
func parseUint(val, typ string) (any, error) {
switch typ {
case "uint":
i, err := strconv.ParseUint(val, 0, 64)
return uint(i), err
case "uint8", "byte":
i, err := strconv.ParseUint(val, 0, 8)
return uint8(i), err
case "uint16":
i, err := strconv.ParseUint(val, 0, 16)
return uint16(i), err
case "uint32":
i, err := strconv.ParseUint(val, 0, 32)
return uint32(i), err
case "uint64":
return strconv.ParseUint(val, 0, 64)
default:
panic("unreachable")
}
}

View File

@@ -0,0 +1,406 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"math"
"strconv"
"testing"
"unicode"
)
func TestUnmarshalMarshal(t *testing.T) {
var tests = []struct {
desc string
in string
reject bool
want string // if different from in
}{
{
desc: "missing version",
in: "int(1234)",
reject: true,
},
{
desc: "malformed string",
in: `go test fuzz v1
string("a"bcad")`,
reject: true,
},
{
desc: "empty value",
in: `go test fuzz v1
int()`,
reject: true,
},
{
desc: "negative uint",
in: `go test fuzz v1
uint(-32)`,
reject: true,
},
{
desc: "int8 too large",
in: `go test fuzz v1
int8(1234456)`,
reject: true,
},
{
desc: "multiplication in int value",
in: `go test fuzz v1
int(20*5)`,
reject: true,
},
{
desc: "double negation",
in: `go test fuzz v1
int(--5)`,
reject: true,
},
{
desc: "malformed bool",
in: `go test fuzz v1
bool(0)`,
reject: true,
},
{
desc: "malformed byte",
in: `go test fuzz v1
byte('aa)`,
reject: true,
},
{
desc: "byte out of range",
in: `go test fuzz v1
byte('☃')`,
reject: true,
},
{
desc: "extra newline",
in: `go test fuzz v1
string("has extra newline")
`,
want: `go test fuzz v1
string("has extra newline")`,
},
{
desc: "trailing spaces",
in: `go test fuzz v1
string("extra")
[]byte("spacing")
`,
want: `go test fuzz v1
string("extra")
[]byte("spacing")`,
},
{
desc: "float types",
in: `go test fuzz v1
float64(0)
float32(0)`,
},
{
desc: "various types",
in: `go test fuzz v1
int(-23)
int8(-2)
int64(2342425)
uint(1)
uint16(234)
uint32(352342)
uint64(123)
rune('œ')
byte('K')
byte('ÿ')
[]byte("hello¿")
[]byte("a")
bool(true)
string("hello\\xbd\\xb2=\\xbc ⌘")
float64(-12.5)
float32(2.5)`,
},
{
desc: "float edge cases",
// The two IEEE 754 bit patterns used for the math.Float{64,32}frombits
// encodings are non-math.NAN quiet-NaN values. Since they are not equal
// to math.NaN(), they should be re-encoded to their bit patterns. They
// are, respectively:
// * math.Float64bits(math.NaN())+1
// * math.Float32bits(float32(math.NaN()))+1
in: `go test fuzz v1
float32(-0)
float64(-0)
float32(+Inf)
float32(-Inf)
float32(NaN)
float64(+Inf)
float64(-Inf)
float64(NaN)
math.Float64frombits(0x7ff8000000000002)
math.Float32frombits(0x7fc00001)`,
},
{
desc: "int variations",
// Although we arbitrarily choose default integer bases (0 or 16), we may
// want to change those arbitrary choices in the future and should not
// break the parser. Verify that integers in the opposite bases still
// parse correctly.
in: `go test fuzz v1
int(0x0)
int32(0x41)
int64(0xfffffffff)
uint32(0xcafef00d)
uint64(0xffffffffffffffff)
uint8(0b0000000)
byte(0x0)
byte('\000')
byte('\u0000')
byte('\'')
math.Float64frombits(9221120237041090562)
math.Float32frombits(2143289345)`,
want: `go test fuzz v1
int(0)
rune('A')
int64(68719476735)
uint32(3405705229)
uint64(18446744073709551615)
byte('\x00')
byte('\x00')
byte('\x00')
byte('\x00')
byte('\'')
math.Float64frombits(0x7ff8000000000002)
math.Float32frombits(0x7fc00001)`,
},
{
desc: "rune validation",
in: `go test fuzz v1
rune(0)
rune(0x41)
rune(-1)
rune(0xfffd)
rune(0xd800)
rune(0x10ffff)
rune(0x110000)
`,
want: `go test fuzz v1
rune('\x00')
rune('A')
int32(-1)
rune('<27>')
int32(55296)
rune('\U0010ffff')
int32(1114112)`,
},
{
desc: "int overflow",
in: `go test fuzz v1
int(0x7fffffffffffffff)
uint(0xffffffffffffffff)`,
want: func() string {
switch strconv.IntSize {
case 32:
return `go test fuzz v1
int(-1)
uint(4294967295)`
case 64:
return `go test fuzz v1
int(9223372036854775807)
uint(18446744073709551615)`
default:
panic("unreachable")
}
}(),
},
{
desc: "windows new line",
in: "go test fuzz v1\r\nint(0)\r\n",
want: "go test fuzz v1\nint(0)",
},
}
for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
vals, err := unmarshalCorpusFile([]byte(test.in))
if test.reject {
if err == nil {
t.Fatalf("unmarshal unexpected success")
}
return
}
if err != nil {
t.Fatalf("unmarshal unexpected error: %v", err)
}
newB := marshalCorpusFile(vals...)
if newB[len(newB)-1] != '\n' {
t.Error("didn't write final newline to corpus file")
}
want := test.want
if want == "" {
want = test.in
}
want += "\n"
got := string(newB)
if got != want {
t.Errorf("unexpected marshaled value\ngot:\n%s\nwant:\n%s", got, want)
}
})
}
}
// BenchmarkMarshalCorpusFile measures the time it takes to serialize byte
// slices of various sizes to a corpus file. The slice contains a repeating
// sequence of bytes 0-255 to mix escaped and non-escaped characters.
func BenchmarkMarshalCorpusFile(b *testing.B) {
buf := make([]byte, 1024*1024)
for i := 0; i < len(buf); i++ {
buf[i] = byte(i)
}
for sz := 1; sz <= len(buf); sz <<= 1 {
sz := sz
b.Run(strconv.Itoa(sz), func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.SetBytes(int64(sz))
marshalCorpusFile(buf[:sz])
}
})
}
}
// BenchmarkUnmarshalCorpusfile measures the time it takes to deserialize
// files encoding byte slices of various sizes. The slice contains a repeating
// sequence of bytes 0-255 to mix escaped and non-escaped characters.
func BenchmarkUnmarshalCorpusFile(b *testing.B) {
buf := make([]byte, 1024*1024)
for i := 0; i < len(buf); i++ {
buf[i] = byte(i)
}
for sz := 1; sz <= len(buf); sz <<= 1 {
sz := sz
data := marshalCorpusFile(buf[:sz])
b.Run(strconv.Itoa(sz), func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.SetBytes(int64(sz))
unmarshalCorpusFile(data)
}
})
}
}
func TestByteRoundTrip(t *testing.T) {
for x := 0; x < 256; x++ {
b1 := byte(x)
buf := marshalCorpusFile(b1)
vs, err := unmarshalCorpusFile(buf)
if err != nil {
t.Fatal(err)
}
b2 := vs[0].(byte)
if b2 != b1 {
t.Fatalf("unmarshaled %v, want %v:\n%s", b2, b1, buf)
}
}
}
func TestInt8RoundTrip(t *testing.T) {
for x := -128; x < 128; x++ {
i1 := int8(x)
buf := marshalCorpusFile(i1)
vs, err := unmarshalCorpusFile(buf)
if err != nil {
t.Fatal(err)
}
i2 := vs[0].(int8)
if i2 != i1 {
t.Fatalf("unmarshaled %v, want %v:\n%s", i2, i1, buf)
}
}
}
func FuzzFloat64RoundTrip(f *testing.F) {
f.Add(math.Float64bits(0))
f.Add(math.Float64bits(math.Copysign(0, -1)))
f.Add(math.Float64bits(math.MaxFloat64))
f.Add(math.Float64bits(math.SmallestNonzeroFloat64))
f.Add(math.Float64bits(math.NaN()))
f.Add(uint64(0x7FF0000000000001)) // signaling NaN
f.Add(math.Float64bits(math.Inf(1)))
f.Add(math.Float64bits(math.Inf(-1)))
f.Fuzz(func(t *testing.T, u1 uint64) {
x1 := math.Float64frombits(u1)
b := marshalCorpusFile(x1)
t.Logf("marshaled math.Float64frombits(0x%x):\n%s", u1, b)
xs, err := unmarshalCorpusFile(b)
if err != nil {
t.Fatal(err)
}
if len(xs) != 1 {
t.Fatalf("unmarshaled %d values", len(xs))
}
x2 := xs[0].(float64)
u2 := math.Float64bits(x2)
if u2 != u1 {
t.Errorf("unmarshaled %v (bits 0x%x)", x2, u2)
}
})
}
func FuzzRuneRoundTrip(f *testing.F) {
f.Add(rune(-1))
f.Add(rune(0xd800))
f.Add(rune(0xdfff))
f.Add(rune(unicode.ReplacementChar))
f.Add(rune(unicode.MaxASCII))
f.Add(rune(unicode.MaxLatin1))
f.Add(rune(unicode.MaxRune))
f.Add(rune(unicode.MaxRune + 1))
f.Add(rune(-0x80000000))
f.Add(rune(0x7fffffff))
f.Fuzz(func(t *testing.T, r1 rune) {
b := marshalCorpusFile(r1)
t.Logf("marshaled rune(0x%x):\n%s", r1, b)
rs, err := unmarshalCorpusFile(b)
if err != nil {
t.Fatal(err)
}
if len(rs) != 1 {
t.Fatalf("unmarshaled %d values", len(rs))
}
r2 := rs[0].(rune)
if r2 != r1 {
t.Errorf("unmarshaled rune(0x%x)", r2)
}
})
}
func FuzzStringRoundTrip(f *testing.F) {
f.Add("")
f.Add("\x00")
f.Add(string([]rune{unicode.ReplacementChar}))
f.Fuzz(func(t *testing.T, s1 string) {
b := marshalCorpusFile(s1)
t.Logf("marshaled %q:\n%s", s1, b)
rs, err := unmarshalCorpusFile(b)
if err != nil {
t.Fatal(err)
}
if len(rs) != 1 {
t.Fatalf("unmarshaled %d values", len(rs))
}
s2 := rs[0].(string)
if s2 != s1 {
t.Errorf("unmarshaled %q", s2)
}
})
}

1102
src/internal/fuzz/fuzz.go Normal file

File diff suppressed because it is too large Load Diff

138
src/internal/fuzz/mem.go Normal file
View File

@@ -0,0 +1,138 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"bytes"
"fmt"
"os"
"unsafe"
)
// sharedMem manages access to a region of virtual memory mapped from a file,
// shared between multiple processes. The region includes space for a header and
// a value of variable length.
//
// When fuzzing, the coordinator creates a sharedMem from a temporary file for
// each worker. This buffer is used to pass values to fuzz between processes.
// Care must be taken to manage access to shared memory across processes;
// sharedMem provides no synchronization on its own. See workerComm for an
// explanation.
type sharedMem struct {
// f is the file mapped into memory.
f *os.File
// region is the mapped region of virtual memory for f. The content of f may
// be read or written through this slice.
region []byte
// removeOnClose is true if the file should be deleted by Close.
removeOnClose bool
// sys contains OS-specific information.
sys sharedMemSys
}
// sharedMemHeader stores metadata in shared memory.
type sharedMemHeader struct {
// count is the number of times the worker has called the fuzz function.
// May be reset by coordinator.
count int64
// valueLen is the number of bytes in region which should be read.
valueLen int
// randState and randInc hold the state of a pseudo-random number generator.
randState, randInc uint64
// rawInMem is true if the region holds raw bytes, which occurs during
// minimization. If true after the worker fails during minimization, this
// indicates that an unrecoverable error occurred, and the region can be
// used to retrieve the raw bytes that caused the error.
rawInMem bool
}
// sharedMemSize returns the size needed for a shared memory buffer that can
// contain values of the given size.
func sharedMemSize(valueSize int) int {
// TODO(jayconrod): set a reasonable maximum size per platform.
return int(unsafe.Sizeof(sharedMemHeader{})) + valueSize
}
// sharedMemTempFile creates a new temporary file of the given size, then maps
// it into memory. The file will be removed when the Close method is called.
func sharedMemTempFile(size int) (m *sharedMem, err error) {
// Create a temporary file.
f, err := os.CreateTemp("", "fuzz-*")
if err != nil {
return nil, err
}
defer func() {
if err != nil {
f.Close()
os.Remove(f.Name())
}
}()
// Resize it to the correct size.
totalSize := sharedMemSize(size)
if err := f.Truncate(int64(totalSize)); err != nil {
return nil, err
}
// Map the file into memory.
removeOnClose := true
return sharedMemMapFile(f, totalSize, removeOnClose)
}
// header returns a pointer to metadata within the shared memory region.
func (m *sharedMem) header() *sharedMemHeader {
return (*sharedMemHeader)(unsafe.Pointer(&m.region[0]))
}
// valueRef returns the value currently stored in shared memory. The returned
// slice points to shared memory; it is not a copy.
func (m *sharedMem) valueRef() []byte {
length := m.header().valueLen
valueOffset := int(unsafe.Sizeof(sharedMemHeader{}))
return m.region[valueOffset : valueOffset+length]
}
// valueCopy returns a copy of the value stored in shared memory.
func (m *sharedMem) valueCopy() []byte {
ref := m.valueRef()
return bytes.Clone(ref)
}
// setValue copies the data in b into the shared memory buffer and sets
// the length. len(b) must be less than or equal to the capacity of the buffer
// (as returned by cap(m.value())).
func (m *sharedMem) setValue(b []byte) {
v := m.valueRef()
if len(b) > cap(v) {
panic(fmt.Sprintf("value length %d larger than shared memory capacity %d", len(b), cap(v)))
}
m.header().valueLen = len(b)
copy(v[:cap(v)], b)
}
// setValueLen sets the length of the shared memory buffer returned by valueRef
// to n, which may be at most the cap of that slice.
//
// Note that we can only store the length in the shared memory header. The full
// slice header contains a pointer, which is likely only valid for one process,
// since each process can map shared memory at a different virtual address.
func (m *sharedMem) setValueLen(n int) {
v := m.valueRef()
if n > cap(v) {
panic(fmt.Sprintf("length %d larger than shared memory capacity %d", n, cap(v)))
}
m.header().valueLen = n
}
// TODO(jayconrod): add method to resize the buffer. We'll need that when the
// mutator can increase input length. Only the coordinator will be able to
// do it, since we'll need to send a message to the worker telling it to
// remap the file.

View File

@@ -0,0 +1,95 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"reflect"
)
func isMinimizable(t reflect.Type) bool {
return t == reflect.TypeOf("") || t == reflect.TypeOf([]byte(nil))
}
func minimizeBytes(v []byte, try func([]byte) bool, shouldStop func() bool) {
tmp := make([]byte, len(v))
// If minimization was successful at any point during minimizeBytes,
// then the vals slice in (*workerServer).minimizeInput will point to
// tmp. Since tmp is altered while making new candidates, we need to
// make sure that it is equal to the correct value, v, before exiting
// this function.
defer copy(tmp, v)
// First, try to cut the tail.
for n := 1024; n != 0; n /= 2 {
for len(v) > n {
if shouldStop() {
return
}
candidate := v[:len(v)-n]
if !try(candidate) {
break
}
// Set v to the new value to continue iterating.
v = candidate
}
}
// Then, try to remove each individual byte.
for i := 0; i < len(v)-1; i++ {
if shouldStop() {
return
}
candidate := tmp[:len(v)-1]
copy(candidate[:i], v[:i])
copy(candidate[i:], v[i+1:])
if !try(candidate) {
continue
}
// Update v to delete the value at index i.
copy(v[i:], v[i+1:])
v = v[:len(candidate)]
// v[i] is now different, so decrement i to redo this iteration
// of the loop with the new value.
i--
}
// Then, try to remove each possible subset of bytes.
for i := 0; i < len(v)-1; i++ {
copy(tmp, v[:i])
for j := len(v); j > i+1; j-- {
if shouldStop() {
return
}
candidate := tmp[:len(v)-j+i]
copy(candidate[i:], v[j:])
if !try(candidate) {
continue
}
// Update v and reset the loop with the new length.
copy(v[i:], v[j:])
v = v[:len(candidate)]
j = len(v)
}
}
// Then, try to make it more simplified and human-readable by trying to replace each
// byte with a printable character.
printableChars := []byte("012789ABCXYZabcxyz !\"#$%&'()*+,.")
for i, b := range v {
if shouldStop() {
return
}
for _, pc := range printableChars {
v[i] = pc
if try(v) {
// Successful. Move on to the next byte in v.
break
}
// Unsuccessful. Revert v[i] back to original value.
v[i] = b
}
}
}

View File

@@ -0,0 +1,182 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin || freebsd || linux || windows
package fuzz
import (
"bytes"
"context"
"errors"
"fmt"
"reflect"
"testing"
"time"
"unicode"
"unicode/utf8"
)
func TestMinimizeInput(t *testing.T) {
type testcase struct {
name string
fn func(CorpusEntry) error
input []any
expected []any
}
cases := []testcase{
{
name: "ones_byte",
fn: func(e CorpusEntry) error {
b := e.Values[0].([]byte)
ones := 0
for _, v := range b {
if v == 1 {
ones++
}
}
if ones == 3 {
return fmt.Errorf("bad %v", e.Values[0])
}
return nil
},
input: []any{[]byte{0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
expected: []any{[]byte{1, 1, 1}},
},
{
name: "single_bytes",
fn: func(e CorpusEntry) error {
b := e.Values[0].([]byte)
if len(b) < 2 {
return nil
}
if len(b) == 2 && b[0] == 1 && b[1] == 2 {
return nil
}
return fmt.Errorf("bad %v", e.Values[0])
},
input: []any{[]byte{1, 2, 3, 4, 5}},
expected: []any{[]byte("00")},
},
{
name: "set_of_bytes",
fn: func(e CorpusEntry) error {
b := e.Values[0].([]byte)
if len(b) < 3 {
return nil
}
if bytes.Equal(b, []byte{0, 1, 2, 3, 4, 5}) || bytes.Equal(b, []byte{0, 4, 5}) {
return fmt.Errorf("bad %v", e.Values[0])
}
return nil
},
input: []any{[]byte{0, 1, 2, 3, 4, 5}},
expected: []any{[]byte{0, 4, 5}},
},
{
name: "non_ascii_bytes",
fn: func(e CorpusEntry) error {
b := e.Values[0].([]byte)
if len(b) == 3 {
return fmt.Errorf("bad %v", e.Values[0])
}
return nil
},
input: []any{[]byte("ท")}, // ท is 3 bytes
expected: []any{[]byte("000")},
},
{
name: "ones_string",
fn: func(e CorpusEntry) error {
b := e.Values[0].(string)
ones := 0
for _, v := range b {
if v == '1' {
ones++
}
}
if ones == 3 {
return fmt.Errorf("bad %v", e.Values[0])
}
return nil
},
input: []any{"001010001000000000000000000"},
expected: []any{"111"},
},
{
name: "string_length",
fn: func(e CorpusEntry) error {
b := e.Values[0].(string)
if len(b) == 5 {
return fmt.Errorf("bad %v", e.Values[0])
}
return nil
},
input: []any{"zzzzz"},
expected: []any{"00000"},
},
{
name: "string_with_letter",
fn: func(e CorpusEntry) error {
b := e.Values[0].(string)
r, _ := utf8.DecodeRune([]byte(b))
if unicode.IsLetter(r) {
return fmt.Errorf("bad %v", e.Values[0])
}
return nil
},
input: []any{"ZZZZZ"},
expected: []any{"A"},
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ws := &workerServer{
fuzzFn: func(e CorpusEntry) (time.Duration, error) {
return time.Second, tc.fn(e)
},
}
mem := &sharedMem{region: make([]byte, 100)} // big enough to hold value and header
vals := tc.input
success, err := ws.minimizeInput(context.Background(), vals, mem, minimizeArgs{})
if !success {
t.Errorf("minimizeInput did not succeed")
}
if err == nil {
t.Fatal("minimizeInput didn't provide an error")
}
if expected := fmt.Sprintf("bad %v", tc.expected[0]); err.Error() != expected {
t.Errorf("unexpected error: got %q, want %q", err, expected)
}
if !reflect.DeepEqual(vals, tc.expected) {
t.Errorf("unexpected results: got %v, want %v", vals, tc.expected)
}
})
}
}
// TestMinimizeFlaky checks that if we're minimizing an interesting
// input and a flaky failure occurs, that minimization was not indicated
// to be successful, and the error isn't returned (since it's flaky).
func TestMinimizeFlaky(t *testing.T) {
ws := &workerServer{fuzzFn: func(e CorpusEntry) (time.Duration, error) {
return time.Second, errors.New("ohno")
}}
mem := &sharedMem{region: make([]byte, 100)} // big enough to hold value and header
vals := []any{[]byte(nil)}
args := minimizeArgs{KeepCoverage: make([]byte, len(coverageSnapshot))}
success, err := ws.minimizeInput(context.Background(), vals, mem, args)
if success {
t.Error("unexpected success")
}
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if count := mem.header().count; count != 1 {
t.Errorf("count: got %d, want 1", count)
}
}

View File

@@ -0,0 +1,293 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"encoding/binary"
"fmt"
"math"
"unsafe"
)
type mutator struct {
r mutatorRand
scratch []byte // scratch slice to avoid additional allocations
}
func newMutator() *mutator {
return &mutator{r: newPcgRand()}
}
func (m *mutator) rand(n int) int {
return m.r.intn(n)
}
func (m *mutator) randByteOrder() binary.ByteOrder {
if m.r.bool() {
return binary.LittleEndian
}
return binary.BigEndian
}
// chooseLen chooses length of range mutation in range [1,n]. It gives
// preference to shorter ranges.
func (m *mutator) chooseLen(n int) int {
switch x := m.rand(100); {
case x < 90:
return m.rand(min(8, n)) + 1
case x < 99:
return m.rand(min(32, n)) + 1
default:
return m.rand(n) + 1
}
}
// mutate performs several mutations on the provided values.
func (m *mutator) mutate(vals []any, maxBytes int) {
// TODO(katiehockman): pull some of these functions into helper methods and
// test that each case is working as expected.
// TODO(katiehockman): perform more types of mutations for []byte.
// maxPerVal will represent the maximum number of bytes that each value be
// allowed after mutating, giving an equal amount of capacity to each line.
// Allow a little wiggle room for the encoding.
maxPerVal := maxBytes/len(vals) - 100
// Pick a random value to mutate.
// TODO: consider mutating more than one value at a time.
i := m.rand(len(vals))
switch v := vals[i].(type) {
case int:
vals[i] = int(m.mutateInt(int64(v), maxInt))
case int8:
vals[i] = int8(m.mutateInt(int64(v), math.MaxInt8))
case int16:
vals[i] = int16(m.mutateInt(int64(v), math.MaxInt16))
case int64:
vals[i] = m.mutateInt(v, maxInt)
case uint:
vals[i] = uint(m.mutateUInt(uint64(v), maxUint))
case uint16:
vals[i] = uint16(m.mutateUInt(uint64(v), math.MaxUint16))
case uint32:
vals[i] = uint32(m.mutateUInt(uint64(v), math.MaxUint32))
case uint64:
vals[i] = m.mutateUInt(v, maxUint)
case float32:
vals[i] = float32(m.mutateFloat(float64(v), math.MaxFloat32))
case float64:
vals[i] = m.mutateFloat(v, math.MaxFloat64)
case bool:
if m.rand(2) == 1 {
vals[i] = !v // 50% chance of flipping the bool
}
case rune: // int32
vals[i] = rune(m.mutateInt(int64(v), math.MaxInt32))
case byte: // uint8
vals[i] = byte(m.mutateUInt(uint64(v), math.MaxUint8))
case string:
if len(v) > maxPerVal {
panic(fmt.Sprintf("cannot mutate bytes of length %d", len(v)))
}
if cap(m.scratch) < maxPerVal {
m.scratch = append(make([]byte, 0, maxPerVal), v...)
} else {
m.scratch = m.scratch[:len(v)]
copy(m.scratch, v)
}
m.mutateBytes(&m.scratch)
vals[i] = string(m.scratch)
case []byte:
if len(v) > maxPerVal {
panic(fmt.Sprintf("cannot mutate bytes of length %d", len(v)))
}
if cap(m.scratch) < maxPerVal {
m.scratch = append(make([]byte, 0, maxPerVal), v...)
} else {
m.scratch = m.scratch[:len(v)]
copy(m.scratch, v)
}
m.mutateBytes(&m.scratch)
vals[i] = m.scratch
default:
panic(fmt.Sprintf("type not supported for mutating: %T", vals[i]))
}
}
func (m *mutator) mutateInt(v, maxValue int64) int64 {
var max int64
for {
max = 100
switch m.rand(2) {
case 0:
// Add a random number
if v >= maxValue {
continue
}
if v > 0 && maxValue-v < max {
// Don't let v exceed maxValue
max = maxValue - v
}
v += int64(1 + m.rand(int(max)))
return v
case 1:
// Subtract a random number
if v <= -maxValue {
continue
}
if v < 0 && maxValue+v < max {
// Don't let v drop below -maxValue
max = maxValue + v
}
v -= int64(1 + m.rand(int(max)))
return v
}
}
}
func (m *mutator) mutateUInt(v, maxValue uint64) uint64 {
var max uint64
for {
max = 100
switch m.rand(2) {
case 0:
// Add a random number
if v >= maxValue {
continue
}
if v > 0 && maxValue-v < max {
// Don't let v exceed maxValue
max = maxValue - v
}
v += uint64(1 + m.rand(int(max)))
return v
case 1:
// Subtract a random number
if v <= 0 {
continue
}
if v < max {
// Don't let v drop below 0
max = v
}
v -= uint64(1 + m.rand(int(max)))
return v
}
}
}
func (m *mutator) mutateFloat(v, maxValue float64) float64 {
var max float64
for {
switch m.rand(4) {
case 0:
// Add a random number
if v >= maxValue {
continue
}
max = 100
if v > 0 && maxValue-v < max {
// Don't let v exceed maxValue
max = maxValue - v
}
v += float64(1 + m.rand(int(max)))
return v
case 1:
// Subtract a random number
if v <= -maxValue {
continue
}
max = 100
if v < 0 && maxValue+v < max {
// Don't let v drop below -maxValue
max = maxValue + v
}
v -= float64(1 + m.rand(int(max)))
return v
case 2:
// Multiply by a random number
absV := math.Abs(v)
if v == 0 || absV >= maxValue {
continue
}
max = 10
if maxValue/absV < max {
// Don't let v go beyond the minimum or maximum value
max = maxValue / absV
}
v *= float64(1 + m.rand(int(max)))
return v
case 3:
// Divide by a random number
if v == 0 {
continue
}
v /= float64(1 + m.rand(10))
return v
}
}
}
type byteSliceMutator func(*mutator, []byte) []byte
var byteSliceMutators = []byteSliceMutator{
byteSliceRemoveBytes,
byteSliceInsertRandomBytes,
byteSliceDuplicateBytes,
byteSliceOverwriteBytes,
byteSliceBitFlip,
byteSliceXORByte,
byteSliceSwapByte,
byteSliceArithmeticUint8,
byteSliceArithmeticUint16,
byteSliceArithmeticUint32,
byteSliceArithmeticUint64,
byteSliceOverwriteInterestingUint8,
byteSliceOverwriteInterestingUint16,
byteSliceOverwriteInterestingUint32,
byteSliceInsertConstantBytes,
byteSliceOverwriteConstantBytes,
byteSliceShuffleBytes,
byteSliceSwapBytes,
}
func (m *mutator) mutateBytes(ptrB *[]byte) {
b := *ptrB
defer func() {
if unsafe.SliceData(*ptrB) != unsafe.SliceData(b) {
panic("data moved to new address")
}
*ptrB = b
}()
for {
mut := byteSliceMutators[m.rand(len(byteSliceMutators))]
if mutated := mut(m, b); mutated != nil {
b = mutated
return
}
}
}
var (
interesting8 = []int8{-128, -1, 0, 1, 16, 32, 64, 100, 127}
interesting16 = []int16{-32768, -129, 128, 255, 256, 512, 1000, 1024, 4096, 32767}
interesting32 = []int32{-2147483648, -100663046, -32769, 32768, 65535, 65536, 100663045, 2147483647}
)
const (
maxUint = uint64(^uint(0))
maxInt = int64(maxUint >> 1)
)
func init() {
for _, v := range interesting8 {
interesting16 = append(interesting16, int16(v))
}
for _, v := range interesting16 {
interesting32 = append(interesting32, int32(v))
}
}

View File

@@ -0,0 +1,117 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"bytes"
"fmt"
"os"
"strconv"
"testing"
)
func BenchmarkMutatorBytes(b *testing.B) {
origEnv := os.Getenv("GODEBUG")
defer func() { os.Setenv("GODEBUG", origEnv) }()
os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv))
m := newMutator()
for _, size := range []int{
1,
10,
100,
1000,
10000,
100000,
} {
b.Run(strconv.Itoa(size), func(b *testing.B) {
buf := make([]byte, size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
// resize buffer to the correct shape and reset the PCG
buf = buf[0:size]
m.r = newPcgRand()
m.mutate([]any{buf}, workerSharedMemSize)
}
})
}
}
func BenchmarkMutatorString(b *testing.B) {
origEnv := os.Getenv("GODEBUG")
defer func() { os.Setenv("GODEBUG", origEnv) }()
os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv))
m := newMutator()
for _, size := range []int{
1,
10,
100,
1000,
10000,
100000,
} {
b.Run(strconv.Itoa(size), func(b *testing.B) {
buf := make([]byte, size)
b.ResetTimer()
for i := 0; i < b.N; i++ {
// resize buffer to the correct shape and reset the PCG
buf = buf[0:size]
m.r = newPcgRand()
m.mutate([]any{string(buf)}, workerSharedMemSize)
}
})
}
}
func BenchmarkMutatorAllBasicTypes(b *testing.B) {
origEnv := os.Getenv("GODEBUG")
defer func() { os.Setenv("GODEBUG", origEnv) }()
os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv))
m := newMutator()
types := []any{
[]byte(""),
string(""),
false,
float32(0),
float64(0),
int(0),
int8(0),
int16(0),
int32(0),
int64(0),
uint8(0),
uint16(0),
uint32(0),
uint64(0),
}
for _, t := range types {
b.Run(fmt.Sprintf("%T", t), func(b *testing.B) {
for i := 0; i < b.N; i++ {
m.r = newPcgRand()
m.mutate([]any{t}, workerSharedMemSize)
}
})
}
}
func TestStringImmutability(t *testing.T) {
v := []any{"hello"}
m := newMutator()
m.mutate(v, 1024)
original := v[0].(string)
originalCopy := make([]byte, len(original))
copy(originalCopy, []byte(original))
for i := 0; i < 25; i++ {
m.mutate(v, 1024)
}
if !bytes.Equal([]byte(original), originalCopy) {
t.Fatalf("string was mutated: got %x, want %x", []byte(original), originalCopy)
}
}

View File

@@ -0,0 +1,313 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
// byteSliceRemoveBytes removes a random chunk of bytes from b.
func byteSliceRemoveBytes(m *mutator, b []byte) []byte {
if len(b) <= 1 {
return nil
}
pos0 := m.rand(len(b))
pos1 := pos0 + m.chooseLen(len(b)-pos0)
copy(b[pos0:], b[pos1:])
b = b[:len(b)-(pos1-pos0)]
return b
}
// byteSliceInsertRandomBytes inserts a chunk of random bytes into b at a random
// position.
func byteSliceInsertRandomBytes(m *mutator, b []byte) []byte {
pos := m.rand(len(b) + 1)
n := m.chooseLen(1024)
if len(b)+n >= cap(b) {
return nil
}
b = b[:len(b)+n]
copy(b[pos+n:], b[pos:])
for i := 0; i < n; i++ {
b[pos+i] = byte(m.rand(256))
}
return b
}
// byteSliceDuplicateBytes duplicates a chunk of bytes in b and inserts it into
// a random position.
func byteSliceDuplicateBytes(m *mutator, b []byte) []byte {
if len(b) <= 1 {
return nil
}
src := m.rand(len(b))
dst := m.rand(len(b))
for dst == src {
dst = m.rand(len(b))
}
n := m.chooseLen(len(b) - src)
// Use the end of the slice as scratch space to avoid doing an
// allocation. If the slice is too small abort and try something
// else.
if len(b)+(n*2) >= cap(b) {
return nil
}
end := len(b)
// Increase the size of b to fit the duplicated block as well as
// some extra working space
b = b[:end+(n*2)]
// Copy the block of bytes we want to duplicate to the end of the
// slice
copy(b[end+n:], b[src:src+n])
// Shift the bytes after the splice point n positions to the right
// to make room for the new block
copy(b[dst+n:end+n], b[dst:end])
// Insert the duplicate block into the splice point
copy(b[dst:], b[end+n:])
b = b[:end+n]
return b
}
// byteSliceOverwriteBytes overwrites a chunk of b with another chunk of b.
func byteSliceOverwriteBytes(m *mutator, b []byte) []byte {
if len(b) <= 1 {
return nil
}
src := m.rand(len(b))
dst := m.rand(len(b))
for dst == src {
dst = m.rand(len(b))
}
n := m.chooseLen(len(b) - src - 1)
copy(b[dst:], b[src:src+n])
return b
}
// byteSliceBitFlip flips a random bit in a random byte in b.
func byteSliceBitFlip(m *mutator, b []byte) []byte {
if len(b) == 0 {
return nil
}
pos := m.rand(len(b))
b[pos] ^= 1 << uint(m.rand(8))
return b
}
// byteSliceXORByte XORs a random byte in b with a random value.
func byteSliceXORByte(m *mutator, b []byte) []byte {
if len(b) == 0 {
return nil
}
pos := m.rand(len(b))
// In order to avoid a no-op (where the random value matches
// the existing value), use XOR instead of just setting to
// the random value.
b[pos] ^= byte(1 + m.rand(255))
return b
}
// byteSliceSwapByte swaps two random bytes in b.
func byteSliceSwapByte(m *mutator, b []byte) []byte {
if len(b) <= 1 {
return nil
}
src := m.rand(len(b))
dst := m.rand(len(b))
for dst == src {
dst = m.rand(len(b))
}
b[src], b[dst] = b[dst], b[src]
return b
}
// byteSliceArithmeticUint8 adds/subtracts from a random byte in b.
func byteSliceArithmeticUint8(m *mutator, b []byte) []byte {
if len(b) == 0 {
return nil
}
pos := m.rand(len(b))
v := byte(m.rand(35) + 1)
if m.r.bool() {
b[pos] += v
} else {
b[pos] -= v
}
return b
}
// byteSliceArithmeticUint16 adds/subtracts from a random uint16 in b.
func byteSliceArithmeticUint16(m *mutator, b []byte) []byte {
if len(b) < 2 {
return nil
}
v := uint16(m.rand(35) + 1)
if m.r.bool() {
v = 0 - v
}
pos := m.rand(len(b) - 1)
enc := m.randByteOrder()
enc.PutUint16(b[pos:], enc.Uint16(b[pos:])+v)
return b
}
// byteSliceArithmeticUint32 adds/subtracts from a random uint32 in b.
func byteSliceArithmeticUint32(m *mutator, b []byte) []byte {
if len(b) < 4 {
return nil
}
v := uint32(m.rand(35) + 1)
if m.r.bool() {
v = 0 - v
}
pos := m.rand(len(b) - 3)
enc := m.randByteOrder()
enc.PutUint32(b[pos:], enc.Uint32(b[pos:])+v)
return b
}
// byteSliceArithmeticUint64 adds/subtracts from a random uint64 in b.
func byteSliceArithmeticUint64(m *mutator, b []byte) []byte {
if len(b) < 8 {
return nil
}
v := uint64(m.rand(35) + 1)
if m.r.bool() {
v = 0 - v
}
pos := m.rand(len(b) - 7)
enc := m.randByteOrder()
enc.PutUint64(b[pos:], enc.Uint64(b[pos:])+v)
return b
}
// byteSliceOverwriteInterestingUint8 overwrites a random byte in b with an interesting
// value.
func byteSliceOverwriteInterestingUint8(m *mutator, b []byte) []byte {
if len(b) == 0 {
return nil
}
pos := m.rand(len(b))
b[pos] = byte(interesting8[m.rand(len(interesting8))])
return b
}
// byteSliceOverwriteInterestingUint16 overwrites a random uint16 in b with an interesting
// value.
func byteSliceOverwriteInterestingUint16(m *mutator, b []byte) []byte {
if len(b) < 2 {
return nil
}
pos := m.rand(len(b) - 1)
v := uint16(interesting16[m.rand(len(interesting16))])
m.randByteOrder().PutUint16(b[pos:], v)
return b
}
// byteSliceOverwriteInterestingUint32 overwrites a random uint16 in b with an interesting
// value.
func byteSliceOverwriteInterestingUint32(m *mutator, b []byte) []byte {
if len(b) < 4 {
return nil
}
pos := m.rand(len(b) - 3)
v := uint32(interesting32[m.rand(len(interesting32))])
m.randByteOrder().PutUint32(b[pos:], v)
return b
}
// byteSliceInsertConstantBytes inserts a chunk of constant bytes into a random position in b.
func byteSliceInsertConstantBytes(m *mutator, b []byte) []byte {
if len(b) <= 1 {
return nil
}
dst := m.rand(len(b))
// TODO(rolandshoemaker,katiehockman): 4096 was mainly picked
// randomly. We may want to either pick a much larger value
// (AFL uses 32768, paired with a similar impl to chooseLen
// which biases towards smaller lengths that grow over time),
// or set the max based on characteristics of the corpus
// (libFuzzer sets a min/max based on the min/max size of
// entries in the corpus and then picks uniformly from
// that range).
n := m.chooseLen(4096)
if len(b)+n >= cap(b) {
return nil
}
b = b[:len(b)+n]
copy(b[dst+n:], b[dst:])
rb := byte(m.rand(256))
for i := dst; i < dst+n; i++ {
b[i] = rb
}
return b
}
// byteSliceOverwriteConstantBytes overwrites a chunk of b with constant bytes.
func byteSliceOverwriteConstantBytes(m *mutator, b []byte) []byte {
if len(b) <= 1 {
return nil
}
dst := m.rand(len(b))
n := m.chooseLen(len(b) - dst)
rb := byte(m.rand(256))
for i := dst; i < dst+n; i++ {
b[i] = rb
}
return b
}
// byteSliceShuffleBytes shuffles a chunk of bytes in b.
func byteSliceShuffleBytes(m *mutator, b []byte) []byte {
if len(b) <= 1 {
return nil
}
dst := m.rand(len(b))
n := m.chooseLen(len(b) - dst)
if n <= 2 {
return nil
}
// Start at the end of the range, and iterate backwards
// to dst, swapping each element with another element in
// dst:dst+n (Fisher-Yates shuffle).
for i := n - 1; i > 0; i-- {
j := m.rand(i + 1)
b[dst+i], b[dst+j] = b[dst+j], b[dst+i]
}
return b
}
// byteSliceSwapBytes swaps two chunks of bytes in b.
func byteSliceSwapBytes(m *mutator, b []byte) []byte {
if len(b) <= 1 {
return nil
}
src := m.rand(len(b))
dst := m.rand(len(b))
for dst == src {
dst = m.rand(len(b))
}
// Choose the random length as len(b) - max(src, dst)
// so that we don't attempt to swap a chunk that extends
// beyond the end of the slice
max := dst
if src > max {
max = src
}
n := m.chooseLen(len(b) - max - 1)
// Check that neither chunk intersect, so that we don't end up
// duplicating parts of the input, rather than swapping them
if src > dst && dst+n >= src || dst > src && src+n >= dst {
return nil
}
// Use the end of the slice as scratch space to avoid doing an
// allocation. If the slice is too small abort and try something
// else.
if len(b)+n >= cap(b) {
return nil
}
end := len(b)
b = b[:end+n]
copy(b[end:], b[dst:dst+n])
copy(b[dst:], b[src:src+n])
copy(b[src:], b[end:])
b = b[:end]
return b
}

View File

@@ -0,0 +1,186 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"bytes"
"testing"
)
type mockRand struct {
values []int
counter int
b bool
}
func (mr *mockRand) uint32() uint32 {
c := mr.values[mr.counter]
mr.counter++
return uint32(c)
}
func (mr *mockRand) intn(n int) int {
c := mr.values[mr.counter]
mr.counter++
return c % n
}
func (mr *mockRand) uint32n(n uint32) uint32 {
c := mr.values[mr.counter]
mr.counter++
return uint32(c) % n
}
func (mr *mockRand) exp2() int {
c := mr.values[mr.counter]
mr.counter++
return c
}
func (mr *mockRand) bool() bool {
b := mr.b
mr.b = !mr.b
return b
}
func (mr *mockRand) save(*uint64, *uint64) {
panic("unimplemented")
}
func (mr *mockRand) restore(uint64, uint64) {
panic("unimplemented")
}
func TestByteSliceMutators(t *testing.T) {
for _, tc := range []struct {
name string
mutator func(*mutator, []byte) []byte
randVals []int
input []byte
expected []byte
}{
{
name: "byteSliceRemoveBytes",
mutator: byteSliceRemoveBytes,
input: []byte{1, 2, 3, 4},
expected: []byte{4},
},
{
name: "byteSliceInsertRandomBytes",
mutator: byteSliceInsertRandomBytes,
input: make([]byte, 4, 8),
expected: []byte{3, 4, 5, 0, 0, 0, 0},
},
{
name: "byteSliceDuplicateBytes",
mutator: byteSliceDuplicateBytes,
input: append(make([]byte, 0, 13), []byte{1, 2, 3, 4}...),
expected: []byte{1, 1, 2, 3, 4, 2, 3, 4},
},
{
name: "byteSliceOverwriteBytes",
mutator: byteSliceOverwriteBytes,
input: []byte{1, 2, 3, 4},
expected: []byte{1, 1, 3, 4},
},
{
name: "byteSliceBitFlip",
mutator: byteSliceBitFlip,
input: []byte{1, 2, 3, 4},
expected: []byte{3, 2, 3, 4},
},
{
name: "byteSliceXORByte",
mutator: byteSliceXORByte,
input: []byte{1, 2, 3, 4},
expected: []byte{3, 2, 3, 4},
},
{
name: "byteSliceSwapByte",
mutator: byteSliceSwapByte,
input: []byte{1, 2, 3, 4},
expected: []byte{2, 1, 3, 4},
},
{
name: "byteSliceArithmeticUint8",
mutator: byteSliceArithmeticUint8,
input: []byte{1, 2, 3, 4},
expected: []byte{255, 2, 3, 4},
},
{
name: "byteSliceArithmeticUint16",
mutator: byteSliceArithmeticUint16,
input: []byte{1, 2, 3, 4},
expected: []byte{1, 3, 3, 4},
},
{
name: "byteSliceArithmeticUint32",
mutator: byteSliceArithmeticUint32,
input: []byte{1, 2, 3, 4},
expected: []byte{2, 2, 3, 4},
},
{
name: "byteSliceArithmeticUint64",
mutator: byteSliceArithmeticUint64,
input: []byte{1, 2, 3, 4, 5, 6, 7, 8},
expected: []byte{2, 2, 3, 4, 5, 6, 7, 8},
},
{
name: "byteSliceOverwriteInterestingUint8",
mutator: byteSliceOverwriteInterestingUint8,
input: []byte{1, 2, 3, 4},
expected: []byte{255, 2, 3, 4},
},
{
name: "byteSliceOverwriteInterestingUint16",
mutator: byteSliceOverwriteInterestingUint16,
input: []byte{1, 2, 3, 4},
expected: []byte{255, 127, 3, 4},
},
{
name: "byteSliceOverwriteInterestingUint32",
mutator: byteSliceOverwriteInterestingUint32,
input: []byte{1, 2, 3, 4},
expected: []byte{250, 0, 0, 250},
},
{
name: "byteSliceInsertConstantBytes",
mutator: byteSliceInsertConstantBytes,
input: append(make([]byte, 0, 8), []byte{1, 2, 3, 4}...),
expected: []byte{3, 3, 3, 1, 2, 3, 4},
},
{
name: "byteSliceOverwriteConstantBytes",
mutator: byteSliceOverwriteConstantBytes,
input: []byte{1, 2, 3, 4},
expected: []byte{3, 3, 3, 4},
},
{
name: "byteSliceShuffleBytes",
mutator: byteSliceShuffleBytes,
input: []byte{1, 2, 3, 4},
expected: []byte{2, 3, 1, 4},
},
{
name: "byteSliceSwapBytes",
mutator: byteSliceSwapBytes,
randVals: []int{0, 2, 0, 2},
input: append(make([]byte, 0, 9), []byte{1, 2, 3, 4}...),
expected: []byte{3, 2, 1, 4},
},
} {
t.Run(tc.name, func(t *testing.T) {
r := &mockRand{values: []int{0, 1, 2, 3, 4, 5}}
if tc.randVals != nil {
r.values = tc.randVals
}
m := &mutator{r: r}
b := tc.mutator(m, tc.input)
if !bytes.Equal(b, tc.expected) {
t.Errorf("got %x, want %x", b, tc.expected)
}
})
}
}

145
src/internal/fuzz/pcg.go Normal file
View File

@@ -0,0 +1,145 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"math/bits"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
)
type mutatorRand interface {
uint32() uint32
intn(int) int
uint32n(uint32) uint32
exp2() int
bool() bool
save(randState, randInc *uint64)
restore(randState, randInc uint64)
}
// The functions in pcg implement a 32 bit PRNG with a 64 bit period: pcg xsh rr
// 64 32. See https://www.pcg-random.org/ for more information. This
// implementation is geared specifically towards the needs of fuzzing: Simple
// creation and use, no reproducibility, no concurrency safety, just the
// necessary methods, optimized for speed.
var globalInc atomic.Uint64 // PCG stream
const multiplier uint64 = 6364136223846793005
// pcgRand is a PRNG. It should not be copied or shared. No Rand methods are
// concurrency safe.
type pcgRand struct {
noCopy noCopy // help avoid mistakes: ask vet to ensure that we don't make a copy
state uint64
inc uint64
}
func godebugSeed() *int {
debug := strings.Split(os.Getenv("GODEBUG"), ",")
for _, f := range debug {
if strings.HasPrefix(f, "fuzzseed=") {
seed, err := strconv.Atoi(strings.TrimPrefix(f, "fuzzseed="))
if err != nil {
panic("malformed fuzzseed")
}
return &seed
}
}
return nil
}
// newPcgRand generates a new, seeded Rand, ready for use.
func newPcgRand() *pcgRand {
r := new(pcgRand)
now := uint64(time.Now().UnixNano())
if seed := godebugSeed(); seed != nil {
now = uint64(*seed)
}
inc := globalInc.Add(1)
r.state = now
r.inc = (inc << 1) | 1
r.step()
r.state += now
r.step()
return r
}
func (r *pcgRand) step() {
r.state *= multiplier
r.state += r.inc
}
func (r *pcgRand) save(randState, randInc *uint64) {
*randState = r.state
*randInc = r.inc
}
func (r *pcgRand) restore(randState, randInc uint64) {
r.state = randState
r.inc = randInc
}
// uint32 returns a pseudo-random uint32.
func (r *pcgRand) uint32() uint32 {
x := r.state
r.step()
return bits.RotateLeft32(uint32(((x>>18)^x)>>27), -int(x>>59))
}
// intn returns a pseudo-random number in [0, n).
// n must fit in a uint32.
func (r *pcgRand) intn(n int) int {
if int(uint32(n)) != n {
panic("large Intn")
}
return int(r.uint32n(uint32(n)))
}
// uint32n returns a pseudo-random number in [0, n).
//
// For implementation details, see:
// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction
// https://lemire.me/blog/2016/06/30/fast-random-shuffling
func (r *pcgRand) uint32n(n uint32) uint32 {
v := r.uint32()
prod := uint64(v) * uint64(n)
low := uint32(prod)
if low < n {
thresh := uint32(-int32(n)) % n
for low < thresh {
v = r.uint32()
prod = uint64(v) * uint64(n)
low = uint32(prod)
}
}
return uint32(prod >> 32)
}
// exp2 generates n with probability 1/2^(n+1).
func (r *pcgRand) exp2() int {
return bits.TrailingZeros32(r.uint32())
}
// bool generates a random bool.
func (r *pcgRand) bool() bool {
return r.uint32()&1 == 0
}
// noCopy may be embedded into structs which must not be copied
// after the first use.
//
// See https://golang.org/issues/8005#issuecomment-190753527
// for details.
type noCopy struct{}
// Lock is a no-op used by -copylocks checker from `go vet`.
func (*noCopy) Lock() {}
func (*noCopy) Unlock() {}

View File

@@ -0,0 +1,71 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
// queue holds a growable sequence of inputs for fuzzing and minimization.
//
// For now, this is a simple ring buffer
// (https://en.wikipedia.org/wiki/Circular_buffer).
//
// TODO(golang.org/issue/46224): use a prioritization algorithm based on input
// size, previous duration, coverage, and any other metrics that seem useful.
type queue struct {
// elems holds a ring buffer.
// The queue is empty when begin = end.
// The queue is full (until grow is called) when end = begin + N - 1 (mod N)
// where N = cap(elems).
elems []any
head, len int
}
func (q *queue) cap() int {
return len(q.elems)
}
func (q *queue) grow() {
oldCap := q.cap()
newCap := oldCap * 2
if newCap == 0 {
newCap = 8
}
newElems := make([]any, newCap)
oldLen := q.len
for i := 0; i < oldLen; i++ {
newElems[i] = q.elems[(q.head+i)%oldCap]
}
q.elems = newElems
q.head = 0
}
func (q *queue) enqueue(e any) {
if q.len+1 > q.cap() {
q.grow()
}
i := (q.head + q.len) % q.cap()
q.elems[i] = e
q.len++
}
func (q *queue) dequeue() (any, bool) {
if q.len == 0 {
return nil, false
}
e := q.elems[q.head]
q.elems[q.head] = nil
q.head = (q.head + 1) % q.cap()
q.len--
return e, true
}
func (q *queue) peek() (any, bool) {
if q.len == 0 {
return nil, false
}
return q.elems[q.head], true
}
func (q *queue) clear() {
*q = queue{}
}

View File

@@ -0,0 +1,58 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import "testing"
func TestQueue(t *testing.T) {
// Zero valued queue should have 0 length and capacity.
var q queue
if n := q.len; n != 0 {
t.Fatalf("empty queue has len %d; want 0", n)
}
if n := q.cap(); n != 0 {
t.Fatalf("empty queue has cap %d; want 0", n)
}
// As we add elements, len should grow.
N := 32
for i := 0; i < N; i++ {
q.enqueue(i)
if n := q.len; n != i+1 {
t.Fatalf("after adding %d elements, queue has len %d", i, n)
}
if v, ok := q.peek(); !ok {
t.Fatalf("couldn't peek after adding %d elements", i)
} else if v.(int) != 0 {
t.Fatalf("after adding %d elements, peek is %d; want 0", i, v)
}
}
// As we remove and add elements, len should shrink and grow.
// We should also remove elements in the same order they were added.
want := 0
for _, r := range []int{1, 2, 3, 5, 8, 13, 21} {
s := make([]int, 0, r)
for i := 0; i < r; i++ {
if got, ok := q.dequeue(); !ok {
t.Fatalf("after removing %d of %d elements, could not dequeue", i+1, r)
} else if got != want {
t.Fatalf("after removing %d of %d elements, got %d; want %d", i+1, r, got, want)
} else {
s = append(s, got.(int))
}
want = (want + 1) % N
if n := q.len; n != N-i-1 {
t.Fatalf("after removing %d of %d elements, len is %d; want %d", i+1, r, n, N-i-1)
}
}
for i, v := range s {
q.enqueue(v)
if n := q.len; n != N-r+i+1 {
t.Fatalf("after adding back %d of %d elements, len is %d; want %d", i+1, r, n, n-r+i+1)
}
}
}
}

View File

@@ -0,0 +1,130 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build darwin || freebsd || linux
package fuzz
import (
"fmt"
"os"
"os/exec"
"syscall"
)
type sharedMemSys struct{}
func sharedMemMapFile(f *os.File, size int, removeOnClose bool) (*sharedMem, error) {
prot := syscall.PROT_READ | syscall.PROT_WRITE
flags := syscall.MAP_FILE | syscall.MAP_SHARED
region, err := syscall.Mmap(int(f.Fd()), 0, size, prot, flags)
if err != nil {
return nil, err
}
return &sharedMem{f: f, region: region, removeOnClose: removeOnClose}, nil
}
// Close unmaps the shared memory and closes the temporary file. If this
// sharedMem was created with sharedMemTempFile, Close also removes the file.
func (m *sharedMem) Close() error {
// Attempt all operations, even if we get an error for an earlier operation.
// os.File.Close may fail due to I/O errors, but we still want to delete
// the temporary file.
var errs []error
errs = append(errs,
syscall.Munmap(m.region),
m.f.Close())
if m.removeOnClose {
errs = append(errs, os.Remove(m.f.Name()))
}
for _, err := range errs {
if err != nil {
return err
}
}
return nil
}
// setWorkerComm configures communication channels on the cmd that will
// run a worker process.
func setWorkerComm(cmd *exec.Cmd, comm workerComm) {
mem := <-comm.memMu
memFile := mem.f
comm.memMu <- mem
cmd.ExtraFiles = []*os.File{comm.fuzzIn, comm.fuzzOut, memFile}
}
// getWorkerComm returns communication channels in the worker process.
func getWorkerComm() (comm workerComm, err error) {
fuzzIn := os.NewFile(3, "fuzz_in")
fuzzOut := os.NewFile(4, "fuzz_out")
memFile := os.NewFile(5, "fuzz_mem")
fi, err := memFile.Stat()
if err != nil {
return workerComm{}, err
}
size := int(fi.Size())
if int64(size) != fi.Size() {
return workerComm{}, fmt.Errorf("fuzz temp file exceeds maximum size")
}
removeOnClose := false
mem, err := sharedMemMapFile(memFile, size, removeOnClose)
if err != nil {
return workerComm{}, err
}
memMu := make(chan *sharedMem, 1)
memMu <- mem
return workerComm{fuzzIn: fuzzIn, fuzzOut: fuzzOut, memMu: memMu}, nil
}
// isInterruptError returns whether an error was returned by a process that
// was terminated by an interrupt signal (SIGINT).
func isInterruptError(err error) bool {
exitErr, ok := err.(*exec.ExitError)
if !ok || exitErr.ExitCode() >= 0 {
return false
}
status := exitErr.Sys().(syscall.WaitStatus)
return status.Signal() == syscall.SIGINT
}
// terminationSignal checks if err is an exec.ExitError with a signal status.
// If it is, terminationSignal returns the signal and true.
// If not, -1 and false.
func terminationSignal(err error) (os.Signal, bool) {
exitErr, ok := err.(*exec.ExitError)
if !ok || exitErr.ExitCode() >= 0 {
return syscall.Signal(-1), false
}
status := exitErr.Sys().(syscall.WaitStatus)
return status.Signal(), status.Signaled()
}
// isCrashSignal returns whether a signal was likely to have been caused by an
// error in the program that received it, triggered by a fuzz input. For
// example, SIGSEGV would be received after a nil pointer dereference.
// Other signals like SIGKILL or SIGHUP are more likely to have been sent by
// another process, and we shouldn't record a crasher if the worker process
// receives one of these.
//
// Note that Go installs its own signal handlers on startup, so some of these
// signals may only be received if signal handlers are changed. For example,
// SIGSEGV is normally transformed into a panic that causes the process to exit
// with status 2 if not recovered, which we handle as a crash.
func isCrashSignal(signal os.Signal) bool {
switch signal {
case
syscall.SIGILL, // illegal instruction
syscall.SIGTRAP, // breakpoint
syscall.SIGABRT, // abort() called
syscall.SIGBUS, // invalid memory access (e.g., misaligned address)
syscall.SIGFPE, // math error, e.g., integer divide by zero
syscall.SIGSEGV, // invalid memory access (e.g., write to read-only)
syscall.SIGPIPE: // sent data to closed pipe or socket
return true
default:
return false
}
}

View File

@@ -0,0 +1,44 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// If you update this constraint, also update internal/platform.FuzzSupported.
//
//go:build !darwin && !freebsd && !linux && !windows
package fuzz
import (
"os"
"os/exec"
)
type sharedMemSys struct{}
func sharedMemMapFile(f *os.File, size int, removeOnClose bool) (*sharedMem, error) {
panic("not implemented")
}
func (m *sharedMem) Close() error {
panic("not implemented")
}
func setWorkerComm(cmd *exec.Cmd, comm workerComm) {
panic("not implemented")
}
func getWorkerComm() (comm workerComm, err error) {
panic("not implemented")
}
func isInterruptError(err error) bool {
panic("not implemented")
}
func terminationSignal(err error) (os.Signal, bool) {
panic("not implemented")
}
func isCrashSignal(signal os.Signal) bool {
panic("not implemented")
}

View File

@@ -0,0 +1,144 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"fmt"
"os"
"os/exec"
"syscall"
"unsafe"
)
type sharedMemSys struct {
mapObj syscall.Handle
}
func sharedMemMapFile(f *os.File, size int, removeOnClose bool) (mem *sharedMem, err error) {
defer func() {
if err != nil {
err = fmt.Errorf("mapping temporary file %s: %w", f.Name(), err)
}
}()
// Create a file mapping object. The object itself is not shared.
mapObj, err := syscall.CreateFileMapping(
syscall.Handle(f.Fd()), // fhandle
nil, // sa
syscall.PAGE_READWRITE, // prot
0, // maxSizeHigh
0, // maxSizeLow
nil, // name
)
if err != nil {
return nil, err
}
// Create a view from the file mapping object.
access := uint32(syscall.FILE_MAP_READ | syscall.FILE_MAP_WRITE)
addr, err := syscall.MapViewOfFile(
mapObj, // handle
access, // access
0, // offsetHigh
0, // offsetLow
uintptr(size), // length
)
if err != nil {
syscall.CloseHandle(mapObj)
return nil, err
}
region := unsafe.Slice((*byte)(unsafe.Pointer(addr)), size)
return &sharedMem{
f: f,
region: region,
removeOnClose: removeOnClose,
sys: sharedMemSys{mapObj: mapObj},
}, nil
}
// Close unmaps the shared memory and closes the temporary file. If this
// sharedMem was created with sharedMemTempFile, Close also removes the file.
func (m *sharedMem) Close() error {
// Attempt all operations, even if we get an error for an earlier operation.
// os.File.Close may fail due to I/O errors, but we still want to delete
// the temporary file.
var errs []error
errs = append(errs,
syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&m.region[0]))),
syscall.CloseHandle(m.sys.mapObj),
m.f.Close())
if m.removeOnClose {
errs = append(errs, os.Remove(m.f.Name()))
}
for _, err := range errs {
if err != nil {
return err
}
}
return nil
}
// setWorkerComm configures communication channels on the cmd that will
// run a worker process.
func setWorkerComm(cmd *exec.Cmd, comm workerComm) {
mem := <-comm.memMu
memFD := mem.f.Fd()
comm.memMu <- mem
syscall.SetHandleInformation(syscall.Handle(comm.fuzzIn.Fd()), syscall.HANDLE_FLAG_INHERIT, 1)
syscall.SetHandleInformation(syscall.Handle(comm.fuzzOut.Fd()), syscall.HANDLE_FLAG_INHERIT, 1)
syscall.SetHandleInformation(syscall.Handle(memFD), syscall.HANDLE_FLAG_INHERIT, 1)
cmd.Env = append(cmd.Env, fmt.Sprintf("GO_TEST_FUZZ_WORKER_HANDLES=%x,%x,%x", comm.fuzzIn.Fd(), comm.fuzzOut.Fd(), memFD))
cmd.SysProcAttr = &syscall.SysProcAttr{AdditionalInheritedHandles: []syscall.Handle{syscall.Handle(comm.fuzzIn.Fd()), syscall.Handle(comm.fuzzOut.Fd()), syscall.Handle(memFD)}}
}
// getWorkerComm returns communication channels in the worker process.
func getWorkerComm() (comm workerComm, err error) {
v := os.Getenv("GO_TEST_FUZZ_WORKER_HANDLES")
if v == "" {
return workerComm{}, fmt.Errorf("GO_TEST_FUZZ_WORKER_HANDLES not set")
}
var fuzzInFD, fuzzOutFD, memFileFD uintptr
if _, err := fmt.Sscanf(v, "%x,%x,%x", &fuzzInFD, &fuzzOutFD, &memFileFD); err != nil {
return workerComm{}, fmt.Errorf("parsing GO_TEST_FUZZ_WORKER_HANDLES=%s: %v", v, err)
}
fuzzIn := os.NewFile(fuzzInFD, "fuzz_in")
fuzzOut := os.NewFile(fuzzOutFD, "fuzz_out")
memFile := os.NewFile(memFileFD, "fuzz_mem")
fi, err := memFile.Stat()
if err != nil {
return workerComm{}, fmt.Errorf("worker checking temp file size: %w", err)
}
size := int(fi.Size())
if int64(size) != fi.Size() {
return workerComm{}, fmt.Errorf("fuzz temp file exceeds maximum size")
}
removeOnClose := false
mem, err := sharedMemMapFile(memFile, size, removeOnClose)
if err != nil {
return workerComm{}, err
}
memMu := make(chan *sharedMem, 1)
memMu <- mem
return workerComm{fuzzIn: fuzzIn, fuzzOut: fuzzOut, memMu: memMu}, nil
}
func isInterruptError(err error) bool {
// On Windows, we can't tell whether the process was interrupted by the error
// returned by Wait. It looks like an ExitError with status 1.
return false
}
// terminationSignal returns -1 and false because Windows doesn't have signals.
func terminationSignal(err error) (os.Signal, bool) {
return syscall.Signal(-1), false
}
// isCrashSignal is not implemented because Windows doesn't have signals.
func isCrashSignal(signal os.Signal) bool {
panic("not implemented: no signals on windows")
}

View File

@@ -0,0 +1,35 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !libfuzzer
package fuzz
import _ "unsafe" // for go:linkname
//go:linkname libfuzzerTraceCmp1 runtime.libfuzzerTraceCmp1
//go:linkname libfuzzerTraceCmp2 runtime.libfuzzerTraceCmp2
//go:linkname libfuzzerTraceCmp4 runtime.libfuzzerTraceCmp4
//go:linkname libfuzzerTraceCmp8 runtime.libfuzzerTraceCmp8
//go:linkname libfuzzerTraceConstCmp1 runtime.libfuzzerTraceConstCmp1
//go:linkname libfuzzerTraceConstCmp2 runtime.libfuzzerTraceConstCmp2
//go:linkname libfuzzerTraceConstCmp4 runtime.libfuzzerTraceConstCmp4
//go:linkname libfuzzerTraceConstCmp8 runtime.libfuzzerTraceConstCmp8
//go:linkname libfuzzerHookStrCmp runtime.libfuzzerHookStrCmp
//go:linkname libfuzzerHookEqualFold runtime.libfuzzerHookEqualFold
func libfuzzerTraceCmp1(arg0, arg1 uint8, fakePC uint) {}
func libfuzzerTraceCmp2(arg0, arg1 uint16, fakePC uint) {}
func libfuzzerTraceCmp4(arg0, arg1 uint32, fakePC uint) {}
func libfuzzerTraceCmp8(arg0, arg1 uint64, fakePC uint) {}
func libfuzzerTraceConstCmp1(arg0, arg1 uint8, fakePC uint) {}
func libfuzzerTraceConstCmp2(arg0, arg1 uint16, fakePC uint) {}
func libfuzzerTraceConstCmp4(arg0, arg1 uint32, fakePC uint) {}
func libfuzzerTraceConstCmp8(arg0, arg1 uint64, fakePC uint) {}
func libfuzzerHookStrCmp(arg0, arg1 string, fakePC uint) {}
func libfuzzerHookEqualFold(arg0, arg1 string, fakePC uint) {}

1195
src/internal/fuzz/worker.go Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,206 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package fuzz
import (
"context"
"errors"
"flag"
"fmt"
"internal/race"
"io"
"os"
"os/signal"
"reflect"
"strconv"
"testing"
"time"
)
var benchmarkWorkerFlag = flag.Bool("benchmarkworker", false, "")
func TestMain(m *testing.M) {
flag.Parse()
if *benchmarkWorkerFlag {
runBenchmarkWorker()
return
}
os.Exit(m.Run())
}
func BenchmarkWorkerFuzzOverhead(b *testing.B) {
if race.Enabled {
b.Skip("TODO(48504): fix and re-enable")
}
origEnv := os.Getenv("GODEBUG")
defer func() { os.Setenv("GODEBUG", origEnv) }()
os.Setenv("GODEBUG", fmt.Sprintf("%s,fuzzseed=123", origEnv))
ws := &workerServer{
fuzzFn: func(_ CorpusEntry) (time.Duration, error) { return time.Second, nil },
workerComm: workerComm{memMu: make(chan *sharedMem, 1)},
}
mem, err := sharedMemTempFile(workerSharedMemSize)
if err != nil {
b.Fatalf("failed to create temporary shared memory file: %s", err)
}
defer func() {
if err := mem.Close(); err != nil {
b.Error(err)
}
}()
initialVal := []any{make([]byte, 32)}
encodedVals := marshalCorpusFile(initialVal...)
mem.setValue(encodedVals)
ws.memMu <- mem
b.ResetTimer()
for i := 0; i < b.N; i++ {
ws.m = newMutator()
mem.setValue(encodedVals)
mem.header().count = 0
ws.fuzz(context.Background(), fuzzArgs{Limit: 1})
}
}
// BenchmarkWorkerPing acts as the coordinator and measures the time it takes
// a worker to respond to N pings. This is a rough measure of our RPC latency.
func BenchmarkWorkerPing(b *testing.B) {
if race.Enabled {
b.Skip("TODO(48504): fix and re-enable")
}
b.SetParallelism(1)
w := newWorkerForTest(b)
for i := 0; i < b.N; i++ {
if err := w.client.ping(context.Background()); err != nil {
b.Fatal(err)
}
}
}
// BenchmarkWorkerFuzz acts as the coordinator and measures the time it takes
// a worker to mutate a given input and call a trivial fuzz function N times.
func BenchmarkWorkerFuzz(b *testing.B) {
if race.Enabled {
b.Skip("TODO(48504): fix and re-enable")
}
b.SetParallelism(1)
w := newWorkerForTest(b)
entry := CorpusEntry{Values: []any{[]byte(nil)}}
entry.Data = marshalCorpusFile(entry.Values...)
for i := int64(0); i < int64(b.N); {
args := fuzzArgs{
Limit: int64(b.N) - i,
Timeout: workerFuzzDuration,
}
_, resp, _, err := w.client.fuzz(context.Background(), entry, args)
if err != nil {
b.Fatal(err)
}
if resp.Err != "" {
b.Fatal(resp.Err)
}
if resp.Count == 0 {
b.Fatal("worker did not make progress")
}
i += resp.Count
}
}
// newWorkerForTest creates and starts a worker process for testing or
// benchmarking. The worker process calls RunFuzzWorker, which responds to
// RPC messages until it's stopped. The process is stopped and cleaned up
// automatically when the test is done.
func newWorkerForTest(tb testing.TB) *worker {
tb.Helper()
c, err := newCoordinator(CoordinateFuzzingOpts{
Types: []reflect.Type{reflect.TypeOf([]byte(nil))},
Log: io.Discard,
})
if err != nil {
tb.Fatal(err)
}
dir := "" // same as self
binPath := os.Args[0] // same as self
args := append(os.Args[1:], "-benchmarkworker")
env := os.Environ() // same as self
w, err := newWorker(c, dir, binPath, args, env)
if err != nil {
tb.Fatal(err)
}
tb.Cleanup(func() {
if err := w.cleanup(); err != nil {
tb.Error(err)
}
})
if err := w.startAndPing(context.Background()); err != nil {
tb.Fatal(err)
}
tb.Cleanup(func() {
if err := w.stop(); err != nil {
tb.Error(err)
}
})
return w
}
func runBenchmarkWorker() {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
defer cancel()
fn := func(CorpusEntry) error { return nil }
if err := RunFuzzWorker(ctx, fn); err != nil && err != ctx.Err() {
panic(err)
}
}
func BenchmarkWorkerMinimize(b *testing.B) {
if race.Enabled {
b.Skip("TODO(48504): fix and re-enable")
}
ws := &workerServer{
workerComm: workerComm{memMu: make(chan *sharedMem, 1)},
}
mem, err := sharedMemTempFile(workerSharedMemSize)
if err != nil {
b.Fatalf("failed to create temporary shared memory file: %s", err)
}
defer func() {
if err := mem.Close(); err != nil {
b.Error(err)
}
}()
ws.memMu <- mem
bytes := make([]byte, 1024)
ctx := context.Background()
for sz := 1; sz <= len(bytes); sz <<= 1 {
sz := sz
input := []any{bytes[:sz]}
encodedVals := marshalCorpusFile(input...)
mem = <-ws.memMu
mem.setValue(encodedVals)
ws.memMu <- mem
b.Run(strconv.Itoa(sz), func(b *testing.B) {
i := 0
ws.fuzzFn = func(_ CorpusEntry) (time.Duration, error) {
if i == 0 {
i++
return time.Second, errors.New("initial failure for deflake")
}
return time.Second, nil
}
for i := 0; i < b.N; i++ {
b.SetBytes(int64(sz))
ws.minimize(ctx, minimizeArgs{})
}
})
}
}