build: separate compiler and libs

This commit is contained in:
Li Jie
2025-01-07 21:49:08 +08:00
parent b0123567cd
commit 1172e5bdce
559 changed files with 190 additions and 176 deletions

View File

@@ -0,0 +1,612 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package parser
import (
"fmt"
"go/ast"
"go/token"
"strings"
)
const debugResolve = false
// resolveFile walks the given file to resolve identifiers within the file
// scope, updating ast.Ident.Obj fields with declaration information.
//
// If declErr is non-nil, it is used to report declaration errors during
// resolution. tok is used to format position in error messages.
func resolveFile(file *ast.File, handle *token.File, declErr func(token.Pos, string)) {
pkgScope := ast.NewScope(nil)
r := &resolver{
handle: handle,
declErr: declErr,
topScope: pkgScope,
pkgScope: pkgScope,
depth: 1,
}
for _, decl := range file.Decls {
ast.Walk(r, decl)
}
r.closeScope()
assert(r.topScope == nil, "unbalanced scopes")
assert(r.labelScope == nil, "unbalanced label scopes")
// resolve global identifiers within the same file
i := 0
for _, ident := range r.unresolved {
// i <= index for current ident
assert(ident.Obj == unresolved, "object already resolved")
ident.Obj = r.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel
if ident.Obj == nil {
r.unresolved[i] = ident
i++
} else if debugResolve {
pos := ident.Obj.Decl.(interface{ Pos() token.Pos }).Pos()
r.trace("resolved %s@%v to package object %v", ident.Name, ident.Pos(), pos)
}
}
file.Scope = r.pkgScope
file.Unresolved = r.unresolved[0:i]
}
const maxScopeDepth int = 1e3
type resolver struct {
handle *token.File
declErr func(token.Pos, string)
// Ordinary identifier scopes
pkgScope *ast.Scope // pkgScope.Outer == nil
topScope *ast.Scope // top-most scope; may be pkgScope
unresolved []*ast.Ident // unresolved identifiers
depth int // scope depth
// Label scopes
// (maintained by open/close LabelScope)
labelScope *ast.Scope // label scope for current function
targetStack [][]*ast.Ident // stack of unresolved labels
}
func (r *resolver) trace(format string, args ...any) {
fmt.Println(strings.Repeat(". ", r.depth) + r.sprintf(format, args...))
}
func (r *resolver) sprintf(format string, args ...any) string {
for i, arg := range args {
switch arg := arg.(type) {
case token.Pos:
args[i] = r.handle.Position(arg)
}
}
return fmt.Sprintf(format, args...)
}
func (r *resolver) openScope(pos token.Pos) {
r.depth++
if r.depth > maxScopeDepth {
panic(bailout{pos: pos, msg: "exceeded max scope depth during object resolution"})
}
if debugResolve {
r.trace("opening scope @%v", pos)
}
r.topScope = ast.NewScope(r.topScope)
}
func (r *resolver) closeScope() {
r.depth--
if debugResolve {
r.trace("closing scope")
}
r.topScope = r.topScope.Outer
}
func (r *resolver) openLabelScope() {
r.labelScope = ast.NewScope(r.labelScope)
r.targetStack = append(r.targetStack, nil)
}
func (r *resolver) closeLabelScope() {
// resolve labels
n := len(r.targetStack) - 1
scope := r.labelScope
for _, ident := range r.targetStack[n] {
ident.Obj = scope.Lookup(ident.Name)
if ident.Obj == nil && r.declErr != nil {
r.declErr(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name))
}
}
// pop label scope
r.targetStack = r.targetStack[0:n]
r.labelScope = r.labelScope.Outer
}
func (r *resolver) declare(decl, data any, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) {
for _, ident := range idents {
if ident.Obj != nil {
panic(fmt.Sprintf("%v: identifier %s already declared or resolved", ident.Pos(), ident.Name))
}
obj := ast.NewObj(kind, ident.Name)
// remember the corresponding declaration for redeclaration
// errors and global variable resolution/typechecking phase
obj.Decl = decl
obj.Data = data
// Identifiers (for receiver type parameters) are written to the scope, but
// never set as the resolved object. See issue #50956.
if _, ok := decl.(*ast.Ident); !ok {
ident.Obj = obj
}
if ident.Name != "_" {
if debugResolve {
r.trace("declaring %s@%v", ident.Name, ident.Pos())
}
if alt := scope.Insert(obj); alt != nil && r.declErr != nil {
prevDecl := ""
if pos := alt.Pos(); pos.IsValid() {
prevDecl = r.sprintf("\n\tprevious declaration at %v", pos)
}
r.declErr(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl))
}
}
}
}
func (r *resolver) shortVarDecl(decl *ast.AssignStmt) {
// Go spec: A short variable declaration may redeclare variables
// provided they were originally declared in the same block with
// the same type, and at least one of the non-blank variables is new.
n := 0 // number of new variables
for _, x := range decl.Lhs {
if ident, isIdent := x.(*ast.Ident); isIdent {
assert(ident.Obj == nil, "identifier already declared or resolved")
obj := ast.NewObj(ast.Var, ident.Name)
// remember corresponding assignment for other tools
obj.Decl = decl
ident.Obj = obj
if ident.Name != "_" {
if debugResolve {
r.trace("declaring %s@%v", ident.Name, ident.Pos())
}
if alt := r.topScope.Insert(obj); alt != nil {
ident.Obj = alt // redeclaration
} else {
n++ // new declaration
}
}
}
}
if n == 0 && r.declErr != nil {
r.declErr(decl.Lhs[0].Pos(), "no new variables on left side of :=")
}
}
// The unresolved object is a sentinel to mark identifiers that have been added
// to the list of unresolved identifiers. The sentinel is only used for verifying
// internal consistency.
var unresolved = new(ast.Object)
// If x is an identifier, resolve attempts to resolve x by looking up
// the object it denotes. If no object is found and collectUnresolved is
// set, x is marked as unresolved and collected in the list of unresolved
// identifiers.
func (r *resolver) resolve(ident *ast.Ident, collectUnresolved bool) {
if ident.Obj != nil {
panic(r.sprintf("%v: identifier %s already declared or resolved", ident.Pos(), ident.Name))
}
// '_' should never refer to existing declarations, because it has special
// handling in the spec.
if ident.Name == "_" {
return
}
for s := r.topScope; s != nil; s = s.Outer {
if obj := s.Lookup(ident.Name); obj != nil {
if debugResolve {
r.trace("resolved %v:%s to %v", ident.Pos(), ident.Name, obj)
}
assert(obj.Name != "", "obj with no name")
// Identifiers (for receiver type parameters) are written to the scope,
// but never set as the resolved object. See issue #50956.
if _, ok := obj.Decl.(*ast.Ident); !ok {
ident.Obj = obj
}
return
}
}
// all local scopes are known, so any unresolved identifier
// must be found either in the file scope, package scope
// (perhaps in another file), or universe scope --- collect
// them so that they can be resolved later
if collectUnresolved {
ident.Obj = unresolved
r.unresolved = append(r.unresolved, ident)
}
}
func (r *resolver) walkExprs(list []ast.Expr) {
for _, node := range list {
ast.Walk(r, node)
}
}
func (r *resolver) walkLHS(list []ast.Expr) {
for _, expr := range list {
expr := unparen(expr)
if _, ok := expr.(*ast.Ident); !ok && expr != nil {
ast.Walk(r, expr)
}
}
}
func (r *resolver) walkStmts(list []ast.Stmt) {
for _, stmt := range list {
ast.Walk(r, stmt)
}
}
func (r *resolver) Visit(node ast.Node) ast.Visitor {
if debugResolve && node != nil {
r.trace("node %T@%v", node, node.Pos())
}
switch n := node.(type) {
// Expressions.
case *ast.Ident:
r.resolve(n, true)
case *ast.FuncLit:
r.openScope(n.Pos())
defer r.closeScope()
r.walkFuncType(n.Type)
r.walkBody(n.Body)
case *ast.SelectorExpr:
ast.Walk(r, n.X)
// Note: don't try to resolve n.Sel, as we don't support qualified
// resolution.
case *ast.StructType:
r.openScope(n.Pos())
defer r.closeScope()
r.walkFieldList(n.Fields, ast.Var)
case *ast.FuncType:
r.openScope(n.Pos())
defer r.closeScope()
r.walkFuncType(n)
case *ast.CompositeLit:
if n.Type != nil {
ast.Walk(r, n.Type)
}
for _, e := range n.Elts {
if kv, _ := e.(*ast.KeyValueExpr); kv != nil {
// See issue #45160: try to resolve composite lit keys, but don't
// collect them as unresolved if resolution failed. This replicates
// existing behavior when resolving during parsing.
if ident, _ := kv.Key.(*ast.Ident); ident != nil {
r.resolve(ident, false)
} else {
ast.Walk(r, kv.Key)
}
ast.Walk(r, kv.Value)
} else {
ast.Walk(r, e)
}
}
case *ast.InterfaceType:
r.openScope(n.Pos())
defer r.closeScope()
r.walkFieldList(n.Methods, ast.Fun)
// Statements
case *ast.LabeledStmt:
r.declare(n, nil, r.labelScope, ast.Lbl, n.Label)
ast.Walk(r, n.Stmt)
case *ast.AssignStmt:
r.walkExprs(n.Rhs)
if n.Tok == token.DEFINE {
r.shortVarDecl(n)
} else {
r.walkExprs(n.Lhs)
}
case *ast.BranchStmt:
// add to list of unresolved targets
if n.Tok != token.FALLTHROUGH && n.Label != nil {
depth := len(r.targetStack) - 1
r.targetStack[depth] = append(r.targetStack[depth], n.Label)
}
case *ast.BlockStmt:
r.openScope(n.Pos())
defer r.closeScope()
r.walkStmts(n.List)
case *ast.IfStmt:
r.openScope(n.Pos())
defer r.closeScope()
if n.Init != nil {
ast.Walk(r, n.Init)
}
ast.Walk(r, n.Cond)
ast.Walk(r, n.Body)
if n.Else != nil {
ast.Walk(r, n.Else)
}
case *ast.CaseClause:
r.walkExprs(n.List)
r.openScope(n.Pos())
defer r.closeScope()
r.walkStmts(n.Body)
case *ast.SwitchStmt:
r.openScope(n.Pos())
defer r.closeScope()
if n.Init != nil {
ast.Walk(r, n.Init)
}
if n.Tag != nil {
// The scope below reproduces some unnecessary behavior of the parser,
// opening an extra scope in case this is a type switch. It's not needed
// for expression switches.
// TODO: remove this once we've matched the parser resolution exactly.
if n.Init != nil {
r.openScope(n.Tag.Pos())
defer r.closeScope()
}
ast.Walk(r, n.Tag)
}
if n.Body != nil {
r.walkStmts(n.Body.List)
}
case *ast.TypeSwitchStmt:
if n.Init != nil {
r.openScope(n.Pos())
defer r.closeScope()
ast.Walk(r, n.Init)
}
r.openScope(n.Assign.Pos())
defer r.closeScope()
ast.Walk(r, n.Assign)
// s.Body consists only of case clauses, so does not get its own
// scope.
if n.Body != nil {
r.walkStmts(n.Body.List)
}
case *ast.CommClause:
r.openScope(n.Pos())
defer r.closeScope()
if n.Comm != nil {
ast.Walk(r, n.Comm)
}
r.walkStmts(n.Body)
case *ast.SelectStmt:
// as for switch statements, select statement bodies don't get their own
// scope.
if n.Body != nil {
r.walkStmts(n.Body.List)
}
case *ast.ForStmt:
r.openScope(n.Pos())
defer r.closeScope()
if n.Init != nil {
ast.Walk(r, n.Init)
}
if n.Cond != nil {
ast.Walk(r, n.Cond)
}
if n.Post != nil {
ast.Walk(r, n.Post)
}
ast.Walk(r, n.Body)
case *ast.RangeStmt:
r.openScope(n.Pos())
defer r.closeScope()
ast.Walk(r, n.X)
var lhs []ast.Expr
if n.Key != nil {
lhs = append(lhs, n.Key)
}
if n.Value != nil {
lhs = append(lhs, n.Value)
}
if len(lhs) > 0 {
if n.Tok == token.DEFINE {
// Note: we can't exactly match the behavior of object resolution
// during the parsing pass here, as it uses the position of the RANGE
// token for the RHS OpPos. That information is not contained within
// the AST.
as := &ast.AssignStmt{
Lhs: lhs,
Tok: token.DEFINE,
TokPos: n.TokPos,
Rhs: []ast.Expr{&ast.UnaryExpr{Op: token.RANGE, X: n.X}},
}
// TODO(rFindley): this walkLHS reproduced the parser resolution, but
// is it necessary? By comparison, for a normal AssignStmt we don't
// walk the LHS in case there is an invalid identifier list.
r.walkLHS(lhs)
r.shortVarDecl(as)
} else {
r.walkExprs(lhs)
}
}
ast.Walk(r, n.Body)
// Declarations
case *ast.GenDecl:
switch n.Tok {
case token.CONST, token.VAR:
for i, spec := range n.Specs {
spec := spec.(*ast.ValueSpec)
kind := ast.Con
if n.Tok == token.VAR {
kind = ast.Var
}
r.walkExprs(spec.Values)
if spec.Type != nil {
ast.Walk(r, spec.Type)
}
r.declare(spec, i, r.topScope, kind, spec.Names...)
}
case token.TYPE:
for _, spec := range n.Specs {
spec := spec.(*ast.TypeSpec)
// Go spec: The scope of a type identifier declared inside a function begins
// at the identifier in the TypeSpec and ends at the end of the innermost
// containing block.
r.declare(spec, nil, r.topScope, ast.Typ, spec.Name)
if spec.TypeParams != nil {
r.openScope(spec.Pos())
r.walkTParams(spec.TypeParams)
r.closeScope()
}
ast.Walk(r, spec.Type)
}
}
case *ast.FuncDecl:
// Open the function scope.
r.openScope(n.Pos())
defer r.closeScope()
r.walkRecv(n.Recv)
// Type parameters are walked normally: they can reference each other, and
// can be referenced by normal parameters.
if n.Type.TypeParams != nil {
r.walkTParams(n.Type.TypeParams)
// TODO(rFindley): need to address receiver type parameters.
}
// Resolve and declare parameters in a specific order to get duplicate
// declaration errors in the correct location.
r.resolveList(n.Type.Params)
r.resolveList(n.Type.Results)
r.declareList(n.Recv, ast.Var)
r.declareList(n.Type.Params, ast.Var)
r.declareList(n.Type.Results, ast.Var)
r.walkBody(n.Body)
if n.Recv == nil && n.Name.Name != "init" {
r.declare(n, nil, r.pkgScope, ast.Fun, n.Name)
}
default:
return r
}
return nil
}
func (r *resolver) walkFuncType(typ *ast.FuncType) {
// typ.TypeParams must be walked separately for FuncDecls.
r.resolveList(typ.Params)
r.resolveList(typ.Results)
r.declareList(typ.Params, ast.Var)
r.declareList(typ.Results, ast.Var)
}
func (r *resolver) resolveList(list *ast.FieldList) {
if list == nil {
return
}
for _, f := range list.List {
if f.Type != nil {
ast.Walk(r, f.Type)
}
}
}
func (r *resolver) declareList(list *ast.FieldList, kind ast.ObjKind) {
if list == nil {
return
}
for _, f := range list.List {
r.declare(f, nil, r.topScope, kind, f.Names...)
}
}
func (r *resolver) walkRecv(recv *ast.FieldList) {
// If our receiver has receiver type parameters, we must declare them before
// trying to resolve the rest of the receiver, and avoid re-resolving the
// type parameter identifiers.
if recv == nil || len(recv.List) == 0 {
return // nothing to do
}
typ := recv.List[0].Type
if ptr, ok := typ.(*ast.StarExpr); ok {
typ = ptr.X
}
var declareExprs []ast.Expr // exprs to declare
var resolveExprs []ast.Expr // exprs to resolve
switch typ := typ.(type) {
case *ast.IndexExpr:
declareExprs = []ast.Expr{typ.Index}
resolveExprs = append(resolveExprs, typ.X)
case *ast.IndexListExpr:
declareExprs = typ.Indices
resolveExprs = append(resolveExprs, typ.X)
default:
resolveExprs = append(resolveExprs, typ)
}
for _, expr := range declareExprs {
if id, _ := expr.(*ast.Ident); id != nil {
r.declare(expr, nil, r.topScope, ast.Typ, id)
} else {
// The receiver type parameter expression is invalid, but try to resolve
// it anyway for consistency.
resolveExprs = append(resolveExprs, expr)
}
}
for _, expr := range resolveExprs {
if expr != nil {
ast.Walk(r, expr)
}
}
// The receiver is invalid, but try to resolve it anyway for consistency.
for _, f := range recv.List[1:] {
if f.Type != nil {
ast.Walk(r, f.Type)
}
}
}
func (r *resolver) walkFieldList(list *ast.FieldList, kind ast.ObjKind) {
if list == nil {
return
}
r.resolveList(list)
r.declareList(list, kind)
}
// walkTParams is like walkFieldList, but declares type parameters eagerly so
// that they may be resolved in the constraint expressions held in the field
// Type.
func (r *resolver) walkTParams(list *ast.FieldList) {
r.declareList(list, ast.Typ)
r.resolveList(list)
}
func (r *resolver) walkBody(body *ast.BlockStmt) {
if body == nil {
return
}
r.openLabelScope()
defer r.closeLabelScope()
r.walkStmts(body.List)
}

View File

@@ -0,0 +1,292 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package textproto implements generic support for text-based request/response
// protocols in the style of HTTP, NNTP, and SMTP.
//
// The package provides:
//
// Error, which represents a numeric error response from
// a server.
//
// Pipeline, to manage pipelined requests and responses
// in a client.
//
// Reader, to read numeric response code lines,
// key: value headers, lines wrapped with leading spaces
// on continuation lines, and whole text blocks ending
// with a dot on a line by itself.
//
// Writer, to write dot-encoded text blocks.
//
// Conn, a convenient packaging of Reader, Writer, and Pipeline for use
// with a single network connection.
package textproto
import (
"bufio"
"errors"
"fmt"
"io"
"io/fs"
"strings"
"syscall"
"unsafe"
"github.com/goplus/llgo/c"
"github.com/goplus/llgo/c/net"
"github.com/goplus/llgo/c/os"
)
// An Error represents a numeric error response from a server.
type Error struct {
Code int
Msg string
}
func (e *Error) Error() string {
return fmt.Sprintf("%03d %s", e.Code, e.Msg)
}
// A ProtocolError describes a protocol violation such
// as an invalid response or a hung-up connection.
type ProtocolError string
func (p ProtocolError) Error() string {
return string(p)
}
// A Conn represents a textual network protocol connection.
// It consists of a Reader and Writer to manage I/O
// and a Pipeline to sequence concurrent requests on the connection.
// These embedded types carry methods with them;
// see the documentation of those types for details.
type Conn struct {
Reader
Writer
Pipeline
conn io.ReadWriteCloser
}
// NewConn returns a new Conn using conn for I/O.
func NewConn(conn io.ReadWriteCloser) *Conn {
return &Conn{
Reader: Reader{R: bufio.NewReader(conn)},
Writer: Writer{W: bufio.NewWriter(conn)},
conn: conn,
}
}
// Close closes the connection.
func (c *Conn) Close() error {
return c.conn.Close()
}
// Dial connects to the given address on the given network using net.Dial
// and then returns a new Conn for the connection.
func Dial(network, addr string) (*Conn, error) {
cconn, err := dialNetWork(network, addr)
if err != nil {
return nil, err
}
return NewConn(cconn), nil
}
type cConn struct {
socketFd c.Int
closed bool
}
func (conn *cConn) Read(p []byte) (n int, err error) {
if conn == nil || conn.closed {
return 0, fs.ErrClosed
}
if len(p) == 0 {
return 0, nil
}
for n < len(p) {
result := os.Read(conn.socketFd, unsafe.Pointer(&p[n:][0]), uintptr(len(p)-n))
if result < 0 {
if os.Errno() == c.Int(syscall.EINTR) {
continue
}
return n, errors.New("read error")
}
if result == 0 {
return n, io.EOF
}
n += result
}
return n, nil
}
func (conn *cConn) Write(p []byte) (n int, err error) {
if conn == nil || conn.closed {
return 0, fs.ErrClosed
}
for n < len(p) {
result := os.Write(conn.socketFd, unsafe.Pointer(&p[n:][0]), uintptr(len(p)-n))
if result < 0 {
if os.Errno() == c.Int(syscall.EINTR) {
continue
}
return n, errors.New("write error")
}
n += result
}
if n < len(p) {
return n, io.ErrShortWrite
}
return n, nil
}
func (conn *cConn) Close() error {
if conn == nil {
return nil
}
if conn.closed {
return fs.ErrClosed
}
conn.closed = true
result := os.Close(conn.socketFd)
if result < 0 {
return errors.New(c.GoString(c.Strerror(os.Errno())))
}
return nil
}
func dialNetWork(network, addr string) (*cConn, error) {
host, port, err := splitAddr(addr)
if err != nil {
return nil, err
}
var hints net.AddrInfo
var res *net.AddrInfo
c.Memset(unsafe.Pointer(&hints), 0, unsafe.Sizeof(hints))
hints.Family = net.AF_UNSPEC
hints.SockType = net.SOCK_STREAM
status := net.Getaddrinfo(c.AllocaCStr(host), c.AllocaCStr(port), &hints, &res)
if status != 0 {
return nil, errors.New("getaddrinfo error")
}
socketFd := net.Socket(res.Family, res.SockType, res.Protocol)
if socketFd == -1 {
net.Freeaddrinfo(res)
return nil, errors.New("socket error")
}
if net.Connect(socketFd, res.Addr, res.AddrLen) == -1 {
os.Close(socketFd)
net.Freeaddrinfo(res)
return nil, errors.New("connect error")
}
net.Freeaddrinfo(res)
return &cConn{
socketFd: socketFd,
}, nil
}
func splitAddr(addr string) (host, port string, err error) {
// Handle IPv6 addresses
if strings.HasPrefix(addr, "[") {
closeBracket := strings.LastIndex(addr, "]")
if closeBracket == -1 {
return "", "", errors.New("invalid IPv6 address: missing closing bracket")
}
host = addr[1:closeBracket]
if len(addr) > closeBracket+1 {
if addr[closeBracket+1] != ':' {
return "", "", errors.New("invalid address: colon missing after IPv6 address")
}
port = addr[closeBracket+2:]
}
} else {
// Handle IPv4 addresses or domain names
parts := strings.Split(addr, ":")
if len(parts) > 2 {
return "", "", errors.New("invalid address: too many colons")
}
host = parts[0]
if len(parts) == 2 {
port = parts[1]
}
}
if host == "" {
return "", "", errors.New("invalid address: host is empty")
}
if port == "" {
port = "80" // Default port is 80
}
return host, port, nil
}
// Cmd is a convenience method that sends a command after
// waiting its turn in the pipeline. The command text is the
// result of formatting format with args and appending \r\n.
// Cmd returns the id of the command, for use with StartResponse and EndResponse.
//
// For example, a client might run a HELP command that returns a dot-body
// by using:
//
// id, err := c.Cmd("HELP")
// if err != nil {
// return nil, err
// }
//
// c.StartResponse(id)
// defer c.EndResponse(id)
//
// if _, _, err = c.ReadCodeLine(110); err != nil {
// return nil, err
// }
// text, err := c.ReadDotBytes()
// if err != nil {
// return nil, err
// }
// return c.ReadCodeLine(250)
func (c *Conn) Cmd(format string, args ...any) (id uint, err error) {
id = c.Next()
c.StartRequest(id)
err = c.PrintfLine(format, args...)
c.EndRequest(id)
if err != nil {
return 0, err
}
return id, nil
}
// TrimString returns s without leading and trailing ASCII space.
func TrimString(s string) string {
for len(s) > 0 && isASCIISpace(s[0]) {
s = s[1:]
}
for len(s) > 0 && isASCIISpace(s[len(s)-1]) {
s = s[:len(s)-1]
}
return s
}
// TrimBytes returns b without leading and trailing ASCII space.
func TrimBytes(b []byte) []byte {
for len(b) > 0 && isASCIISpace(b[0]) {
b = b[1:]
}
for len(b) > 0 && isASCIISpace(b[len(b)-1]) {
b = b[:len(b)-1]
}
return b
}
func isASCIISpace(b byte) bool {
return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}
func isASCIILetter(b byte) bool {
b |= 0x20 // make lower case
return 'a' <= b && b <= 'z'
}

View File

@@ -0,0 +1,884 @@
/*
* Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package build
import (
"bytes"
"debug/macho"
"fmt"
"go/ast"
"go/build"
"go/constant"
"go/token"
"go/types"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strings"
"unsafe"
"golang.org/x/tools/go/ssa"
"github.com/goplus/llgo/compiler/cl"
"github.com/goplus/llgo/compiler/internal/env"
"github.com/goplus/llgo/compiler/internal/packages"
"github.com/goplus/llgo/compiler/internal/typepatch"
"github.com/goplus/llgo/compiler/ssa/abi"
xenv "github.com/goplus/llgo/xtool/env"
"github.com/goplus/llgo/xtool/env/llvm"
llssa "github.com/goplus/llgo/compiler/ssa"
clangCheck "github.com/goplus/llgo/xtool/clang/check"
)
type Mode int
const (
ModeBuild Mode = iota
ModeInstall
ModeRun
ModeCmpTest
ModeGen
)
const (
debugBuild = packages.DebugPackagesLoad
)
func needLLFile(mode Mode) bool {
return mode != ModeBuild
}
type Config struct {
BinPath string
AppExt string // ".exe" on Windows, empty on Unix
OutFile string // only valid for ModeBuild when len(pkgs) == 1
RunArgs []string // only valid for ModeRun
Mode Mode
GenExpect bool // only valid for ModeCmpTest
}
func NewDefaultConf(mode Mode) *Config {
bin := os.Getenv("GOBIN")
if bin == "" {
gopath, err := envGOPATH()
if err != nil {
panic(fmt.Errorf("cannot get GOPATH: %v", err))
}
bin = filepath.Join(gopath, "bin")
}
if err := os.MkdirAll(bin, 0755); err != nil {
panic(fmt.Errorf("cannot create bin directory: %v", err))
}
conf := &Config{
BinPath: bin,
Mode: mode,
AppExt: DefaultAppExt(),
}
return conf
}
func envGOPATH() (string, error) {
if gopath := os.Getenv("GOPATH"); gopath != "" {
return gopath, nil
}
home, err := os.UserHomeDir()
if err != nil {
return "", err
}
return filepath.Join(home, "go"), nil
}
func DefaultAppExt() string {
if runtime.GOOS == "windows" {
return ".exe"
}
return ""
}
// -----------------------------------------------------------------------------
const (
loadFiles = packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles
loadImports = loadFiles | packages.NeedImports
loadTypes = loadImports | packages.NeedTypes | packages.NeedTypesSizes
loadSyntax = loadTypes | packages.NeedSyntax | packages.NeedTypesInfo
)
func Do(args []string, conf *Config) ([]Package, error) {
flags, patterns, verbose := ParseArgs(args, buildFlags)
flags = append(flags, "-tags", "llgo")
cfg := &packages.Config{
Mode: loadSyntax | packages.NeedDeps | packages.NeedModule | packages.NeedExportFile,
BuildFlags: flags,
Fset: token.NewFileSet(),
}
if len(overlayFiles) > 0 {
cfg.Overlay = make(map[string][]byte)
for file, src := range overlayFiles {
overlay := unsafe.Slice(unsafe.StringData(src), len(src))
cfg.Overlay[filepath.Join(runtime.GOROOT(), "src", file)] = overlay
}
}
cl.EnableDebugSymbols(IsDebugEnabled())
llssa.Initialize(llssa.InitAll)
target := &llssa.Target{
GOOS: build.Default.GOOS,
GOARCH: build.Default.GOARCH,
}
prog := llssa.NewProgram(target)
sizes := prog.TypeSizes
dedup := packages.NewDeduper()
dedup.SetPreload(func(pkg *types.Package, files []*ast.File) {
if canSkipToBuild(pkg.Path()) {
return
}
cl.ParsePkgSyntax(prog, pkg, files)
})
if patterns == nil {
patterns = []string{"."}
}
initial, err := packages.LoadEx(dedup, sizes, cfg, patterns...)
check(err)
mode := conf.Mode
if len(initial) == 1 && len(initial[0].CompiledGoFiles) > 0 {
if mode == ModeBuild {
mode = ModeInstall
}
} else if mode == ModeRun {
if len(initial) > 1 {
return nil, fmt.Errorf("cannot run multiple packages")
} else {
return nil, fmt.Errorf("no Go files in matched packages")
}
}
altPkgPaths := altPkgs(initial, llssa.PkgRuntime)
// Use LLGoROOT as default implementation if `github.com/goplus/llgo` is not
// imported in user's go.mod. This ensures compilation works without import
// while allowing `github.com/goplus/llgo` upgrades via go.mod.
//
// WARNING(lijie): This approach cannot guarantee compatibility between `llgo`
// executable and runtime. This is a known design limitation that needs to be
// addressed in future improvements. The runtime should be:
// 1. Released and fully tested with the `llgo` compiler across different Go
// compiler versions and user-specified go versions in go.mod
// 2. Not be dependent on `github.com/goplus/llgo/c` library. Current runtime directly
// depends on it, causing version conflicts: using LLGoROOT makes user's specified
// version ineffective, while not using it leaves runtime unable to follow compiler
// updates. Since `github.com/goplus/llgo/c/*` contains many application libraries
// that may change frequently, a possible solution is to have both depend on a
// stable and limited c core API.
if !llgoPkgImported(initial) {
cfg.Dir = env.LLGoROOT()
}
altPkgs, err := packages.LoadEx(dedup, sizes, cfg, altPkgPaths...)
check(err)
noRt := 1
prog.SetRuntime(func() *types.Package {
noRt = 0
return altPkgs[0].Types
})
prog.SetPython(func() *types.Package {
return dedup.Check(llssa.PkgPython).Types
})
buildMode := ssaBuildMode
if IsDebugEnabled() {
buildMode |= ssa.GlobalDebug
}
if !IsOptimizeEnabled() {
buildMode |= ssa.NaiveForm
}
progSSA := ssa.NewProgram(initial[0].Fset, buildMode)
patches := make(cl.Patches, len(altPkgPaths))
altSSAPkgs(progSSA, patches, altPkgs[1:], verbose)
env := llvm.New("")
os.Setenv("PATH", env.BinDir()+":"+os.Getenv("PATH")) // TODO(xsw): check windows
ctx := &context{env, cfg, progSSA, prog, dedup, patches, make(map[string]none), initial, mode, 0}
pkgs := buildAllPkgs(ctx, initial, verbose)
if mode == ModeGen {
for _, pkg := range pkgs {
if pkg.Package == initial[0] {
return []*aPackage{pkg}, nil
}
}
return nil, fmt.Errorf("initial package not found")
}
dpkg := buildAllPkgs(ctx, altPkgs[noRt:], verbose)
var linkArgs []string
for _, pkg := range dpkg {
linkArgs = append(linkArgs, pkg.LinkArgs...)
}
if mode != ModeBuild {
nErr := 0
for _, pkg := range initial {
if pkg.Name == "main" {
nErr += linkMainPkg(ctx, pkg, pkgs, linkArgs, conf, mode, verbose)
}
}
if nErr > 0 {
os.Exit(nErr)
}
}
return dpkg, nil
}
func llgoPkgImported(pkgs []*packages.Package) bool {
for _, pkg := range pkgs {
for _, imp := range pkg.Imports {
if imp.Module != nil && imp.Module.Path == env.LLGoCompilerPkg {
return true
}
}
}
return false
}
func setNeedRuntimeOrPyInit(pkg *packages.Package, needRuntime, needPyInit bool) {
v := []byte{'0', '0'}
if needRuntime {
v[0] = '1'
}
if needPyInit {
v[1] = '1'
}
pkg.ID = string(v) // just use pkg.ID to mark it needs runtime
}
func isNeedRuntimeOrPyInit(pkg *packages.Package) (needRuntime, needPyInit bool) {
if len(pkg.ID) == 2 {
return pkg.ID[0] == '1', pkg.ID[1] == '1'
}
return
}
const (
ssaBuildMode = ssa.SanityCheckFunctions | ssa.InstantiateGenerics
)
type context struct {
env *llvm.Env
conf *packages.Config
progSSA *ssa.Program
prog llssa.Program
dedup packages.Deduper
patches cl.Patches
built map[string]none
initial []*packages.Package
mode Mode
nLibdir int
}
func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs []*aPackage) {
prog := ctx.prog
pkgs, errPkgs := allPkgs(ctx, initial, verbose)
for _, errPkg := range errPkgs {
for _, err := range errPkg.Errors {
fmt.Fprintln(os.Stderr, err)
}
fmt.Fprintln(os.Stderr, "cannot build SSA for package", errPkg)
}
if len(errPkgs) > 0 {
os.Exit(1)
}
built := ctx.built
for _, aPkg := range pkgs {
pkg := aPkg.Package
if _, ok := built[pkg.PkgPath]; ok {
pkg.ExportFile = ""
continue
}
built[pkg.PkgPath] = none{}
switch kind, param := cl.PkgKindOf(pkg.Types); kind {
case cl.PkgDeclOnly:
// skip packages that only contain declarations
// and set no export file
pkg.ExportFile = ""
case cl.PkgLinkIR, cl.PkgLinkExtern, cl.PkgPyModule:
if len(pkg.GoFiles) > 0 {
cgoLdflags, err := buildPkg(ctx, aPkg, verbose)
if err != nil {
panic(err)
}
linkParts := concatPkgLinkFiles(ctx, pkg, verbose)
allParts := append(linkParts, cgoLdflags...)
allParts = append(allParts, pkg.ExportFile)
aPkg.LinkArgs = allParts
} else {
// panic("todo")
// TODO(xsw): support packages out of llgo
pkg.ExportFile = ""
}
if kind == cl.PkgLinkExtern {
// need to be linked with external library
// format: ';' separated alternative link methods. e.g.
// link: $LLGO_LIB_PYTHON; $(pkg-config --libs python3-embed); -lpython3
altParts := strings.Split(param, ";")
expdArgs := make([]string, 0, len(altParts))
for _, param := range altParts {
param = strings.TrimSpace(param)
if strings.ContainsRune(param, '$') {
expdArgs = append(expdArgs, xenv.ExpandEnvToArgs(param)...)
ctx.nLibdir++
} else {
fields := strings.Fields(param)
expdArgs = append(expdArgs, fields...)
}
if len(expdArgs) > 0 {
break
}
}
if len(expdArgs) == 0 {
panic(fmt.Sprintf("'%s' cannot locate the external library", param))
}
pkgLinkArgs := make([]string, 0, 3)
if expdArgs[0][0] == '-' {
pkgLinkArgs = append(pkgLinkArgs, expdArgs...)
} else {
linkFile := expdArgs[0]
dir, lib := filepath.Split(linkFile)
pkgLinkArgs = append(pkgLinkArgs, "-l"+lib)
if dir != "" {
pkgLinkArgs = append(pkgLinkArgs, "-L"+dir)
ctx.nLibdir++
}
}
if err := clangCheck.CheckLinkArgs(pkgLinkArgs); err != nil {
panic(fmt.Sprintf("test link args '%s' failed\n\texpanded to: %v\n\tresolved to: %v\n\terror: %v", param, expdArgs, pkgLinkArgs, err))
}
aPkg.LinkArgs = append(aPkg.LinkArgs, pkgLinkArgs...)
}
default:
cgoLdflags, err := buildPkg(ctx, aPkg, verbose)
if err != nil {
panic(err)
}
aPkg.LinkArgs = append(cgoLdflags, pkg.ExportFile)
setNeedRuntimeOrPyInit(pkg, prog.NeedRuntime, prog.NeedPyInit)
}
}
return
}
func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, linkArgs []string, conf *Config, mode Mode, verbose bool) (nErr int) {
pkgPath := pkg.PkgPath
name := path.Base(pkgPath)
app := conf.OutFile
if app == "" {
app = filepath.Join(conf.BinPath, name+conf.AppExt)
}
args := make([]string, 0, len(pkg.Imports)+len(linkArgs)+16)
args = append(
args,
"-o", app,
"-fuse-ld=lld",
"-Wno-override-module",
// "-O2", // FIXME: This will cause TestFinalizer in _test/bdwgc.go to fail on macOS.
)
switch runtime.GOOS {
case "darwin": // ld64.lld (macOS)
args = append(
args,
"-rpath", "@loader_path",
"-rpath", "@loader_path/../lib",
"-Xlinker", "-dead_strip",
)
case "windows": // lld-link (Windows)
// TODO: Add options for Windows.
default: // ld.lld (Unix), wasm-ld (WebAssembly)
args = append(
args,
"-rpath", "$ORIGIN",
"-rpath", "$ORIGIN/../lib",
"-fdata-sections",
"-ffunction-sections",
"-Xlinker", "--gc-sections",
"-lm",
"-latomic",
"-lpthread", // libpthread is built-in since glibc 2.34 (2021-08-01); we need to support earlier versions.
)
}
needRuntime := false
needPyInit := false
pkgsMap := make(map[*packages.Package]*aPackage, len(pkgs))
for _, v := range pkgs {
pkgsMap[v.Package] = v
}
packages.Visit([]*packages.Package{pkg}, nil, func(p *packages.Package) {
if p.ExportFile != "" { // skip packages that only contain declarations
aPkg := pkgsMap[p]
args = append(args, aPkg.LinkArgs...)
need1, need2 := isNeedRuntimeOrPyInit(p)
if !needRuntime {
needRuntime = need1
}
if !needPyInit {
needPyInit = need2
}
}
})
var aPkg *aPackage
for _, v := range pkgs {
if v.Package == pkg { // found this package
aPkg = v
break
}
}
dirty := false
if needRuntime {
args = append(args, linkArgs...)
} else {
dirty = true
fn := aPkg.LPkg.FuncOf(cl.RuntimeInit)
fn.MakeBody(1).Return()
}
if needPyInit {
dirty = aPkg.LPkg.PyInit()
}
if dirty && needLLFile(mode) {
lpkg := aPkg.LPkg
os.WriteFile(pkg.ExportFile, []byte(lpkg.String()), 0644)
}
if verbose || mode != ModeRun {
fmt.Fprintln(os.Stderr, "#", pkgPath)
}
defer func() {
if e := recover(); e != nil {
nErr = 1
}
}()
// add rpath and find libs
exargs := make([]string, 0, ctx.nLibdir<<1)
libs := make([]string, 0, ctx.nLibdir*3)
for _, arg := range args {
if strings.HasPrefix(arg, "-L") {
exargs = append(exargs, "-rpath", arg[2:])
} else if strings.HasPrefix(arg, "-l") {
libs = append(libs, arg[2:])
}
}
args = append(args, exargs...)
if IsDebugEnabled() {
args = append(args, "-gdwarf-4")
}
// TODO(xsw): show work
if verbose {
fmt.Fprintln(os.Stderr, "clang", args)
}
err := ctx.env.Clang().Exec(args...)
check(err)
if runtime.GOOS == "darwin" {
dylibDeps := make([]string, 0, len(libs))
for _, lib := range libs {
dylibDep := findDylibDep(app, lib)
if dylibDep != "" {
dylibDeps = append(dylibDeps, dylibDep)
}
}
err := ctx.env.InstallNameTool().ChangeToRpath(app, dylibDeps...)
check(err)
}
switch mode {
case ModeRun:
cmd := exec.Command(app, conf.RunArgs...)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()
if s := cmd.ProcessState; s != nil {
os.Exit(s.ExitCode())
}
case ModeCmpTest:
cmpTest(filepath.Dir(pkg.GoFiles[0]), pkgPath, app, conf.GenExpect, conf.RunArgs)
}
return
}
func buildPkg(ctx *context, aPkg *aPackage, verbose bool) (cgoLdflags []string, err error) {
pkg := aPkg.Package
pkgPath := pkg.PkgPath
if debugBuild || verbose {
fmt.Fprintln(os.Stderr, pkgPath)
}
if canSkipToBuild(pkgPath) {
pkg.ExportFile = ""
return
}
var syntax = pkg.Syntax
if altPkg := aPkg.AltPkg; altPkg != nil {
syntax = append(syntax, altPkg.Syntax...)
}
showDetail := verbose && pkgExists(ctx.initial, pkg)
if showDetail {
llssa.SetDebug(llssa.DbgFlagAll)
cl.SetDebug(cl.DbgFlagAll)
}
ret, externs, err := cl.NewPackageEx(ctx.prog, ctx.patches, aPkg.SSA, syntax)
if showDetail {
llssa.SetDebug(0)
cl.SetDebug(0)
}
check(err)
aPkg.LPkg = ret
cgoLdflags, err = buildCgo(ctx, aPkg, aPkg.Package.Syntax, externs, verbose)
if needLLFile(ctx.mode) {
pkg.ExportFile += ".ll"
os.WriteFile(pkg.ExportFile, []byte(ret.String()), 0644)
if debugBuild || verbose {
fmt.Fprintf(os.Stderr, "==> Export %s: %s\n", aPkg.PkgPath, pkg.ExportFile)
}
if IsCheckEnable() {
if err, msg := llcCheck(ctx.env, pkg.ExportFile); err != nil {
fmt.Fprintf(os.Stderr, "==> lcc %v: %v\n%v\n", pkg.PkgPath, pkg.ExportFile, msg)
}
}
}
return
}
func llcCheck(env *llvm.Env, exportFile string) (err error, msg string) {
bin := filepath.Join(env.BinDir(), "llc")
cmd := exec.Command(bin, "-filetype=null", exportFile)
var buf bytes.Buffer
cmd.Stderr = &buf
if err = cmd.Run(); err != nil {
msg = buf.String()
}
return
}
const (
altPkgPathPrefix = abi.PatchPathPrefix
)
func altPkgs(initial []*packages.Package, alts ...string) []string {
packages.Visit(initial, nil, func(p *packages.Package) {
if p.Types != nil && !p.IllTyped {
if _, ok := hasAltPkg[p.PkgPath]; ok {
alts = append(alts, altPkgPathPrefix+p.PkgPath)
}
}
})
return alts
}
func altSSAPkgs(prog *ssa.Program, patches cl.Patches, alts []*packages.Package, verbose bool) {
packages.Visit(alts, nil, func(p *packages.Package) {
if typs := p.Types; typs != nil && !p.IllTyped {
if debugBuild || verbose {
log.Println("==> BuildSSA", p.PkgPath)
}
pkgSSA := prog.CreatePackage(typs, p.Syntax, p.TypesInfo, true)
if strings.HasPrefix(p.PkgPath, altPkgPathPrefix) {
path := p.PkgPath[len(altPkgPathPrefix):]
patches[path] = cl.Patch{Alt: pkgSSA, Types: typepatch.Clone(typs)}
if debugBuild || verbose {
log.Println("==> Patching", path)
}
}
}
})
prog.Build()
}
type aPackage struct {
*packages.Package
SSA *ssa.Package
AltPkg *packages.Cached
LPkg llssa.Package
LinkArgs []string
}
type Package = *aPackage
func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aPackage, errs []*packages.Package) {
prog := ctx.progSSA
built := ctx.built
packages.Visit(initial, nil, func(p *packages.Package) {
if p.Types != nil && !p.IllTyped {
pkgPath := p.PkgPath
if _, ok := built[pkgPath]; ok || strings.HasPrefix(pkgPath, altPkgPathPrefix) {
return
}
var altPkg *packages.Cached
var ssaPkg = createSSAPkg(prog, p, verbose)
if _, ok := hasAltPkg[pkgPath]; ok {
if altPkg = ctx.dedup.Check(altPkgPathPrefix + pkgPath); altPkg == nil {
return
}
}
all = append(all, &aPackage{p, ssaPkg, altPkg, nil, nil})
} else {
errs = append(errs, p)
}
})
return
}
func createSSAPkg(prog *ssa.Program, p *packages.Package, verbose bool) *ssa.Package {
pkgSSA := prog.ImportedPackage(p.PkgPath)
if pkgSSA == nil {
if debugBuild || verbose {
log.Println("==> BuildSSA", p.PkgPath)
}
pkgSSA = prog.CreatePackage(p.Types, p.Syntax, p.TypesInfo, true)
pkgSSA.Build() // TODO(xsw): build concurrently
}
return pkgSSA
}
var (
// TODO(xsw): complete build flags
buildFlags = map[string]bool{
"-C": true, // -C dir: Change to dir before running the command
"-a": false, // -a: force rebuilding of packages that are already up-to-date
"-n": false, // -n: print the commands but do not run them
"-p": true, // -p n: the number of programs to run in parallel
"-race": false, // -race: enable data race detection
"-cover": false, // -cover: enable coverage analysis
"-covermode": true, // -covermode mode: set the mode for coverage analysis
"-v": false, // -v: print the names of packages as they are compiled
"-work": false, // -work: print the name of the temporary work directory and do not delete it when exiting
"-x": false, // -x: print the commands
"-tags": true, // -tags 'tag,list': a space-separated list of build tags to consider satisfied during the build
"-pkgdir": true, // -pkgdir dir: install and load all packages from dir instead of the usual locations
"-ldflags": true, // --ldflags 'flag list': arguments to pass on each go tool link invocation
}
)
const llgoDebug = "LLGO_DEBUG"
const llgoOptimize = "LLGO_OPTIMIZE"
const llgoCheck = "LLGO_CHECK"
func isEnvOn(env string, defVal bool) bool {
envVal := strings.ToLower(os.Getenv(env))
if envVal == "" {
return defVal
}
return envVal == "1" || envVal == "true" || envVal == "on"
}
func IsDebugEnabled() bool {
return isEnvOn(llgoDebug, false)
}
func IsOptimizeEnabled() bool {
return isEnvOn(llgoOptimize, true)
}
func IsCheckEnable() bool {
return isEnvOn(llgoCheck, false)
}
func ParseArgs(args []string, swflags map[string]bool) (flags, patterns []string, verbose bool) {
n := len(args)
for i := 0; i < n; i++ {
arg := args[i]
if strings.HasPrefix(arg, "-") {
checkFlag(arg, &i, &verbose, swflags)
} else {
patterns = append([]string{}, args[i:]...)
flags = append([]string{}, args[:i]...)
return
}
}
return
}
func SkipFlagArgs(args []string) int {
n := len(args)
for i := 0; i < n; i++ {
arg := args[i]
if strings.HasPrefix(arg, "-") {
checkFlag(arg, &i, nil, buildFlags)
} else {
return i
}
}
return -1
}
func checkFlag(arg string, i *int, verbose *bool, swflags map[string]bool) {
if pos := strings.IndexByte(arg, '='); pos > 0 {
if verbose != nil && arg == "-v=true" {
*verbose = true
}
} else if hasarg, ok := swflags[arg]; ok {
if hasarg {
*i++
} else if verbose != nil && arg == "-v" {
*verbose = true
}
} else {
panic("unknown flag: " + arg)
}
}
func concatPkgLinkFiles(ctx *context, pkg *packages.Package, verbose bool) (parts []string) {
llgoPkgLinkFiles(ctx, pkg, func(linkFile string) {
parts = append(parts, linkFile)
}, verbose)
return
}
// const LLGoFiles = "file1; file2; ..."
func llgoPkgLinkFiles(ctx *context, pkg *packages.Package, procFile func(linkFile string), verbose bool) {
if o := pkg.Types.Scope().Lookup("LLGoFiles"); o != nil {
val := o.(*types.Const).Val()
if val.Kind() == constant.String {
clFiles(ctx, constant.StringVal(val), pkg, procFile, verbose)
}
}
}
// files = "file1; file2; ..."
// files = "$(pkg-config --cflags xxx): file1; file2; ..."
func clFiles(ctx *context, files string, pkg *packages.Package, procFile func(linkFile string), verbose bool) {
dir := filepath.Dir(pkg.GoFiles[0])
expFile := pkg.ExportFile
args := make([]string, 0, 16)
if strings.HasPrefix(files, "$") { // has cflags
if pos := strings.IndexByte(files, ':'); pos > 0 {
cflags := xenv.ExpandEnvToArgs(files[:pos])
files = files[pos+1:]
args = append(args, cflags...)
}
}
for _, file := range strings.Split(files, ";") {
cFile := filepath.Join(dir, strings.TrimSpace(file))
clFile(ctx, args, cFile, expFile, procFile, verbose)
}
}
func clFile(ctx *context, args []string, cFile, expFile string, procFile func(linkFile string), verbose bool) {
llFile := expFile + filepath.Base(cFile) + ".ll"
args = append(args, "-emit-llvm", "-S", "-o", llFile, "-c", cFile)
if verbose {
fmt.Fprintln(os.Stderr, "clang", args)
}
err := ctx.env.Clang().Exec(args...)
check(err)
procFile(llFile)
}
func pkgExists(initial []*packages.Package, pkg *packages.Package) bool {
for _, v := range initial {
if v == pkg {
return true
}
}
return false
}
func canSkipToBuild(pkgPath string) bool {
if _, ok := hasAltPkg[pkgPath]; ok {
return false
}
switch pkgPath {
case "unsafe":
return true
default:
return strings.HasPrefix(pkgPath, "internal/") ||
strings.HasPrefix(pkgPath, "runtime/internal/")
}
}
// findDylibDep finds the dylib dependency in the executable. It returns empty
// string if not found.
func findDylibDep(exe, lib string) string {
file, err := macho.Open(exe)
check(err)
defer file.Close()
for _, load := range file.Loads {
if dylib, ok := load.(*macho.Dylib); ok {
if strings.HasPrefix(filepath.Base(dylib.Name), fmt.Sprintf("lib%s.", lib)) {
return dylib.Name
}
}
}
return ""
}
type none struct{}
var hasAltPkg = map[string]none{
"crypto/hmac": {},
"crypto/md5": {},
"crypto/rand": {},
"crypto/sha1": {},
"crypto/sha256": {},
"crypto/sha512": {},
"crypto/subtle": {},
"fmt": {},
"hash/crc32": {},
"internal/abi": {},
"internal/bytealg": {},
"internal/itoa": {},
"internal/filepathlite": {},
"internal/oserror": {},
"internal/race": {},
"internal/reflectlite": {},
"internal/stringslite": {},
"internal/syscall/execenv": {},
"internal/syscall/unix": {},
"math": {},
"math/big": {},
"math/cmplx": {},
"math/rand": {},
"reflect": {},
"sync": {},
"sync/atomic": {},
"syscall": {},
"time": {},
"os": {},
"os/exec": {},
"runtime": {},
"io": {},
}
func check(err error) {
if err != nil {
panic(err)
}
}
// -----------------------------------------------------------------------------

View File

@@ -0,0 +1,400 @@
/*
* Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package build
import (
"encoding/json"
"fmt"
"go/ast"
"go/token"
"go/types"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"github.com/goplus/llgo/compiler/internal/buildtags"
llssa "github.com/goplus/llgo/compiler/ssa"
"github.com/goplus/llgo/xtool/safesplit"
)
type cgoDecl struct {
tag string
cflags []string
ldflags []string
}
type cgoPreamble struct {
goFile string
src string
}
const (
cgoHeader = `
#include <stdlib.h>
static void* _Cmalloc(size_t size) {
return malloc(size);
}
`
)
func buildCgo(ctx *context, pkg *aPackage, files []*ast.File, externs []string, verbose bool) (cgoLdflags []string, err error) {
cfiles, preambles, cdecls, err := parseCgo_(pkg, files)
if err != nil {
return
}
tagUsed := make(map[string]bool)
for _, cdecl := range cdecls {
if cdecl.tag != "" {
tagUsed[cdecl.tag] = false
}
}
buildtags.CheckTags(ctx.conf.BuildFlags, tagUsed)
cflags := []string{}
ldflags := []string{}
for _, cdecl := range cdecls {
if cdecl.tag == "" || tagUsed[cdecl.tag] {
if len(cdecl.cflags) > 0 {
cflags = append(cflags, cdecl.cflags...)
}
if len(cdecl.ldflags) > 0 {
ldflags = append(ldflags, cdecl.ldflags...)
}
}
}
incDirs := make(map[string]none)
for _, preamble := range preambles {
dir, _ := filepath.Split(preamble.goFile)
if _, ok := incDirs[dir]; !ok {
incDirs[dir] = none{}
cflags = append(cflags, "-I"+dir)
}
}
for _, cfile := range cfiles {
clFile(ctx, cflags, cfile, pkg.ExportFile, func(linkFile string) {
cgoLdflags = append(cgoLdflags, linkFile)
}, verbose)
}
re := regexp.MustCompile(`^(_cgo_[^_]+_(Cfunc|Cmacro)_)(.*)$`)
cgoSymbols := make(map[string]string)
mallocFix := false
for _, symbolName := range externs {
lastPart := symbolName
lastDot := strings.LastIndex(symbolName, ".")
if lastDot != -1 {
lastPart = symbolName[lastDot+1:]
}
if strings.HasPrefix(lastPart, "__cgo_") {
// func ptr var: main.__cgo_func_name
cgoSymbols[symbolName] = lastPart
} else if m := re.FindStringSubmatch(symbolName); len(m) > 0 {
prefix := m[1] // _cgo_hash_(Cfunc|Cmacro)_
name := m[3] // remaining part
cgoSymbols[symbolName] = name
// fix missing _cgo_9113e32b6599_Cfunc__Cmalloc
if !mallocFix && m[2] == "Cfunc" {
mallocName := prefix + "_Cmalloc"
cgoSymbols[mallocName] = "_Cmalloc"
mallocFix = true
}
}
}
for _, preamble := range preambles {
tmpFile, err := os.CreateTemp("", "-cgo-*.c")
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %v", err)
}
tmpName := tmpFile.Name()
defer os.Remove(tmpName)
code := cgoHeader + "\n\n" + preamble.src
externDecls, err := genExternDeclsByClang(pkg, code, cflags, cgoSymbols)
if err != nil {
return nil, err
}
if err = os.WriteFile(tmpName, []byte(code+"\n\n"+externDecls), 0644); err != nil {
return nil, err
}
clFile(ctx, cflags, tmpName, pkg.ExportFile, func(linkFile string) {
cgoLdflags = append(cgoLdflags, linkFile)
}, verbose)
}
for _, ldflag := range ldflags {
cgoLdflags = append(cgoLdflags, safesplit.SplitPkgConfigFlags(ldflag)...)
}
return
}
// clangASTNode represents a node in clang's AST
type clangASTNode struct {
Kind string `json:"kind"`
Name string `json:"name,omitempty"`
Inner []clangASTNode `json:"inner,omitempty"`
}
func genExternDeclsByClang(pkg *aPackage, src string, cflags []string, cgoSymbols map[string]string) (string, error) {
tmpSrc, err := os.CreateTemp("", "cgo-src-*.c")
if err != nil {
return "", err
}
defer os.Remove(tmpSrc.Name())
if err := os.WriteFile(tmpSrc.Name(), []byte(src), 0644); err != nil {
return "", err
}
symbolNames := make(map[string]bool)
if err := getFuncNames(tmpSrc.Name(), cflags, symbolNames); err != nil {
return "", err
}
macroNames := make(map[string]bool)
if err := getMacroNames(tmpSrc.Name(), cflags, macroNames); err != nil {
return "", err
}
b := strings.Builder{}
var toRemove []string
for cgoName, symbolName := range cgoSymbols {
if strings.HasPrefix(symbolName, "__cgo_") {
gofuncName := strings.Replace(cgoName, ".__cgo_", ".", 1)
gofn := pkg.LPkg.FuncOf(gofuncName)
cgoVar := pkg.LPkg.VarOf(cgoName)
if gofn != nil {
cgoVar.ReplaceAllUsesWith(gofn.Expr)
} else {
cfuncName := symbolName[len("__cgo_"):]
cfn := pkg.LPkg.NewFunc(cfuncName, types.NewSignatureType(nil, nil, nil, nil, nil, false), llssa.InC)
cgoVar.ReplaceAllUsesWith(cfn.Expr)
}
toRemove = append(toRemove, cgoName)
} else {
usePtr := ""
if symbolNames[symbolName] {
usePtr = "*"
} else if !macroNames[symbolName] {
continue
}
/* template:
typeof(fputs)* _cgo_1574167f3838_Cfunc_fputs;
__attribute__((constructor))
static void _init__cgo_1574167f3838_Cfunc_fputs() {
_cgo_1574167f3838_Cfunc_fputs = fputs;
}*/
b.WriteString(fmt.Sprintf(`
typeof(%s)%s %s;
__attribute__((constructor))
static void _init_%s() {
%s = %s;
}
`,
symbolName, usePtr, cgoName,
cgoName,
cgoName, symbolName))
toRemove = append(toRemove, cgoName)
}
}
for _, funcName := range toRemove {
delete(cgoSymbols, funcName)
}
return b.String(), nil
}
func getMacroNames(file string, cflags []string, macroNames map[string]bool) error {
args := append([]string{"-dM", "-E"}, cflags...)
args = append(args, file)
cmd := exec.Command("clang", args...)
output, err := cmd.Output()
if err != nil {
return err
}
for _, line := range strings.Split(string(output), "\n") {
if strings.HasPrefix(line, "#define ") {
define := strings.TrimPrefix(line, "#define ")
parts := strings.SplitN(define, " ", 2)
if len(parts) > 1 {
macroNames[parts[0]] = true
}
}
}
return nil
}
func getFuncNames(file string, cflags []string, symbolNames map[string]bool) error {
args := append([]string{"-Xclang", "-ast-dump=json", "-fsyntax-only"}, cflags...)
args = append(args, file)
cmd := exec.Command("clang", args...)
output, err := cmd.Output()
if err != nil {
return err
}
var astRoot clangASTNode
if err := json.Unmarshal(output, &astRoot); err != nil {
return err
}
extractFuncNames(&astRoot, symbolNames)
return nil
}
func extractFuncNames(node *clangASTNode, funcNames map[string]bool) {
for _, inner := range node.Inner {
if inner.Kind == "FunctionDecl" && inner.Name != "" {
funcNames[inner.Name] = true
}
}
}
func parseCgo_(pkg *aPackage, files []*ast.File) (cfiles []string, preambles []cgoPreamble, cdecls []cgoDecl, err error) {
dirs := make(map[string]none)
for _, file := range files {
pos := pkg.Fset.Position(file.Name.NamePos)
dir, _ := filepath.Split(pos.Filename)
dirs[dir] = none{}
}
for dir := range dirs {
matches, err := filepath.Glob(filepath.Join(dir, "*.c"))
if err != nil {
continue
}
for _, match := range matches {
if strings.HasSuffix(match, "_test.c") {
continue
}
if fi, err := os.Stat(match); err == nil && !fi.IsDir() {
cfiles = append(cfiles, match)
}
}
}
for _, file := range files {
for _, decl := range file.Decls {
switch decl := decl.(type) {
case *ast.GenDecl:
if decl.Tok == token.IMPORT {
if doc := decl.Doc; doc != nil && len(decl.Specs) == 1 {
spec := decl.Specs[0].(*ast.ImportSpec)
if spec.Path.Value == "\"unsafe\"" {
pos := pkg.Fset.Position(doc.Pos())
preamble, flags, err := parseCgoPreamble(pos, doc.Text())
if err != nil {
panic(err)
}
preambles = append(preambles, preamble)
cdecls = append(cdecls, flags...)
}
}
}
}
}
}
return
}
func parseCgoPreamble(pos token.Position, text string) (preamble cgoPreamble, decls []cgoDecl, err error) {
b := strings.Builder{}
fline := pos.Line
fname := pos.Filename
b.WriteString(fmt.Sprintf("#line %d %q\n", fline, fname))
for _, line := range strings.Split(text, "\n") {
fline++
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "#cgo ") {
var cgoDecls []cgoDecl
cgoDecls, err = parseCgoDecl(line)
if err != nil {
return
}
decls = append(decls, cgoDecls...)
b.WriteString(fmt.Sprintf("#line %d %q\n", fline, fname))
} else {
b.WriteString(line)
b.WriteString("\n")
}
}
preamble = cgoPreamble{
goFile: pos.Filename,
src: b.String(),
}
return
}
// Parse cgo directive like:
// #cgo pkg-config: python3
// #cgo windows CFLAGS: -IC:/Python312/include
// #cgo windows LDFLAGS: -LC:/Python312/libs -lpython312
// #cgo CFLAGS: -I/usr/include/python3.12
// #cgo LDFLAGS: -L/usr/lib/python3.12/config-3.12-x86_64-linux-gnu -lpython3.12
func parseCgoDecl(line string) (cgoDecls []cgoDecl, err error) {
idx := strings.Index(line, ":")
if idx == -1 {
err = fmt.Errorf("invalid cgo format: %v", line)
return
}
decl := strings.TrimSpace(line[:idx])
arg := strings.TrimSpace(line[idx+1:])
// Split on first space to remove #cgo
parts := strings.SplitN(decl, " ", 2)
if len(parts) < 2 {
err = fmt.Errorf("invalid cgo directive: %v", line)
return
}
// Process remaining part
remaining := strings.TrimSpace(parts[1])
var tag, flag string
// Split on last space to get flag
if lastSpace := strings.LastIndex(remaining, " "); lastSpace != -1 {
tag = strings.TrimSpace(remaining[:lastSpace])
flag = strings.TrimSpace(remaining[lastSpace+1:])
} else {
flag = remaining
}
switch flag {
case "pkg-config":
ldflags, e := exec.Command("pkg-config", "--libs", arg).Output()
if e != nil {
err = fmt.Errorf("pkg-config: %v", e)
return
}
cflags, e := exec.Command("pkg-config", "--cflags", arg).Output()
if e != nil {
err = fmt.Errorf("pkg-config: %v", e)
return
}
cgoDecls = append(cgoDecls, cgoDecl{
tag: tag,
cflags: safesplit.SplitPkgConfigFlags(string(cflags)),
ldflags: safesplit.SplitPkgConfigFlags(string(ldflags)),
})
case "CFLAGS":
cgoDecls = append(cgoDecls, cgoDecl{
tag: tag,
cflags: safesplit.SplitPkgConfigFlags(arg),
})
case "LDFLAGS":
cgoDecls = append(cgoDecls, cgoDecl{
tag: tag,
ldflags: safesplit.SplitPkgConfigFlags(arg),
})
}
return
}

View File

@@ -0,0 +1,85 @@
/*
* Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package build
import (
"fmt"
"os"
"path"
"path/filepath"
"github.com/goplus/llgo/compiler/internal/packages"
)
var (
// TODO(xsw): complete clean flags
cleanFlags = map[string]bool{
"-v": false, // -v: print the paths of packages as they are clean
}
)
func Clean(args []string, conf *Config) {
flags, patterns, verbose := ParseArgs(args, cleanFlags)
cfg := &packages.Config{
Mode: loadSyntax | packages.NeedExportFile,
BuildFlags: flags,
}
if patterns == nil {
patterns = []string{"."}
}
initial, err := packages.LoadEx(nil, nil, cfg, patterns...)
check(err)
cleanPkgs(initial, verbose)
for _, pkg := range initial {
if pkg.Name == "main" {
cleanMainPkg(pkg, conf, verbose)
}
}
}
func cleanMainPkg(pkg *packages.Package, conf *Config, verbose bool) {
pkgPath := pkg.PkgPath
name := path.Base(pkgPath)
fname := name + conf.AppExt
app := filepath.Join(conf.BinPath, fname)
removeFile(app, verbose)
if len(pkg.CompiledGoFiles) > 0 {
dir := filepath.Dir(pkg.CompiledGoFiles[0])
buildApp := filepath.Join(dir, fname)
removeFile(buildApp, verbose)
}
}
func cleanPkgs(initial []*packages.Package, verbose bool) {
packages.Visit(initial, nil, func(p *packages.Package) {
file := p.ExportFile + ".ll"
removeFile(file, verbose)
})
}
func removeFile(file string, verbose bool) {
if _, err := os.Stat(file); os.IsNotExist(err) {
return
}
if verbose {
fmt.Fprintln(os.Stderr, "Remove", file)
}
os.Remove(file)
}

View File

@@ -0,0 +1,111 @@
/*
* Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package build
import (
"bytes"
"errors"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
)
func cmpTest(dir, pkgPath, llApp string, genExpect bool, runArgs []string) {
var llgoOut, llgoErr bytes.Buffer
var llgoRunErr = runApp(runArgs, dir, &llgoOut, &llgoErr, llApp)
llgoExpect := formatExpect(llgoOut.Bytes(), llgoErr.Bytes(), llgoRunErr)
llgoExpectFile := filepath.Join(dir, "llgo.expect")
if genExpect {
if _, err := os.Stat(llgoExpectFile); !errors.Is(err, os.ErrNotExist) {
fatal(fmt.Errorf("llgo.expect file already exists: %s", llgoExpectFile))
}
if err := os.WriteFile(llgoExpectFile, llgoExpect, 0644); err != nil {
fatal(err)
}
return
}
if b, err := os.ReadFile(llgoExpectFile); err == nil {
checkEqual("llgo.expect", llgoExpect, b)
return
} else if !errors.Is(err, os.ErrNotExist) {
fatal(err)
}
var goOut, goErr bytes.Buffer
var goRunErr = runApp(runArgs, dir, &goOut, &goErr, "go", "run", pkgPath)
checkEqual("output", llgoOut.Bytes(), goOut.Bytes())
checkEqual("stderr", llgoErr.Bytes(), goErr.Bytes())
checkEqualRunErr(llgoRunErr, goRunErr)
}
func formatExpect(stdout, stderr []byte, runErr error) []byte {
var exitCode int
if runErr != nil {
if ee, ok := runErr.(*exec.ExitError); ok {
exitCode = ee.ExitCode()
} else { // This should never happen, but just in case.
exitCode = 255
}
}
return []byte(fmt.Sprintf("#stdout\n%s\n#stderr\n%s\n#exit %d\n", stdout, stderr, exitCode))
}
func checkEqualRunErr(llgoRunErr, goRunErr error) {
if llgoRunErr == goRunErr {
return
}
fmt.Fprintln(os.Stderr, "=> Exit:", llgoRunErr)
fmt.Fprintln(os.Stderr, "\n=> Expected Exit:", goRunErr)
}
func checkEqual(prompt string, a, expected []byte) {
if bytes.Equal(a, expected) {
return
}
fmt.Fprintln(os.Stderr, "=> Result of", prompt)
os.Stderr.Write(a)
fmt.Fprintln(os.Stderr, "\n=> Expected", prompt)
os.Stderr.Write(expected)
fatal(errors.New("checkEqual: unexpected " + prompt))
}
func runApp(runArgs []string, dir string, stdout, stderr io.Writer, app string, args ...string) error {
if len(runArgs) > 0 {
if len(args) > 0 {
args = append(args, runArgs...)
} else {
args = runArgs
}
}
cmd := exec.Command(app, args...)
cmd.Dir = dir
cmd.Stdout = stdout
cmd.Stderr = stderr
return cmd.Run()
}
func fatal(err error) {
log.Panicln(err)
}

View File

@@ -0,0 +1,17 @@
package build
import (
_ "embed"
)
//go:embed _overlay/go/parser/resolver.go
var go_parser_resolver string
//go:embed _overlay/net/textproto/textproto.go
var net_textproto string
var overlayFiles = map[string]string{
"math/exp_amd64.go": "package math;",
"go/parser/resolver.go": go_parser_resolver,
"net/textproto/textproto.go": net_textproto,
}