split llfiles and ldflags and extract compileAndLinkLLFiles to prepare multi-phase build

internal/build/build.go

@@ -307,14 +307,12 @@ func Do(args []string, conf *Config) ([]Package, error) {
 
 	dpkg, err := buildAllPkgs(ctx, altPkgs[noRt:], verbose)
 	check(err)
-	var linkArgs []string
+	allPkgs := append([]*aPackage{}, pkgs...)
-	for _, pkg := range dpkg {
+	allPkgs = append(allPkgs, dpkg...)
-		linkArgs = append(linkArgs, pkg.LinkArgs...)
-	}
 
 	for _, pkg := range initial {
 		if needLink(pkg, mode) {
-			linkMainPkg(ctx, pkg, pkgs, linkArgs, conf, mode, verbose)
+			linkMainPkg(ctx, pkg, allPkgs, conf, mode, verbose)
 		}
 	}
 
@@ -406,16 +404,10 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs
 			pkg.ExportFile = ""
 		case cl.PkgLinkIR, cl.PkgLinkExtern, cl.PkgPyModule:
 			if len(pkg.GoFiles) > 0 {
-				cgoLdflags, err := buildPkg(ctx, aPkg, verbose)
+				err := buildPkg(ctx, aPkg, verbose)
 				if err != nil {
-					panic(err)
+					return nil, err
 				}
-				linkParts := concatPkgLinkFiles(ctx, pkg, verbose)
-				allParts := append(linkParts, cgoLdflags...)
-				if pkg.ExportFile != "" {
-					allParts = append(allParts, pkg.ExportFile)
-				}
-				aPkg.LinkArgs = allParts
 			} else {
 				// panic("todo")
 				// TODO(xsw): support packages out of llgo
@@ -462,16 +454,9 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs
 				aPkg.LinkArgs = append(aPkg.LinkArgs, pkgLinkArgs...)
 			}
 		default:
-			cgoLdflags, err := buildPkg(ctx, aPkg, verbose)
+			err := buildPkg(ctx, aPkg, verbose)
 			if err != nil {
-				panic(err)
+				return nil, err
-			}
-			if pkg.ExportFile != "" {
-				aPkg.LinkArgs = append(cgoLdflags, pkg.ExportFile)
-			}
-			aPkg.LinkArgs = append(aPkg.LinkArgs, concatPkgLinkFiles(ctx, pkg, verbose)...)
-			if aPkg.AltPkg != nil {
-				aPkg.LinkArgs = append(aPkg.LinkArgs, concatPkgLinkFiles(ctx, aPkg.AltPkg.Package, verbose)...)
 			}
 			setNeedRuntimeOrPyInit(ctx, pkg, aPkg.LPkg.NeedRuntime, aPkg.LPkg.NeedPyInit)
 		}
@@ -479,7 +464,7 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs
 	return
 }
 
-func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, linkArgs []string, conf *Config, mode Mode, verbose bool) {
+func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, conf *Config, mode Mode, verbose bool) {
 	pkgPath := pkg.PkgPath
 	name := path.Base(pkgPath)
 	app := conf.OutFile
@@ -497,24 +482,21 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, linkArgs
 		app += conf.AppExt
 	}
 
-	// Start with output file argument
-	args := make([]string, 0, len(pkg.Imports)+len(linkArgs)+16)
-	args = append(args, "-o", app)
-
-	// Add common linker arguments based on target OS and architecture
-	targetTriple := llvmTarget.GetTargetTriple(conf.Goos, conf.Goarch)
-	args = append(args, buildLdflags(conf.Goos, conf.Goarch, targetTriple)...)
-
 	needRuntime := false
 	needPyInit := false
 	pkgsMap := make(map[*packages.Package]*aPackage, len(pkgs))
+	allPkgs := []*packages.Package{pkg}
 	for _, v := range pkgs {
 		pkgsMap[v.Package] = v
+		allPkgs = append(allPkgs, v.Package)
 	}
-	packages.Visit([]*packages.Package{pkg}, nil, func(p *packages.Package) {
+	var llFiles []string
-		if p.ExportFile != "" { // skip packages that only contain declarations
+	var linkArgs []string
-			aPkg := pkgsMap[p]
+	packages.Visit(allPkgs, nil, func(p *packages.Package) {
-			args = append(args, aPkg.LinkArgs...)
+		aPkg := pkgsMap[p]
+		if p.ExportFile != "" && aPkg != nil { // skip packages that only contain declarations
+			linkArgs = append(linkArgs, aPkg.LinkArgs...)
+			llFiles = append(llFiles, aPkg.LLFiles...)
 			need1, need2 := isNeedRuntimeOrPyInit(ctx, p)
 			if !needRuntime {
 				needRuntime = need1
@@ -525,32 +507,15 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, linkArgs
 		}
 	})
 	entryLLFile, err := genMainModuleFile(conf, llssa.PkgRuntime, pkg.PkgPath, needRuntime, needPyInit)
-	if err != nil {
+	check(err)
-		panic(err)
-	}
 	// defer os.Remove(entryLLFile)
-	args = append(args, entryLLFile)
+	llFiles = append(llFiles, entryLLFile)
-
-	var aPkg *aPackage
-	for _, v := range pkgs {
-		if v.Package == pkg { // found this package
-			aPkg = v
-			break
-		}
-	}
-
-	args = append(args, linkArgs...)
-
-	if ctx.output {
-		lpkg := aPkg.LPkg
-		os.WriteFile(pkg.ExportFile, []byte(lpkg.String()), 0644)
-	}
 
 	// add rpath and find libs
 	exargs := make([]string, 0, ctx.nLibdir<<1)
 	libs := make([]string, 0, ctx.nLibdir*3)
 	if IsRpathChangeEnabled() {
-		for _, arg := range args {
+		for _, arg := range linkArgs {
 			if strings.HasPrefix(arg, "-L") {
 				exargs = append(exargs, "-rpath", arg[2:])
 			} else if strings.HasPrefix(arg, "-l") {
@@ -558,20 +523,12 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, linkArgs
 			}
 		}
 	}
-	args = append(args, exargs...)
+	linkArgs = append(linkArgs, exargs...)
-	if IsDbgSymsEnabled() {
-		args = append(args, "-gdwarf-4")
-	}
 
-	args = append(args, ctx.crossCompile.CCFLAGS...)
+	err = compileAndLinkLLFiles(ctx, app, llFiles, linkArgs, verbose)
-	args = append(args, ctx.crossCompile.LDFLAGS...)
-
-	cmd := ctx.env.Clang()
-	cmd.Verbose = verbose
-	err = cmd.Link(args...)
 	check(err)
 
-	if IsRpathChangeEnabled() && conf.Goos == "darwin" {
+	if IsRpathChangeEnabled() && ctx.buildConf.Goos == "darwin" {
 		dylibDeps := make([]string, 0, len(libs))
 		for _, lib := range libs {
 			dylibDep := findDylibDep(app, lib)
@@ -623,6 +580,27 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, linkArgs
 	}
 }
 
+func compileAndLinkLLFiles(ctx *context, app string, llFiles, linkArgs []string, verbose bool) error {
+	buildArgs := []string{"-o", app}
+	buildArgs = append(buildArgs, linkArgs...)
+
+	// Add common linker arguments based on target OS and architecture
+	targetTriple := llvmTarget.GetTargetTriple(ctx.buildConf.Goos, ctx.buildConf.Goarch)
+	buildArgs = append(buildArgs, buildLdflags(ctx.buildConf.Goos, ctx.buildConf.Goarch, targetTriple)...)
+
+	if IsDbgSymsEnabled() {
+		buildArgs = append(buildArgs, "-gdwarf-4")
+	}
+
+	buildArgs = append(buildArgs, ctx.crossCompile.CCFLAGS...)
+	buildArgs = append(buildArgs, ctx.crossCompile.LDFLAGS...)
+	buildArgs = append(buildArgs, llFiles...)
+
+	cmd := ctx.env.Clang()
+	cmd.Verbose = verbose
+	return cmd.Link(buildArgs...)
+}
+
 func buildCflags(goos, goarch, targetTriple string) []string {
 	args := []string{}
 	if goarch == "wasm" {
@@ -823,7 +801,7 @@ func is32Bits(goarch string) bool {
 	return goarch == "386" || goarch == "arm" || goarch == "mips" || goarch == "wasm"
 }
 
-func buildPkg(ctx *context, aPkg *aPackage, verbose bool) (cgoLdflags []string, err error) {
+func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error {
 	pkg := aPkg.Package
 	pkgPath := pkg.PkgPath
 	if debugBuild || verbose {
@@ -831,7 +809,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) (cgoLdflags []string,
 	}
 	if llruntime.SkipToBuild(pkgPath) {
 		pkg.ExportFile = ""
-		return
+		return nil
 	}
 	var syntax = pkg.Syntax
 	if altPkg := aPkg.AltPkg; altPkg != nil {
@@ -850,17 +828,26 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) (cgoLdflags []string,
 	}
 	check(err)
 	aPkg.LPkg = ret
-	cgoLdflags, err = buildCgo(ctx, aPkg, aPkg.Package.Syntax, externs, verbose)
+	cgoLLFiles, cgoLdflags, err := buildCgo(ctx, aPkg, aPkg.Package.Syntax, externs, verbose)
+	if err != nil {
+		return fmt.Errorf("build cgo of %v failed: %v", pkgPath, err)
+	}
+	aPkg.LLFiles = append(aPkg.LLFiles, cgoLLFiles...)
+	aPkg.LLFiles = append(aPkg.LLFiles, concatPkgLinkFiles(ctx, pkg, verbose)...)
+	aPkg.LinkArgs = append(aPkg.LinkArgs, cgoLdflags...)
 	if aPkg.AltPkg != nil {
-		altLdflags, e := buildCgo(ctx, aPkg, aPkg.AltPkg.Syntax, externs, verbose)
+		altLLFiles, altLdflags, e := buildCgo(ctx, aPkg, aPkg.AltPkg.Syntax, externs, verbose)
 		if e != nil {
-			return nil, fmt.Errorf("build cgo of %v failed: %v", pkgPath, e)
+			return fmt.Errorf("build cgo of %v failed: %v", pkgPath, e)
 		}
-		cgoLdflags = append(cgoLdflags, altLdflags...)
+		aPkg.LLFiles = append(aPkg.LLFiles, altLLFiles...)
+		aPkg.LLFiles = append(aPkg.LLFiles, concatPkgLinkFiles(ctx, aPkg.AltPkg.Package, verbose)...)
+		aPkg.LinkArgs = append(aPkg.LinkArgs, altLdflags...)
 	}
 	if pkg.ExportFile != "" {
 		pkg.ExportFile += ".ll"
 		os.WriteFile(pkg.ExportFile, []byte(ret.String()), 0644)
+		aPkg.LLFiles = append(aPkg.LLFiles, pkg.ExportFile)
 		if debugBuild || verbose {
 			fmt.Fprintf(os.Stderr, "==> Export %s: %s\n", aPkg.PkgPath, pkg.ExportFile)
 		}
@@ -870,7 +857,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) (cgoLdflags []string,
 			}
 		}
 	}
-	return
+	return nil
 }
 
 func llcCheck(env *llvm.Env, exportFile string) (err error, msg string) {
@@ -925,6 +912,7 @@ type aPackage struct {
 	LPkg   llssa.Package
 
 	LinkArgs []string
+	LLFiles  []string
 }
 
 type Package = *aPackage
@@ -945,7 +933,7 @@ func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aP
 					return
 				}
 			}
-			all = append(all, &aPackage{p, ssaPkg, altPkg, nil, nil})
+			all = append(all, &aPackage{p, ssaPkg, altPkg, nil, nil, nil})
 		} else {
 			errs = append(errs, p)
 		}

internal/build/cgo.go

@@ -53,7 +53,7 @@ static void* _Cmalloc(size_t size) {
 `
 )
 
-func buildCgo(ctx *context, pkg *aPackage, files []*ast.File, externs []string, verbose bool) (cgoLdflags []string, err error) {
+func buildCgo(ctx *context, pkg *aPackage, files []*ast.File, externs []string, verbose bool) (llfiles, cgoLdflags []string, err error) {
 	cfiles, preambles, cdecls, err := parseCgo_(pkg, files)
 	if err != nil {
 		return
@@ -87,7 +87,7 @@ func buildCgo(ctx *context, pkg *aPackage, files []*ast.File, externs []string,
 	}
 	for _, cfile := range cfiles {
 		clFile(ctx, cflags, cfile, pkg.ExportFile, func(linkFile string) {
-			cgoLdflags = append(cgoLdflags, linkFile)
+			llfiles = append(llfiles, linkFile)
 		}, verbose)
 	}
 	re := regexp.MustCompile(`^(_cgo_[^_]+_(Cfunc|Cmacro)_)(.*)$`)
@@ -117,20 +117,20 @@ func buildCgo(ctx *context, pkg *aPackage, files []*ast.File, externs []string,
 	for _, preamble := range preambles {
 		tmpFile, err := os.CreateTemp("", "-cgo-*.c")
 		if err != nil {
-			return nil, fmt.Errorf("failed to create temp file: %v", err)
+			return nil, nil, fmt.Errorf("failed to create temp file: %v", err)
 		}
 		tmpName := tmpFile.Name()
 		defer os.Remove(tmpName)
 		code := cgoHeader + "\n\n" + preamble.src
 		externDecls, err := genExternDeclsByClang(pkg, code, cflags, cgoSymbols)
 		if err != nil {
-			return nil, fmt.Errorf("failed to generate extern decls: %v", err)
+			return nil, nil, fmt.Errorf("failed to generate extern decls: %v", err)
 		}
 		if err = os.WriteFile(tmpName, []byte(code+"\n\n"+externDecls), 0644); err != nil {
-			return nil, fmt.Errorf("failed to write temp file: %v", err)
+			return nil, nil, fmt.Errorf("failed to write temp file: %v", err)
 		}
 		clFile(ctx, cflags, tmpName, pkg.ExportFile, func(linkFile string) {
-			cgoLdflags = append(cgoLdflags, linkFile)
+			llfiles = append(llfiles, linkFile)
 		}, verbose)
 	}
 	for _, ldflag := range ldflags {