supports binary-format, only esp* supported for now

This commit is contained in:
Li Jie
2025-08-22 20:42:43 +08:00
parent 1f193c8533
commit ecaf7c8ac6
10 changed files with 568 additions and 15 deletions

View File

@@ -42,6 +42,7 @@ import (
"github.com/goplus/llgo/internal/clang" "github.com/goplus/llgo/internal/clang"
"github.com/goplus/llgo/internal/crosscompile" "github.com/goplus/llgo/internal/crosscompile"
"github.com/goplus/llgo/internal/env" "github.com/goplus/llgo/internal/env"
"github.com/goplus/llgo/internal/firmware"
"github.com/goplus/llgo/internal/mockable" "github.com/goplus/llgo/internal/mockable"
"github.com/goplus/llgo/internal/packages" "github.com/goplus/llgo/internal/packages"
"github.com/goplus/llgo/internal/typepatch" "github.com/goplus/llgo/internal/typepatch"
@@ -629,23 +630,76 @@ func compileExtraFiles(ctx *context, verbose bool) ([]string, error) {
return objFiles, nil return objFiles, nil
} }
func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global llssa.Package, conf *Config, mode Mode, verbose bool) { // generateOutputFilenames generates the final output filename (app) and intermediate filename (orgApp)
pkgPath := pkg.PkgPath // based on configuration and build context.
name := path.Base(pkgPath) func generateOutputFilenames(outFile, binPath, appExt, binExt, pkgName string, mode Mode, isMultiplePkgs bool) (app, orgApp string, err error) {
app := conf.OutFile if outFile == "" {
if app == "" { if mode == ModeBuild && isMultiplePkgs {
if mode == ModeBuild && len(ctx.initial) > 1 {
// For multiple packages in ModeBuild mode, use temporary file // For multiple packages in ModeBuild mode, use temporary file
tmpFile, err := os.CreateTemp("", name+"*"+conf.AppExt) name := pkgName
check(err) if binExt != "" {
name += "*" + binExt
} else {
name += "*" + appExt
}
tmpFile, err := os.CreateTemp("", name)
if err != nil {
return "", "", err
}
app = tmpFile.Name() app = tmpFile.Name()
tmpFile.Close() tmpFile.Close()
} else { } else {
app = filepath.Join(conf.BinPath, name+conf.AppExt) app = filepath.Join(binPath, pkgName+appExt)
}
orgApp = app
} else {
// outFile is not empty, use it as base part
base := outFile
if binExt != "" {
// If binExt has value, use temporary file as orgApp for firmware conversion
tmpFile, err := os.CreateTemp("", "llgo-*"+appExt)
if err != nil {
return "", "", err
}
orgApp = tmpFile.Name()
tmpFile.Close()
// Check if base already ends with binExt, if so, don't add it again
if strings.HasSuffix(base, binExt) {
app = base
} else {
app = base + binExt
}
} else {
// No binExt, use base + AppExt directly
if filepath.Ext(base) == "" {
app = base + appExt
} else {
app = base
}
orgApp = app
} }
} else if filepath.Ext(app) == "" {
app += conf.AppExt
} }
return app, orgApp, nil
}
func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global llssa.Package, conf *Config, mode Mode, verbose bool) {
pkgPath := pkg.PkgPath
name := path.Base(pkgPath)
binFmt := ctx.crossCompile.BinaryFormat
binExt := firmware.BinaryExt(binFmt)
// app: converted firmware output file or executable file
// orgApp: before converted output file
app, orgApp, err := generateOutputFilenames(
conf.OutFile,
conf.BinPath,
conf.AppExt,
binExt,
name,
mode,
len(ctx.initial) > 1,
)
check(err)
needRuntime := false needRuntime := false
needPyInit := false needPyInit := false
@@ -701,9 +755,15 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l
linkArgs = append(linkArgs, exargs...) linkArgs = append(linkArgs, exargs...)
} }
err = linkObjFiles(ctx, app, objFiles, linkArgs, verbose) err = linkObjFiles(ctx, orgApp, objFiles, linkArgs, verbose)
check(err) check(err)
if orgApp != app {
fmt.Printf("cross compile: %#v\n", ctx.crossCompile)
err = firmware.MakeFirmwareImage(orgApp, app, ctx.crossCompile.BinaryFormat)
check(err)
}
switch mode { switch mode {
case ModeTest: case ModeTest:
cmd := exec.Command(app, conf.RunArgs...) cmd := exec.Command(app, conf.RunArgs...)

View File

@@ -8,6 +8,7 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"strings"
"testing" "testing"
"github.com/goplus/llgo/internal/mockable" "github.com/goplus/llgo/internal/mockable"
@@ -93,3 +94,173 @@ func TestExtest(t *testing.T) {
func TestCmpTest(t *testing.T) { func TestCmpTest(t *testing.T) {
mockRun([]string{"../../cl/_testgo/runtest"}, &Config{Mode: ModeCmpTest}) mockRun([]string{"../../cl/_testgo/runtest"}, &Config{Mode: ModeCmpTest})
} }
func TestGenerateOutputFilenames(t *testing.T) {
tests := []struct {
name string
outFile string
binPath string
appExt string
binExt string
pkgName string
mode Mode
isMultiplePkgs bool
wantAppSuffix string
wantOrgAppDiff bool // true if orgApp should be different from app
wantErr bool
}{
{
name: "empty outFile, single package",
outFile: "",
binPath: "/usr/local/bin",
appExt: "",
binExt: "",
pkgName: "hello",
mode: ModeBuild,
isMultiplePkgs: false,
wantAppSuffix: "/usr/local/bin/hello",
wantOrgAppDiff: false,
},
{
name: "empty outFile with appExt",
outFile: "",
binPath: "/usr/local/bin",
appExt: ".exe",
binExt: "",
pkgName: "hello",
mode: ModeBuild,
isMultiplePkgs: false,
wantAppSuffix: "/usr/local/bin/hello.exe",
wantOrgAppDiff: false,
},
{
name: "outFile without binExt",
outFile: "myapp",
binPath: "/usr/local/bin",
appExt: ".exe",
binExt: "",
pkgName: "hello",
mode: ModeBuild,
isMultiplePkgs: false,
wantAppSuffix: "myapp.exe",
wantOrgAppDiff: false,
},
{
name: "outFile with existing extension, no binExt",
outFile: "myapp.exe",
binPath: "/usr/local/bin",
appExt: ".exe",
binExt: "",
pkgName: "hello",
mode: ModeBuild,
isMultiplePkgs: false,
wantAppSuffix: "myapp.exe",
wantOrgAppDiff: false,
},
{
name: "outFile with binExt, different from existing extension",
outFile: "myapp",
binPath: "/usr/local/bin",
appExt: ".exe",
binExt: ".bin",
pkgName: "hello",
mode: ModeBuild,
isMultiplePkgs: false,
wantAppSuffix: "myapp.bin",
wantOrgAppDiff: true,
},
{
name: "outFile already ends with binExt",
outFile: "t.bin",
binPath: "/usr/local/bin",
appExt: ".exe",
binExt: ".bin",
pkgName: "hello",
mode: ModeBuild,
isMultiplePkgs: false,
wantAppSuffix: "t.bin",
wantOrgAppDiff: true,
},
{
name: "outFile with full path already ends with binExt",
outFile: "/path/to/t.bin",
binPath: "/usr/local/bin",
appExt: ".exe",
binExt: ".bin",
pkgName: "hello",
mode: ModeBuild,
isMultiplePkgs: false,
wantAppSuffix: "/path/to/t.bin",
wantOrgAppDiff: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
app, orgApp, err := generateOutputFilenames(
tt.outFile,
tt.binPath,
tt.appExt,
tt.binExt,
tt.pkgName,
tt.mode,
tt.isMultiplePkgs,
)
if (err != nil) != tt.wantErr {
t.Errorf("generateOutputFilenames() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.wantAppSuffix != "" {
if app != tt.wantAppSuffix {
t.Errorf("generateOutputFilenames() app = %v, want %v", app, tt.wantAppSuffix)
}
}
if tt.wantOrgAppDiff {
if app == orgApp {
t.Errorf("generateOutputFilenames() orgApp should be different from app, but both are %v", app)
}
// Clean up temp file
if orgApp != "" && strings.Contains(orgApp, "llgo-") {
os.Remove(orgApp)
}
} else {
if app != orgApp {
t.Errorf("generateOutputFilenames() orgApp = %v, want %v (same as app)", orgApp, app)
}
}
})
}
}
func TestGenerateOutputFilenames_EdgeCases(t *testing.T) {
// Test case where outFile has same extension as binExt
app, orgApp, err := generateOutputFilenames(
"firmware.bin",
"/usr/local/bin",
".exe",
".bin",
"esp32app",
ModeBuild,
false,
)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if app != "firmware.bin" {
t.Errorf("Expected app to be 'firmware.bin', got '%s'", app)
}
if app == orgApp {
t.Errorf("Expected orgApp to be different from app when binExt is present, but both are '%s'", app)
}
// Clean up temp file
if orgApp != "" && strings.Contains(orgApp, "llgo-") {
os.Remove(orgApp)
}
}

View File

@@ -29,6 +29,8 @@ type Export struct {
ExtraFiles []string // Extra files to compile and link (e.g., .s, .c files) ExtraFiles []string // Extra files to compile and link (e.g., .s, .c files)
ClangRoot string // Root directory of custom clang installation ClangRoot string // Root directory of custom clang installation
ClangBinPath string // Path to clang binary directory ClangBinPath string // Path to clang binary directory
BinaryFormat string // Binary format (e.g., "elf", "esp", "uf2")
} }
const wasiSdkUrl = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-macos.tar.gz" const wasiSdkUrl = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-macos.tar.gz"
@@ -447,6 +449,7 @@ func useTarget(targetName string) (export Export, err error) {
export.GOOS = config.GOOS export.GOOS = config.GOOS
export.GOARCH = config.GOARCH export.GOARCH = config.GOARCH
export.ExtraFiles = config.ExtraFiles export.ExtraFiles = config.ExtraFiles
export.BinaryFormat = config.BinaryFormat
// Build environment map for template variable expansion // Build environment map for template variable expansion
envs := buildEnvMap(env.LLGoROOT()) envs := buildEnvMap(env.LLGoROOT())

188
internal/firmware/esp.go Normal file
View File

@@ -0,0 +1,188 @@
package firmware
import (
"bytes"
"crypto/sha256"
"debug/elf"
"encoding/binary"
"fmt"
"os"
"sort"
"strings"
)
// From tinygo/builder/esp.go
type espImageSegment struct {
addr uint32
data []byte
}
// makeESPFirmareImage converts an input ELF file to an image file for an ESP32 or
// ESP8266 chip. This is a special purpose image format just for the ESP chip
// family, and is parsed by the on-chip mask ROM bootloader.
//
// The following documentation has been used:
// https://github.com/espressif/esptool/wiki/Firmware-Image-Format
// https://github.com/espressif/esp-idf/blob/8fbb63c2a701c22ccf4ce249f43aded73e134a34/components/bootloader_support/include/esp_image_format.h#L58
// https://github.com/espressif/esptool/blob/master/esptool.py
func makeESPFirmareImage(infile, outfile, format string) error {
inf, err := elf.Open(infile)
if err != nil {
return err
}
defer inf.Close()
// Load all segments to be written to the image. These are actually ELF
// sections, not true ELF segments (similar to how esptool does it).
var segments []*espImageSegment
for _, section := range inf.Sections {
if section.Type != elf.SHT_PROGBITS || section.Size == 0 || section.Flags&elf.SHF_ALLOC == 0 {
continue
}
data, err := section.Data()
if err != nil {
return fmt.Errorf("failed to read section data: %w", err)
}
for len(data)%4 != 0 {
// Align segment to 4 bytes.
data = append(data, 0)
}
if uint64(uint32(section.Addr)) != section.Addr {
return fmt.Errorf("section address too big: 0x%x", section.Addr)
}
segments = append(segments, &espImageSegment{
addr: uint32(section.Addr),
data: data,
})
}
// Sort the segments by address. This is what esptool does too.
sort.SliceStable(segments, func(i, j int) bool { return segments[i].addr < segments[j].addr })
// Calculate checksum over the segment data. This is used in the image
// footer.
checksum := uint8(0xef)
for _, segment := range segments {
for _, b := range segment.data {
checksum ^= b
}
}
// Write first to an in-memory buffer, primarily so that we can easily
// calculate a hash over the entire image.
// An added benefit is that we don't need to check for errors all the time.
outf := &bytes.Buffer{}
// Separate esp32 and esp32-img. The -img suffix indicates we should make an
// image, not just a binary to be flashed at 0x1000 for example.
chip := format
makeImage := false
if strings.HasSuffix(format, "-img") {
makeImage = true
chip = format[:len(format)-len("-img")]
}
if makeImage {
// The bootloader starts at 0x1000, or 4096.
// TinyGo doesn't use a separate bootloader and runs the entire
// application in the bootloader location.
outf.Write(make([]byte, 4096))
}
// Chip IDs. Source:
// https://github.com/espressif/esp-idf/blob/v4.3/components/bootloader_support/include/esp_app_format.h#L22
chip_id := map[string]uint16{
"esp32": 0x0000,
"esp32c3": 0x0005,
}[chip]
// Image header.
switch chip {
case "esp32", "esp32c3":
// Header format:
// https://github.com/espressif/esp-idf/blob/v4.3/components/bootloader_support/include/esp_app_format.h#L71
// Note: not adding a SHA256 hash as the binary is modified by
// esptool.py while flashing and therefore the hash won't be valid
// anymore.
binary.Write(outf, binary.LittleEndian, struct {
magic uint8
segment_count uint8
spi_mode uint8
spi_speed_size uint8
entry_addr uint32
wp_pin uint8
spi_pin_drv [3]uint8
chip_id uint16
min_chip_rev uint8
reserved [8]uint8
hash_appended bool
}{
magic: 0xE9,
segment_count: byte(len(segments)),
spi_mode: 2, // ESP_IMAGE_SPI_MODE_DIO
spi_speed_size: 0x1f, // ESP_IMAGE_SPI_SPEED_80M, ESP_IMAGE_FLASH_SIZE_2MB
entry_addr: uint32(inf.Entry),
wp_pin: 0xEE, // disable WP pin
chip_id: chip_id,
hash_appended: true, // add a SHA256 hash
})
case "esp8266":
// Header format:
// https://github.com/espressif/esptool/wiki/Firmware-Image-Format
// Basically a truncated version of the ESP32 header.
binary.Write(outf, binary.LittleEndian, struct {
magic uint8
segment_count uint8
spi_mode uint8
spi_speed_size uint8
entry_addr uint32
}{
magic: 0xE9,
segment_count: byte(len(segments)),
spi_mode: 0, // irrelevant, replaced by esptool when flashing
spi_speed_size: 0x20, // spi_speed, spi_size: replaced by esptool when flashing
entry_addr: uint32(inf.Entry),
})
default:
return fmt.Errorf("builder: unknown binary format %#v, expected esp32 or esp8266", format)
}
// Write all segments to the image.
// https://github.com/espressif/esptool/wiki/Firmware-Image-Format#segment
for _, segment := range segments {
binary.Write(outf, binary.LittleEndian, struct {
addr uint32
length uint32
}{
addr: segment.addr,
length: uint32(len(segment.data)),
})
outf.Write(segment.data)
}
// Footer, including checksum.
// The entire image size must be a multiple of 16, so pad the image to one
// byte less than that before writing the checksum.
outf.Write(make([]byte, 15-outf.Len()%16))
outf.WriteByte(checksum)
if chip != "esp8266" {
// SHA256 hash (to protect against image corruption, not for security).
hash := sha256.Sum256(outf.Bytes())
outf.Write(hash[:])
}
// QEMU (or more precisely, qemu-system-xtensa from Espressif) expects the
// image to be a certain size.
if makeImage {
// Use a default image size of 4MB.
grow := 4096*1024 - outf.Len()
if grow > 0 {
outf.Write(make([]byte, grow))
}
}
// Write the image to the output file.
return os.WriteFile(outfile, outf.Bytes(), 0666)
}

12
internal/firmware/ext.go Normal file
View File

@@ -0,0 +1,12 @@
package firmware
import "strings"
// BinaryExt returns the binary file extension based on the binary format
// Returns ".bin" for ESP-based formats, "" for others
func BinaryExt(binaryFormat string) string {
if strings.HasPrefix(binaryFormat, "esp") {
return ".bin"
}
return ""
}

View File

@@ -0,0 +1,28 @@
package firmware
import "testing"
func TestBinaryExt(t *testing.T) {
tests := []struct {
name string
binaryFormat string
expected string
}{
{"ESP32", "esp32", ".bin"},
{"ESP8266", "esp8266", ".bin"},
{"ESP32C3", "esp32c3", ".bin"},
{"UF2", "uf2", ""},
{"ELF", "elf", ""},
{"Empty", "", ""},
{"NRF-DFU", "nrf-dfu", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := BinaryExt(tt.binaryFormat)
if result != tt.expected {
t.Errorf("BinaryExt() = %q, want %q", result, tt.expected)
}
})
}
}

View File

@@ -0,0 +1,14 @@
package firmware
import (
"fmt"
"strings"
)
func MakeFirmwareImage(infile, outfile, format string) error {
fmt.Printf("Creating firmware image from %s to %s with format %s\n", infile, outfile, format)
if strings.HasPrefix(format, "esp") {
return makeESPFirmareImage(infile, outfile, format)
}
return fmt.Errorf("unsupported firmware format: %s", format)
}

View File

@@ -26,6 +26,33 @@ type Config struct {
CodeModel string `json:"code-model"` CodeModel string `json:"code-model"`
TargetABI string `json:"target-abi"` TargetABI string `json:"target-abi"`
RelocationModel string `json:"relocation-model"` RelocationModel string `json:"relocation-model"`
// Binary and firmware configuration
BinaryFormat string `json:"binary-format"`
// Flash and deployment configuration
FlashCommand string `json:"flash-command"`
FlashMethod string `json:"flash-method"`
Flash1200BpsReset string `json:"flash-1200-bps-reset"`
// Mass storage device configuration
MSDVolumeName []string `json:"msd-volume-name"`
MSDFirmwareName string `json:"msd-firmware-name"`
// UF2 configuration
UF2FamilyID string `json:"uf2-family-id"`
// Device-specific configuration
RP2040BootPatch bool `json:"rp2040-boot-patch"`
// Debug and emulation configuration
Emulator string `json:"emulator"`
GDB []string `json:"gdb"`
// OpenOCD configuration
OpenOCDInterface string `json:"openocd-interface"`
OpenOCDTransport string `json:"openocd-transport"`
OpenOCDTarget string `json:"openocd-target"`
} }
// RawConfig represents the raw JSON configuration before inheritance resolution // RawConfig represents the raw JSON configuration before inheritance resolution

View File

@@ -149,6 +149,39 @@ func (l *Loader) mergeConfig(dst, src *Config) {
if src.RelocationModel != "" { if src.RelocationModel != "" {
dst.RelocationModel = src.RelocationModel dst.RelocationModel = src.RelocationModel
} }
if src.BinaryFormat != "" {
dst.BinaryFormat = src.BinaryFormat
}
if src.FlashCommand != "" {
dst.FlashCommand = src.FlashCommand
}
if src.FlashMethod != "" {
dst.FlashMethod = src.FlashMethod
}
if src.Flash1200BpsReset != "" {
dst.Flash1200BpsReset = src.Flash1200BpsReset
}
if src.MSDFirmwareName != "" {
dst.MSDFirmwareName = src.MSDFirmwareName
}
if src.UF2FamilyID != "" {
dst.UF2FamilyID = src.UF2FamilyID
}
if src.RP2040BootPatch {
dst.RP2040BootPatch = src.RP2040BootPatch
}
if src.Emulator != "" {
dst.Emulator = src.Emulator
}
if src.OpenOCDInterface != "" {
dst.OpenOCDInterface = src.OpenOCDInterface
}
if src.OpenOCDTransport != "" {
dst.OpenOCDTransport = src.OpenOCDTransport
}
if src.OpenOCDTarget != "" {
dst.OpenOCDTarget = src.OpenOCDTarget
}
// Merge slices (append, don't replace) // Merge slices (append, don't replace)
if len(src.BuildTags) > 0 { if len(src.BuildTags) > 0 {
@@ -163,6 +196,12 @@ func (l *Loader) mergeConfig(dst, src *Config) {
if len(src.ExtraFiles) > 0 { if len(src.ExtraFiles) > 0 {
dst.ExtraFiles = append(dst.ExtraFiles, src.ExtraFiles...) dst.ExtraFiles = append(dst.ExtraFiles, src.ExtraFiles...)
} }
if len(src.MSDVolumeName) > 0 {
dst.MSDVolumeName = append(dst.MSDVolumeName, src.MSDVolumeName...)
}
if len(src.GDB) > 0 {
dst.GDB = append(dst.GDB, src.GDB...)
}
} }
// GetTargetsDir returns the targets directory path // GetTargetsDir returns the targets directory path

View File

@@ -61,7 +61,8 @@ func TestLoaderLoadRaw(t *testing.T) {
"goarch": "arm", "goarch": "arm",
"build-tags": ["test", "embedded"], "build-tags": ["test", "embedded"],
"cflags": ["-Os", "-g"], "cflags": ["-Os", "-g"],
"ldflags": ["--gc-sections"] "ldflags": ["--gc-sections"],
"binary-format": "uf2"
}` }`
configPath := filepath.Join(tempDir, "test-target.json") configPath := filepath.Join(tempDir, "test-target.json")
@@ -87,6 +88,9 @@ func TestLoaderLoadRaw(t *testing.T) {
if len(config.BuildTags) != 2 || config.BuildTags[0] != "test" || config.BuildTags[1] != "embedded" { if len(config.BuildTags) != 2 || config.BuildTags[0] != "test" || config.BuildTags[1] != "embedded" {
t.Errorf("Expected build-tags [test, embedded], got %v", config.BuildTags) t.Errorf("Expected build-tags [test, embedded], got %v", config.BuildTags)
} }
if config.BinaryFormat != "uf2" {
t.Errorf("Expected binary-format 'uf2', got '%s'", config.BinaryFormat)
}
} }
func TestLoaderInheritance(t *testing.T) { func TestLoaderInheritance(t *testing.T) {
@@ -99,7 +103,8 @@ func TestLoaderInheritance(t *testing.T) {
"goos": "linux", "goos": "linux",
"goarch": "arm", "goarch": "arm",
"cflags": ["-Os"], "cflags": ["-Os"],
"ldflags": ["--gc-sections"] "ldflags": ["--gc-sections"],
"binary-format": "elf"
}` }`
// Create child config that inherits from parent // Create child config that inherits from parent
@@ -108,7 +113,8 @@ func TestLoaderInheritance(t *testing.T) {
"cpu": "cortex-m4", "cpu": "cortex-m4",
"build-tags": ["child"], "build-tags": ["child"],
"cflags": ["-O2"], "cflags": ["-O2"],
"ldflags": ["-g"] "ldflags": ["-g"],
"binary-format": "uf2"
}` }`
parentPath := filepath.Join(tempDir, "parent.json") parentPath := filepath.Join(tempDir, "parent.json")
@@ -143,6 +149,11 @@ func TestLoaderInheritance(t *testing.T) {
t.Errorf("Expected overridden cpu 'cortex-m4', got '%s'", config.CPU) t.Errorf("Expected overridden cpu 'cortex-m4', got '%s'", config.CPU)
} }
// Check binary-format override
if config.BinaryFormat != "uf2" {
t.Errorf("Expected overridden binary-format 'uf2', got '%s'", config.BinaryFormat)
}
// Check merged arrays // Check merged arrays
expectedCFlags := []string{"-Os", "-O2"} expectedCFlags := []string{"-Os", "-O2"}
if len(config.CFlags) != 2 || config.CFlags[0] != "-Os" || config.CFlags[1] != "-O2" { if len(config.CFlags) != 2 || config.CFlags[0] != "-Os" || config.CFlags[1] != "-O2" {