From ecaf7c8ac62ece91b00c809b531dbb733129b1cf Mon Sep 17 00:00:00 2001 From: Li Jie Date: Fri, 22 Aug 2025 20:42:43 +0800 Subject: [PATCH] supports binary-format, only esp* supported for now --- internal/build/build.go | 84 ++++++++++-- internal/build/build_test.go | 171 +++++++++++++++++++++++ internal/crosscompile/crosscompile.go | 3 + internal/firmware/esp.go | 188 ++++++++++++++++++++++++++ internal/firmware/ext.go | 12 ++ internal/firmware/ext_test.go | 28 ++++ internal/firmware/firmware.go | 14 ++ internal/targets/config.go | 27 ++++ internal/targets/loader.go | 39 ++++++ internal/targets/targets_test.go | 17 ++- 10 files changed, 568 insertions(+), 15 deletions(-) create mode 100644 internal/firmware/esp.go create mode 100644 internal/firmware/ext.go create mode 100644 internal/firmware/ext_test.go create mode 100644 internal/firmware/firmware.go diff --git a/internal/build/build.go b/internal/build/build.go index f7cbed63..fee0f3cd 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -42,6 +42,7 @@ import ( "github.com/goplus/llgo/internal/clang" "github.com/goplus/llgo/internal/crosscompile" "github.com/goplus/llgo/internal/env" + "github.com/goplus/llgo/internal/firmware" "github.com/goplus/llgo/internal/mockable" "github.com/goplus/llgo/internal/packages" "github.com/goplus/llgo/internal/typepatch" @@ -629,23 +630,76 @@ func compileExtraFiles(ctx *context, verbose bool) ([]string, error) { return objFiles, nil } -func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global llssa.Package, conf *Config, mode Mode, verbose bool) { - pkgPath := pkg.PkgPath - name := path.Base(pkgPath) - app := conf.OutFile - if app == "" { - if mode == ModeBuild && len(ctx.initial) > 1 { +// generateOutputFilenames generates the final output filename (app) and intermediate filename (orgApp) +// based on configuration and build context. +func generateOutputFilenames(outFile, binPath, appExt, binExt, pkgName string, mode Mode, isMultiplePkgs bool) (app, orgApp string, err error) { + if outFile == "" { + if mode == ModeBuild && isMultiplePkgs { // For multiple packages in ModeBuild mode, use temporary file - tmpFile, err := os.CreateTemp("", name+"*"+conf.AppExt) - check(err) + name := pkgName + if binExt != "" { + name += "*" + binExt + } else { + name += "*" + appExt + } + tmpFile, err := os.CreateTemp("", name) + if err != nil { + return "", "", err + } app = tmpFile.Name() tmpFile.Close() } else { - app = filepath.Join(conf.BinPath, name+conf.AppExt) + app = filepath.Join(binPath, pkgName+appExt) + } + orgApp = app + } else { + // outFile is not empty, use it as base part + base := outFile + if binExt != "" { + // If binExt has value, use temporary file as orgApp for firmware conversion + tmpFile, err := os.CreateTemp("", "llgo-*"+appExt) + if err != nil { + return "", "", err + } + orgApp = tmpFile.Name() + tmpFile.Close() + // Check if base already ends with binExt, if so, don't add it again + if strings.HasSuffix(base, binExt) { + app = base + } else { + app = base + binExt + } + } else { + // No binExt, use base + AppExt directly + if filepath.Ext(base) == "" { + app = base + appExt + } else { + app = base + } + orgApp = app } - } else if filepath.Ext(app) == "" { - app += conf.AppExt } + return app, orgApp, nil +} + +func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global llssa.Package, conf *Config, mode Mode, verbose bool) { + pkgPath := pkg.PkgPath + name := path.Base(pkgPath) + binFmt := ctx.crossCompile.BinaryFormat + binExt := firmware.BinaryExt(binFmt) + + // app: converted firmware output file or executable file + // orgApp: before converted output file + app, orgApp, err := generateOutputFilenames( + conf.OutFile, + conf.BinPath, + conf.AppExt, + binExt, + name, + mode, + len(ctx.initial) > 1, + ) + check(err) needRuntime := false needPyInit := false @@ -701,9 +755,15 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l linkArgs = append(linkArgs, exargs...) } - err = linkObjFiles(ctx, app, objFiles, linkArgs, verbose) + err = linkObjFiles(ctx, orgApp, objFiles, linkArgs, verbose) check(err) + if orgApp != app { + fmt.Printf("cross compile: %#v\n", ctx.crossCompile) + err = firmware.MakeFirmwareImage(orgApp, app, ctx.crossCompile.BinaryFormat) + check(err) + } + switch mode { case ModeTest: cmd := exec.Command(app, conf.RunArgs...) diff --git a/internal/build/build_test.go b/internal/build/build_test.go index f2f8730f..b43ff6a4 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "os" + "strings" "testing" "github.com/goplus/llgo/internal/mockable" @@ -93,3 +94,173 @@ func TestExtest(t *testing.T) { func TestCmpTest(t *testing.T) { mockRun([]string{"../../cl/_testgo/runtest"}, &Config{Mode: ModeCmpTest}) } + +func TestGenerateOutputFilenames(t *testing.T) { + tests := []struct { + name string + outFile string + binPath string + appExt string + binExt string + pkgName string + mode Mode + isMultiplePkgs bool + wantAppSuffix string + wantOrgAppDiff bool // true if orgApp should be different from app + wantErr bool + }{ + { + name: "empty outFile, single package", + outFile: "", + binPath: "/usr/local/bin", + appExt: "", + binExt: "", + pkgName: "hello", + mode: ModeBuild, + isMultiplePkgs: false, + wantAppSuffix: "/usr/local/bin/hello", + wantOrgAppDiff: false, + }, + { + name: "empty outFile with appExt", + outFile: "", + binPath: "/usr/local/bin", + appExt: ".exe", + binExt: "", + pkgName: "hello", + mode: ModeBuild, + isMultiplePkgs: false, + wantAppSuffix: "/usr/local/bin/hello.exe", + wantOrgAppDiff: false, + }, + { + name: "outFile without binExt", + outFile: "myapp", + binPath: "/usr/local/bin", + appExt: ".exe", + binExt: "", + pkgName: "hello", + mode: ModeBuild, + isMultiplePkgs: false, + wantAppSuffix: "myapp.exe", + wantOrgAppDiff: false, + }, + { + name: "outFile with existing extension, no binExt", + outFile: "myapp.exe", + binPath: "/usr/local/bin", + appExt: ".exe", + binExt: "", + pkgName: "hello", + mode: ModeBuild, + isMultiplePkgs: false, + wantAppSuffix: "myapp.exe", + wantOrgAppDiff: false, + }, + { + name: "outFile with binExt, different from existing extension", + outFile: "myapp", + binPath: "/usr/local/bin", + appExt: ".exe", + binExt: ".bin", + pkgName: "hello", + mode: ModeBuild, + isMultiplePkgs: false, + wantAppSuffix: "myapp.bin", + wantOrgAppDiff: true, + }, + { + name: "outFile already ends with binExt", + outFile: "t.bin", + binPath: "/usr/local/bin", + appExt: ".exe", + binExt: ".bin", + pkgName: "hello", + mode: ModeBuild, + isMultiplePkgs: false, + wantAppSuffix: "t.bin", + wantOrgAppDiff: true, + }, + { + name: "outFile with full path already ends with binExt", + outFile: "/path/to/t.bin", + binPath: "/usr/local/bin", + appExt: ".exe", + binExt: ".bin", + pkgName: "hello", + mode: ModeBuild, + isMultiplePkgs: false, + wantAppSuffix: "/path/to/t.bin", + wantOrgAppDiff: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + app, orgApp, err := generateOutputFilenames( + tt.outFile, + tt.binPath, + tt.appExt, + tt.binExt, + tt.pkgName, + tt.mode, + tt.isMultiplePkgs, + ) + + if (err != nil) != tt.wantErr { + t.Errorf("generateOutputFilenames() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if tt.wantAppSuffix != "" { + if app != tt.wantAppSuffix { + t.Errorf("generateOutputFilenames() app = %v, want %v", app, tt.wantAppSuffix) + } + } + + if tt.wantOrgAppDiff { + if app == orgApp { + t.Errorf("generateOutputFilenames() orgApp should be different from app, but both are %v", app) + } + // Clean up temp file + if orgApp != "" && strings.Contains(orgApp, "llgo-") { + os.Remove(orgApp) + } + } else { + if app != orgApp { + t.Errorf("generateOutputFilenames() orgApp = %v, want %v (same as app)", orgApp, app) + } + } + }) + } +} + +func TestGenerateOutputFilenames_EdgeCases(t *testing.T) { + // Test case where outFile has same extension as binExt + app, orgApp, err := generateOutputFilenames( + "firmware.bin", + "/usr/local/bin", + ".exe", + ".bin", + "esp32app", + ModeBuild, + false, + ) + + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if app != "firmware.bin" { + t.Errorf("Expected app to be 'firmware.bin', got '%s'", app) + } + + if app == orgApp { + t.Errorf("Expected orgApp to be different from app when binExt is present, but both are '%s'", app) + } + + // Clean up temp file + if orgApp != "" && strings.Contains(orgApp, "llgo-") { + os.Remove(orgApp) + } +} diff --git a/internal/crosscompile/crosscompile.go b/internal/crosscompile/crosscompile.go index 8e4c8ec5..e6c75b0c 100644 --- a/internal/crosscompile/crosscompile.go +++ b/internal/crosscompile/crosscompile.go @@ -29,6 +29,8 @@ type Export struct { ExtraFiles []string // Extra files to compile and link (e.g., .s, .c files) ClangRoot string // Root directory of custom clang installation ClangBinPath string // Path to clang binary directory + + BinaryFormat string // Binary format (e.g., "elf", "esp", "uf2") } const wasiSdkUrl = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-macos.tar.gz" @@ -447,6 +449,7 @@ func useTarget(targetName string) (export Export, err error) { export.GOOS = config.GOOS export.GOARCH = config.GOARCH export.ExtraFiles = config.ExtraFiles + export.BinaryFormat = config.BinaryFormat // Build environment map for template variable expansion envs := buildEnvMap(env.LLGoROOT()) diff --git a/internal/firmware/esp.go b/internal/firmware/esp.go new file mode 100644 index 00000000..7318e066 --- /dev/null +++ b/internal/firmware/esp.go @@ -0,0 +1,188 @@ +package firmware + +import ( + "bytes" + "crypto/sha256" + "debug/elf" + "encoding/binary" + "fmt" + "os" + "sort" + "strings" +) + +// From tinygo/builder/esp.go + +type espImageSegment struct { + addr uint32 + data []byte +} + +// makeESPFirmareImage converts an input ELF file to an image file for an ESP32 or +// ESP8266 chip. This is a special purpose image format just for the ESP chip +// family, and is parsed by the on-chip mask ROM bootloader. +// +// The following documentation has been used: +// https://github.com/espressif/esptool/wiki/Firmware-Image-Format +// https://github.com/espressif/esp-idf/blob/8fbb63c2a701c22ccf4ce249f43aded73e134a34/components/bootloader_support/include/esp_image_format.h#L58 +// https://github.com/espressif/esptool/blob/master/esptool.py +func makeESPFirmareImage(infile, outfile, format string) error { + inf, err := elf.Open(infile) + if err != nil { + return err + } + defer inf.Close() + + // Load all segments to be written to the image. These are actually ELF + // sections, not true ELF segments (similar to how esptool does it). + var segments []*espImageSegment + for _, section := range inf.Sections { + if section.Type != elf.SHT_PROGBITS || section.Size == 0 || section.Flags&elf.SHF_ALLOC == 0 { + continue + } + data, err := section.Data() + if err != nil { + return fmt.Errorf("failed to read section data: %w", err) + } + for len(data)%4 != 0 { + // Align segment to 4 bytes. + data = append(data, 0) + } + if uint64(uint32(section.Addr)) != section.Addr { + return fmt.Errorf("section address too big: 0x%x", section.Addr) + } + segments = append(segments, &espImageSegment{ + addr: uint32(section.Addr), + data: data, + }) + } + + // Sort the segments by address. This is what esptool does too. + sort.SliceStable(segments, func(i, j int) bool { return segments[i].addr < segments[j].addr }) + + // Calculate checksum over the segment data. This is used in the image + // footer. + checksum := uint8(0xef) + for _, segment := range segments { + for _, b := range segment.data { + checksum ^= b + } + } + + // Write first to an in-memory buffer, primarily so that we can easily + // calculate a hash over the entire image. + // An added benefit is that we don't need to check for errors all the time. + outf := &bytes.Buffer{} + + // Separate esp32 and esp32-img. The -img suffix indicates we should make an + // image, not just a binary to be flashed at 0x1000 for example. + chip := format + makeImage := false + if strings.HasSuffix(format, "-img") { + makeImage = true + chip = format[:len(format)-len("-img")] + } + + if makeImage { + // The bootloader starts at 0x1000, or 4096. + // TinyGo doesn't use a separate bootloader and runs the entire + // application in the bootloader location. + outf.Write(make([]byte, 4096)) + } + + // Chip IDs. Source: + // https://github.com/espressif/esp-idf/blob/v4.3/components/bootloader_support/include/esp_app_format.h#L22 + chip_id := map[string]uint16{ + "esp32": 0x0000, + "esp32c3": 0x0005, + }[chip] + + // Image header. + switch chip { + case "esp32", "esp32c3": + // Header format: + // https://github.com/espressif/esp-idf/blob/v4.3/components/bootloader_support/include/esp_app_format.h#L71 + // Note: not adding a SHA256 hash as the binary is modified by + // esptool.py while flashing and therefore the hash won't be valid + // anymore. + binary.Write(outf, binary.LittleEndian, struct { + magic uint8 + segment_count uint8 + spi_mode uint8 + spi_speed_size uint8 + entry_addr uint32 + wp_pin uint8 + spi_pin_drv [3]uint8 + chip_id uint16 + min_chip_rev uint8 + reserved [8]uint8 + hash_appended bool + }{ + magic: 0xE9, + segment_count: byte(len(segments)), + spi_mode: 2, // ESP_IMAGE_SPI_MODE_DIO + spi_speed_size: 0x1f, // ESP_IMAGE_SPI_SPEED_80M, ESP_IMAGE_FLASH_SIZE_2MB + entry_addr: uint32(inf.Entry), + wp_pin: 0xEE, // disable WP pin + chip_id: chip_id, + hash_appended: true, // add a SHA256 hash + }) + case "esp8266": + // Header format: + // https://github.com/espressif/esptool/wiki/Firmware-Image-Format + // Basically a truncated version of the ESP32 header. + binary.Write(outf, binary.LittleEndian, struct { + magic uint8 + segment_count uint8 + spi_mode uint8 + spi_speed_size uint8 + entry_addr uint32 + }{ + magic: 0xE9, + segment_count: byte(len(segments)), + spi_mode: 0, // irrelevant, replaced by esptool when flashing + spi_speed_size: 0x20, // spi_speed, spi_size: replaced by esptool when flashing + entry_addr: uint32(inf.Entry), + }) + default: + return fmt.Errorf("builder: unknown binary format %#v, expected esp32 or esp8266", format) + } + + // Write all segments to the image. + // https://github.com/espressif/esptool/wiki/Firmware-Image-Format#segment + for _, segment := range segments { + binary.Write(outf, binary.LittleEndian, struct { + addr uint32 + length uint32 + }{ + addr: segment.addr, + length: uint32(len(segment.data)), + }) + outf.Write(segment.data) + } + + // Footer, including checksum. + // The entire image size must be a multiple of 16, so pad the image to one + // byte less than that before writing the checksum. + outf.Write(make([]byte, 15-outf.Len()%16)) + outf.WriteByte(checksum) + + if chip != "esp8266" { + // SHA256 hash (to protect against image corruption, not for security). + hash := sha256.Sum256(outf.Bytes()) + outf.Write(hash[:]) + } + + // QEMU (or more precisely, qemu-system-xtensa from Espressif) expects the + // image to be a certain size. + if makeImage { + // Use a default image size of 4MB. + grow := 4096*1024 - outf.Len() + if grow > 0 { + outf.Write(make([]byte, grow)) + } + } + + // Write the image to the output file. + return os.WriteFile(outfile, outf.Bytes(), 0666) +} diff --git a/internal/firmware/ext.go b/internal/firmware/ext.go new file mode 100644 index 00000000..ea5b70ef --- /dev/null +++ b/internal/firmware/ext.go @@ -0,0 +1,12 @@ +package firmware + +import "strings" + +// BinaryExt returns the binary file extension based on the binary format +// Returns ".bin" for ESP-based formats, "" for others +func BinaryExt(binaryFormat string) string { + if strings.HasPrefix(binaryFormat, "esp") { + return ".bin" + } + return "" +} diff --git a/internal/firmware/ext_test.go b/internal/firmware/ext_test.go new file mode 100644 index 00000000..2c594e56 --- /dev/null +++ b/internal/firmware/ext_test.go @@ -0,0 +1,28 @@ +package firmware + +import "testing" + +func TestBinaryExt(t *testing.T) { + tests := []struct { + name string + binaryFormat string + expected string + }{ + {"ESP32", "esp32", ".bin"}, + {"ESP8266", "esp8266", ".bin"}, + {"ESP32C3", "esp32c3", ".bin"}, + {"UF2", "uf2", ""}, + {"ELF", "elf", ""}, + {"Empty", "", ""}, + {"NRF-DFU", "nrf-dfu", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := BinaryExt(tt.binaryFormat) + if result != tt.expected { + t.Errorf("BinaryExt() = %q, want %q", result, tt.expected) + } + }) + } +} diff --git a/internal/firmware/firmware.go b/internal/firmware/firmware.go new file mode 100644 index 00000000..fff89fc2 --- /dev/null +++ b/internal/firmware/firmware.go @@ -0,0 +1,14 @@ +package firmware + +import ( + "fmt" + "strings" +) + +func MakeFirmwareImage(infile, outfile, format string) error { + fmt.Printf("Creating firmware image from %s to %s with format %s\n", infile, outfile, format) + if strings.HasPrefix(format, "esp") { + return makeESPFirmareImage(infile, outfile, format) + } + return fmt.Errorf("unsupported firmware format: %s", format) +} diff --git a/internal/targets/config.go b/internal/targets/config.go index 8bb01356..e30e8fab 100644 --- a/internal/targets/config.go +++ b/internal/targets/config.go @@ -26,6 +26,33 @@ type Config struct { CodeModel string `json:"code-model"` TargetABI string `json:"target-abi"` RelocationModel string `json:"relocation-model"` + + // Binary and firmware configuration + BinaryFormat string `json:"binary-format"` + + // Flash and deployment configuration + FlashCommand string `json:"flash-command"` + FlashMethod string `json:"flash-method"` + Flash1200BpsReset string `json:"flash-1200-bps-reset"` + + // Mass storage device configuration + MSDVolumeName []string `json:"msd-volume-name"` + MSDFirmwareName string `json:"msd-firmware-name"` + + // UF2 configuration + UF2FamilyID string `json:"uf2-family-id"` + + // Device-specific configuration + RP2040BootPatch bool `json:"rp2040-boot-patch"` + + // Debug and emulation configuration + Emulator string `json:"emulator"` + GDB []string `json:"gdb"` + + // OpenOCD configuration + OpenOCDInterface string `json:"openocd-interface"` + OpenOCDTransport string `json:"openocd-transport"` + OpenOCDTarget string `json:"openocd-target"` } // RawConfig represents the raw JSON configuration before inheritance resolution diff --git a/internal/targets/loader.go b/internal/targets/loader.go index e535e6d8..ebcd0c07 100644 --- a/internal/targets/loader.go +++ b/internal/targets/loader.go @@ -149,6 +149,39 @@ func (l *Loader) mergeConfig(dst, src *Config) { if src.RelocationModel != "" { dst.RelocationModel = src.RelocationModel } + if src.BinaryFormat != "" { + dst.BinaryFormat = src.BinaryFormat + } + if src.FlashCommand != "" { + dst.FlashCommand = src.FlashCommand + } + if src.FlashMethod != "" { + dst.FlashMethod = src.FlashMethod + } + if src.Flash1200BpsReset != "" { + dst.Flash1200BpsReset = src.Flash1200BpsReset + } + if src.MSDFirmwareName != "" { + dst.MSDFirmwareName = src.MSDFirmwareName + } + if src.UF2FamilyID != "" { + dst.UF2FamilyID = src.UF2FamilyID + } + if src.RP2040BootPatch { + dst.RP2040BootPatch = src.RP2040BootPatch + } + if src.Emulator != "" { + dst.Emulator = src.Emulator + } + if src.OpenOCDInterface != "" { + dst.OpenOCDInterface = src.OpenOCDInterface + } + if src.OpenOCDTransport != "" { + dst.OpenOCDTransport = src.OpenOCDTransport + } + if src.OpenOCDTarget != "" { + dst.OpenOCDTarget = src.OpenOCDTarget + } // Merge slices (append, don't replace) if len(src.BuildTags) > 0 { @@ -163,6 +196,12 @@ func (l *Loader) mergeConfig(dst, src *Config) { if len(src.ExtraFiles) > 0 { dst.ExtraFiles = append(dst.ExtraFiles, src.ExtraFiles...) } + if len(src.MSDVolumeName) > 0 { + dst.MSDVolumeName = append(dst.MSDVolumeName, src.MSDVolumeName...) + } + if len(src.GDB) > 0 { + dst.GDB = append(dst.GDB, src.GDB...) + } } // GetTargetsDir returns the targets directory path diff --git a/internal/targets/targets_test.go b/internal/targets/targets_test.go index bfe5c677..bf2407e0 100644 --- a/internal/targets/targets_test.go +++ b/internal/targets/targets_test.go @@ -61,7 +61,8 @@ func TestLoaderLoadRaw(t *testing.T) { "goarch": "arm", "build-tags": ["test", "embedded"], "cflags": ["-Os", "-g"], - "ldflags": ["--gc-sections"] + "ldflags": ["--gc-sections"], + "binary-format": "uf2" }` configPath := filepath.Join(tempDir, "test-target.json") @@ -87,6 +88,9 @@ func TestLoaderLoadRaw(t *testing.T) { if len(config.BuildTags) != 2 || config.BuildTags[0] != "test" || config.BuildTags[1] != "embedded" { t.Errorf("Expected build-tags [test, embedded], got %v", config.BuildTags) } + if config.BinaryFormat != "uf2" { + t.Errorf("Expected binary-format 'uf2', got '%s'", config.BinaryFormat) + } } func TestLoaderInheritance(t *testing.T) { @@ -99,7 +103,8 @@ func TestLoaderInheritance(t *testing.T) { "goos": "linux", "goarch": "arm", "cflags": ["-Os"], - "ldflags": ["--gc-sections"] + "ldflags": ["--gc-sections"], + "binary-format": "elf" }` // Create child config that inherits from parent @@ -108,7 +113,8 @@ func TestLoaderInheritance(t *testing.T) { "cpu": "cortex-m4", "build-tags": ["child"], "cflags": ["-O2"], - "ldflags": ["-g"] + "ldflags": ["-g"], + "binary-format": "uf2" }` parentPath := filepath.Join(tempDir, "parent.json") @@ -143,6 +149,11 @@ func TestLoaderInheritance(t *testing.T) { t.Errorf("Expected overridden cpu 'cortex-m4', got '%s'", config.CPU) } + // Check binary-format override + if config.BinaryFormat != "uf2" { + t.Errorf("Expected overridden binary-format 'uf2', got '%s'", config.BinaryFormat) + } + // Check merged arrays expectedCFlags := []string{"-Os", "-O2"} if len(config.CFlags) != 2 || config.CFlags[0] != "-Os" || config.CFlags[1] != "-O2" {