diff --git a/internal/crosscompile/crosscompile.go b/internal/crosscompile/crosscompile.go index 6518409d..41458418 100644 --- a/internal/crosscompile/crosscompile.go +++ b/internal/crosscompile/crosscompile.go @@ -33,7 +33,10 @@ type Export struct { BinaryFormat string // Binary format (e.g., "elf", "esp", "uf2") } -const wasiSdkUrl = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-macos.tar.gz" +const ( + wasiSdkUrl = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-macos.tar.gz" + wasiMacosSubdir = "wasi-sdk-25.0-x86_64-macos" +) const ( espClangBaseUrl = "https://github.com/goplus/espressif-llvm-project-prebuilt/releases/download/19.1.2_20250820" @@ -305,7 +308,7 @@ func use(goos, goarch string, wasiThreads bool) (export Export, err error) { // If not exists in LLGoROOT, download and use cached wasiSdkRoot if _, err = os.Stat(wasiSdkRoot); err != nil { sdkDir := filepath.Join(cacheDir(), llvm.GetTargetTriple(goos, goarch)) - if wasiSdkRoot, err = checkDownloadAndExtract(wasiSdkUrl, sdkDir); err != nil { + if wasiSdkRoot, err = checkDownloadAndExtractWasiSDK(sdkDir); err != nil { return } } diff --git a/internal/crosscompile/fetch.go b/internal/crosscompile/fetch.go index c0c4e457..3584b38f 100644 --- a/internal/crosscompile/fetch.go +++ b/internal/crosscompile/fetch.go @@ -13,6 +13,71 @@ import ( "syscall" ) +// checkDownloadAndExtractWasiSDK downloads and extracts WASI SDK +func checkDownloadAndExtractWasiSDK(dir string) (wasiSdkRoot string, err error) { + wasiSdkRoot = filepath.Join(dir, wasiMacosSubdir) + + // Check if already exists + if _, err := os.Stat(wasiSdkRoot); err == nil { + return wasiSdkRoot, nil + } + + // Create lock file path for the parent directory (dir) since that's what we're operating on + lockPath := dir + ".lock" + lockFile, err := acquireLock(lockPath) + if err != nil { + return "", fmt.Errorf("failed to acquire lock: %w", err) + } + defer releaseLock(lockFile) + + // Double-check after acquiring lock + if _, err := os.Stat(wasiSdkRoot); err == nil { + return wasiSdkRoot, nil + } + + err = downloadAndExtractArchive(wasiSdkUrl, dir, "WASI SDK") + return wasiSdkRoot, err +} + +// checkDownloadAndExtractESPClang downloads and extracts ESP Clang binaries and libraries +func checkDownloadAndExtractESPClang(platformSuffix, dir string) error { + // Check if already exists + if _, err := os.Stat(dir); err == nil { + return nil + } + + // Create lock file path for the final destination + lockPath := dir + ".lock" + lockFile, err := acquireLock(lockPath) + if err != nil { + return fmt.Errorf("failed to acquire lock: %w", err) + } + defer releaseLock(lockFile) + + // Double-check after acquiring lock + if _, err := os.Stat(dir); err == nil { + return nil + } + + clangUrl := fmt.Sprintf("%s/clang-esp-%s-%s.tar.xz", espClangBaseUrl, espClangVersion, platformSuffix) + description := fmt.Sprintf("ESP Clang %s-%s", espClangVersion, platformSuffix) + + // Use temporary extraction directory for ESP Clang special handling + tempExtractDir := dir + ".extract" + if err := downloadAndExtractArchive(clangUrl, tempExtractDir, description); err != nil { + return err + } + defer os.RemoveAll(tempExtractDir) + + // ESP Clang needs special handling: move esp-clang subdirectory to final destination + espClangDir := filepath.Join(tempExtractDir, "esp-clang") + if err := os.Rename(espClangDir, dir); err != nil { + return fmt.Errorf("failed to rename esp-clang directory: %w", err) + } + + return nil +} + // acquireLock creates and locks a file to prevent concurrent operations func acquireLock(lockPath string) (*os.File, error) { // Ensure the parent directory exists @@ -43,51 +108,49 @@ func releaseLock(lockFile *os.File) error { return nil } -func checkDownloadAndExtract(url, dir string) (wasiSdkRoot string, err error) { - wasiSdkRoot = filepath.Join(dir, "wasi-sdk-25.0-x86_64-macos") - if _, err := os.Stat(dir); err == nil { - return wasiSdkRoot, nil +// downloadAndExtractArchive downloads and extracts an archive to the destination directory (without locking) +func downloadAndExtractArchive(url, destDir, description string) error { + fmt.Fprintf(os.Stderr, "Downloading %s...\n", description) + + // Use temporary extraction directory + tempDir := destDir + ".temp" + os.RemoveAll(tempDir) + if err := os.MkdirAll(tempDir, 0755); err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + defer os.RemoveAll(tempDir) + + // Download the archive + urlPath := strings.Split(url, "/") + filename := urlPath[len(urlPath)-1] + localFile := filepath.Join(tempDir, filename) + if err := downloadFile(url, localFile); err != nil { + return fmt.Errorf("failed to download %s from %s: %w", description, url, err) } - // Create lock file path - lockPath := dir + ".lock" - lockFile, err := acquireLock(lockPath) - if err != nil { - return "", fmt.Errorf("failed to acquire lock: %w", err) - } - defer releaseLock(lockFile) - - if _, err = os.Stat(dir); err != nil { - os.RemoveAll(dir) - tempDir := dir + ".temp" - os.RemoveAll(tempDir) - if err := os.MkdirAll(tempDir, 0755); err != nil { - return "", fmt.Errorf("failed to create temporary directory: %w", err) - } - - urlPath := strings.Split(url, "/") - filename := urlPath[len(urlPath)-1] - localFile := filepath.Join(tempDir, filename) - if err = downloadFile(url, localFile); err != nil { - return "", fmt.Errorf("failed to download file: %w", err) - } - defer os.Remove(localFile) - - if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") { - err = extractTarGz(localFile, tempDir) - } else if strings.HasSuffix(filename, ".tar.xz") { - err = extractTarXz(localFile, tempDir) - } else { - return "", fmt.Errorf("unsupported archive format: %s", filename) - } + // Extract the archive + fmt.Fprintf(os.Stderr, "Extracting %s...\n", description) + if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") { + err := extractTarGz(localFile, tempDir) if err != nil { - return "", fmt.Errorf("failed to extract archive: %w", err) + return fmt.Errorf("failed to extract %s archive: %w", description, err) } - if err = os.Rename(tempDir, dir); err != nil { - return "", fmt.Errorf("failed to rename directory: %w", err) + } else if strings.HasSuffix(filename, ".tar.xz") { + err := extractTarXz(localFile, tempDir) + if err != nil { + return fmt.Errorf("failed to extract %s archive: %w", description, err) } + } else { + return fmt.Errorf("unsupported archive format: %s", filename) } - return + + // Rename temp directory to target directory + if err := os.Rename(tempDir, destDir); err != nil { + return fmt.Errorf("failed to rename directory: %w", err) + } + + fmt.Fprintf(os.Stderr, "%s downloaded and extracted successfully.\n", description) + return nil } func downloadFile(url, filepath string) error { @@ -160,57 +223,3 @@ func extractTarXz(tarXzFile, dest string) error { cmd := exec.Command("tar", "-xf", tarXzFile, "-C", dest) return cmd.Run() } - -// checkDownloadAndExtractESPClang downloads and extracts ESP Clang binaries and libraries -func checkDownloadAndExtractESPClang(platformSuffix, dir string) error { - // Create lock file path - lockPath := dir + ".lock" - lockFile, err := acquireLock(lockPath) - if err != nil { - return fmt.Errorf("failed to acquire lock: %w", err) - } - defer releaseLock(lockFile) - - // Check again after acquiring lock (double-check pattern) - if _, err := os.Stat(dir); err == nil { - return nil // Already exists, nothing to do - } - - // Create download temp directory - downloadDir := dir + "-download" - os.RemoveAll(downloadDir) - if err := os.MkdirAll(downloadDir, 0755); err != nil { - return fmt.Errorf("failed to create download directory: %w", err) - } - defer os.RemoveAll(downloadDir) - - // Download clang binary package - fmt.Fprintf(os.Stderr, "Downloading ESP Clang %s-%s...\n", espClangVersion, platformSuffix) - clangUrl := fmt.Sprintf("%s/clang-esp-%s-%s.tar.xz", espClangBaseUrl, espClangVersion, platformSuffix) - clangFile := filepath.Join(downloadDir, fmt.Sprintf("clang-%s-%s.tar.xz", espClangVersion, platformSuffix)) - if err := downloadFile(clangUrl, clangFile); err != nil { - return fmt.Errorf("failed to download clang from %s: %w", clangUrl, err) - } - - // Create extract temp directory - extractDir := dir + "-extract" - os.RemoveAll(extractDir) - if err := os.MkdirAll(extractDir, 0755); err != nil { - return fmt.Errorf("failed to create extract directory: %w", err) - } - defer os.RemoveAll(extractDir) - - // Extract both packages to extract directory - fmt.Fprintln(os.Stderr, "Extracting ESP Clang...") - if err := extractTarXz(clangFile, extractDir); err != nil { - return fmt.Errorf("failed to extract clang: %w", err) - } - - // Rename esp-clang directory to final destination - espClangDir := filepath.Join(extractDir, "esp-clang") - if err := os.Rename(espClangDir, dir); err != nil { - return fmt.Errorf("failed to rename esp-clang directory: %w", err) - } - fmt.Fprintln(os.Stderr, "ESP Clang downloaded and extracted successfully.") - return nil -} diff --git a/internal/crosscompile/fetch_test.go b/internal/crosscompile/fetch_test.go new file mode 100644 index 00000000..a6d95605 --- /dev/null +++ b/internal/crosscompile/fetch_test.go @@ -0,0 +1,363 @@ +package crosscompile + +import ( + "archive/tar" + "compress/gzip" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// Helper function to create a test tar.gz archive +func createTestTarGz(t *testing.T, files map[string]string) string { + tempFile, err := os.CreateTemp("", "test*.tar.gz") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer tempFile.Close() + + gzw := gzip.NewWriter(tempFile) + defer gzw.Close() + + tw := tar.NewWriter(gzw) + defer tw.Close() + + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("Failed to write tar header: %v", err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatalf("Failed to write tar content: %v", err) + } + } + + return tempFile.Name() +} + +// Helper function to create a test HTTP server +func createTestServer(t *testing.T, files map[string]string) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/") + if content, exists := files[path]; exists { + w.Header().Set("Content-Type", "application/octet-stream") + w.Write([]byte(content)) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) +} + +func TestAcquireAndReleaseLock(t *testing.T) { + tempDir := t.TempDir() + lockPath := filepath.Join(tempDir, "test.lock") + + // Test acquiring lock + lockFile, err := acquireLock(lockPath) + if err != nil { + t.Fatalf("Failed to acquire lock: %v", err) + } + + // Check lock file exists + if _, err := os.Stat(lockPath); os.IsNotExist(err) { + t.Error("Lock file should exist") + } + + // Test releasing lock + if err := releaseLock(lockFile); err != nil { + t.Errorf("Failed to release lock: %v", err) + } + + // Check lock file is removed + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Error("Lock file should be removed after release") + } +} + +func TestAcquireLockConcurrency(t *testing.T) { + tempDir := t.TempDir() + lockPath := filepath.Join(tempDir, "concurrent.lock") + + var wg sync.WaitGroup + var results []int + var resultsMu sync.Mutex + + // Start multiple goroutines trying to acquire the same lock + for i := 0; i < 5; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + + lockFile, err := acquireLock(lockPath) + if err != nil { + t.Errorf("Goroutine %d failed to acquire lock: %v", id, err) + return + } + + // Hold the lock for a short time + resultsMu.Lock() + results = append(results, id) + resultsMu.Unlock() + + time.Sleep(10 * time.Millisecond) + + if err := releaseLock(lockFile); err != nil { + t.Errorf("Goroutine %d failed to release lock: %v", id, err) + } + }(i) + } + + wg.Wait() + + // All goroutines should have successfully acquired and released the lock + if len(results) != 5 { + t.Errorf("Expected 5 successful lock acquisitions, got %d", len(results)) + } +} + +func TestDownloadFile(t *testing.T) { + // Create test server + testContent := "test file content" + server := createTestServer(t, map[string]string{ + "test.txt": testContent, + }) + defer server.Close() + + tempDir := t.TempDir() + targetFile := filepath.Join(tempDir, "downloaded.txt") + + // Test successful download + err := downloadFile(server.URL+"/test.txt", targetFile) + if err != nil { + t.Fatalf("Failed to download file: %v", err) + } + + // Check file content + content, err := os.ReadFile(targetFile) + if err != nil { + t.Fatalf("Failed to read downloaded file: %v", err) + } + + if string(content) != testContent { + t.Errorf("Expected content %q, got %q", testContent, string(content)) + } + + // Test download failure (404) + err = downloadFile(server.URL+"/nonexistent.txt", targetFile) + if err == nil { + t.Error("Expected error for non-existent file, got nil") + } +} + +func TestExtractTarGz(t *testing.T) { + // Create test archive + files := map[string]string{ + "test-dir/file1.txt": "content of file1", + "test-dir/file2.txt": "content of file2", + "file3.txt": "content of file3", + } + + archivePath := createTestTarGz(t, files) + defer os.Remove(archivePath) + + // Extract to temp directory + tempDir := t.TempDir() + err := extractTarGz(archivePath, tempDir) + if err != nil { + t.Fatalf("Failed to extract tar.gz: %v", err) + } + + // Check extracted files + for name, expectedContent := range files { + filePath := filepath.Join(tempDir, name) + content, err := os.ReadFile(filePath) + if err != nil { + t.Errorf("Failed to read extracted file %s: %v", name, err) + continue + } + if string(content) != expectedContent { + t.Errorf("File %s: expected content %q, got %q", name, expectedContent, string(content)) + } + } +} + +func TestExtractTarGzPathTraversal(t *testing.T) { + // Create a malicious archive with path traversal + tempFile, err := os.CreateTemp("", "malicious*.tar.gz") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } + defer tempFile.Close() + defer os.Remove(tempFile.Name()) + + gzw := gzip.NewWriter(tempFile) + tw := tar.NewWriter(gzw) + + // Add a file with path traversal attack + hdr := &tar.Header{ + Name: "../../../etc/passwd", + Mode: 0644, + Size: 5, + Typeflag: tar.TypeReg, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("Failed to write tar header: %v", err) + } + if _, err := tw.Write([]byte("pwned")); err != nil { + t.Fatalf("Failed to write tar content: %v", err) + } + + // Close writers to flush all data + if err := tw.Close(); err != nil { + t.Fatalf("Failed to close tar writer: %v", err) + } + if err := gzw.Close(); err != nil { + t.Fatalf("Failed to close gzip writer: %v", err) + } + if err := tempFile.Close(); err != nil { + t.Fatalf("Failed to close temp file: %v", err) + } + + tempDir := t.TempDir() + err = extractTarGz(tempFile.Name(), tempDir) + if err == nil { + t.Error("Expected error for path traversal attack, got nil") + } + if !strings.Contains(err.Error(), "illegal file path") { + t.Errorf("Expected 'illegal file path' error, got: %v", err) + } +} + +func TestDownloadAndExtractArchive(t *testing.T) { + // Create test archive + files := map[string]string{ + "test-app/bin/app": "#!/bin/bash\necho hello", + "test-app/lib/lib.so": "fake library content", + "test-app/README": "This is a test application", + } + + archivePath := createTestTarGz(t, files) + defer os.Remove(archivePath) + + // Create test server to serve the archive + archiveContent, err := os.ReadFile(archivePath) + if err != nil { + t.Fatalf("Failed to read test archive: %v", err) + } + + server := createTestServer(t, map[string]string{ + "test-app.tar.gz": string(archiveContent), + }) + defer server.Close() + + // Test download and extract + tempDir := t.TempDir() + destDir := filepath.Join(tempDir, "extracted") + + err = downloadAndExtractArchive(server.URL+"/test-app.tar.gz", destDir, "Test App") + if err != nil { + t.Fatalf("Failed to download and extract: %v", err) + } + + // Check extracted files + for name, expectedContent := range files { + filePath := filepath.Join(destDir, name) + content, err := os.ReadFile(filePath) + if err != nil { + t.Errorf("Failed to read extracted file %s: %v", name, err) + continue + } + if string(content) != expectedContent { + t.Errorf("File %s: expected content %q, got %q", name, expectedContent, string(content)) + } + } +} + +func TestDownloadAndExtractArchiveUnsupportedFormat(t *testing.T) { + server := createTestServer(t, map[string]string{ + "test.zip": "fake zip content", + }) + defer server.Close() + + tempDir := t.TempDir() + destDir := filepath.Join(tempDir, "extracted") + + err := downloadAndExtractArchive(server.URL+"/test.zip", destDir, "Test Archive") + if err == nil { + t.Error("Expected error for unsupported format, got nil") + } + if !strings.Contains(err.Error(), "unsupported archive format") { + t.Errorf("Expected 'unsupported archive format' error, got: %v", err) + } +} + +// Mock test for WASI SDK (without actual download) +func TestWasiSDKExtractionLogic(t *testing.T) { + tempDir := t.TempDir() + + // Create fake WASI SDK directory structure + wasiSdkDir := filepath.Join(tempDir, wasiMacosSubdir) + binDir := filepath.Join(wasiSdkDir, "bin") + err := os.MkdirAll(binDir, 0755) + if err != nil { + t.Fatalf("Failed to create fake WASI SDK structure: %v", err) + } + + // Create fake clang binary + clangPath := filepath.Join(binDir, "clang") + err = os.WriteFile(clangPath, []byte("fake clang"), 0755) + if err != nil { + t.Fatalf("Failed to create fake clang: %v", err) + } + + // Test that function returns correct path for existing SDK + sdkRoot, err := checkDownloadAndExtractWasiSDK(tempDir) + if err != nil { + t.Fatalf("checkDownloadAndExtractWasiSDK failed: %v", err) + } + + expectedRoot := filepath.Join(tempDir, wasiMacosSubdir) + if sdkRoot != expectedRoot { + t.Errorf("Expected SDK root %q, got %q", expectedRoot, sdkRoot) + } +} + +// Test ESP Clang extraction logic with existing directory +func TestESPClangExtractionLogic(t *testing.T) { + tempDir := t.TempDir() + espClangDir := filepath.Join(tempDir, "esp-clang") + + // Create fake ESP Clang directory structure + binDir := filepath.Join(espClangDir, "bin") + err := os.MkdirAll(binDir, 0755) + if err != nil { + t.Fatalf("Failed to create fake ESP Clang structure: %v", err) + } + + // Create fake clang binary + clangPath := filepath.Join(binDir, "clang") + err = os.WriteFile(clangPath, []byte("fake esp clang"), 0755) + if err != nil { + t.Fatalf("Failed to create fake esp clang: %v", err) + } + + // Test that function skips download for existing directory + err = checkDownloadAndExtractESPClang("linux", espClangDir) + if err != nil { + t.Fatalf("checkDownloadAndExtractESPClang failed: %v", err) + } + + // Check that the directory still exists and has the right content + if _, err := os.Stat(clangPath); os.IsNotExist(err) { + t.Error("ESP Clang binary should exist") + } +}