refine and test internal/crosscompile

internal/crosscompile/crosscompile.go

@@ -33,7 +33,10 @@ type Export struct {
 	BinaryFormat string // Binary format (e.g., "elf", "esp", "uf2")
 }
 
-const wasiSdkUrl = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-macos.tar.gz"
+const (
+	wasiSdkUrl      = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-macos.tar.gz"
+	wasiMacosSubdir = "wasi-sdk-25.0-x86_64-macos"
+)
 
 const (
 	espClangBaseUrl = "https://github.com/goplus/espressif-llvm-project-prebuilt/releases/download/19.1.2_20250820"
@@ -305,7 +308,7 @@ func use(goos, goarch string, wasiThreads bool) (export Export, err error) {
 		// If not exists in LLGoROOT, download and use cached wasiSdkRoot
 		if _, err = os.Stat(wasiSdkRoot); err != nil {
 			sdkDir := filepath.Join(cacheDir(), llvm.GetTargetTriple(goos, goarch))
-			if wasiSdkRoot, err = checkDownloadAndExtract(wasiSdkUrl, sdkDir); err != nil {
+			if wasiSdkRoot, err = checkDownloadAndExtractWasiSDK(sdkDir); err != nil {
 				return
 			}
 		}

internal/crosscompile/fetch.go

@@ -13,6 +13,71 @@ import (
 	"syscall"
 )
 
+// checkDownloadAndExtractWasiSDK downloads and extracts WASI SDK
+func checkDownloadAndExtractWasiSDK(dir string) (wasiSdkRoot string, err error) {
+	wasiSdkRoot = filepath.Join(dir, wasiMacosSubdir)
+
+	// Check if already exists
+	if _, err := os.Stat(wasiSdkRoot); err == nil {
+		return wasiSdkRoot, nil
+	}
+
+	// Create lock file path for the parent directory (dir) since that's what we're operating on
+	lockPath := dir + ".lock"
+	lockFile, err := acquireLock(lockPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to acquire lock: %w", err)
+	}
+	defer releaseLock(lockFile)
+
+	// Double-check after acquiring lock
+	if _, err := os.Stat(wasiSdkRoot); err == nil {
+		return wasiSdkRoot, nil
+	}
+
+	err = downloadAndExtractArchive(wasiSdkUrl, dir, "WASI SDK")
+	return wasiSdkRoot, err
+}
+
+// checkDownloadAndExtractESPClang downloads and extracts ESP Clang binaries and libraries
+func checkDownloadAndExtractESPClang(platformSuffix, dir string) error {
+	// Check if already exists
+	if _, err := os.Stat(dir); err == nil {
+		return nil
+	}
+
+	// Create lock file path for the final destination
+	lockPath := dir + ".lock"
+	lockFile, err := acquireLock(lockPath)
+	if err != nil {
+		return fmt.Errorf("failed to acquire lock: %w", err)
+	}
+	defer releaseLock(lockFile)
+
+	// Double-check after acquiring lock
+	if _, err := os.Stat(dir); err == nil {
+		return nil
+	}
+
+	clangUrl := fmt.Sprintf("%s/clang-esp-%s-%s.tar.xz", espClangBaseUrl, espClangVersion, platformSuffix)
+	description := fmt.Sprintf("ESP Clang %s-%s", espClangVersion, platformSuffix)
+
+	// Use temporary extraction directory for ESP Clang special handling
+	tempExtractDir := dir + ".extract"
+	if err := downloadAndExtractArchive(clangUrl, tempExtractDir, description); err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempExtractDir)
+
+	// ESP Clang needs special handling: move esp-clang subdirectory to final destination
+	espClangDir := filepath.Join(tempExtractDir, "esp-clang")
+	if err := os.Rename(espClangDir, dir); err != nil {
+		return fmt.Errorf("failed to rename esp-clang directory: %w", err)
+	}
+
+	return nil
+}
+
 // acquireLock creates and locks a file to prevent concurrent operations
 func acquireLock(lockPath string) (*os.File, error) {
 	// Ensure the parent directory exists
@@ -43,51 +108,49 @@ func releaseLock(lockFile *os.File) error {
 	return nil
 }
 
-func checkDownloadAndExtract(url, dir string) (wasiSdkRoot string, err error) {
-	wasiSdkRoot = filepath.Join(dir, "wasi-sdk-25.0-x86_64-macos")
-	if _, err := os.Stat(dir); err == nil {
-		return wasiSdkRoot, nil
+// downloadAndExtractArchive downloads and extracts an archive to the destination directory (without locking)
+func downloadAndExtractArchive(url, destDir, description string) error {
+	fmt.Fprintf(os.Stderr, "Downloading %s...\n", description)
+
+	// Use temporary extraction directory
+	tempDir := destDir + ".temp"
+	os.RemoveAll(tempDir)
+	if err := os.MkdirAll(tempDir, 0755); err != nil {
+		return fmt.Errorf("failed to create temporary directory: %w", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	// Download the archive
+	urlPath := strings.Split(url, "/")
+	filename := urlPath[len(urlPath)-1]
+	localFile := filepath.Join(tempDir, filename)
+	if err := downloadFile(url, localFile); err != nil {
+		return fmt.Errorf("failed to download %s from %s: %w", description, url, err)
 	}
 
-	// Create lock file path
-	lockPath := dir + ".lock"
-	lockFile, err := acquireLock(lockPath)
-	if err != nil {
-		return "", fmt.Errorf("failed to acquire lock: %w", err)
-	}
-	defer releaseLock(lockFile)
-
-	if _, err = os.Stat(dir); err != nil {
-		os.RemoveAll(dir)
-		tempDir := dir + ".temp"
-		os.RemoveAll(tempDir)
-		if err := os.MkdirAll(tempDir, 0755); err != nil {
-			return "", fmt.Errorf("failed to create temporary directory: %w", err)
-		}
-
-		urlPath := strings.Split(url, "/")
-		filename := urlPath[len(urlPath)-1]
-		localFile := filepath.Join(tempDir, filename)
-		if err = downloadFile(url, localFile); err != nil {
-			return "", fmt.Errorf("failed to download file: %w", err)
-		}
-		defer os.Remove(localFile)
-
-		if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") {
-			err = extractTarGz(localFile, tempDir)
-		} else if strings.HasSuffix(filename, ".tar.xz") {
-			err = extractTarXz(localFile, tempDir)
-		} else {
-			return "", fmt.Errorf("unsupported archive format: %s", filename)
-		}
+	// Extract the archive
+	fmt.Fprintf(os.Stderr, "Extracting %s...\n", description)
+	if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") {
+		err := extractTarGz(localFile, tempDir)
 		if err != nil {
-			return "", fmt.Errorf("failed to extract archive: %w", err)
+			return fmt.Errorf("failed to extract %s archive: %w", description, err)
 		}
-		if err = os.Rename(tempDir, dir); err != nil {
-			return "", fmt.Errorf("failed to rename directory: %w", err)
+	} else if strings.HasSuffix(filename, ".tar.xz") {
+		err := extractTarXz(localFile, tempDir)
+		if err != nil {
+			return fmt.Errorf("failed to extract %s archive: %w", description, err)
 		}
+	} else {
+		return fmt.Errorf("unsupported archive format: %s", filename)
 	}
-	return
+
+	// Rename temp directory to target directory
+	if err := os.Rename(tempDir, destDir); err != nil {
+		return fmt.Errorf("failed to rename directory: %w", err)
+	}
+
+	fmt.Fprintf(os.Stderr, "%s downloaded and extracted successfully.\n", description)
+	return nil
 }
 
 func downloadFile(url, filepath string) error {
@@ -160,57 +223,3 @@ func extractTarXz(tarXzFile, dest string) error {
 	cmd := exec.Command("tar", "-xf", tarXzFile, "-C", dest)
 	return cmd.Run()
 }
-
-// checkDownloadAndExtractESPClang downloads and extracts ESP Clang binaries and libraries
-func checkDownloadAndExtractESPClang(platformSuffix, dir string) error {
-	// Create lock file path
-	lockPath := dir + ".lock"
-	lockFile, err := acquireLock(lockPath)
-	if err != nil {
-		return fmt.Errorf("failed to acquire lock: %w", err)
-	}
-	defer releaseLock(lockFile)
-
-	// Check again after acquiring lock (double-check pattern)
-	if _, err := os.Stat(dir); err == nil {
-		return nil // Already exists, nothing to do
-	}
-
-	// Create download temp directory
-	downloadDir := dir + "-download"
-	os.RemoveAll(downloadDir)
-	if err := os.MkdirAll(downloadDir, 0755); err != nil {
-		return fmt.Errorf("failed to create download directory: %w", err)
-	}
-	defer os.RemoveAll(downloadDir)
-
-	// Download clang binary package
-	fmt.Fprintf(os.Stderr, "Downloading ESP Clang %s-%s...\n", espClangVersion, platformSuffix)
-	clangUrl := fmt.Sprintf("%s/clang-esp-%s-%s.tar.xz", espClangBaseUrl, espClangVersion, platformSuffix)
-	clangFile := filepath.Join(downloadDir, fmt.Sprintf("clang-%s-%s.tar.xz", espClangVersion, platformSuffix))
-	if err := downloadFile(clangUrl, clangFile); err != nil {
-		return fmt.Errorf("failed to download clang from %s: %w", clangUrl, err)
-	}
-
-	// Create extract temp directory
-	extractDir := dir + "-extract"
-	os.RemoveAll(extractDir)
-	if err := os.MkdirAll(extractDir, 0755); err != nil {
-		return fmt.Errorf("failed to create extract directory: %w", err)
-	}
-	defer os.RemoveAll(extractDir)
-
-	// Extract both packages to extract directory
-	fmt.Fprintln(os.Stderr, "Extracting ESP Clang...")
-	if err := extractTarXz(clangFile, extractDir); err != nil {
-		return fmt.Errorf("failed to extract clang: %w", err)
-	}
-
-	// Rename esp-clang directory to final destination
-	espClangDir := filepath.Join(extractDir, "esp-clang")
-	if err := os.Rename(espClangDir, dir); err != nil {
-		return fmt.Errorf("failed to rename esp-clang directory: %w", err)
-	}
-	fmt.Fprintln(os.Stderr, "ESP Clang downloaded and extracted successfully.")
-	return nil
-}

internal/crosscompile/fetch_test.go

@@ -0,0 +1,363 @@
+package crosscompile
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// Helper function to create a test tar.gz archive
+func createTestTarGz(t *testing.T, files map[string]string) string {
+	tempFile, err := os.CreateTemp("", "test*.tar.gz")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %v", err)
+	}
+	defer tempFile.Close()
+
+	gzw := gzip.NewWriter(tempFile)
+	defer gzw.Close()
+
+	tw := tar.NewWriter(gzw)
+	defer tw.Close()
+
+	for name, content := range files {
+		hdr := &tar.Header{
+			Name: name,
+			Mode: 0644,
+			Size: int64(len(content)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatalf("Failed to write tar header: %v", err)
+		}
+		if _, err := tw.Write([]byte(content)); err != nil {
+			t.Fatalf("Failed to write tar content: %v", err)
+		}
+	}
+
+	return tempFile.Name()
+}
+
+// Helper function to create a test HTTP server
+func createTestServer(t *testing.T, files map[string]string) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		path := strings.TrimPrefix(r.URL.Path, "/")
+		if content, exists := files[path]; exists {
+			w.Header().Set("Content-Type", "application/octet-stream")
+			w.Write([]byte(content))
+		} else {
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+}
+
+func TestAcquireAndReleaseLock(t *testing.T) {
+	tempDir := t.TempDir()
+	lockPath := filepath.Join(tempDir, "test.lock")
+
+	// Test acquiring lock
+	lockFile, err := acquireLock(lockPath)
+	if err != nil {
+		t.Fatalf("Failed to acquire lock: %v", err)
+	}
+
+	// Check lock file exists
+	if _, err := os.Stat(lockPath); os.IsNotExist(err) {
+		t.Error("Lock file should exist")
+	}
+
+	// Test releasing lock
+	if err := releaseLock(lockFile); err != nil {
+		t.Errorf("Failed to release lock: %v", err)
+	}
+
+	// Check lock file is removed
+	if _, err := os.Stat(lockPath); !os.IsNotExist(err) {
+		t.Error("Lock file should be removed after release")
+	}
+}
+
+func TestAcquireLockConcurrency(t *testing.T) {
+	tempDir := t.TempDir()
+	lockPath := filepath.Join(tempDir, "concurrent.lock")
+
+	var wg sync.WaitGroup
+	var results []int
+	var resultsMu sync.Mutex
+
+	// Start multiple goroutines trying to acquire the same lock
+	for i := 0; i < 5; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+
+			lockFile, err := acquireLock(lockPath)
+			if err != nil {
+				t.Errorf("Goroutine %d failed to acquire lock: %v", id, err)
+				return
+			}
+
+			// Hold the lock for a short time
+			resultsMu.Lock()
+			results = append(results, id)
+			resultsMu.Unlock()
+
+			time.Sleep(10 * time.Millisecond)
+
+			if err := releaseLock(lockFile); err != nil {
+				t.Errorf("Goroutine %d failed to release lock: %v", id, err)
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	// All goroutines should have successfully acquired and released the lock
+	if len(results) != 5 {
+		t.Errorf("Expected 5 successful lock acquisitions, got %d", len(results))
+	}
+}
+
+func TestDownloadFile(t *testing.T) {
+	// Create test server
+	testContent := "test file content"
+	server := createTestServer(t, map[string]string{
+		"test.txt": testContent,
+	})
+	defer server.Close()
+
+	tempDir := t.TempDir()
+	targetFile := filepath.Join(tempDir, "downloaded.txt")
+
+	// Test successful download
+	err := downloadFile(server.URL+"/test.txt", targetFile)
+	if err != nil {
+		t.Fatalf("Failed to download file: %v", err)
+	}
+
+	// Check file content
+	content, err := os.ReadFile(targetFile)
+	if err != nil {
+		t.Fatalf("Failed to read downloaded file: %v", err)
+	}
+
+	if string(content) != testContent {
+		t.Errorf("Expected content %q, got %q", testContent, string(content))
+	}
+
+	// Test download failure (404)
+	err = downloadFile(server.URL+"/nonexistent.txt", targetFile)
+	if err == nil {
+		t.Error("Expected error for non-existent file, got nil")
+	}
+}
+
+func TestExtractTarGz(t *testing.T) {
+	// Create test archive
+	files := map[string]string{
+		"test-dir/file1.txt": "content of file1",
+		"test-dir/file2.txt": "content of file2",
+		"file3.txt":          "content of file3",
+	}
+
+	archivePath := createTestTarGz(t, files)
+	defer os.Remove(archivePath)
+
+	// Extract to temp directory
+	tempDir := t.TempDir()
+	err := extractTarGz(archivePath, tempDir)
+	if err != nil {
+		t.Fatalf("Failed to extract tar.gz: %v", err)
+	}
+
+	// Check extracted files
+	for name, expectedContent := range files {
+		filePath := filepath.Join(tempDir, name)
+		content, err := os.ReadFile(filePath)
+		if err != nil {
+			t.Errorf("Failed to read extracted file %s: %v", name, err)
+			continue
+		}
+		if string(content) != expectedContent {
+			t.Errorf("File %s: expected content %q, got %q", name, expectedContent, string(content))
+		}
+	}
+}
+
+func TestExtractTarGzPathTraversal(t *testing.T) {
+	// Create a malicious archive with path traversal
+	tempFile, err := os.CreateTemp("", "malicious*.tar.gz")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %v", err)
+	}
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	gzw := gzip.NewWriter(tempFile)
+	tw := tar.NewWriter(gzw)
+
+	// Add a file with path traversal attack
+	hdr := &tar.Header{
+		Name:     "../../../etc/passwd",
+		Mode:     0644,
+		Size:     5,
+		Typeflag: tar.TypeReg,
+	}
+	if err := tw.WriteHeader(hdr); err != nil {
+		t.Fatalf("Failed to write tar header: %v", err)
+	}
+	if _, err := tw.Write([]byte("pwned")); err != nil {
+		t.Fatalf("Failed to write tar content: %v", err)
+	}
+
+	// Close writers to flush all data
+	if err := tw.Close(); err != nil {
+		t.Fatalf("Failed to close tar writer: %v", err)
+	}
+	if err := gzw.Close(); err != nil {
+		t.Fatalf("Failed to close gzip writer: %v", err)
+	}
+	if err := tempFile.Close(); err != nil {
+		t.Fatalf("Failed to close temp file: %v", err)
+	}
+
+	tempDir := t.TempDir()
+	err = extractTarGz(tempFile.Name(), tempDir)
+	if err == nil {
+		t.Error("Expected error for path traversal attack, got nil")
+	}
+	if !strings.Contains(err.Error(), "illegal file path") {
+		t.Errorf("Expected 'illegal file path' error, got: %v", err)
+	}
+}
+
+func TestDownloadAndExtractArchive(t *testing.T) {
+	// Create test archive
+	files := map[string]string{
+		"test-app/bin/app":    "#!/bin/bash\necho hello",
+		"test-app/lib/lib.so": "fake library content",
+		"test-app/README":     "This is a test application",
+	}
+
+	archivePath := createTestTarGz(t, files)
+	defer os.Remove(archivePath)
+
+	// Create test server to serve the archive
+	archiveContent, err := os.ReadFile(archivePath)
+	if err != nil {
+		t.Fatalf("Failed to read test archive: %v", err)
+	}
+
+	server := createTestServer(t, map[string]string{
+		"test-app.tar.gz": string(archiveContent),
+	})
+	defer server.Close()
+
+	// Test download and extract
+	tempDir := t.TempDir()
+	destDir := filepath.Join(tempDir, "extracted")
+
+	err = downloadAndExtractArchive(server.URL+"/test-app.tar.gz", destDir, "Test App")
+	if err != nil {
+		t.Fatalf("Failed to download and extract: %v", err)
+	}
+
+	// Check extracted files
+	for name, expectedContent := range files {
+		filePath := filepath.Join(destDir, name)
+		content, err := os.ReadFile(filePath)
+		if err != nil {
+			t.Errorf("Failed to read extracted file %s: %v", name, err)
+			continue
+		}
+		if string(content) != expectedContent {
+			t.Errorf("File %s: expected content %q, got %q", name, expectedContent, string(content))
+		}
+	}
+}
+
+func TestDownloadAndExtractArchiveUnsupportedFormat(t *testing.T) {
+	server := createTestServer(t, map[string]string{
+		"test.zip": "fake zip content",
+	})
+	defer server.Close()
+
+	tempDir := t.TempDir()
+	destDir := filepath.Join(tempDir, "extracted")
+
+	err := downloadAndExtractArchive(server.URL+"/test.zip", destDir, "Test Archive")
+	if err == nil {
+		t.Error("Expected error for unsupported format, got nil")
+	}
+	if !strings.Contains(err.Error(), "unsupported archive format") {
+		t.Errorf("Expected 'unsupported archive format' error, got: %v", err)
+	}
+}
+
+// Mock test for WASI SDK (without actual download)
+func TestWasiSDKExtractionLogic(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create fake WASI SDK directory structure
+	wasiSdkDir := filepath.Join(tempDir, wasiMacosSubdir)
+	binDir := filepath.Join(wasiSdkDir, "bin")
+	err := os.MkdirAll(binDir, 0755)
+	if err != nil {
+		t.Fatalf("Failed to create fake WASI SDK structure: %v", err)
+	}
+
+	// Create fake clang binary
+	clangPath := filepath.Join(binDir, "clang")
+	err = os.WriteFile(clangPath, []byte("fake clang"), 0755)
+	if err != nil {
+		t.Fatalf("Failed to create fake clang: %v", err)
+	}
+
+	// Test that function returns correct path for existing SDK
+	sdkRoot, err := checkDownloadAndExtractWasiSDK(tempDir)
+	if err != nil {
+		t.Fatalf("checkDownloadAndExtractWasiSDK failed: %v", err)
+	}
+
+	expectedRoot := filepath.Join(tempDir, wasiMacosSubdir)
+	if sdkRoot != expectedRoot {
+		t.Errorf("Expected SDK root %q, got %q", expectedRoot, sdkRoot)
+	}
+}
+
+// Test ESP Clang extraction logic with existing directory
+func TestESPClangExtractionLogic(t *testing.T) {
+	tempDir := t.TempDir()
+	espClangDir := filepath.Join(tempDir, "esp-clang")
+
+	// Create fake ESP Clang directory structure
+	binDir := filepath.Join(espClangDir, "bin")
+	err := os.MkdirAll(binDir, 0755)
+	if err != nil {
+		t.Fatalf("Failed to create fake ESP Clang structure: %v", err)
+	}
+
+	// Create fake clang binary
+	clangPath := filepath.Join(binDir, "clang")
+	err = os.WriteFile(clangPath, []byte("fake esp clang"), 0755)
+	if err != nil {
+		t.Fatalf("Failed to create fake esp clang: %v", err)
+	}
+
+	// Test that function skips download for existing directory
+	err = checkDownloadAndExtractESPClang("linux", espClangDir)
+	if err != nil {
+		t.Fatalf("checkDownloadAndExtractESPClang failed: %v", err)
+	}
+
+	// Check that the directory still exists and has the right content
+	if _, err := os.Stat(clangPath); os.IsNotExist(err) {
+		t.Error("ESP Clang binary should exist")
+	}
+}