diff --git a/.github/codecov.yml b/.github/codecov.yml index 60703b4d..318a3749 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -8,3 +8,4 @@ coverage: - "internal/packages" - "internal/typepatch" - "internal/github" + - "xtool/cppkg" diff --git a/go.mod b/go.mod index 6a6f3ed0..c1b78658 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,8 @@ require ( ) require ( - golang.org/x/mod v0.23.0 // indirect + github.com/goccy/go-yaml v1.17.1 + golang.org/x/mod v0.23.0 golang.org/x/sync v0.11.0 // indirect ) diff --git a/go.sum b/go.sum index 3b87ba80..d7ae6fe2 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= +github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/goplus/gogen v1.17.3 h1:Xhoj2KQw4feRdPEtOYjTUe9lSvNIoxBG4urhdjf+fUg= diff --git a/xtool/cppkg/command.go b/xtool/cppkg/command.go new file mode 100644 index 00000000..a72a986a --- /dev/null +++ b/xtool/cppkg/command.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cppkg + +import ( + "os" + "os/exec" + "strings" +) + +var ( + // ErrNotFound is the error resulting if a path search failed to find + // an executable file. + ErrNotFound = exec.ErrNotFound +) + +// Tool represents a tool that can be executed. +type Tool struct { + cmd string + installs [][]string +} + +// NewTool creates a new Tool instance with the specified tool and install commands. +func NewTool(cmd string, installs []string) *Tool { + inst := make([][]string, len(installs)) + for i, install := range installs { + inst[i] = strings.Split(install, " ") + } + return &Tool{ + cmd: cmd, + installs: inst, + } +} + +// New creates a new command with the specified arguments. +func (p *Tool) New(quietInstall bool, args ...string) (cmd *exec.Cmd, err error) { + app, err := p.Get(quietInstall) + if err != nil { + return + } + return exec.Command(app, args...), nil +} + +// Get retrieves the path of the command. +// If the command is not found, it attempts to install it using the specified +// install commands. +func (p *Tool) Get(quietInstall bool) (app string, err error) { + app, err = exec.LookPath(p.cmd) + if err == nil { + return + } + amPath, install, err := p.getAppManager() + if err != nil { + return + } + c := exec.Command(amPath, install[1:]...) + c.Stdout = os.Stdout + c.Stderr = os.Stderr + if err = c.Run(); err != nil { + return + } + return exec.LookPath(p.cmd) +} + +func (p *Tool) getAppManager() (amPath string, install []string, err error) { + for _, install = range p.installs { + am := install[0] + if amPath, err = exec.LookPath(am); err == nil { + return + } + } + err = ErrNotFound + return +} diff --git a/xtool/cppkg/conan.go b/xtool/cppkg/conan.go new file mode 100644 index 00000000..7492354e --- /dev/null +++ b/xtool/cppkg/conan.go @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cppkg + +import ( + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "time" + + "github.com/goccy/go-yaml" + "github.com/goplus/llgo/internal/github" + "github.com/qiniu/x/httputil" +) + +var conanCmd = NewTool("conan", []string{ + "brew install conan", + "apt-get install conan", +}) + +type conandata struct { + Sources map[string]any `yaml:"sources"` +} + +func replaceVer(src any, fromVer, toVer string) any { + switch src := src.(type) { + case map[string]any: + doReplace(src, fromVer, toVer) + case []any: + for _, u := range src { + doReplace(u.(map[string]any), fromVer, toVer) + } + } + return src +} + +func doReplace(src map[string]any, fromVer, toVer string) { + switch url := src["url"].(type) { + case string: + src["url"] = strings.ReplaceAll(url, fromVer, toVer) + delete(src, "sha256") + // TODO(xsw): src["sha256"] = hash + case []any: + for i, u := range url { + url[i] = strings.ReplaceAll(u.(string), fromVer, toVer) + } + delete(src, "sha256") + // TODO(xsw): src["sha256"] = hash + } +} + +type githubRelease struct { + PublishedAt string +} + +func getRelease(pkg *Package, tagPattern string) (ret *githubRelease, err error) { + if tagPattern == "" { + return nil, errors.New("dynamic tag") + } + ver := strings.Replace(tagPattern, "*", pkg.Version, 1) + gr, err := github.GetRelease(pkg.Path, ver) + if err == nil { + ret = &githubRelease{PublishedAt: gr.PublishedAt} + return + } + t, err := github.GetTag(pkg.Path, ver) + if err != nil { + return + } + c, err := github.GetCommit(pkg.Path, t.Commit.URL) + if err == nil { + ret = &githubRelease{PublishedAt: c.Commit.Author.Date} + } + return +} + +// Install installs the specified package using Conan. +func (p *Manager) Install(pkg *Package, flags int) (err error) { + outDir := p.outDir(pkg) + os.MkdirAll(outDir, os.ModePerm) + + var rev string + var gr *githubRelease + var conandataYml, conanfilePy []byte + + conanfileDir := p.conanfileDir(pkg.Path, pkg.Folder) + pkgVer := pkg.Version + template := pkg.Template + if template != nil { + gr, err = getRelease(pkg, template.Tag) + if err != nil { + return + } + + err = copyDirR(conanfileDir, outDir) + if err != nil { + return + } + + conanfilePy, err = os.ReadFile(outDir + "/conanfile.py") + if err != nil { + return + } + + conandataFile := outDir + "/conandata.yml" + conandataYml, err = os.ReadFile(conandataFile) + if err != nil { + return + } + var cd conandata + err = yaml.Unmarshal(conandataYml, &cd) + if err != nil { + return + } + fromVer := template.FromVer + source, ok := cd.Sources[fromVer] + if !ok { + return ErrVersionNotFound + } + cd.Sources = map[string]any{ + pkgVer: replaceVer(source, fromVer, pkgVer), + } + conandataYml, err = yaml.Marshal(cd) + if err != nil { + return + } + err = os.WriteFile(conandataFile, conandataYml, os.ModePerm) + if err != nil { + return + } + rev = recipeRevision(pkg, gr, conandataYml) + conanfileDir = outDir + } + + outFile := outDir + "/out.json" + out, err := os.Create(outFile) + if err == nil { + defer out.Close() + } else { + out = os.Stdout + } + + nameAndVer := pkg.Name + "/" + pkgVer + if template == nil { + return conanInstall(nameAndVer, outDir, conanfileDir, out, flags) + } + + logFile := "" + if flags&LogRevertProxy != 0 { + logFile = outDir + "/rp.log" + } + return remoteProxy(flags, logFile, func() error { + return conanInstall(nameAndVer, outDir, conanfileDir, out, flags) + }, func(mux *http.ServeMux) { + base := "/v2/conans/" + nameAndVer + revbase := base + "/_/_/revisions/" + rev + mux.HandleFunc(base+"/_/_/latest", func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("Cache-Control", "public,max-age=300") + httputil.Reply(w, http.StatusOK, map[string]any{ + "revision": rev, + "time": gr.PublishedAt, + }) + }) + mux.HandleFunc(revbase+"/files", func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("Cache-Control", "public,max-age=3600") + empty := map[string]any{} + httputil.Reply(w, http.StatusOK, map[string]any{ + "files": map[string]any{ + "conan_export.tgz": empty, + "conanmanifest.txt": empty, + "conanfile.py": empty, + }, + }) + }) + mux.HandleFunc(revbase+"/files/conanfile.py", func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("Cache-Control", "public,max-age=3600") + h.Set("Content-Disposition", `attachment; filename="conanfile.py"`) + httputil.ReplyWith(w, http.StatusOK, "text/x-python", conanfilePy) + }) + const conanmanifest = "%d\nconandata.yml: %s\nconanfile.py: %s\n" + mux.HandleFunc(revbase+"/files/conanmanifest.txt", func(w http.ResponseWriter, r *http.Request) { + mtime, err := unixTime(gr.PublishedAt) + if err != nil { + replyError(w, err) + return + } + h := w.Header() + h.Set("Cache-Control", "public,max-age=3600") + h.Set("Content-Disposition", `attachment; filename="conanmanifest.txt"`) + data := fmt.Sprintf(conanmanifest, mtime, md5Of(conandataYml), md5Of(conanfilePy)) + httputil.ReplyWithStream(w, http.StatusOK, "text/plain", strings.NewReader(data), int64(len(data))) + }) + mux.HandleFunc(revbase+"/files/conan_export.tgz", func(w http.ResponseWriter, r *http.Request) { + conanExportTgz, err := tgzOfConandata(outDir) + if err != nil { + replyError(w, err) + return + } + h := w.Header() + h.Set("Cache-Control", "public,max-age=3600") + h.Set("Content-Disposition", `attachment; filename="conan_export.tgz"`) + httputil.ReplyWith(w, http.StatusOK, "application/x-gzip", conanExportTgz) + }) + }) +} + +func (p *Manager) outDir(pkg *Package) string { + return p.cacheDir + "/build/" + pkg.Name + "@" + pkg.Version +} + +func (p *Manager) conanfileDir(pkgPath, pkgFolder string) string { + root := p.indexRoot() + return root + "/" + pkgPath + "/" + pkgFolder +} + +func conanInstall(pkg, outDir, conanfileDir string, out io.Writer, flags int) (err error) { + args := make([]string, 0, 12) + args = append(args, "install", + "--requires", pkg, + "--generator", "PkgConfigDeps", + "--build", "missing", + "--format", "json", + "--output-folder", outDir, + ) + quietInstall := flags&ToolQuietInstall != 0 + cmd, err := conanCmd.New(quietInstall, args...) + if err != nil { + return + } + cmd.Dir = conanfileDir + cmd.Stderr = os.Stderr + cmd.Stdout = out + err = cmd.Run() + return +} + +func recipeRevision(_ *Package, _ *githubRelease, conandataYml []byte) string { + return md5Of(conandataYml) +} + +func md5Of(data []byte) string { + h := md5.New() + h.Write(data) + return hex.EncodeToString(h.Sum(nil)) +} + +func tgzOfConandata(outDir string) (_ []byte, err error) { + cmd := exec.Command("tar", "-czf", "conan_export.tgz", "conandata.yml") + cmd.Dir = outDir + err = cmd.Run() + if err != nil { + return + } + return os.ReadFile(outDir + "/conan_export.tgz") +} + +func unixTime(tstr string) (ret int64, err error) { + t, err := time.Parse(time.RFC3339, tstr) + if err == nil { + ret = t.Unix() + } + return +} + +func copyDirR(srcDir, destDir string) error { + if cp, err := exec.LookPath("cp"); err == nil { + return exec.Command(cp, "-r", "-p", srcDir+"/", destDir).Run() + } + if cp, err := exec.LookPath("xcopy"); err == nil { + // TODO(xsw): check xcopy + return exec.Command(cp, "/E", "/I", "/Y", srcDir+"/", destDir).Run() + } + return errors.New("copy command not found") +} diff --git a/xtool/cppkg/cppkg.go b/xtool/cppkg/cppkg.go new file mode 100644 index 00000000..e5bbfde7 --- /dev/null +++ b/xtool/cppkg/cppkg.go @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cppkg + +import ( + "strings" +) + +const ( + // DefaultFlags is the default flags for package installation. + DefaultFlags = IndexAutoUpdate | ToolQuietInstall +) + +// Main is the main entry point for the cppkg package. +// pkgAndVer: 7bitcoder/7bitconf@1.2.0 +func Main(pkgAndVer string, flags int) { + pkgPath, ver := parsePkgVer(pkgAndVer) + if ver == "" { + panic("TODO: get latest version") + } + + m, err := New("") + check(err) + + pkg, err := m.Lookup(pkgPath, ver, flags) + check(err) + + err = m.Install(pkg, flags) + check(err) +} + +func parsePkgVer(pkg string) (string, string) { + parts := strings.SplitN(pkg, "@", 2) + if len(parts) == 1 { + return parts[0], "" + } + return parts[0], parts[1] +} + +func check(err error) { + if err != nil { + panic(err) + } +} diff --git a/xtool/cppkg/manager.go b/xtool/cppkg/manager.go new file mode 100644 index 00000000..4ff5aae8 --- /dev/null +++ b/xtool/cppkg/manager.go @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cppkg + +import ( + "errors" + "os" + + "github.com/goccy/go-yaml" + "golang.org/x/mod/semver" +) + +var gitCmd = NewTool("git", []string{ + "brew install git", + "apt-get install git", +}) + +// Manager represents a package manager for C/C++ packages. +type Manager struct { + cacheDir string +} + +func New(cacheDir string) (ret *Manager, err error) { + if cacheDir == "" { + cacheDir, err = os.UserCacheDir() + if err != nil { + return + } + cacheDir += "/cppkg" + } + os.MkdirAll(cacheDir, os.ModePerm) + ret = &Manager{ + cacheDir: cacheDir, + } + return +} + +type version struct { + Folder string `yaml:"folder"` +} + +// Template represents a template for package versions. +type Template struct { + FromVer string `yaml:"from"` + Folder string `yaml:"folder"` + Tag string `yaml:"tag,omitempty"` // pattern with *, empty if dynamic tag +} + +type config struct { + PkgName string `yaml:"name"` + Versions map[string]version `yaml:"versions"` + Template Template `yaml:"template"` +} + +// Package represents a C/C++ package. +type Package struct { + Name string + Path string + Version string + Folder string + Template *Template +} + +var ( + // ErrVersionNotFound is returned when the specified version is not found. + ErrVersionNotFound = errors.New("version not found") +) + +const ( + // IndexAutoUpdate is a flag to automatically update the index. + IndexAutoUpdate = 1 << iota + + // ToolQuietInstall is a flag to suppress output during installation. + ToolQuietInstall + + // LogRevertProxy is a flag to log revert proxy. + LogRevertProxy +) + +// Lookup looks up a package by its path and version. +func (p *Manager) Lookup(pkgPath, ver string, flags int) (_ *Package, err error) { + root := p.indexRoot() + err = indexUpate(root, flags) + if err != nil { + return + } + pkgDir := root + "/" + pkgPath + confFile := pkgDir + "/config.yml" + b, err := os.ReadFile(confFile) + if err != nil { + return + } + var conf config + err = yaml.Unmarshal(b, &conf) + if err != nil { + return + } + if v, ok := conf.Versions[ver]; ok { + return &Package{conf.PkgName, pkgPath, ver, v.Folder, nil}, nil + } + if compareVer(ver, conf.Template.FromVer) < 0 { + err = ErrVersionNotFound + return + } + folder := conf.Template.Folder + return &Package{conf.PkgName, pkgPath, ver, folder, &conf.Template}, nil +} + +func (p *Manager) indexRoot() string { + return p.cacheDir + "/index" +} + +func indexUpate(root string, flags int) (err error) { + if _, err = os.Stat(root + "/.git"); os.IsNotExist(err) { + os.RemoveAll(root) + return indexInit(root, flags) + } + if flags&IndexAutoUpdate != 0 { + quietInstall := flags&ToolQuietInstall != 0 + git, e := gitCmd.New(quietInstall, "pull", "--ff-only", "origin", "main") + if e != nil { + return e + } + git.Dir = root + git.Stdout = os.Stdout + git.Stderr = os.Stderr + err = git.Run() + } + return +} + +func indexInit(root string, flags int) (err error) { + quietInstall := flags&ToolQuietInstall != 0 + git, err := gitCmd.New(quietInstall, "clone", "https://github.com/goplus/cppkg.git", root) + if err != nil { + return + } + git.Stdout = os.Stdout + git.Stderr = os.Stderr + err = git.Run() + return +} + +func compareVer(v1, v2 string) int { + return semver.Compare("v"+v1, "v"+v2) +} diff --git a/xtool/cppkg/revertproxy.go b/xtool/cppkg/revertproxy.go new file mode 100644 index 00000000..ffc688d0 --- /dev/null +++ b/xtool/cppkg/revertproxy.go @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cppkg + +import ( + "bytes" + "encoding/json" + "io" + stdlog "log" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "strings" +) + +type rtHandler func(req *http.Request) (resp *http.Response, err error) + +func (p rtHandler) RoundTrip(req *http.Request) (*http.Response, error) { + return p(req) +} + +type teeReader struct { + rc io.ReadCloser + b bytes.Buffer + req *http.Request + resp *http.Response + log *stdlog.Logger +} + +func (p *teeReader) Read(b []byte) (n int, err error) { + n, err = p.rc.Read(b) + p.b.Write(b[:n]) + return +} + +func (p *teeReader) Close() error { + err := p.rc.Close() + if log := p.log; log != nil { + resp := *p.resp + resp.Body = io.NopCloser(&p.b) + var b bytes.Buffer + p.req.Write(&b) + resp.Write(&b) + log.Print(b.String()) + } + return err +} + +type response = httptest.ResponseRecorder + +func newResponse() *response { + return httptest.NewRecorder() +} + +type revertProxy = httptest.Server +type rpFunc = func(mux *http.ServeMux) + +const ( + passThrough = http.StatusNotFound +) + +func replyError(w http.ResponseWriter, _ error) { + w.WriteHeader(passThrough) +} + +func startRevertProxy(endpoint string, f rpFunc, log *stdlog.Logger) (_ *revertProxy, err error) { + rpURL, err := url.Parse(endpoint) + if err != nil { + return + } + var mux *http.ServeMux + if f != nil { + mux = http.NewServeMux() + f(mux) + } + proxy := httptest.NewServer(&httputil.ReverseProxy{ + Rewrite: func(r *httputil.ProxyRequest) { + r.SetURL(rpURL) + }, + Transport: rtHandler(func(req *http.Request) (resp *http.Response, err error) { + if mux != nil { + w := newResponse() + mux.ServeHTTP(w, req) + if w.Code != passThrough { + resp = w.Result() + } + } + if resp == nil { + resp, err = http.DefaultTransport.RoundTrip(req) + } + if err == nil && resp.Body != nil { + resp.Body = &teeReader{ + rc: resp.Body, + req: req, + resp: resp, + log: log, + } + } + return + }), + }) + return proxy, nil +} + +const ( + conanCenter = "conancenter" + conanEndpoint = "https://center2.conan.io" +) + +type remoteList []struct { + Name string `json:"name"` + URL string `json:"url"` +} + +func remoteProxy(flags int, logFile string, f func() error, rpf rpFunc) (err error) { + quietInstall := flags&ToolQuietInstall != 0 + app, err := conanCmd.Get(quietInstall) + if err != nil { + return + } + + endpoint := conanEndpoint + cmd := exec.Command(app, "remote", "list", "-f", "json") + if b, err := cmd.Output(); err == nil { + var rl remoteList + if json.Unmarshal(b, &rl) == nil { + for _, r := range rl { + if r.Name == conanCenter && strings.HasPrefix(r.URL, "https://") { + endpoint = r.URL + break + } + } + } + } + defer func() { + exec.Command(app, "remote", "add", "--force", conanCenter, endpoint).Run() + }() + + var log *stdlog.Logger + if logFile != "" { + f, err := os.Create(logFile) + if err == nil { + defer f.Close() + log = stdlog.New(f, "", stdlog.LstdFlags) + } + } + rp, err := startRevertProxy(conanEndpoint, rpf, log) + if err != nil { + return + } + defer rp.Close() + + err = exec.Command(app, "remote", "add", "--force", conanCenter, rp.URL).Run() + if err != nil { + return + } + + return f() +}