package: xtool/cppkg

This commit is contained in:
xushiwei
2025-05-03 23:13:10 +08:00
parent 604ce47d5e
commit e93e7126b6
8 changed files with 784 additions and 1 deletions

1
.github/codecov.yml vendored
View File

@@ -8,3 +8,4 @@ coverage:
- "internal/packages" - "internal/packages"
- "internal/typepatch" - "internal/typepatch"
- "internal/github" - "internal/github"
- "xtool/cppkg"

3
go.mod
View File

@@ -15,7 +15,8 @@ require (
) )
require ( require (
golang.org/x/mod v0.23.0 // indirect github.com/goccy/go-yaml v1.17.1
golang.org/x/mod v0.23.0
golang.org/x/sync v0.11.0 // indirect golang.org/x/sync v0.11.0 // indirect
) )

2
go.sum
View File

@@ -1,3 +1,5 @@
github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY=
github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/goplus/gogen v1.17.3 h1:Xhoj2KQw4feRdPEtOYjTUe9lSvNIoxBG4urhdjf+fUg= github.com/goplus/gogen v1.17.3 h1:Xhoj2KQw4feRdPEtOYjTUe9lSvNIoxBG4urhdjf+fUg=

88
xtool/cppkg/command.go Normal file
View File

@@ -0,0 +1,88 @@
/*
* Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cppkg
import (
"os"
"os/exec"
"strings"
)
var (
// ErrNotFound is the error resulting if a path search failed to find
// an executable file.
ErrNotFound = exec.ErrNotFound
)
// Tool represents a tool that can be executed.
type Tool struct {
cmd string
installs [][]string
}
// NewTool creates a new Tool instance with the specified tool and install commands.
func NewTool(cmd string, installs []string) *Tool {
inst := make([][]string, len(installs))
for i, install := range installs {
inst[i] = strings.Split(install, " ")
}
return &Tool{
cmd: cmd,
installs: inst,
}
}
// New creates a new command with the specified arguments.
func (p *Tool) New(quietInstall bool, args ...string) (cmd *exec.Cmd, err error) {
app, err := p.Get(quietInstall)
if err != nil {
return
}
return exec.Command(app, args...), nil
}
// Get retrieves the path of the command.
// If the command is not found, it attempts to install it using the specified
// install commands.
func (p *Tool) Get(quietInstall bool) (app string, err error) {
app, err = exec.LookPath(p.cmd)
if err == nil {
return
}
amPath, install, err := p.getAppManager()
if err != nil {
return
}
c := exec.Command(amPath, install[1:]...)
c.Stdout = os.Stdout
c.Stderr = os.Stderr
if err = c.Run(); err != nil {
return
}
return exec.LookPath(p.cmd)
}
func (p *Tool) getAppManager() (amPath string, install []string, err error) {
for _, install = range p.installs {
am := install[0]
if amPath, err = exec.LookPath(am); err == nil {
return
}
}
err = ErrNotFound
return
}

297
xtool/cppkg/conan.go Normal file
View File

@@ -0,0 +1,297 @@
/*
* Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cppkg
import (
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"strings"
"time"
"github.com/goccy/go-yaml"
"github.com/goplus/llgo/internal/github"
"github.com/qiniu/x/httputil"
)
var conanCmd = NewTool("conan", []string{
"brew install conan",
"apt-get install conan",
})
type conandata struct {
Sources map[string]any `yaml:"sources"`
}
func replaceVer(src any, fromVer, toVer string) any {
switch src := src.(type) {
case map[string]any:
doReplace(src, fromVer, toVer)
case []any:
for _, u := range src {
doReplace(u.(map[string]any), fromVer, toVer)
}
}
return src
}
func doReplace(src map[string]any, fromVer, toVer string) {
switch url := src["url"].(type) {
case string:
src["url"] = strings.ReplaceAll(url, fromVer, toVer)
delete(src, "sha256")
// TODO(xsw): src["sha256"] = hash
case []any:
for i, u := range url {
url[i] = strings.ReplaceAll(u.(string), fromVer, toVer)
}
delete(src, "sha256")
// TODO(xsw): src["sha256"] = hash
}
}
type githubRelease struct {
PublishedAt string
}
func getRelease(pkg *Package, tagPattern string) (ret *githubRelease, err error) {
if tagPattern == "" {
return nil, errors.New("dynamic tag")
}
ver := strings.Replace(tagPattern, "*", pkg.Version, 1)
gr, err := github.GetRelease(pkg.Path, ver)
if err == nil {
ret = &githubRelease{PublishedAt: gr.PublishedAt}
return
}
t, err := github.GetTag(pkg.Path, ver)
if err != nil {
return
}
c, err := github.GetCommit(pkg.Path, t.Commit.URL)
if err == nil {
ret = &githubRelease{PublishedAt: c.Commit.Author.Date}
}
return
}
// Install installs the specified package using Conan.
func (p *Manager) Install(pkg *Package, flags int) (err error) {
outDir := p.outDir(pkg)
os.MkdirAll(outDir, os.ModePerm)
var rev string
var gr *githubRelease
var conandataYml, conanfilePy []byte
conanfileDir := p.conanfileDir(pkg.Path, pkg.Folder)
pkgVer := pkg.Version
template := pkg.Template
if template != nil {
gr, err = getRelease(pkg, template.Tag)
if err != nil {
return
}
err = copyDirR(conanfileDir, outDir)
if err != nil {
return
}
conanfilePy, err = os.ReadFile(outDir + "/conanfile.py")
if err != nil {
return
}
conandataFile := outDir + "/conandata.yml"
conandataYml, err = os.ReadFile(conandataFile)
if err != nil {
return
}
var cd conandata
err = yaml.Unmarshal(conandataYml, &cd)
if err != nil {
return
}
fromVer := template.FromVer
source, ok := cd.Sources[fromVer]
if !ok {
return ErrVersionNotFound
}
cd.Sources = map[string]any{
pkgVer: replaceVer(source, fromVer, pkgVer),
}
conandataYml, err = yaml.Marshal(cd)
if err != nil {
return
}
err = os.WriteFile(conandataFile, conandataYml, os.ModePerm)
if err != nil {
return
}
rev = recipeRevision(pkg, gr, conandataYml)
conanfileDir = outDir
}
outFile := outDir + "/out.json"
out, err := os.Create(outFile)
if err == nil {
defer out.Close()
} else {
out = os.Stdout
}
nameAndVer := pkg.Name + "/" + pkgVer
if template == nil {
return conanInstall(nameAndVer, outDir, conanfileDir, out, flags)
}
logFile := ""
if flags&LogRevertProxy != 0 {
logFile = outDir + "/rp.log"
}
return remoteProxy(flags, logFile, func() error {
return conanInstall(nameAndVer, outDir, conanfileDir, out, flags)
}, func(mux *http.ServeMux) {
base := "/v2/conans/" + nameAndVer
revbase := base + "/_/_/revisions/" + rev
mux.HandleFunc(base+"/_/_/latest", func(w http.ResponseWriter, r *http.Request) {
h := w.Header()
h.Set("Cache-Control", "public,max-age=300")
httputil.Reply(w, http.StatusOK, map[string]any{
"revision": rev,
"time": gr.PublishedAt,
})
})
mux.HandleFunc(revbase+"/files", func(w http.ResponseWriter, r *http.Request) {
h := w.Header()
h.Set("Cache-Control", "public,max-age=3600")
empty := map[string]any{}
httputil.Reply(w, http.StatusOK, map[string]any{
"files": map[string]any{
"conan_export.tgz": empty,
"conanmanifest.txt": empty,
"conanfile.py": empty,
},
})
})
mux.HandleFunc(revbase+"/files/conanfile.py", func(w http.ResponseWriter, r *http.Request) {
h := w.Header()
h.Set("Cache-Control", "public,max-age=3600")
h.Set("Content-Disposition", `attachment; filename="conanfile.py"`)
httputil.ReplyWith(w, http.StatusOK, "text/x-python", conanfilePy)
})
const conanmanifest = "%d\nconandata.yml: %s\nconanfile.py: %s\n"
mux.HandleFunc(revbase+"/files/conanmanifest.txt", func(w http.ResponseWriter, r *http.Request) {
mtime, err := unixTime(gr.PublishedAt)
if err != nil {
replyError(w, err)
return
}
h := w.Header()
h.Set("Cache-Control", "public,max-age=3600")
h.Set("Content-Disposition", `attachment; filename="conanmanifest.txt"`)
data := fmt.Sprintf(conanmanifest, mtime, md5Of(conandataYml), md5Of(conanfilePy))
httputil.ReplyWithStream(w, http.StatusOK, "text/plain", strings.NewReader(data), int64(len(data)))
})
mux.HandleFunc(revbase+"/files/conan_export.tgz", func(w http.ResponseWriter, r *http.Request) {
conanExportTgz, err := tgzOfConandata(outDir)
if err != nil {
replyError(w, err)
return
}
h := w.Header()
h.Set("Cache-Control", "public,max-age=3600")
h.Set("Content-Disposition", `attachment; filename="conan_export.tgz"`)
httputil.ReplyWith(w, http.StatusOK, "application/x-gzip", conanExportTgz)
})
})
}
func (p *Manager) outDir(pkg *Package) string {
return p.cacheDir + "/build/" + pkg.Name + "@" + pkg.Version
}
func (p *Manager) conanfileDir(pkgPath, pkgFolder string) string {
root := p.indexRoot()
return root + "/" + pkgPath + "/" + pkgFolder
}
func conanInstall(pkg, outDir, conanfileDir string, out io.Writer, flags int) (err error) {
args := make([]string, 0, 12)
args = append(args, "install",
"--requires", pkg,
"--generator", "PkgConfigDeps",
"--build", "missing",
"--format", "json",
"--output-folder", outDir,
)
quietInstall := flags&ToolQuietInstall != 0
cmd, err := conanCmd.New(quietInstall, args...)
if err != nil {
return
}
cmd.Dir = conanfileDir
cmd.Stderr = os.Stderr
cmd.Stdout = out
err = cmd.Run()
return
}
func recipeRevision(_ *Package, _ *githubRelease, conandataYml []byte) string {
return md5Of(conandataYml)
}
func md5Of(data []byte) string {
h := md5.New()
h.Write(data)
return hex.EncodeToString(h.Sum(nil))
}
func tgzOfConandata(outDir string) (_ []byte, err error) {
cmd := exec.Command("tar", "-czf", "conan_export.tgz", "conandata.yml")
cmd.Dir = outDir
err = cmd.Run()
if err != nil {
return
}
return os.ReadFile(outDir + "/conan_export.tgz")
}
func unixTime(tstr string) (ret int64, err error) {
t, err := time.Parse(time.RFC3339, tstr)
if err == nil {
ret = t.Unix()
}
return
}
func copyDirR(srcDir, destDir string) error {
if cp, err := exec.LookPath("cp"); err == nil {
return exec.Command(cp, "-r", "-p", srcDir+"/", destDir).Run()
}
if cp, err := exec.LookPath("xcopy"); err == nil {
// TODO(xsw): check xcopy
return exec.Command(cp, "/E", "/I", "/Y", srcDir+"/", destDir).Run()
}
return errors.New("copy command not found")
}

58
xtool/cppkg/cppkg.go Normal file
View File

@@ -0,0 +1,58 @@
/*
* Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cppkg
import (
"strings"
)
const (
// DefaultFlags is the default flags for package installation.
DefaultFlags = IndexAutoUpdate | ToolQuietInstall
)
// Main is the main entry point for the cppkg package.
// pkgAndVer: 7bitcoder/7bitconf@1.2.0
func Main(pkgAndVer string, flags int) {
pkgPath, ver := parsePkgVer(pkgAndVer)
if ver == "" {
panic("TODO: get latest version")
}
m, err := New("")
check(err)
pkg, err := m.Lookup(pkgPath, ver, flags)
check(err)
err = m.Install(pkg, flags)
check(err)
}
func parsePkgVer(pkg string) (string, string) {
parts := strings.SplitN(pkg, "@", 2)
if len(parts) == 1 {
return parts[0], ""
}
return parts[0], parts[1]
}
func check(err error) {
if err != nil {
panic(err)
}
}

160
xtool/cppkg/manager.go Normal file
View File

@@ -0,0 +1,160 @@
/*
* Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cppkg
import (
"errors"
"os"
"github.com/goccy/go-yaml"
"golang.org/x/mod/semver"
)
var gitCmd = NewTool("git", []string{
"brew install git",
"apt-get install git",
})
// Manager represents a package manager for C/C++ packages.
type Manager struct {
cacheDir string
}
func New(cacheDir string) (ret *Manager, err error) {
if cacheDir == "" {
cacheDir, err = os.UserCacheDir()
if err != nil {
return
}
cacheDir += "/cppkg"
}
os.MkdirAll(cacheDir, os.ModePerm)
ret = &Manager{
cacheDir: cacheDir,
}
return
}
type version struct {
Folder string `yaml:"folder"`
}
// Template represents a template for package versions.
type Template struct {
FromVer string `yaml:"from"`
Folder string `yaml:"folder"`
Tag string `yaml:"tag,omitempty"` // pattern with *, empty if dynamic tag
}
type config struct {
PkgName string `yaml:"name"`
Versions map[string]version `yaml:"versions"`
Template Template `yaml:"template"`
}
// Package represents a C/C++ package.
type Package struct {
Name string
Path string
Version string
Folder string
Template *Template
}
var (
// ErrVersionNotFound is returned when the specified version is not found.
ErrVersionNotFound = errors.New("version not found")
)
const (
// IndexAutoUpdate is a flag to automatically update the index.
IndexAutoUpdate = 1 << iota
// ToolQuietInstall is a flag to suppress output during installation.
ToolQuietInstall
// LogRevertProxy is a flag to log revert proxy.
LogRevertProxy
)
// Lookup looks up a package by its path and version.
func (p *Manager) Lookup(pkgPath, ver string, flags int) (_ *Package, err error) {
root := p.indexRoot()
err = indexUpate(root, flags)
if err != nil {
return
}
pkgDir := root + "/" + pkgPath
confFile := pkgDir + "/config.yml"
b, err := os.ReadFile(confFile)
if err != nil {
return
}
var conf config
err = yaml.Unmarshal(b, &conf)
if err != nil {
return
}
if v, ok := conf.Versions[ver]; ok {
return &Package{conf.PkgName, pkgPath, ver, v.Folder, nil}, nil
}
if compareVer(ver, conf.Template.FromVer) < 0 {
err = ErrVersionNotFound
return
}
folder := conf.Template.Folder
return &Package{conf.PkgName, pkgPath, ver, folder, &conf.Template}, nil
}
func (p *Manager) indexRoot() string {
return p.cacheDir + "/index"
}
func indexUpate(root string, flags int) (err error) {
if _, err = os.Stat(root + "/.git"); os.IsNotExist(err) {
os.RemoveAll(root)
return indexInit(root, flags)
}
if flags&IndexAutoUpdate != 0 {
quietInstall := flags&ToolQuietInstall != 0
git, e := gitCmd.New(quietInstall, "pull", "--ff-only", "origin", "main")
if e != nil {
return e
}
git.Dir = root
git.Stdout = os.Stdout
git.Stderr = os.Stderr
err = git.Run()
}
return
}
func indexInit(root string, flags int) (err error) {
quietInstall := flags&ToolQuietInstall != 0
git, err := gitCmd.New(quietInstall, "clone", "https://github.com/goplus/cppkg.git", root)
if err != nil {
return
}
git.Stdout = os.Stdout
git.Stderr = os.Stderr
err = git.Run()
return
}
func compareVer(v1, v2 string) int {
return semver.Compare("v"+v1, "v"+v2)
}

176
xtool/cppkg/revertproxy.go Normal file
View File

@@ -0,0 +1,176 @@
/*
* Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cppkg
import (
"bytes"
"encoding/json"
"io"
stdlog "log"
"net/http"
"net/http/httptest"
"net/http/httputil"
"net/url"
"os"
"os/exec"
"strings"
)
type rtHandler func(req *http.Request) (resp *http.Response, err error)
func (p rtHandler) RoundTrip(req *http.Request) (*http.Response, error) {
return p(req)
}
type teeReader struct {
rc io.ReadCloser
b bytes.Buffer
req *http.Request
resp *http.Response
log *stdlog.Logger
}
func (p *teeReader) Read(b []byte) (n int, err error) {
n, err = p.rc.Read(b)
p.b.Write(b[:n])
return
}
func (p *teeReader) Close() error {
err := p.rc.Close()
if log := p.log; log != nil {
resp := *p.resp
resp.Body = io.NopCloser(&p.b)
var b bytes.Buffer
p.req.Write(&b)
resp.Write(&b)
log.Print(b.String())
}
return err
}
type response = httptest.ResponseRecorder
func newResponse() *response {
return httptest.NewRecorder()
}
type revertProxy = httptest.Server
type rpFunc = func(mux *http.ServeMux)
const (
passThrough = http.StatusNotFound
)
func replyError(w http.ResponseWriter, _ error) {
w.WriteHeader(passThrough)
}
func startRevertProxy(endpoint string, f rpFunc, log *stdlog.Logger) (_ *revertProxy, err error) {
rpURL, err := url.Parse(endpoint)
if err != nil {
return
}
var mux *http.ServeMux
if f != nil {
mux = http.NewServeMux()
f(mux)
}
proxy := httptest.NewServer(&httputil.ReverseProxy{
Rewrite: func(r *httputil.ProxyRequest) {
r.SetURL(rpURL)
},
Transport: rtHandler(func(req *http.Request) (resp *http.Response, err error) {
if mux != nil {
w := newResponse()
mux.ServeHTTP(w, req)
if w.Code != passThrough {
resp = w.Result()
}
}
if resp == nil {
resp, err = http.DefaultTransport.RoundTrip(req)
}
if err == nil && resp.Body != nil {
resp.Body = &teeReader{
rc: resp.Body,
req: req,
resp: resp,
log: log,
}
}
return
}),
})
return proxy, nil
}
const (
conanCenter = "conancenter"
conanEndpoint = "https://center2.conan.io"
)
type remoteList []struct {
Name string `json:"name"`
URL string `json:"url"`
}
func remoteProxy(flags int, logFile string, f func() error, rpf rpFunc) (err error) {
quietInstall := flags&ToolQuietInstall != 0
app, err := conanCmd.Get(quietInstall)
if err != nil {
return
}
endpoint := conanEndpoint
cmd := exec.Command(app, "remote", "list", "-f", "json")
if b, err := cmd.Output(); err == nil {
var rl remoteList
if json.Unmarshal(b, &rl) == nil {
for _, r := range rl {
if r.Name == conanCenter && strings.HasPrefix(r.URL, "https://") {
endpoint = r.URL
break
}
}
}
}
defer func() {
exec.Command(app, "remote", "add", "--force", conanCenter, endpoint).Run()
}()
var log *stdlog.Logger
if logFile != "" {
f, err := os.Create(logFile)
if err == nil {
defer f.Close()
log = stdlog.New(f, "", stdlog.LstdFlags)
}
}
rp, err := startRevertProxy(conanEndpoint, rpf, log)
if err != nil {
return
}
defer rp.Close()
err = exec.Command(app, "remote", "add", "--force", conanCenter, rp.URL).Run()
if err != nil {
return
}
return f()
}