From 81e943f1c5b25b85adba989b83926662b9743ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3=20=D0=91=D0=BE=D1=80=D0=BE=D0=B4?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Fri, 6 Mar 2026 19:13:29 +0200 Subject: [PATCH] working commit --- pkg/auxutar/utar.go | 72 ++++++++++++----- pkg/digest/digest.go | 147 ++++++++++++++++++++++++++++++++++ pkg/repocli/attic/authbas.go | 20 ----- pkg/repocli/imager.go | 14 +++- pkg/repocli/imager_test.go | 2 +- pkg/repocli/pathupload.go | 45 ----------- pkg/repocli/pullimage.go | 51 +++++++++--- pkg/repocli/pullimage_test.go | 4 +- pkg/rpccli/client.go | 131 ++++++++++++++++++++++++++++++ pkg/rpccli/repo.go | 56 +++++++++++++ 10 files changed, 438 insertions(+), 104 deletions(-) create mode 100644 pkg/digest/digest.go delete mode 100644 pkg/repocli/attic/authbas.go delete mode 100644 pkg/repocli/pathupload.go create mode 100644 pkg/rpccli/client.go create mode 100644 pkg/rpccli/repo.go diff --git a/pkg/auxutar/utar.go b/pkg/auxutar/utar.go index 4ac905f..689adb9 100644 --- a/pkg/auxutar/utar.go +++ b/pkg/auxutar/utar.go @@ -15,16 +15,15 @@ import ( "os" "path/filepath" "strings" + "time" ) -// TODO: file and dir modes - func Archive(srcdir, dstpath string) error { var err error srcdir = filepath.Clean(srcdir) dstpath = filepath.Clean(dstpath) - err = os.MkdirAll(filepath.Dir(dstpath), 0755) + err = os.MkdirAll(filepath.Dir(dstpath), 0750) if err != nil { return err } @@ -50,17 +49,23 @@ func Archive(srcdir, dstpath string) error { } header.Name = strings.TrimPrefix(filename, filepath.Clean(srcdir)) header.Name = strings.TrimPrefix(header.Name, string(filepath.Separator)) - err = tarWriter.WriteHeader(header) if err != nil { return err } - file, err := os.Open(filename) - if err != nil { + wrapfunc := func() error { + file, err := os.Open(filename) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(tarWriter, file) + if err != nil { + return err + } return err } - defer file.Close() - _, err = io.Copy(tarWriter, file) + err = wrapfunc() if err != nil { return err } @@ -98,31 +103,54 @@ func Unarchive(filename, dstdir string) error { } target := filepath.Join(dstdir, header.Name) target = filepath.Clean(target) - //fileInfo := header.FileInfo() + + fileInfo := header.FileInfo() + switch header.Typeflag { case tar.TypeDir: _, err := os.Stat(target) - if err != nil { - err := os.MkdirAll(target, 0755) + if err != nil && err == os.ErrNotExist { + err := os.MkdirAll(target, fileInfo.Mode()) if err != nil { return err } - + } + if err != nil { + return err } case tar.TypeReg: - err := os.MkdirAll(filepath.Dir(target), 0755) + wrapfunc := func() error { + dir := filepath.Dir(target) + _, err := os.Stat(dir) + if err != nil && err == os.ErrNotExist { + err := os.MkdirAll(dir, 0750) + if err != nil { + return err + } + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fileInfo.Mode()) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(file, tarReader) + if err != nil { + return err + } + return err + } + err = wrapfunc() if err != nil { return err } - file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode)) - if err != nil { - return err - } - _, err = io.Copy(file, tarReader) - if err != nil { - return err - } - file.Close() + } + err = os.Chtimes(target, time.Now(), fileInfo.ModTime()) + if err != nil { + return err + } + err = file.Chmod(fileInfo.Mode()) + if err != nil { + return err } } return err diff --git a/pkg/digest/digest.go b/pkg/digest/digest.go new file mode 100644 index 0000000..686cf3c --- /dev/null +++ b/pkg/digest/digest.go @@ -0,0 +1,147 @@ +package digest + +import ( + "crypto/sha256" + "crypto/sha512" + "hash" + + "encoding/hex" + "errors" + "strings" +) + +type Algorithm int + +const ( + Undefined Algorithm = iota + SHA256 + SHA384 + SHA512 +) + +type Digest struct { + algorithm Algorithm + decoded []byte +} + +func NewDigest(algorithm Algorithm, payload []byte) *Digest { + var sum []byte + var hasher hash.Hash + switch algorithm { + case SHA512: + hasher = sha512.New() + case SHA384: + hasher = sha512.New384() + default: + hasher = sha256.New() + } + hasher.Write(payload) + sum = hasher.Sum(nil) + digest := &Digest{ + algorithm: algorithm, + decoded: sum, + } + return digest +} + +func ParseDigest(str string) (*Digest, error) { + var err error + digest := &Digest{} + + str = strings.ToLower(str) + str = strings.TrimPrefix(str, "sha256:") + str = strings.TrimPrefix(str, "sha384:") + str = strings.TrimPrefix(str, "sha512:") + + decoded, err := hex.DecodeString(str) + if err != nil { + err := errors.New("Can't decode digest") + return digest, err + } + digest.decoded = decoded + switch len(decoded) { + case 32: + digest.algorithm = SHA256 + case 48: + digest.algorithm = SHA384 + case 64: + digest.algorithm = SHA512 + default: + err = errors.New("Unknown digest type") + return digest, err + } + return digest, err +} + +func (dig *Digest) Encoded() string { + return hex.EncodeToString(dig.decoded) +} + +func (dig *Digest) Algorithm() Algorithm { + return dig.algorithm +} + +func (dig *Digest) Prefix() string { + var prefix string + switch dig.algorithm { + case SHA256: + prefix = "sha256" + case SHA384: + prefix = "sha384" + case SHA512: + prefix = "sha512" + } + return prefix +} + +func (dig *Digest) String() string { + var prefix string + switch dig.algorithm { + case SHA256: + prefix = "sha256" + case SHA384: + prefix = "sha384" + case SHA512: + prefix = "sha512" + } + hexx := hex.EncodeToString(dig.decoded) + return prefix + ":" + hexx +} + +func xxxDigestType(digest string) Algorithm { + var err error + var algorithm Algorithm + digest = strings.ToLower(digest) + digest = strings.TrimPrefix(digest, "sha256:") + digest = strings.TrimPrefix(digest, "sha384:") + digest = strings.TrimPrefix(digest, "sha512:") + decoded, err := hex.DecodeString(digest) + if err != nil { + return Undefined + } + switch len(decoded) { + case 32: + algorithm = SHA256 + case 48: + algorithm = SHA384 + case 64: + algorithm = SHA512 + default: + algorithm = Undefined + } + return algorithm +} + +func xxxSHA256Digest(src []byte) string { + hasher := sha256.New() + hasher.Write(src) + sum := hasher.Sum(nil) + return "sha256:" + hex.EncodeToString(sum) +} + +func xxxSHA512Digest(src []byte) string { + hasher := sha512.New() + hasher.Write(src) + sum := hasher.Sum(nil) + return "sha512:" + hex.EncodeToString(sum) +} diff --git a/pkg/repocli/attic/authbas.go b/pkg/repocli/attic/authbas.go deleted file mode 100644 index 3bba0a0..0000000 --- a/pkg/repocli/attic/authbas.go +++ /dev/null @@ -1,20 +0,0 @@ -package repocli - -import ( - "encoding/base64" -) - -type xxxAuthenticator interface { - MakeHeader(user, pass string) (key, value string, err error) -} - -type BasicAuthenticator struct{} - -func xxxNewBasicAuthenticator() *BasicAuthenticator { - return &BasicAuthenticator{} -} - -func (auth *BasicAuthenticator) MakeHeader(user, pass string) (string, string, error) { - pair := base64.StdEncoding.EncodeToString([]byte(user + ":" + pass)) - return "Authorization", "Basic " + pair, nil -} diff --git a/pkg/repocli/imager.go b/pkg/repocli/imager.go index 69ffac6..9519a51 100644 --- a/pkg/repocli/imager.go +++ b/pkg/repocli/imager.go @@ -130,6 +130,18 @@ func (ima *Imager) ReadManifest(ctx context.Context, digstr string) (bool, strin } func (ima *Imager) WriteManifest(ctx context.Context, digest, mime string, payload []byte) error { + err := ima.writeManifest(ctx, digest, mime, payload) + if err != nil { + return err + } + err = ima.writeIndex(ctx) + if err != nil { + return err + } + return err +} + +func (ima *Imager) writeManifest(ctx context.Context, digest, mime string, payload []byte) error { var err error digobj, err := ocidigest.Parse(digest) if err != nil { @@ -169,7 +181,7 @@ func (ima *Imager) WriteManifest(ctx context.Context, digest, mime string, paylo return err } -func (ima *Imager) WriteIndex(ctx context.Context) error { +func (ima *Imager) writeIndex(ctx context.Context) error { indexdat, err := json.Marshal(ima.index) if err != nil { return err diff --git a/pkg/repocli/imager_test.go b/pkg/repocli/imager_test.go index 3bd6761..ae643fa 100644 --- a/pkg/repocli/imager_test.go +++ b/pkg/repocli/imager_test.go @@ -34,7 +34,7 @@ func TestDigest(t *testing.T) { err := imager.WriteLayer(ctx, digest, size, reader) require.NoError(t, err) - err = imager.WriteIndex(ctx) + err = imager.writeIndex(ctx) require.NoError(t, err) } diff --git a/pkg/repocli/pathupload.go b/pkg/repocli/pathupload.go deleted file mode 100644 index f1bc1b6..0000000 --- a/pkg/repocli/pathupload.go +++ /dev/null @@ -1,45 +0,0 @@ -package repocli - -import ( - "context" - "fmt" - "io" - "net/http" - "strconv" -) - -func (cli *Client) PatchUpload(ctx context.Context, rawrepo string, src io.Reader, uploc string, size int64) (string, error) { - var err error - var ouloc string - - ref, err := NewRepository(rawrepo) - if err != nil { - return ouloc, err - } - uri, err := ref.Patch(uploc) - if err != nil { - return ouloc, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodPatch, uri, src) - if err != nil { - return ouloc, err - } - req.Header.Set("User-Agent", cli.userAgent) - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) - resp, err := cli.httpClient.Do(req) - if err != nil { - return ouloc, err - } - resp.Body.Close() - if resp.StatusCode != http.StatusAccepted { - err = fmt.Errorf("Upload not accepted, code %d", resp.StatusCode) - return ouloc, err - } - ouloc = resp.Header.Get("Location") - if ouloc == "" { - err := fmt.Errorf("Empty blob location declaration") - return ouloc, err - } - return ouloc, err -} diff --git a/pkg/repocli/pullimage.go b/pkg/repocli/pullimage.go index 6097ce3..389b100 100644 --- a/pkg/repocli/pullimage.go +++ b/pkg/repocli/pullimage.go @@ -5,7 +5,7 @@ import ( "encoding/json" "errors" "fmt" - "io" + "os" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -28,7 +28,7 @@ func NewDownloader(client *Client) *Downloader { } } -func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string) error { +func (down *Downloader) Pull(ctx context.Context, rawref, dir, osname, arch string) error { var err error ref, err := ParseReference(rawref) if err != nil { @@ -46,6 +46,7 @@ func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string) return err } + var digstr string if mime == MediaTypeOIIv1 || mime == MediaTypeDDMLv2 { var index ocispec.Index err = json.Unmarshal(man, &index) @@ -55,9 +56,10 @@ func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string) for _, descr := range index.Manifests { if descr.Platform != nil { cond := descr.Platform.Architecture == arch - cond = cond && descr.Platform.OS == os + cond = cond && descr.Platform.OS == osname if cond { tag = descr.Digest.String() + digstr = descr.Digest.String() } } } @@ -81,24 +83,47 @@ func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string) if err != nil { return err } - //"oci-layout" - //"index.json" + // Write image manifest + imager := NewEmptyImager(dir) + err = imager.WriteManifest(ctx, digstr, mime, man) + if err != nil { + return err + } layers := make([]ocispec.Descriptor, 0) layers = append(layers, manifest.Config) layers = append(layers, manifest.Layers...) for _, layer := range layers { - digest := layer.Digest.String() - exist, err := down.cli.GetBlob(ctx, rawrepo, io.Discard, digest) + wrapfunc := func() error { + tmpfile, err := os.CreateTemp(dir, "*.bin") + if err != nil { + return err + } + defer tmpfile.Close() + tmpname := tmpfile.Name() + defer os.Remove(tmpname) + + digstr := layer.Digest.String() + exist, err := down.cli.GetBlob(ctx, rawrepo, tmpfile, digstr) + if err != nil { + return err + } + if !exist { + err = errors.New("Layer not found") + return err + } + tmpfile.Seek(0, 0) + size := layer.Size + err = imager.WriteLayer(ctx, digstr, size, tmpfile) + if err != nil { + return err + } + return err + } + err = wrapfunc() if err != nil { return err } - if !exist { - err = errors.New("Layer not found") - return err - } - fmt.Printf("Layer type: %s\n", layer.MediaType) - } return err } diff --git a/pkg/repocli/pullimage_test.go b/pkg/repocli/pullimage_test.go index 948fcc3..31e4bd2 100644 --- a/pkg/repocli/pullimage_test.go +++ b/pkg/repocli/pullimage_test.go @@ -9,7 +9,7 @@ import ( "time" ) -func xxxTestPullImage(t *testing.T) { +func TestPullImage(t *testing.T) { ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) cli := NewClientWithTransport(nil, nil) @@ -19,7 +19,7 @@ func xxxTestPullImage(t *testing.T) { require.NotNil(t, down) rawref := "mirror.gcr.io/alpine:3.20.0" - destdir := "qwert" + destdir := "aaa" err := down.Pull(ctx, rawref, destdir, "linux", "amd64") require.NoError(t, err) } diff --git a/pkg/rpccli/client.go b/pkg/rpccli/client.go new file mode 100644 index 0000000..a054b00 --- /dev/null +++ b/pkg/rpccli/client.go @@ -0,0 +1,131 @@ +package rpccli + +import ( + "crypto/tls" + "encoding/base64" + "net/http" +) + +type Client struct { + httpClient *http.Client + userAgent string +} + +func NewClient(transport http.RoundTripper, mwFuncs ...MiddlewareFunc) *Client { + if transport == nil { + transport = NewDefaultTransport() + } + for _, mwFunc := range mwFuncs { + transport = mwFunc(transport) + } + httpClient := &http.Client{ + Transport: transport, + } + + return &Client{ + httpClient: httpClient, + userAgent: "ociClient/1.0", + } +} + +func (cli *Client) SetTransport(transport http.RoundTripper) { + cli.httpClient.Transport = transport +} + +type MiddlewareFunc func(next http.RoundTripper) http.RoundTripper + +func (cli *Client) UseMiddleware(mwFunc MiddlewareFunc) { + cli.httpClient.Transport = mwFunc(cli.httpClient.Transport) +} + +// BasicAuthMiddleware +func NewBasicAuthMiddleware(user, pass string) MiddlewareFunc { + return func(next http.RoundTripper) http.RoundTripper { + return newBasicAuthMW(next, user, pass) + } +} + +type BasicAuthMW struct { + user, pass string + next http.RoundTripper +} + +func newBasicAuthMW(next http.RoundTripper, user, pass string) *BasicAuthMW { + return &BasicAuthMW{ + user: user, + pass: pass, + next: next, + } +} + +func (tran BasicAuthMW) RoundTrip(req *http.Request) (*http.Response, error) { + if tran.user != "" && tran.pass != "" { + pair := base64.StdEncoding.EncodeToString([]byte(tran.user + ":" + tran.pass)) + req.Header.Set("Authorization", "Basic "+pair) + } + return tran.next.RoundTrip(req) +} + +// BearerAuthMiddleware +func NewBearerAuthMiddleware(token string) MiddlewareFunc { + return func(next http.RoundTripper) http.RoundTripper { + return newBearerAuthMW(next, token) + } +} + +type BearerAuthMW struct { + token string + next http.RoundTripper +} + +func newBearerAuthMW(next http.RoundTripper, token string) *BearerAuthMW { + return &BearerAuthMW{ + token: token, + next: next, + } +} + +func (tran BearerAuthMW) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("Authorization", "Bearer "+tran.token) + return tran.next.RoundTrip(req) +} + +// DefaultTransport +type DefaultTransport struct { + transport http.RoundTripper +} + +func NewDefaultTransport() *DefaultTransport { + return &DefaultTransport{ + transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } +} + +func (wrap *DefaultTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return wrap.transport.RoundTrip(req) +} + +// ExampleMiddleware +func NewExampleMiddleware() MiddlewareFunc { + return func(next http.RoundTripper) http.RoundTripper { + return newExampleTransport(next) + } +} + +type ExampleTransport struct { + next http.RoundTripper +} + +func newExampleTransport(next http.RoundTripper) *ExampleTransport { + return &ExampleTransport{ + next: next, + } +} + +func (tran ExampleTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return tran.next.RoundTrip(req) +} diff --git a/pkg/rpccli/repo.go b/pkg/rpccli/repo.go new file mode 100644 index 0000000..9d29906 --- /dev/null +++ b/pkg/rpccli/repo.go @@ -0,0 +1,56 @@ +package rpccli + +import ( + "net/url" + "path" + "strconv" + "strings" +) + +type Referer struct { + urlobj *url.URL + user, pass string + obj, oper string +} + +func NewReferer(hostname, object, operation string) (*Referer, error) { + repo := &Referer{ + obj: object, + oper: operation, + } + if !strings.Contains(hostname, "://") { + hostname = "https://" + hostname + } + urlobj, err := url.Parse(hostname) + if err != nil { + return repo, err + } + if urlobj.User != nil { + repo.user = urlobj.User.Username() + repo.pass, _ = urlobj.User.Password() + urlobj.User = nil + } + repo.resource = path.Join("/", urlobj.Path) + urlobj.Path = "/" + repo.urlobj = urlobj + return repo, err +} + +func (repo *Referer) Raw() string { + return path.Join(repo.urlobj.Host, "/v3/api/", repo.obj, repo.oper) +} + +func (repo *Referer) URI(object, operation) string { + curl := repo.urlobj.JoinPath("/v3/api/", object, operation) + return curl.String() +} + +func (repo *Referer) Userinfo() (string, string) { + return repo.user, repo.pass +} + +func (repo *Referer) SetUserinfo(user, pass string) { + if user != "" && pass != "" { + repo.user, repo.pass = user, pass + } +}