repocli: first version of pull & push image

This commit is contained in:
2026-03-10 21:19:19 +02:00
parent f85cda868a
commit 9bc3b77144
13 changed files with 61 additions and 90 deletions
+1 -1
View File
@@ -52,7 +52,7 @@ func (hand *Handler) CheckAccess(rctx *router.Context) (bool, string, error) {
accountID = terms.AnonymousID accountID = terms.AnonymousID
hand.logg.Debugf("URL: %s", rctx.URL().String()) //hand.logg.Debugf("URL: %s", rctx.URL().String())
authHeader := rctx.GetHeader("Authorization") authHeader := rctx.GetHeader("Authorization")
hand.logg.Debugf("Authorization: %s", authHeader) hand.logg.Debugf("Authorization: %s", authHeader)
if authHeader != "" { if authHeader != "" {
+1 -1
View File
@@ -62,7 +62,7 @@ func (hand *Handler) BlobExists(rctx *router.Context) {
func (hand *Handler) PostUpload(rctx *router.Context) { func (hand *Handler) PostUpload(rctx *router.Context) {
name, _ := rctx.GetSubpath("name") name, _ := rctx.GetSubpath("name")
hand.DumpHeaders("PostUploads", rctx) //hand.DumpHeaders("PostUploads", rctx)
authorization := rctx.GetHeader("Authorization") authorization := rctx.GetHeader("Authorization")
if authorization == "" { if authorization == "" {
+6
View File
@@ -88,6 +88,7 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
} }
incomingManifestBytes := buffer.Bytes() incomingManifestBytes := buffer.Bytes()
if int64(len(incomingManifestBytes)) != contentLength { if int64(len(incomingManifestBytes)) != contentLength {
err = fmt.Errorf("Mismatch Content-Length and received manifest size: %d vs %d", err = fmt.Errorf("Mismatch Content-Length and received manifest size: %d vs %d",
contentLength, len(incomingManifestBytes)) contentLength, len(incomingManifestBytes))
@@ -163,11 +164,14 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
if err != nil { if err != nil {
return res, http.StatusInternalServerError, err return res, http.StatusInternalServerError, err
} }
// Starting manifest and blobs transaction // Starting manifest and blobs transaction
err = oper.mdb.UpdateManifestWithBlobs(ctx, &incomingManifestDescr, addedBlobDescrs, uselessBlobDescrs) err = oper.mdb.UpdateManifestWithBlobs(ctx, &incomingManifestDescr, addedBlobDescrs, uselessBlobDescrs)
if err != nil { if err != nil {
return res, http.StatusInternalServerError, err return res, http.StatusInternalServerError, err
} }
//goto end
for _, blob := range uselessBlobDescrs { for _, blob := range uselessBlobDescrs {
exists, _, err := oper.store.BlobExists(blob.Digest) exists, _, err := oper.store.BlobExists(blob.Digest)
if err != nil { if err != nil {
@@ -201,6 +205,8 @@ func (oper *Operator) PutManifest(ctx context.Context, params *PutManifestParams
} }
} }
//end:
res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference) res.Location = fmt.Sprintf(`/v2/%s/manifests/%s`, params.Name, params.Reference)
return res, http.StatusCreated, err return res, http.StatusCreated, err
} }
+2 -1
View File
@@ -129,7 +129,8 @@ func (db *Database) UpdateManifestWithBlobs(ctx context.Context, manifest *descr
// Manifest // Manifest
request = `UPDATE manifests SET contentType = $1, payload = $2, digest = $3, updated_at = $4, updated_by = $5 request = `UPDATE manifests SET contentType = $1, payload = $2, digest = $3, updated_at = $4, updated_by = $5
WHERE name = $6 AND reference = $7, architecture = $8, os = $9, variant = $10` WHERE name = $6 AND reference = $7
AND architecture = $8 AND os = $9 AND variant = $10`
_, err = tx.Exec(request, manifest.ContentType, manifest.Payload, manifest.Digest, _, err = tx.Exec(request, manifest.ContentType, manifest.Payload, manifest.Digest,
manifest.UpdatedAt, manifest.UpdatedBy, manifest.UpdatedAt, manifest.UpdatedBy,
manifest.Name, manifest.Reference, manifest.Name, manifest.Reference,
+1 -1
View File
@@ -97,7 +97,7 @@ func (svc *Service) Build() error {
svc.rout.Post(`/v2/{name}/blobs/uploads/`, svc.hand.PostUpload) svc.rout.Post(`/v2/{name}/blobs/uploads/`, svc.hand.PostUpload)
svc.rout.Patch(`/v2/{name}/blobs/uploads/{reference}`, svc.hand.PatchUpload) svc.rout.Patch(`/v2/{name}/blobs/uploads/{reference}`, svc.hand.PatchUpload)
svc.rout.Put(`/v2/{name}/uploads/{reference}`, svc.hand.PutUpload) svc.rout.Put(`/v2/{name}/blobs/uploads/{reference}`, svc.hand.PutUpload)
svc.rout.Get(`/v2/{name}/blobs/{digest}`, svc.hand.GetBlob) svc.rout.Get(`/v2/{name}/blobs/{digest}`, svc.hand.GetBlob)
svc.rout.Delete(`/v2/{name}/blobs/{digest}`, svc.hand.DeleteBlob) svc.rout.Delete(`/v2/{name}/blobs/{digest}`, svc.hand.DeleteBlob)
-37
View File
@@ -1,37 +0,0 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package client
import (
"net/url"
"path"
"strings"
)
func repackServiceURI(fileuri string) (string, string, string, error) {
var err error
var res, username, password string
if !strings.Contains(fileuri, "://") {
fileuri = "https://" + fileuri
}
uri, err := url.Parse(fileuri)
if err != nil {
return res, username, password, err
}
uri.Path = path.Clean(uri.Path)
if uri.User != nil {
username = uri.User.Username()
password, _ = uri.User.Password()
}
uri.User = nil
//uri.Scheme = "https"
res = uri.String()
return res, username, password, err
}
+16 -15
View File
@@ -13,61 +13,62 @@ import (
ocidigest "github.com/opencontainers/go-digest" ocidigest "github.com/opencontainers/go-digest"
) )
func (cli *Client) GetRawManifest(ctx context.Context, rawrepo string) (bool, string, []byte, error) { func (cli *Client) GetRawManifest(ctx context.Context, rawrepo string) (bool, string, []byte, string, error) {
var err error var err error
var exist bool var exist bool
var mime string var mime string
var man []byte var man []byte
var digstr string
fmt.Printf("=== %s\n", rawrepo) fmt.Printf("=== %s\n", rawrepo)
ref, err := NewReferer(rawrepo) ref, err := NewReferer(rawrepo)
if err != nil { if err != nil {
return exist, mime, man, err return exist, mime, man, digstr, err
} }
uri := ref.ManifestEP() uri := ref.ManifestEP()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
if err != nil { if err != nil {
return exist, mime, man, err return exist, mime, man, digstr, err
} }
req.Header.Set("User-Agent", cli.userAgent) req.Header.Set("User-Agent", cli.userAgent)
req.Header.Set("Accept", "*/*") req.Header.Set("Accept", "*/*")
resp, err := cli.httpClient.Do(req) resp, err := cli.httpClient.Do(req)
if err != nil { if err != nil {
return exist, mime, man, err return exist, mime, man, digstr, err
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound { if resp.StatusCode == http.StatusNotFound {
return exist, mime, man, err return exist, mime, man, digstr, err
} }
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
err := fmt.Errorf("Unxected response code %s", resp.Status) err := fmt.Errorf("Unxected response code %s", resp.Status)
return exist, mime, man, err return exist, mime, man, digstr, err
} }
digstr := resp.Header.Get("Docker-Content-Digest") digstr = resp.Header.Get("Docker-Content-Digest")
if digstr == "" { if digstr == "" {
err := fmt.Errorf("Empty digest declaration") err := fmt.Errorf("Empty digest declaration")
return exist, mime, man, err return exist, mime, man, digstr, err
} }
contentLength := resp.Header.Get("Content-Length") contentLength := resp.Header.Get("Content-Length")
if contentLength == "" { if contentLength == "" {
err = errors.New("Empty Content-Length header") err = errors.New("Empty Content-Length header")
return exist, mime, man, err return exist, mime, man, digstr, err
} }
manSize, err := strconv.ParseInt(contentLength, 10, 64) manSize, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil { if err != nil {
return exist, mime, man, err return exist, mime, man, digstr, err
} }
mime = resp.Header.Get("Content-Type") mime = resp.Header.Get("Content-Type")
if mime == "" { if mime == "" {
err := fmt.Errorf("Empty MIME type declaration") err := fmt.Errorf("Empty MIME type declaration")
return exist, mime, man, err return exist, mime, man, digstr, err
} }
digstr = strings.ToLower(digstr) digstr = strings.ToLower(digstr)
digobj, err := ocidigest.Parse(digstr) digobj, err := ocidigest.Parse(digstr)
if err != nil { if err != nil {
return exist, mime, man, err return exist, mime, man, digstr, err
} }
verifier := digobj.Verifier() verifier := digobj.Verifier()
buffer := bytes.NewBuffer(nil) buffer := bytes.NewBuffer(nil)
@@ -75,13 +76,13 @@ func (cli *Client) GetRawManifest(ctx context.Context, rawrepo string) (bool, st
recSize, err := Copy(ctx, mwriter, resp.Body) recSize, err := Copy(ctx, mwriter, resp.Body)
if manSize != recSize { if manSize != recSize {
err := fmt.Errorf("Mismatch declared and actual size") err := fmt.Errorf("Mismatch declared and actual size")
return exist, mime, man, err return exist, mime, man, digstr, err
} }
man = buffer.Bytes() man = buffer.Bytes()
if !verifier.Verified() { if !verifier.Verified() {
err := fmt.Errorf("Mismatch digest declaration and actual") err := fmt.Errorf("Mismatch digest declaration and actual")
return exist, mime, man, err return exist, mime, man, digstr, err
} }
exist = true exist = true
return exist, mime, man, err return exist, mime, man, digstr, err
} }
+3 -2
View File
@@ -11,7 +11,7 @@ import (
"time" "time"
) )
func xxxTestClientGetManifest(t *testing.T) { func TestClientGetManifest(t *testing.T) {
rawrepo := "mirror.gcr.io/alpine" rawrepo := "mirror.gcr.io/alpine"
tags := []string{ tags := []string{
"3.20.0", "3.20.0",
@@ -20,11 +20,12 @@ func xxxTestClientGetManifest(t *testing.T) {
for _, tag := range tags { for _, tag := range tags {
cli := NewClient(nil, nil) cli := NewClient(nil, nil)
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
exist, mime, man, err := cli.GetRawManifest(ctx, rawrepo+":"+tag) exist, mime, man, digstr, err := cli.GetRawManifest(ctx, rawrepo+":"+tag)
require.NoError(t, err) require.NoError(t, err)
require.True(t, exist) require.True(t, exist)
fmt.Printf("Type: %s\n", mime) fmt.Printf("Type: %s\n", mime)
fmt.Printf("Digest: %s\n", digstr)
buffer := bytes.NewBuffer(nil) buffer := bytes.NewBuffer(nil)
err = json.Indent(buffer, man, " ", " ") err = json.Indent(buffer, man, " ", " ")
require.NoError(t, err) require.NoError(t, err)
+17 -21
View File
@@ -4,7 +4,6 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"os" "os"
ocispec "github.com/opencontainers/image-spec/specs-go/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -35,9 +34,8 @@ func (down *Loader) Pull(ctx context.Context, rawref, dir, osname, arch string)
return err return err
} }
rawrepo := ref.RawRepo() rawrepo := ref.RawRepo()
tag := ref.Tag()
exist, mime, man, err := down.cli.GetRawManifest(ctx, ref.Raw()) exist, mime, man, digstr, err := down.cli.GetRawManifest(ctx, ref.Raw())
if err != nil { if err != nil {
return err return err
} }
@@ -45,8 +43,7 @@ func (down *Loader) Pull(ctx context.Context, rawref, dir, osname, arch string)
err = errors.New("Manifest not found") err = errors.New("Manifest not found")
return err return err
} }
tag := ref.Tag()
var digstr string
if mime == MediaTypeOIIv1 || mime == MediaTypeDDMLv2 { if mime == MediaTypeOIIv1 || mime == MediaTypeDDMLv2 {
var index ocispec.Index var index ocispec.Index
err = json.Unmarshal(man, &index) err = json.Unmarshal(man, &index)
@@ -63,25 +60,24 @@ func (down *Loader) Pull(ctx context.Context, rawref, dir, osname, arch string)
} }
} }
} }
} if tag == "" {
if digstr == "" { err = errors.New("Manifest for required arch and OS not found")
err = errors.New("Manifest for required arch and OS not found") return err
return err }
}
fmt.Printf("Tag: %s\n", tag) exist, mime, man, digstr, err = down.cli.GetRawManifest(ctx, ref.RawWithTag(tag))
exist, mime, man, err = down.cli.GetRawManifest(ctx, ref.RawWithTag(digstr)) if err != nil {
if err != nil { return err
return err }
} if !exist {
fmt.Printf("Mime: %s\n", mime) err = errors.New("Manifest not found")
if !exist { return err
err = errors.New("Manifest not found") }
return err } else if mime != MediaTypeOIMv1 && mime != MediaTypeDDMv2 {
}
if mime != MediaTypeOIMv1 && mime != MediaTypeDDMv2 {
err = errors.New("Unknown manifest media type") err = errors.New("Unknown manifest media type")
return err return err
} }
var manifest ocispec.Manifest var manifest ocispec.Manifest
err = json.Unmarshal(man, &manifest) err = json.Unmarshal(man, &manifest)
if err != nil { if err != nil {
+1
View File
@@ -22,4 +22,5 @@ func xxxTestPullImage(t *testing.T) {
destdir := "aaa" destdir := "aaa"
err := down.Pull(ctx, rawref, destdir, "linux", "amd64") err := down.Pull(ctx, rawref, destdir, "linux", "amd64")
require.NoError(t, err) require.NoError(t, err)
} }
+5 -3
View File
@@ -21,7 +21,7 @@ func (down *Loader) Push(ctx context.Context, rawref, dir, osname, arch string)
index := imager.Index() index := imager.Index()
for _, descr := range index.Manifests { for _, descr := range index.Manifests {
digstr := descr.Digest.String() digstr := descr.Digest.String()
mime := descr.MediaType
_, _, mandata, err := imager.ReadManifest(ctx, digstr) _, _, mandata, err := imager.ReadManifest(ctx, digstr)
if err != nil { if err != nil {
return err return err
@@ -54,8 +54,10 @@ func (down *Loader) Push(ctx context.Context, rawref, dir, osname, arch string)
return err return err
} }
} }
err = down.cli.PutManifest(ctx, ref.Raw(), mandata, mime)
if err != nil {
return err
}
} }
return err return err
} }
+5 -5
View File
@@ -9,17 +9,17 @@ import (
ocidigest "github.com/opencontainers/go-digest" ocidigest "github.com/opencontainers/go-digest"
) )
func (cli *Client) PutManifest(ctx context.Context, rawrepo, tag string, man []byte, mime string) error { func (cli *Client) PutManifest(ctx context.Context, rawref string, man []byte, mime string) error {
var err error var err error
ref, err := NewReferer(rawrepo) ref, err := NewReferer(rawref)
if err != nil { if err != nil {
return err return err
} }
uri := ref.ManifestEP() uri := ref.ManifestEP()
buffer := bytes.NewBuffer(man) reader := bytes.NewReader(man)
req, err := http.NewRequestWithContext(ctx, http.MethodPut, uri, buffer) req, err := http.NewRequestWithContext(ctx, http.MethodPut, uri, reader)
if err != nil { if err != nil {
return err return err
} }
@@ -33,7 +33,7 @@ func (cli *Client) PutManifest(ctx context.Context, rawrepo, tag string, man []b
return err return err
} }
resp.Body.Close() resp.Body.Close()
if resp.StatusCode != http.StatusAccepted { if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusCreated {
err = fmt.Errorf("Manifest not accepted, code %d", resp.StatusCode) err = fmt.Errorf("Manifest not accepted, code %d", resp.StatusCode)
return err return err
} }
+3 -3
View File
@@ -11,9 +11,9 @@
package terms package terms
const ( const (
AsFinePath string = "asFinePath" xxxAsFinePath string = "asFinePath"
AsPrefix string = "asPrefix" xxxAsPrefix string = "asPrefix"
AsRegexp string = "asRegexp" xxxAsRegexp string = "asRegexp"
) )
const ( const (