diff --git a/app/handler/blob.go b/app/handler/blob.go index 268fcfc..6dbb7e4 100644 --- a/app/handler/blob.go +++ b/app/handler/blob.go @@ -3,10 +3,17 @@ package handler import ( "mstore/app/operator" "mstore/app/router" + + "sigs.k8s.io/yaml" ) -// HEAD /v2//blobs/ 200 404 +func (hand *Handler) DumpHeaders(message string, rctx *router.Context) { + headers := rctx.GetHeaders() + yamlData, _ := yaml.Marshal(headers) + hand.logg.Debugf("%s:\n%s\n", message, string(yamlData)) +} +// 1 HEAD /v2//blobs/ 200 404 func (hand *Handler) BlobExists(rctx *router.Context) { name, _ := rctx.GetSubpath("name") digest, _ := rctx.GetSubpath("digest") @@ -27,7 +34,7 @@ func (hand *Handler) BlobExists(rctx *router.Context) { rctx.SetStatus(code) } -// POST /v2//blobs/uploads/ 202 404 +// 2 POST /v2//blobs/uploads/ 202 404 func (hand *Handler) PostUpload(rctx *router.Context) { name, _ := rctx.GetSubpath("name") digest := rctx.GetQuery("digest") @@ -56,9 +63,11 @@ func (hand *Handler) PostUpload(rctx *router.Context) { // POST /v2//blobs/uploads/?digest= 201/202 404/400 // POST /v2//blobs/uploads/?mount=&from= 201 404 -// PATCH /v2//blobs/uploads/ 202 404/416 +// 3 PATCH /v2//blobs/uploads/ 202 404/416 func (hand *Handler) PatchUpload(rctx *router.Context) { + hand.DumpHeaders("PatchUpload headers", rctx) + contentLength := rctx.GetHeader("Content-Length") contentType := rctx.GetHeader("Content-Type") name, _ := rctx.GetSubpath("name") @@ -80,3 +89,35 @@ func (hand *Handler) PatchUpload(rctx *router.Context) { rctx.SetHeader("Location", res.Location) rctx.SetStatus(code) } + +// 4 PUT /v2//blobs/uploads/?digest= 202 404/416 +// +// PUT /v2//uploads/?digest= 202 404/416 +func (hand *Handler) PutUpload(rctx *router.Context) { + + hand.DumpHeaders("PutUpload headers", rctx) + + contentType := rctx.GetHeader("Content-Type") + contentLength := rctx.GetHeader("Content-Length") + contentRange := rctx.GetHeader("Content-Range") + + name, _ := rctx.GetSubpath("name") + reference, _ := rctx.GetSubpath("reference") + + digest := rctx.GetQuery("digest") + + params := &operator.PutUploadParams{ + ContentLength: contentLength, + ContentType: contentType, + ContentRange: contentRange, + Name: name, + Reference: reference, + Digest: digest, + } + res, code, err := hand.oper.PutUpload(rctx.Ctx, params) + if err != nil { + hand.logg.Errorf("PutUpload error: %v", err) + } + rctx.SetHeader("Location", res.Location) + rctx.SetStatus(code) +} diff --git a/app/operator/blob.go b/app/operator/blob.go index 6a9c040..3e090e8 100644 --- a/app/operator/blob.go +++ b/app/operator/blob.go @@ -128,26 +128,88 @@ func (oper *Operator) PatchUpload(ctx context.Context, params *PatchUploadParams err = fmt.Errorf("Wrong Conten-Type header: %s", params.ContentType) return res, http.StatusBadRequest, err } - if params.ContentLength == "" { - err = fmt.Errorf("Empty Content-length header") - return res, http.StatusBadRequest, err - } - contentLength, err := strconv.ParseInt(params.ContentLength, 10, 64) - if err != nil { - err = fmt.Errorf("Wrong Content-length header") - return res, http.StatusBadRequest, err + var contentLength int64 + // podman & github.com/containers/image don't set Content-length header for docker transport + if params.ContentLength != "" { + contentLength, err = strconv.ParseInt(params.ContentLength, 10, 64) + if err != nil { + err = fmt.Errorf("Wrong Content-length header") + return res, http.StatusBadRequest, err + } } recsize, err := oper.store.WriteUpload(params.Reference, params.Reader) if err != nil { return res, http.StatusInternalServerError, err } - if recsize != contentLength { + if contentLength != 0 && recsize != contentLength { oper.store.RemoveUpload(params.Reference) err = fmt.Errorf("Mismatch upload recorded size and content length") return res, http.StatusInternalServerError, err } res.Location = fmt.Sprintf("/v2/%s/uploads/%s", params.Name, params.Reference) - return res, http.StatusAccepted, err // http.StatusCreated + return res, http.StatusAccepted, err +} + +type PutUploadParams struct { + ContentType string + ContentLength string + ContentRange string + Name string + Reference string + Digest string + Reader io.Reader +} +type PutUploadResult struct { + Location string +} + +func (oper *Operator) PutUpload(ctx context.Context, params *PutUploadParams) (*PutUploadResult, int, error) { + var err error + res := &PutUploadResult{} + oper.logg.Debugf("Call PutUpload") + + if params.Reference == "" { + err = fmt.Errorf("Empty reference") + return res, http.StatusBadRequest, err + } + if params.Name == "" { + err = fmt.Errorf("Empty name") + return res, http.StatusBadRequest, err + } + if params.Digest == "" { + err = fmt.Errorf("Empty digest") + return res, http.StatusBadRequest, err + } + if params.ContentType != "application/octet-stream" { + err = fmt.Errorf("Wrong conten type: %s", params.ContentType) + return res, http.StatusBadRequest, err + } + + var contentLength int64 + if params.ContentLength != "" { + contentLength, err = strconv.ParseInt(params.ContentLength, 10, 64) + if err != nil { + err = fmt.Errorf("Cannot convert Content-Length=%s to integer: %v", params.ContentLength, err) + return res, http.StatusBadRequest, err + } + + } + if contentLength != 0 { + // TODO + err = fmt.Errorf("Unexpected Content-Length header: %s", params.ContentLength) + return res, http.StatusInternalServerError, err + + Content - Range + } + + err = oper.store.LinkUpload(params.Reference, params.Digest) + if err != nil { + err = fmt.Errorf("Failed to link upload %s, err: %v", params.Reference, err) + return res, http.StatusInternalServerError, err + } + + res.Location = fmt.Sprintf("/v2/%s/blobs/%s", params.Name, params.Digest) + return res, http.StatusCreated, err } diff --git a/app/router/context.go b/app/router/context.go index a05243f..e44ad1e 100644 --- a/app/router/context.go +++ b/app/router/context.go @@ -25,6 +25,7 @@ func NewContext(writer http.ResponseWriter, request *http.Request) *Context { return rctx } +// Request func (rctx *Context) GetSubpath(key string) (string, bool) { value, exists := rctx.PathMap[key] return value, exists @@ -34,6 +35,19 @@ func (rctx *Context) GetQuery(key string) string { return rctx.Request.URL.Query().Get(key) } +func (rctx *Context) GetHeader(key string) string { + return rctx.Request.Header.Get(key) +} + +func (rctx *Context) GetHeaders() http.Header { + return rctx.Request.Header +} + +func (rctx *Context) GetContext() context.Context { + return rctx.Request.Context() +} + +// Response func (rctx *Context) SetHeader(key, value string) { rctx.Writer.Header().Set(key, value) } @@ -52,11 +66,3 @@ func (rctx *Context) SendText(payload string) { rctx.Writer.Header().Set("Content-Type", "text/plain") rctx.Writer.Write([]byte(payload)) } - -func (rctx *Context) GetHeader(key string) string { - return rctx.Request.Header.Get(key) -} - -func (rctx *Context) GetContext() context.Context { - return rctx.Request.Context() -} diff --git a/app/service/service.go b/app/service/service.go index 0fdfcba..a519439 100644 --- a/app/service/service.go +++ b/app/service/service.go @@ -189,6 +189,30 @@ func (svc *Service) Build() error { svc.rout.Patch(`/v2/{name}/blobs/uploads/{reference}`, svc.hand.PatchUpload) + // To close the session, issue a PUT request to a url in the following format, + // and with the following headers (and optional body, depending on whether or not + // the final chunk was uploaded already via a PATCH request): + // + // ?digest= + // + // Content-Length: + // Content-Range: + // Content-Type: application/octet-stream + // + // OPTIONAL: + // + // The closing PUT request MUST include the of the whole blob + // (not the final chunk) as a query parameter. + // + // The response to a successful closing of the session MUST be 201 Created, + // and MUST contain the following header: + // + // Location: + // + // Here, is a pullable blob URL. + + svc.rout.Put(`/v2/{name}/uploads/{reference}`, svc.hand.PutUpload) + svc.rout.NotFound(svc.hand.NotFound) selector := svc.rout.Selector() diff --git a/app/storage/storage.go b/app/storage/storage.go index 6240554..308eab8 100644 --- a/app/storage/storage.go +++ b/app/storage/storage.go @@ -139,17 +139,36 @@ func (store *Storage) DeleteFile(collection, filename string) error { } const ( - upsubdir = "uploads" + upsubdir = "uploads" + blobsubdir = "blobs" ) func (store *Storage) makeUppath(upname string) string { - return filepath.Join(store.basepath, upsubdir, upname) + return filepath.Join(store.basepath, upsubdir, upname) + ".bin" +} + +func (store *Storage) makeUpsubdir() string { + return filepath.Join(store.basepath, upsubdir) +} + +func (store *Storage) makeBlobpath(upname string) string { + return filepath.Join(store.basepath, blobsubdir, upname) + ".bin" +} + +func (store *Storage) makeBlobsubdir() string { + return filepath.Join(store.basepath, blobsubdir) } func (store *Storage) WriteUpload(digest string, source io.Reader) (int64, error) { var err error var recsize int64 + uploadDir := store.makeUpsubdir() + err = os.MkdirAll(uploadDir, 0750) + if err != nil { + return recsize, err + } + uploadPath := store.makeUppath(digest) uploadFile, err := os.OpenFile(uploadPath, os.O_WRONLY|os.O_CREATE, 0644) if err != nil { @@ -166,9 +185,31 @@ func (store *Storage) WriteUpload(digest string, source io.Reader) (int64, error return recsize, err } -func (st *Storage) RemoveUpload(digest string) error { +func (store *Storage) LinkUpload(reference, digest string) error { var err error - uploadPath := st.makeUppath(digest) + uploadPath := store.makeUppath(reference) + blobPath := store.makeBlobpath(digest) + + blobdir := store.makeBlobsubdir() + err = os.MkdirAll(blobdir, 0750) + if err != nil { + return err + } + + err = os.Link(uploadPath, blobPath) + if err != nil { + return err + } + err = os.Remove(uploadPath) + if err != nil { + return err + } + return err +} + +func (store *Storage) RemoveUpload(digest string) error { + var err error + uploadPath := store.makeUppath(digest) err = os.Remove(uploadPath) if err != nil { return err diff --git a/pkg/auxhttp/crange.go b/pkg/auxhttp/crange.go new file mode 100644 index 0000000..4869e73 --- /dev/null +++ b/pkg/auxhttp/crange.go @@ -0,0 +1,88 @@ +package auxhttp + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +func ParseOCIContentRange(ref string) (int64, int64, error) { + var start, end int64 + var err error + + startstop := strings.TrimSpace(ref) + tmp := strings.SplitN(startstop, "-", 2) + if len(tmp) != 2 { + err = fmt.Errorf("Wrong Content-Range definition, strange range def %v", startstop) + return start, end, err + } + startstr := tmp[0] + endstr := tmp[1] + + start, err = strconv.ParseInt(startstr, 10, 64) + if err != nil { + err = fmt.Errorf("Wrong Content-Range definition, strange start %s", startstr) + return start, end, err + } + end, err = strconv.ParseInt(endstr, 10, 64) + if err != nil { + err = fmt.Errorf("Wrong Content-Range definition, strange end %s", endstr) + return start, end, err + } + return start, end, err +} + +func ParseContentRange(ref string) (int64, int64, int64, error) { + var start, end, total int64 + var err error + + const bytesUnit = "bytes" + + tmp := strings.SplitN(ref, " ", 2) + if len(tmp) != 2 { + err := fmt.Errorf("Wrong Content-Range definition, len is only %d", len(tmp)) + return start, end, total, err + } + unit := strings.ToLower(tmp[0]) + if unit != bytesUnit { + err = fmt.Errorf("Wrong Content-Range definition, strange length def %s", unit) + return start, end, total, err + } + ranger := strings.TrimSpace(tmp[1]) + + tmp = strings.Split(ranger, "/") + if len(tmp) != 2 { + err = fmt.Errorf("Wrong Content-Range definition, dont found total") + return start, end, total, err + } + totalstr := tmp[1] + if totalstr == "*" { + total = 0 + } else { + total, err = strconv.ParseInt(totalstr, 10, 64) + if err != nil { + return start, end, total, err + } + } + startstop := tmp[0] + tmp = strings.SplitN(startstop, "-", 2) + if len(tmp) != 2 { + err = fmt.Errorf("Wrong Content-Range definition, strange range def %v", startstop) + return start, end, total, err + } + startstr := tmp[0] + endstr := tmp[1] + + start, err = strconv.ParseInt(startstr, 10, 64) + if err != nil { + err = fmt.Errorf("Wrong Content-Range definition, strange start %s", startstr) + return start, end, total, err + } + end, err = strconv.ParseInt(endstr, 10, 64) + if err != nil { + err = fmt.Errorf("Wrong Content-Range definition, strange end %s", endstr) + return start, end, total, err + } + return start, end, total, err +}