diff --git a/app/handler/aaafunc.go b/app/handler/aaafunc.go index f24099c..0d2a6f9 100644 --- a/app/handler/aaafunc.go +++ b/app/handler/aaafunc.go @@ -52,7 +52,9 @@ func (hand *Handler) CheckAccess(rctx *router.Context) (bool, string, error) { accountID = terms.AnonymousID + hand.logg.Debugf("URL: %s", rctx.URL().String()) authHeader := rctx.GetHeader("Authorization") + hand.logg.Debugf("Authorization: %s", authHeader) if authHeader != "" { username, password, err = auxhttp.ParseBasicAuth(authHeader) if err != nil { diff --git a/app/handler/blob.go b/app/handler/blob.go index e196b1a..1972514 100644 --- a/app/handler/blob.go +++ b/app/handler/blob.go @@ -57,6 +57,8 @@ func (hand *Handler) BlobExists(rctx *router.Context) { } // POST /v2//blobs/uploads/ 202 404 +// POST /v2//blobs/uploads/?digest= 201/202 404/400 +// POST /v2//blobs/uploads/?mount=&from= 201 404 func (hand *Handler) PostUpload(rctx *router.Context) { name, _ := rctx.GetSubpath("name") @@ -102,8 +104,7 @@ func (hand *Handler) PostUpload(rctx *router.Context) { rctx.SetStatus(code) } -// POST /v2//blobs/uploads/?digest= 201/202 404/400 -// POST /v2//blobs/uploads/?mount=&from= 201 404 + // PATCH /v2//blobs/uploads/ 202 404/416 func (hand *Handler) PatchUpload(rctx *router.Context) { diff --git a/app/operator/account.go b/app/operator/account.go index 570305d..71af671 100644 --- a/app/operator/account.go +++ b/app/operator/account.go @@ -256,9 +256,7 @@ type ListAccountsResult struct { func (oper *Operator) ListAccounts(ctx context.Context, params *ListAccountsParams) (*ListAccountsResult, error) { var err error - res := &ListAccountsResult{ - Accounts: make([]descr.AccountShort, 0), - } + res := &ListAccountsResult{} accountDescrs, err := oper.mdb.ReducedListAccounts(ctx) if err != nil { diff --git a/app/operator/file.go b/app/operator/file.go index 11691bc..6e81dde 100644 --- a/app/operator/file.go +++ b/app/operator/file.go @@ -24,7 +24,7 @@ import ( "mstore/pkg/auxtool" "mstore/pkg/auxuuid" "mstore/pkg/descr" - "mstore/pkg/terms" + "mstore/pkg/filecli" ) // FileInfo @@ -39,11 +39,10 @@ type FileInfoResult struct { ContentType string ContentSize string ContentDigest string - - ContentCreatedAt string - ContentCreatedBy string - ContentUpdatedAt string - ContentUpdatedBy string + ContentCreatedAt string + ContentCreatedBy string + ContentUpdatedAt string + ContentUpdatedBy string } func cleanFilepath(filename string) (string, error) { @@ -299,122 +298,10 @@ func (oper *Operator) DeleteFile(ctx context.Context, operatorID string, params return code, res, err } -// ListFiles -type ListFilesParams struct { - Filepath string - PathAs string `param:"pathAs"` -} -type ListFilesResult struct { - Files []descr.File `json:"files,omitempty"` -} - -func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) { - var err error - res := &ListFilesResult{ - Files: make([]descr.File, 0), - } - switch params.PathAs { - case terms.AsRegexp: - files, err := oper.listFilesWithRegex(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - case terms.AsPrefix: - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - files, err := oper.listFilesWithPrefix(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - default: // Fine - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - params.Filepath, err = cleanFilepath(params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - - files, err := oper.listFilesInOneCollection(ctx, params.Filepath) - if err != nil { - code := http.StatusInternalServerError - return code, res, err - } - res.Files = files - - } - code := http.StatusOK - return code, res, err -} - -func (oper *Operator) listFilesInOneCollection(ctx context.Context, collection string) ([]descr.File, error) { - var err error - res := make([]descr.File, 0) - files, err := oper.mdb.ListFilesByCollection(ctx, collection) - if err != nil { - return res, err - } - res = files - return res, err -} - -func (oper *Operator) listFilesWithPrefix(ctx context.Context, prefix string) ([]descr.File, error) { - var err error - res := make([]descr.File, 0) - files, err := oper.mdb.ListAllFiles(ctx) - if err != nil { - return res, err - } - for _, file := range files { - fullpath := filepath.Join(file.Collection, file.Name) - if strings.HasPrefix(fullpath, prefix) { - res = append(res, file) - } - } - return res, err -} - -func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]descr.File, error) { - var err error - res := make([]descr.File, 0) - - re, err := regexp.Compile(regex) - if err != nil { - return res, err - } - files, err := oper.mdb.ListAllFiles(ctx) - if err != nil { - return res, err - } - for _, file := range files { - fullpath := filepath.Join(file.Collection, file.Name) - if re.MatchString(fullpath) { - res = append(res, file) - } - } - return res, err -} - // ListCollections type ListCollectionsParams struct { - Path string - PathAS string `param:"pathAs"` + Path string + PathType string `param:"pathType"` } type ListCollectionsResult struct { Collections []string `json:"collection,omitempty"` @@ -427,14 +314,14 @@ func (oper *Operator) ListCollections(ctx context.Context, operatorID string, pa } collectionList := make([]string, 0) - switch param.PathAS { - case terms.AsRegexp: + switch param.PathType { + case filecli.PathTypeRegexp: collectionList, err = oper.listCollectionsWithRegexp(ctx, param.Path) if err != nil { code := http.StatusInternalServerError return code, res, err } - case terms.AsPrefix: + case filecli.PathTypePrefix: param.Path, err = cleanFilepath(param.Path) if err != nil { code := http.StatusInternalServerError @@ -545,9 +432,9 @@ func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error) // DeleteColletion type DeleteColletionParams struct { - Path string - PathAs string `param:"pathAs"` - DryRun bool `param:"dryRun"` + Path string + PathType string `param:"pathType"` + DryRun bool `param:"dryRun"` } type DeleteColletionResult struct { Files []descr.File `json:"files,omitempty"` @@ -558,8 +445,8 @@ func (oper *Operator) DeleteColletion(ctx context.Context, operatorID string, pa res := &DeleteColletionResult{ Files: make([]descr.File, 0), } - switch param.PathAs { - case terms.AsRegexp: + switch param.PathType { + case filecli.PathTypeRegexp: collections, err := oper.listCollectionsWithRegexp(ctx, param.Path) if err != nil { code := http.StatusInternalServerError @@ -576,7 +463,7 @@ func (oper *Operator) DeleteColletion(ctx context.Context, operatorID string, pa } res.Files = allfiles - case terms.AsPrefix: + case filecli.PathTypePrefix: param.Path, err = cleanFilepath(param.Path) if err != nil { code := http.StatusInternalServerError @@ -598,7 +485,7 @@ func (oper *Operator) DeleteColletion(ctx context.Context, operatorID string, pa allfiles = append(allfiles, files...) } res.Files = allfiles - default: // Fine + default: param.Path, err = cleanFilepath(param.Path) if err != nil { code := http.StatusInternalServerError @@ -641,3 +528,112 @@ func (oper *Operator) deleteFilesInCollection(ctx context.Context, collection st } return res, err } + +// ListFiles +type ListFilesParams struct { + Filepath string + PathType string `param:"pathType"` +} +type ListFilesResult struct { + Files []descr.File `json:"files,omitempty"` +} + +func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) { + var err error + res := &ListFilesResult{} + switch params.PathType { + case filecli.PathTypeRegexp: + files, err := oper.listFilesWithRegex(ctx, params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res.Files = files + case filecli.PathTypePrefix: + params.Filepath, err = cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + params.Filepath, err = cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + + files, err := oper.listFilesWithPrefix(ctx, params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res.Files = files + default: + params.Filepath, err = cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + params.Filepath, err = cleanFilepath(params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + files, err := oper.listFilesInCollection(ctx, params.Filepath) + if err != nil { + code := http.StatusInternalServerError + return code, res, err + } + res.Files = files + + } + code := http.StatusOK + return code, res, err +} + +func (oper *Operator) listFilesInCollection(ctx context.Context, collection string) ([]descr.File, error) { + var err error + res := make([]descr.File, 0) + files, err := oper.mdb.ListFilesByCollection(ctx, collection) + if err != nil { + return res, err + } + res = files + return res, err +} + +func (oper *Operator) listFilesWithPrefix(ctx context.Context, prefix string) ([]descr.File, error) { + var err error + res := make([]descr.File, 0) + files, err := oper.mdb.ListAllFiles(ctx) + if err != nil { + return res, err + } + for _, file := range files { + fullpath := filepath.Join(file.Collection, file.Name) + if strings.HasPrefix(fullpath, prefix) { + res = append(res, file) + } + } + return res, err +} + +func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]descr.File, error) { + var err error + res := make([]descr.File, 0) + + re, err := regexp.Compile(regex) + if err != nil { + return res, err + } + files, err := oper.mdb.ListAllFiles(ctx) + if err != nil { + return res, err + } + for _, file := range files { + fullpath := filepath.Join(file.Collection, file.Name) + if re.MatchString(fullpath) { + res = append(res, file) + } + } + return res, err +} diff --git a/app/operator/manifest.go b/app/operator/manifest.go index cd8b04e..c9683c8 100644 --- a/app/operator/manifest.go +++ b/app/operator/manifest.go @@ -321,17 +321,27 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams if !exists { return res, http.StatusNotFound, err } + /* + index, indexBytes, err := indexFromManigestDescrs(manifestDescrs) + if err != nil { + return res, http.StatusInternalServerError, err + } + indexDigest := auxoci.SHA256DigestFromString(indexBytes) + res.DockerContentDigest = indexDigest.String() - index, indexBytes, err := indexFromManigestDescrs(manifestDescrs) - if err != nil { - return res, http.StatusInternalServerError, err - } - indexDigest := auxoci.SHA256DigestFromString(indexBytes) - res.DockerContentDigest = indexDigest.String() + res.ContentLength = strconv.FormatInt(int64(len(indexBytes)), 10) + res.ContentType = index.MediaType + res.Payload = string(indexBytes) + */ + manifestDescr = manifestDescrs[0] + + manifestDigest := auxoci.SHA256DigestFromString(manifestDescr.Payload) + res.DockerContentDigest = manifestDigest.String() + + res.ContentLength = strconv.FormatInt(int64(len(manifestDescr.Payload)), 10) + res.ContentType = manifestDescr.ContentType + res.Payload = manifestDescr.Payload - res.ContentLength = strconv.FormatInt(int64(len(indexBytes)), 10) - res.ContentType = index.MediaType - res.Payload = string(indexBytes) } return res, http.StatusOK, err } diff --git a/app/router/context.go b/app/router/context.go index ce32d08..acdb814 100644 --- a/app/router/context.go +++ b/app/router/context.go @@ -15,6 +15,7 @@ import ( "encoding/json" "io" "net/http" + "net/url" "strconv" ) @@ -29,11 +30,10 @@ type Context struct { } func NewContext(writer http.ResponseWriter, request *http.Request) *Context { - ctx := context.Background() rctx := &Context{ Writer: writer, Request: request, - Ctx: ctx, + Ctx: request.Context(), PathMap: make(map[string]string), Bools: make(map[string]bool), Strings: make(map[string]string), @@ -66,6 +66,10 @@ func (rctx *Context) GetSubpath(key string) (string, bool) { return value, exists } +func (rctx *Context) URL() *url.URL { + return rctx.Request.URL +} + func (rctx *Context) GetQuery(key string) string { return rctx.Request.URL.Query().Get(key) } @@ -83,7 +87,6 @@ func (rctx *Context) GetContext() context.Context { } // Binding - const emptyJSON = "{}" func (rctx *Context) BindJSON(obj any) error { diff --git a/cmd/mstorectl/command/filecmd.go b/cmd/mstorectl/command/filecmd.go index b226afe..ade6315 100644 --- a/cmd/mstorectl/command/filecmd.go +++ b/cmd/mstorectl/command/filecmd.go @@ -11,6 +11,7 @@ package command import ( "context" + "encoding/json" "errors" "fmt" "io/fs" @@ -18,19 +19,17 @@ import ( "os" "path" "path/filepath" - "slices" "strings" "time" "github.com/spf13/cobra" "github.com/spf13/viper" - "mstore/pkg/client" + "mstore/pkg/filecli" "mstore/pkg/descr" - "mstore/pkg/terms" ) -func (util *FileUtil) CreateFileCmds() *cobra.Command { +func (util *FileUtil) MakeFileCmds() *cobra.Command { var subCmd = &cobra.Command{ Use: "files", Short: "File operations", @@ -127,7 +126,7 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command { return subCmd } -func (util *FileUtil) CreateCollectionCmds() *cobra.Command { +func (util *FileUtil) MakeCollectionCmds() *cobra.Command { var subCmd = &cobra.Command{ Use: "collections", Short: "Colletion operations", @@ -176,8 +175,8 @@ type FileUtil struct { listFilesParams ListFilesParams importFilesParams ImportFilesParams exportFilesParams ExportFilesParams - listCollectionsParams ListCollectionsParams deleteCollectionParams DeleteCollectionParams + listCollectionsParams ListCollectionsParams commonFileParams CommonFileParams } @@ -188,41 +187,6 @@ type CommonFileParams struct { SkipTLSVerify bool } -// FileInfo -type FileInfoParams struct { - Filepath string -} -type FileInfoResult struct { - File *descr.File `yaml:"file,omitempty"` -} - -func (util *FileUtil) FileInfo(cmd *cobra.Command, args []string) { - util.fileInfoParams.Filepath = args[0] - res, err := util.fileInfo(&util.commonFileParams, &util.fileInfoParams) - printResponse(res, err) -} - -func (util *FileUtil) fileInfo(common *CommonFileParams, params *FileInfoParams) (*FileInfoResult, error) { - var err error - res := &FileInfoResult{} - params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password) - if err != nil { - return res, err - } - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - exists, opres, err := client.NewClient(common.SkipTLSVerify).FileInfo(ctx, params.Filepath) - if err != nil { - return res, err - } - if !exists { - err = fmt.Errorf("File %s not exists", params.Filepath) - return res, err - } - res.File = opres - return res, err -} - // PutFile type PutFileParams struct { Source string @@ -240,13 +204,27 @@ func (util *FileUtil) PutFile(cmd *cobra.Command, args []string) { func (util *FileUtil) putFile(common *CommonFileParams, params *PutFileParams) (*PutFileResult, error) { var err error res := &PutFileResult{} - params.Dest, err = packUserinfo(params.Dest, common.Username, common.Password) + + file, err := os.OpenFile(params.Dest, os.O_RDONLY, 0) + if err != nil { + return res, err + } + defer file.Close() + stat, err := file.Stat() if err != nil { return res, err } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) - err = client.NewClient(common.SkipTLSVerify).PutFile(ctx, params.Source, params.Dest) + ref, err := filecli.ParsePath(params.Dest) + if err != nil { + return res, err + } + ref.SetUserinfo(common.Username, common.Password) + + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + err = cli.PutFile(ctx, ref.Raw(), file, stat.Size()) if err != nil { return res, err } @@ -271,13 +249,27 @@ type GetFileResult struct{} func (util *FileUtil) getFile(common *CommonFileParams, params *GetFileParams) (*GetFileResult, error) { var err error res := &GetFileResult{} - params.Dest, err = packUserinfo(params.Source, common.Username, common.Password) + + err = os.MkdirAll(filepath.Dir(params.Dest), 0750) if err != nil { return res, err } + file, err := os.OpenFile(params.Dest, os.O_WRONLY|os.O_CREATE, 0640) + if err != nil { + return res, err + } + defer file.Close() + timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) - _, err = client.NewClient(common.SkipTLSVerify).GetFile(ctx, params.Dest, params.Source) + ref, err := filecli.ParsePath(params.Source) + if err != nil { + return res, err + } + ref.SetUserinfo(common.Username, common.Password) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + _, err = cli.GetFile(ctx, ref.Raw(), file) if err != nil { return res, err } @@ -299,71 +291,23 @@ func (util *FileUtil) DeleteFile(cmd *cobra.Command, args []string) { func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFileParams) (*DeleteFileResult, error) { var err error res := &DeleteFileResult{} - params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password) + ref, err := filecli.ParsePath(params.Filepath) if err != nil { return res, err } timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) - err = client.NewClient(common.SkipTLSVerify).DeleteFile(ctx, params.Filepath) + ref.SetUserinfo(common.Username, common.Password) + + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + _, err = cli.DeleteFile(ctx, ref.Raw()) if err != nil { return res, err } return res, err } -// ListFiles -type ListFilesParams struct { - Filepath string - Detail bool - AsPrefix bool - AsRegexp bool -} - -type ListFilesResult struct { - Files []descr.File `yaml:"files,omitempty"` - Filenames []string `yaml:"filenames,omitempty"` -} - -func (util *FileUtil) ListFiles(cmd *cobra.Command, args []string) { - util.listFilesParams.Filepath = args[0] - res, err := util.listFiles(&util.commonFileParams, &util.listFilesParams) - printResponse(res, err) -} -func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParams) (*ListFilesResult, error) { - var err error - res := &ListFilesResult{} - params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password) - if err != nil { - return res, err - } - if params.AsRegexp { - params.AsPrefix = false - } - var pathUsage string - switch { - case params.AsRegexp: - pathUsage = terms.AsRegexp - case params.AsPrefix: - pathUsage = terms.AsPrefix - default: - pathUsage = terms.AsFinePath - } - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - files, err := client.NewClient(common.SkipTLSVerify).ListFiles(ctx, params.Filepath, pathUsage) - if err != nil { - return res, err - } - - if params.Detail { - res.Files = files - } else { - res.Filenames = makeFilelistFromFiles(files) - } - return res, err -} - // ImportFiles type ImportFilesParams struct { Source string @@ -397,25 +341,39 @@ func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesP if err != nil { return err } - if infoItem.Mode() == fs.ModeDevice { - return nil - } - if infoItem.Mode() == fs.ModeNamedPipe { - return nil - } - if infoItem.Mode() == fs.ModeCharDevice { + mode := infoItem.Mode() + skip := mode == fs.ModeDevice + skip = skip || mode == fs.ModeNamedPipe + skip = skip || mode == fs.ModeCharDevice + if skip { return nil } if !infoItem.IsDir() { - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) relPath, _ := strings.CutPrefix(walkPath, params.Source) dest, _ := url.JoinPath(params.Dest, relPath) + ref, err := filecli.ParsePath(dest) if err != nil { putErrors = append(putErrors, err) return nil } - err = client.NewClient(common.SkipTLSVerify).PutFile(ctx, walkPath, dest) + file, err := os.OpenFile(dest, os.O_RDONLY, 0) + if err != nil { + putErrors = append(putErrors, err) + return nil + } + defer file.Close() + stat, err := file.Stat() + if err != nil { + putErrors = append(putErrors, err) + return nil + } + timeout := time.Duration(common.Timeout) * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref.SetUserinfo(common.Username, common.Password) + user, pass := ref.Userinfo() + mw := filecli.NewBasicAuthMiddleware(user, pass) + cli := filecli.NewClient(nil, mw) + err = cli.PutFile(ctx, ref.Raw(), file, stat.Size()) if err != nil { putErrors = append(putErrors, err) fmt.Printf("- %s: error: %v \n", walkPath, err) @@ -459,57 +417,62 @@ func (util *FileUtil) ExportFiles(cmd *cobra.Command, args []string) { func (util *FileUtil) exportFiles(common *CommonFileParams, params *ExportFilesParams) (*ExportFilesResult, error) { var err error res := &ExportFilesResult{} - params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password) + + + timeout := time.Duration(common.Timeout) * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref, err := filecli.ParsePath(params.Filepath) if err != nil { return res, err } - if params.AsRegexp { - params.AsPrefix = false + ref.SetUserinfo(common.Username, common.Password) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + list, err := cli.ListFiles(ctx, ref.Raw()) + if err != nil { + return res, err } - var pathUsage string - switch { - case params.AsRegexp: - pathUsage = terms.AsRegexp - case params.AsPrefix: - pathUsage = terms.AsPrefix - default: - pathUsage = terms.AsFinePath - } - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - files, err := client.NewClient(common.SkipTLSVerify).ListFiles(ctx, params.Filepath, pathUsage) + files := make([]descr.File, 0) + err = json.Unmarshal(list, &files) if err != nil { return res, err } - exportedFiles := make([]descr.File, 0) - for _, file := range files { - destdir := filepath.Join(params.Dest, file.Collection) + exported := make([]descr.File, 0) + for _, descr := range files { + destdir := filepath.Join(params.Dest, descr.Collection) err = os.MkdirAll(destdir, 0750) if err != nil { return res, err } - srcpath, err := makeFileURI(params.Filepath, file.Collection, file.Name) + destpath := filepath.Join(params.Dest, descr.Collection, descr.Name) + destfile, err := os.OpenFile(destpath, os.O_WRONLY|os.O_CREATE, 0640) + if err != nil { + return res, err + } + defer destfile.Close() + + timeout := time.Duration(common.Timeout) * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref.SetResource(filepath.Join(descr.Collection, descr.Name)) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + _, err = cli.GetFile(ctx, ref.Raw(), destfile) if err != nil { - return res, err - } - destpath := filepath.Join(params.Dest, file.Collection, file.Name) - timeout := time.Duration(common.Timeout) * time.Second - ctx, _ := context.WithTimeout(context.Background(), timeout) - _, err = client.NewClient(common.SkipTLSVerify).GetFile(ctx, srcpath, destpath) - if err != nil { - fmt.Printf("- %s: error %v\n", srcpath, err) - //return res, err + + fmt.Printf("- %s: error %v\n", ref.Raw(), err) err = nil } else { - fmt.Printf("- %s: ok\n", srcpath) - exportedFiles = append(exportedFiles, file) + fmt.Printf("- %s: ok\n", ref.Raw()) + exported = append(exported, descr) } } if params.Detail { - res.Files = exportedFiles + res.Files = exported } else { - res.Filenames = makeFilelistFromFiles(exportedFiles) + files := descr.NewFiles() + files.Set(exported) + res.Filenames = files.List() } return res, err } @@ -548,29 +511,26 @@ func (util *FileUtil) listCollections(common *CommonFileParams, params *ListColl res := &ListCollectionsResult{ Collections: make([]string, 0), } - params.Path, err = packUserinfo(params.Path, common.Username, common.Password) - if err != nil { - return res, err - } - if params.AsRegexp { - params.AsPrefix = false - } - var pathUsage string - switch { - case params.AsRegexp: - pathUsage = terms.AsRegexp - case params.AsPrefix: - pathUsage = terms.AsPrefix - default: - pathUsage = terms.AsFinePath - } + timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) - collecions, err := client.NewClient(common.SkipTLSVerify).ListCollections(ctx, params.Path, pathUsage) + ref, err := filecli.ParsePath(params.Path) if err != nil { return res, err } - res.Collections = collecions + ref.SetUserinfo(common.Username, common.Password) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + list, err := cli.ListCollections(ctx, ref.Raw()) + if err != nil { + return res, err + } + collections := make([]string, 0) + err = json.Unmarshal(list, collections) + if err != nil { + return res, err + } + res.Collections = collections return res, err } @@ -599,36 +559,28 @@ func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteC res := &DeleteCollectionResult{ Filenames: make([]string, 0), } - params.Path, err = packUserinfo(params.Path, common.Username, common.Password) - if err != nil { - return res, err - } + timeout := time.Duration(common.Timeout) * time.Second ctx, _ := context.WithTimeout(context.Background(), timeout) - var pathUsage string - switch { - case params.AsPrefix: - pathUsage = terms.AsPrefix - default: - pathUsage = terms.AsFinePath - } - files, err := client.NewClient(common.SkipTLSVerify).DeleteCollection(ctx, params.Path, pathUsage, params.DryRun) + ref, err := filecli.ParsePath(params.Path) if err != nil { return res, err } - if params.Detail { - res.Files = files + ref.SetUserinfo(common.Username, common.Password) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + list, err := cli.DeleteCollection(ctx, ref.Raw()) + + files := descr.NewFiles() + err = json.Unmarshal(list, files.ArrayPtr()) + if err != nil { + return res, err + } + if !params.Detail { + res.Filenames = files.List() } else { - res.Filenames = makeFilelistFromFiles(files) + res.Files = files.Array() } return res, err } -func makeFilelistFromFiles(files []descr.File) []string { - res := make([]string, 0, len(files)) - for _, file := range files { - res = append(res, filepath.Join(file.Collection, file.Name)) - } - slices.Sort(res) - return res -} diff --git a/cmd/mstorectl/command/imagecmd.go b/cmd/mstorectl/command/imagecmd.go index e035e00..0a9f010 100644 --- a/cmd/mstorectl/command/imagecmd.go +++ b/cmd/mstorectl/command/imagecmd.go @@ -11,6 +11,8 @@ package command import ( "context" + "encoding/json" + "fmt" "net/url" "os" "path" @@ -21,6 +23,7 @@ import ( "github.com/spf13/viper" "mstore/pkg/client" + "mstore/pkg/repocli" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -268,33 +271,55 @@ type ImageManifestParams struct { } type ImageManifestResult struct { - ImageManifest any `json:"imageManifest"` + Index *ocispec.Index `json:"index,omitempty"` + Manifest *ocispec.Manifest `json:"manifest,omitempty"` } func (util *ImageUtil) ImageManifest(cmd *cobra.Command, args []string) { util.imageManifestParams.Imagepath = args[0] - res, err := util.imageInfo(&util.commonImageParams, &util.imageManifestParams) + res, err := util.imageManifest(&util.commonImageParams, &util.imageManifestParams) printResponse(res, err) } -func (util *ImageUtil) imageInfo(common *CommonImageParams, params *ImageManifestParams) (*ImageManifestResult, error) { +func (util *ImageUtil) imageManifest(common *CommonImageParams, params *ImageManifestParams) (*ImageManifestResult, error) { var err error - res := &ImageManifestResult{} - ctx := context.Background() - - cli := client.NewClient(common.SkipTLSVerify) - timeout := time.Duration(common.Timeout) * time.Second - + res := &ImageManifestResult{ + Index: &ocispec.Index{}, + Manifest: &ocispec.Manifest{}, + } params.Imagepath, err = packUserinfo(params.Imagepath, common.Username, common.Password) if err != nil { return res, err } - ctx, _ = context.WithTimeout(ctx, timeout) - opres, err := cli.ImageManifest(ctx, params.Imagepath) + timeout := time.Duration(common.Timeout) * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref, err := repocli.ParseReference(params.Imagepath) if err != nil { return res, err } - res.ImageManifest = opres + mw := repocli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := repocli.NewClientWithTransport(nil, mw) + exists, mime, man, err := cli.GetManifest(ctx, ref.Repo(), ref.Tag()) + if !exists { + err = fmt.Errorf("Manifest not found") + return res, err + + } + switch mime { + case repocli.MediaTypeDDMLv2, repocli.MediaTypeOIIv1: + err = json.Unmarshal(man, res.Index) + if err != nil { + return res, err + } + case repocli.MediaTypeDDMv2, repocli.MediaTypeOIMv1: + err = json.Unmarshal(man, res.Manifest) + if err != nil { + return res, err + } + default: + err = fmt.Errorf("Unknown content type: %s", mime) + return res, err + } return res, err } diff --git a/go.mod b/go.mod index 5ec7b23..e3305fc 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module mstore go 1.25.6 require ( - github.com/google/go-containerregistry v0.21.0 + github.com/google/go-containerregistry v0.21.2 github.com/google/uuid v1.6.0 github.com/jmoiron/sqlx v1.4.0 github.com/mattn/go-sqlite3 v1.14.34 diff --git a/go.sum b/go.sum index 85f4b9c..f12d4de 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-containerregistry v0.21.0 h1:ocqxUOczFwAZQBMNE7kuzfqvDe0VWoZxQMOesXreCDI= -github.com/google/go-containerregistry v0.21.0/go.mod h1:ctO5aCaewH4AK1AumSF5DPW+0+R+d2FmylMJdp5G7p0= +github.com/google/go-containerregistry v0.21.2 h1:vYaMU4nU55JJGFC9JR/s8NZcTjbE9DBBbvusTW9NeS0= +github.com/google/go-containerregistry v0.21.2/go.mod h1:ctO5aCaewH4AK1AumSF5DPW+0+R+d2FmylMJdp5G7p0= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/pkg/descr/file.go b/pkg/descr/file.go index ef91832..9125709 100644 --- a/pkg/descr/file.go +++ b/pkg/descr/file.go @@ -9,6 +9,10 @@ */ package descr +import ( + "path" +) + type File struct { ID string `db:"id" json:"id,omitempty" yaml:"id,omitempty"` Collection string `db:"collection" json:"collection,omitempty" yaml:"collection,omitempty"` @@ -21,3 +25,44 @@ type File struct { CreatedBy string `db:"created_by" json:"createdBy,omitempty" yaml:"createdBy,omitempty"` UpdatedBy string `db:"updated_by" json:"updatedBy,omitempty" yaml:"updatedBy,omitempty"` } + + + +type Files struct { + files []File +} + +func NewFiles() *Files { + return &Files{ + files: make([]File, 0), + } +} + +func xxxNewFiles(files []File) *Files { + return &Files{ + files: files, + } +} + +func (fi *Files) Set(files []File) { + fi.files = files +} + +func (fi Files) ArrayPtr() *[]File { + return &fi.files +} + +func (fi Files) Array() []File { + return fi.files +} + + + +func (fi Files) List() []string { + list := make([]string, 0) + for _, file := range fi.files { + list = append(list, path.Join(file.Collection, file.Name)) + } + return list +} + diff --git a/pkg/filecli/client.go b/pkg/filecli/client.go index 5670e4b..645a554 100644 --- a/pkg/filecli/client.go +++ b/pkg/filecli/client.go @@ -11,13 +11,17 @@ type Client struct { userAgent string } -func NewClient(transport http.RoundTripper, mwFunc ...MiddlewareFunc) *Client { +func NewClient(transport http.RoundTripper, mwFuncs ...MiddlewareFunc) *Client { if transport == nil { transport = NewDefaultTransport() } + for _, mwFunc := range mwFuncs { + transport = mwFunc(transport) + } httpClient := &http.Client{ Transport: transport, } + return &Client{ httpClient: httpClient, userAgent: "ociClient/1.0", @@ -34,26 +38,6 @@ func (cli *Client) UseMiddleware(mwFunc MiddlewareFunc) { cli.httpClient.Transport = mwFunc(cli.httpClient.Transport) } -// ExampleMiddleware -func NewExampleMiddleware() MiddlewareFunc { - return func(next http.RoundTripper) http.RoundTripper { - return newExampleTransport(next) - } -} - -type ExampleTransport struct { - next http.RoundTripper -} - -func newExampleTransport(next http.RoundTripper) *ExampleTransport { - return &ExampleTransport{ - next: next, - } -} - -func (tran ExampleTransport) RoundTrip(req *http.Request) (*http.Response, error) { - return tran.next.RoundTrip(req) -} // BasicAuthMiddleware func NewBasicAuthMiddleware(user, pass string) MiddlewareFunc { @@ -76,8 +60,10 @@ func newBasicAuthMW(next http.RoundTripper, user, pass string) *BasicAuthMW { } func (tran BasicAuthMW) RoundTrip(req *http.Request) (*http.Response, error) { - pair := base64.StdEncoding.EncodeToString([]byte(tran.user + ":" + tran.pass)) - req.Header.Set("Authorization", "Basic "+pair) + if tran.user != "" && tran.pass != "" { + pair := base64.StdEncoding.EncodeToString([]byte(tran.user + ":" + tran.pass)) + req.Header.Set("Authorization", "Basic "+pair) + } return tran.next.RoundTrip(req) } @@ -123,3 +109,25 @@ func NewDefaultTransport() *DefaultTransport { func (wrap *DefaultTransport) RoundTrip(req *http.Request) (*http.Response, error) { return wrap.transport.RoundTrip(req) } + + +// ExampleMiddleware +func NewExampleMiddleware() MiddlewareFunc { + return func(next http.RoundTripper) http.RoundTripper { + return newExampleTransport(next) + } +} + +type ExampleTransport struct { + next http.RoundTripper +} + +func newExampleTransport(next http.RoundTripper) *ExampleTransport { + return &ExampleTransport{ + next: next, + } +} + +func (tran ExampleTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return tran.next.RoundTrip(req) +} diff --git a/pkg/filecli/delcoll.go b/pkg/filecli/delcoll.go new file mode 100644 index 0000000..e7e30c6 --- /dev/null +++ b/pkg/filecli/delcoll.go @@ -0,0 +1,51 @@ +package filecli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strconv" +) + +func (cli *Client) DeleteCollection(ctx context.Context, rawpath string) ([]byte, error) { + var err error + var list []byte + + ref, err := ParsePath(rawpath) + if err != nil { + return list, err + } + uri := ref.Collection() + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, uri, nil) + if err != nil { + return list, err + } + req.Header.Set("User-Agent", cli.userAgent) + req.Header.Set("Accept", "*/*") + resp, err := cli.httpClient.Do(req) + if err != nil { + return list, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return list, err + } + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("Unxected response code %s", resp.Status) + return list, err + } + contentLength := resp.Header.Get("Content-Length") + blobSize, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil { + return list, err + } + buffer := bytes.NewBuffer(nil) + recSize, err := Copy(ctx, buffer, resp.Body) + if blobSize != recSize { + err := fmt.Errorf("Mismatch declared and actual body size, %d and %d", blobSize, recSize) + return list, err + } + list = buffer.Bytes() + return list, err +} diff --git a/pkg/filecli/fileexist.go b/pkg/filecli/fileinfo.go similarity index 60% rename from pkg/filecli/fileexist.go rename to pkg/filecli/fileinfo.go index 14632b3..ef27c6f 100644 --- a/pkg/filecli/fileexist.go +++ b/pkg/filecli/fileinfo.go @@ -7,44 +7,45 @@ import ( "strconv" ) -func (cli *Client) FileExists(ctx context.Context, rawpath string) (bool, int64, error) { +func (cli *Client) FileInfo(ctx context.Context, rawpath string) (bool, int64, string, error) { var err error var exist bool var size int64 + var digest string ref, err := ParsePath(rawpath) if err != nil { - return exist, size, err + return exist, size, digest, err } uri := ref.File() fmt.Println(uri) req, err := http.NewRequestWithContext(ctx, http.MethodHead, uri, nil) if err != nil { - return exist, size, err + return exist, size, digest, err } req.Header.Set("User-Agent", cli.userAgent) req.Header.Set("Accept", "*/*") resp, err := cli.httpClient.Do(req) if err != nil { - return exist, size, err + return exist, size, digest, err } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { - return exist, size, err + return exist, size, digest, err } if resp.StatusCode != http.StatusOK { err := fmt.Errorf("Unxected response code %s", resp.Status) - return exist, size, err + return exist, size, digest, err } - contentLength := resp.Header.Get("Content-Length") + contentLength := resp.Header.Get("Content-Size") size, err = strconv.ParseInt(contentLength, 10, 64) if err != nil { - return exist, size, err + return exist, size, digest, err } - + digest = resp.Header.Get("Content-Digest") exist = true - return exist, size, err + return exist, size, digest, err } diff --git a/pkg/filecli/getfile.go b/pkg/filecli/getfile.go index 6f854e5..3826cf3 100644 --- a/pkg/filecli/getfile.go +++ b/pkg/filecli/getfile.go @@ -12,7 +12,7 @@ func (cli *Client) GetFile(ctx context.Context, rawpath string, writer io.Writer var err error var exist bool - ref, err :=ParsePath(rawpath) + ref, err := ParsePath(rawpath) if err != nil { return exist, err } diff --git a/pkg/filecli/listcoll.go b/pkg/filecli/listcoll.go new file mode 100644 index 0000000..9046af5 --- /dev/null +++ b/pkg/filecli/listcoll.go @@ -0,0 +1,53 @@ +package filecli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strconv" +) + +func (cli *Client) ListCollections(ctx context.Context, rawpath string) ([]byte, error) { + var err error + var list []byte + + ref, err := ParsePath(rawpath) + if err != nil { + return list, err + } + uri := ref.Collections() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) + if err != nil { + return list, err + } + req.Header.Set("User-Agent", cli.userAgent) + req.Header.Set("Accept", "*/*") + resp, err := cli.httpClient.Do(req) + if err != nil { + return list, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return list, err + } + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("Unexpected response code %s", resp.Status) + return list, err + } + contentLength := resp.Header.Get("Content-Length") + blobSize, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil { + return list, err + } + buffer := bytes.NewBuffer(nil) + recSize, err := Copy(ctx, buffer, resp.Body) + if blobSize != recSize { + err := fmt.Errorf("Mismatch declared and actual body size, %d and %d", blobSize, recSize) + return list, err + } + list = buffer.Bytes() + return list, err +} diff --git a/pkg/filecli/listfiles.go b/pkg/filecli/listfiles.go new file mode 100644 index 0000000..b215d73 --- /dev/null +++ b/pkg/filecli/listfiles.go @@ -0,0 +1,53 @@ +package filecli + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strconv" +) + +func (cli *Client) ListFiles(ctx context.Context, rawpath string) ([]byte, error) { + var err error + var list []byte + + ref, err := ParsePath(rawpath) + if err != nil { + return list, err + } + uri := ref.Files() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil) + if err != nil { + return list, err + } + req.Header.Set("User-Agent", cli.userAgent) + req.Header.Set("Accept", "*/*") + resp, err := cli.httpClient.Do(req) + if err != nil { + return list, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return list, err + } + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("Unexpected response code %s", resp.Status) + return list, err + } + contentLength := resp.Header.Get("Content-Length") + blobSize, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil { + return list, err + } + buffer := bytes.NewBuffer(nil) + recSize, err := Copy(ctx, buffer, resp.Body) + if blobSize != recSize { + err := fmt.Errorf("Mismatch declared and actual body size, %d and %d", blobSize, recSize) + return list, err + } + list = buffer.Bytes() + return list, err +} diff --git a/pkg/filecli/putfile.go b/pkg/filecli/putfile.go index c21cf51..d2f9b66 100644 --- a/pkg/filecli/putfile.go +++ b/pkg/filecli/putfile.go @@ -21,14 +21,14 @@ func (cli *Client) PutFile(ctx context.Context, rawpath string, src io.Reader, s } req.Header.Set("User-Agent", cli.userAgent) req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) + req.Header.Set("Content-Size", strconv.FormatInt(size, 10)) resp, err := cli.httpClient.Do(req) if err != nil { return err } resp.Body.Close() if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("File not accepted, code %d", resp.StatusCode) + err = fmt.Errorf("File not accepted, code %s", resp.Status) return err } return err diff --git a/pkg/filecli/repo.go b/pkg/filecli/repo.go index 0c436be..3c75b6f 100644 --- a/pkg/filecli/repo.go +++ b/pkg/filecli/repo.go @@ -2,17 +2,27 @@ package filecli import ( "net/url" + "path" "strings" ) +const ( + PathTypeIdentic = "identic" + PathTypePrefix = "prefix" + PathTypeRegexp = "regexp" +) + type Repository struct { urlobj *url.URL user, pass string - resource string + resource string + values url.Values } func ParsePath(rawpath string) (*Repository, error) { - repo := &Repository{} + repo := &Repository{ + values: url.Values{}, + } if !strings.Contains(rawpath, "://") { rawpath = "https://" + rawpath } @@ -25,14 +35,30 @@ func ParsePath(rawpath string) (*Repository, error) { repo.pass, _ = urlobj.User.Password() urlobj.User = nil } - repo.resource = repo.urlobj.Path + repo.resource = urlobj.Path + urlobj.Path = "/" repo.urlobj = urlobj - repo.urlobj.Path = "/" - repo.urlobj = urlobj - + repo.values = urlobj.Query() return repo, err } +func (repo *Repository) Raw() string { + res := path.Join(repo.urlobj.Host, repo.resource) + query := repo.values.Encode() + if query != "" { + return res + "?" + query + } + return res +} + +func (repo *Repository) SetResource(resource string) { + repo.resource = resource +} + +func (repo *Repository) PathType(typ string) { + repo.values.Set("pathType", typ) +} + func (repo *Repository) File() string { curl := repo.urlobj.JoinPath("/v3/api/file", repo.resource) return curl.String() @@ -43,7 +69,22 @@ func (repo *Repository) Files() string { return curl.String() } +func (repo *Repository) Collection() string { + curl := repo.urlobj.JoinPath("/v3/api/collection", repo.resource) + return curl.String() +} + +func (repo *Repository) Collections() string { + curl := repo.urlobj.JoinPath("/v3/api/collections", repo.resource) + return curl.String() +} func (repo *Repository) Userinfo() (string, string) { return repo.user, repo.pass } + +func (repo *Repository) SetUserinfo(user, pass string) { + if user != "" && pass != "" { + repo.user, repo.pass = user, pass + } +} diff --git a/pkg/repocli/client.go b/pkg/repocli/client.go index 0983196..795aff9 100644 --- a/pkg/repocli/client.go +++ b/pkg/repocli/client.go @@ -87,8 +87,10 @@ func newBasicAuthMW(next http.RoundTripper, user, pass string) *BasicAuthMW { } func (tran BasicAuthMW) RoundTrip(req *http.Request) (*http.Response, error) { - pair := base64.StdEncoding.EncodeToString([]byte(tran.user + ":" + tran.pass)) - req.Header.Set("Authorization", "Basic "+pair) + if tran.user != "" && tran.pass != "" { + pair := base64.StdEncoding.EncodeToString([]byte(tran.user + ":" + tran.pass)) + req.Header.Set("Authorization", "Basic "+pair) + } return tran.next.RoundTrip(req) } diff --git a/pkg/repocli/client_test.go b/pkg/repocli/client_test.go index efcb220..b651a02 100644 --- a/pkg/repocli/client_test.go +++ b/pkg/repocli/client_test.go @@ -12,7 +12,7 @@ import ( "time" ) -func xxxTestClientGetManifest(t *testing.T) { +func TestClientGetManifest(t *testing.T) { rawrepo := "mirror.gcr.io/alpine" tags := []string{ "3.20.0", diff --git a/pkg/repocli/mimetyp.go b/pkg/repocli/mimetyp.go index c5da23f..83b12fa 100644 --- a/pkg/repocli/mimetyp.go +++ b/pkg/repocli/mimetyp.go @@ -1,6 +1,6 @@ package repocli const ( - MediaTypeDDMLv2 = "application/vnd.docker.distribution.manifest.list.v2+json" - MediaTypeDDMv2 = "application/vnd.docker.distribution.manifest.v2+json" + //MediaTypeDDMLv2 = "application/vnd.docker.distribution.manifest.list.v2+json" + //MediaTypeDDMv2 = "application/vnd.docker.distribution.manifest.v2+json" ) diff --git a/pkg/repocli/pullimage.go b/pkg/repocli/pullimage.go index 20883b1..6097ce3 100644 --- a/pkg/repocli/pullimage.go +++ b/pkg/repocli/pullimage.go @@ -12,9 +12,9 @@ import ( const ( MediaTypeOIIv1 = "application/vnd.oci.image.index.v1+json" - MediatypeDDMLv2 = "application/vnd.docker.distribution.manifest.list.v2+json" + MediaTypeDDMLv2 = "application/vnd.docker.distribution.manifest.list.v2+json" - MediatypeDDMv2 = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeDDMv2 = "application/vnd.docker.distribution.manifest.v2+json" MediaTypeOIMv1 = "application/vnd.oci.image.manifest.v1+json" ) @@ -30,7 +30,7 @@ func NewDownloader(client *Client) *Downloader { func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string) error { var err error - ref, err := NewReference(rawref) + ref, err := ParseReference(rawref) if err != nil { return err } @@ -46,7 +46,7 @@ func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string) return err } - if mime == MediaTypeOIIv1 || mime == MediatypeDDMLv2 { + if mime == MediaTypeOIIv1 || mime == MediaTypeDDMLv2 { var index ocispec.Index err = json.Unmarshal(man, &index) if err != nil { @@ -72,7 +72,7 @@ func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string) err = errors.New("Manifest not found") return err } - if mime != MediaTypeOIMv1 && mime != MediatypeDDMv2 { + if mime != MediaTypeOIMv1 && mime != MediaTypeDDMv2 { err = errors.New("Unknown manifest media type") return err } diff --git a/pkg/repocli/reference.go b/pkg/repocli/reference.go index a08a749..768a943 100644 --- a/pkg/repocli/reference.go +++ b/pkg/repocli/reference.go @@ -13,7 +13,7 @@ type Reference struct { base, tag string } -func NewReference(rawref string) (*Reference, error) { +func ParseReference(rawref string) (*Reference, error) { ref := &Reference{} if !strings.Contains(rawref, "://") { rawref = "https://" + rawref @@ -54,3 +54,7 @@ func (ref *Reference) Repo() string { func (ref *Reference) Tag() string { return ref.tag } + +func (ref *Reference) Userinfo() (string, string) { + return ref.user, ref.pass +} diff --git a/test/account_test.go b/test/attic/account_test.go similarity index 100% rename from test/account_test.go rename to test/attic/account_test.go diff --git a/test/attic/file_test.go b/test/attic/file_test.go new file mode 100644 index 0000000..0da2bc8 --- /dev/null +++ b/test/attic/file_test.go @@ -0,0 +1,163 @@ +/* + * Copyright 2026 Oleg Borodin + * + * This work is published and licensed under a Creative Commons + * Attribution-NonCommercial-NoDerivatives 4.0 International License. + * + * Distribution of this work is permitted, but commercial use and + * modifications are strictly prohibited. + */ +package test + +import ( + "context" + "fmt" + "math/rand" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "mstore/app/server" + "mstore/pkg/client" + "mstore/pkg/terms" + + "github.com/stretchr/testify/require" +) + +func TestFileLife(t *testing.T) { + var srvport int64 = 10250 + srvdir := t.TempDir() + srvaddr := fmt.Sprintf("mstore:mstore@127.0.0.1:%d", srvport) + + srv, err := server.NewServer() + require.NoError(t, err) + { + err = srv.Configure() + require.NoError(t, err) + + useTmpDir := true + if useTmpDir { + srv.SetDatadir(srvdir) + srv.SetLogdir(srvdir) + srv.SetRundir(srvdir) + } + srv.SetPort(srvport) + + err = srv.Build() + require.NoError(t, err) + + var svcWG sync.WaitGroup + errPipe := make(chan error, 5) + + startFunc := func() { + err := srv.Service().Run() + errPipe <- err + svcWG.Done() + } + + stopFunc := func() { + srv.Service().Stop() + svcWG.Wait() + err = <-errPipe + require.NoError(t, err) + } + defer stopFunc() + + svcWG.Add(1) + go startFunc() + time.Sleep(1 * time.Second) + } + { + // ServiceHello + fmt.Printf("=== ServiceHello ===\n") + cli := client.NewClient(true) + ctx := context.Background() + helloRes, err := cli.ServiceHello(ctx, srvaddr+"/hello", 1*time.Second) + require.NoError(t, err) + require.True(t, helloRes) + } + + filesize := 32 + { + // PutFile + tmpdir := t.TempDir() + tmpfile := filepath.Join(tmpdir, "foo.bin") + + filedata := make([]byte, filesize) + _, err = rand.Read(filedata) + require.NoError(t, err) + + err := os.WriteFile(tmpfile, filedata, 0666) + require.NoError(t, err) + + fmt.Printf("=== PutFile ===\n") + cli := client.NewClient(true) + ctx := context.Background() + ctx, _ = context.WithTimeout(ctx, 1*time.Second) + + err = cli.PutFile(ctx, tmpfile, srvaddr+"/foo.bin") + require.NoError(t, err) + } + { + // FileInfo + fmt.Printf("=== FileInfo ===\n") + cli := client.NewClient(true) + ctx := context.Background() + ctx, _ = context.WithTimeout(ctx, 1*time.Second) + + exists, file, err := cli.FileInfo(ctx, srvaddr+"/foo.bin") + require.NoError(t, err) + require.True(t, exists) + require.NotNil(t, file) + } + { + // GetFile + fmt.Printf("=== GetFile ===\n") + cli := client.NewClient(true) + ctx := context.Background() + ctx, _ = context.WithTimeout(ctx, 1*time.Second) + + tmpdir := t.TempDir() + tmpfile := filepath.Join(tmpdir, "foo.bin") + + recsize, err := cli.GetFile(ctx, srvaddr+"/foo.bin", tmpfile) + require.NoError(t, err) + require.Equal(t, int64(filesize), recsize) + } + { + // ListFiles + fmt.Printf("=== ListFiles ===\n") + cli := client.NewClient(true) + ctx := context.Background() + ctx, _ = context.WithTimeout(ctx, 1*time.Second) + + files, err := cli.ListFiles(ctx, srvaddr+"/", terms.AsFinePath) + require.NoError(t, err) + require.NotZero(t, len(files)) + } + { + // DeleteFile + fmt.Printf("=== DeleteFile ===\n") + cli := client.NewClient(true) + ctx := context.Background() + ctx, _ = context.WithTimeout(ctx, 1*time.Second) + + err = cli.DeleteFile(ctx, srvaddr+"/foo.bin") + require.NoError(t, err) + } + { + // !FileInfo + fmt.Printf("=== FileInfo ===\n") + cli := client.NewClient(true) + ctx := context.Background() + ctx, _ = context.WithTimeout(ctx, 1*time.Second) + + exists, _, err := cli.FileInfo(ctx, srvaddr+"/foo.bin") + require.NoError(t, err) + require.False(t, exists) + + } + +} diff --git a/test/image_test.go b/test/attic/image_test.go similarity index 100% rename from test/image_test.go rename to test/attic/image_test.go diff --git a/test/file_test.go b/test/file_test.go index 0da2bc8..8cbffdc 100644 --- a/test/file_test.go +++ b/test/file_test.go @@ -12,16 +12,17 @@ package test import ( "context" "fmt" - "math/rand" - "os" - "path/filepath" + //"math/rand" + //"os" + //"path/filepath" "sync" "testing" "time" + "bytes" "mstore/app/server" "mstore/pkg/client" - "mstore/pkg/terms" + "mstore/pkg/filecli" "github.com/stretchr/testify/require" ) @@ -29,7 +30,7 @@ import ( func TestFileLife(t *testing.T) { var srvport int64 = 10250 srvdir := t.TempDir() - srvaddr := fmt.Sprintf("mstore:mstore@127.0.0.1:%d", srvport) + srvaddr := fmt.Sprintf("127.0.0.1:%d", srvport) srv, err := server.NewServer() require.NoError(t, err) @@ -78,86 +79,114 @@ func TestFileLife(t *testing.T) { require.NoError(t, err) require.True(t, helloRes) } - - filesize := 32 + fileaddr := srvaddr+"/foo2/bare.bin" + filedata := []byte("Hello, World") + filesize := int64(len(filedata)) + user, pass := "mstore", "mstore" + diraddr := srvaddr+"/foo" { // PutFile - tmpdir := t.TempDir() - tmpfile := filepath.Join(tmpdir, "foo.bin") - - filedata := make([]byte, filesize) - _, err = rand.Read(filedata) - require.NoError(t, err) - - err := os.WriteFile(tmpfile, filedata, 0666) - require.NoError(t, err) - + file := bytes.NewReader(filedata) + size := int64(len(filedata)) fmt.Printf("=== PutFile ===\n") - cli := client.NewClient(true) - ctx := context.Background() - ctx, _ = context.WithTimeout(ctx, 1*time.Second) + timeout := 10 * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref, err := filecli.ParsePath(fileaddr) + require.NoError(t, err) + fmt.Printf("Raw: %s\n", ref.Raw()) + ref.SetUserinfo(user, pass) - err = cli.PutFile(ctx, tmpfile, srvaddr+"/foo.bin") - require.NoError(t, err) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + err = cli.PutFile(ctx, ref.Raw(), file, size) + require.NoError(t, err) + } + { + // GetFile + file := bytes.NewBuffer(nil) + fmt.Printf("=== GetFile ===\n") + timeout := 10 * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref, err := filecli.ParsePath(fileaddr) + require.NoError(t, err) + fmt.Printf("Raw: %s\n", ref.Raw()) + ref.SetUserinfo(user, pass) + + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + exist, err := cli.GetFile(ctx, ref.Raw(), file) + require.NoError(t, err) + require.True(t, exist) } { // FileInfo fmt.Printf("=== FileInfo ===\n") - cli := client.NewClient(true) - ctx := context.Background() - ctx, _ = context.WithTimeout(ctx, 1*time.Second) + timeout := 10 * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref, err := filecli.ParsePath(fileaddr) + require.NoError(t, err) + fmt.Printf("Raw: %s\n", ref.Raw()) + ref.SetUserinfo(user, pass) - exists, file, err := cli.FileInfo(ctx, srvaddr+"/foo.bin") - require.NoError(t, err) - require.True(t, exists) - require.NotNil(t, file) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + exist, size, digest, err := cli.FileInfo(ctx, ref.Raw()) + require.NoError(t, err) + require.True(t, exist) + require.Equal(t, size, filesize) + fmt.Printf("Digest: %s\n", digest) } { - // GetFile - fmt.Printf("=== GetFile ===\n") - cli := client.NewClient(true) - ctx := context.Background() - ctx, _ = context.WithTimeout(ctx, 1*time.Second) + // FileList + fmt.Printf("=== FileList ===\n") + timeout := 10 * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref, err := filecli.ParsePath(diraddr) + require.NoError(t, err) + ref.SetUserinfo(user, pass) + ref.PathType(filecli.PathTypePrefix) + fmt.Printf("Raw: %s\n", ref.Raw()) - tmpdir := t.TempDir() - tmpfile := filepath.Join(tmpdir, "foo.bin") - - recsize, err := cli.GetFile(ctx, srvaddr+"/foo.bin", tmpfile) - require.NoError(t, err) - require.Equal(t, int64(filesize), recsize) - } - { - // ListFiles - fmt.Printf("=== ListFiles ===\n") - cli := client.NewClient(true) - ctx := context.Background() - ctx, _ = context.WithTimeout(ctx, 1*time.Second) - - files, err := cli.ListFiles(ctx, srvaddr+"/", terms.AsFinePath) - require.NoError(t, err) - require.NotZero(t, len(files)) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + list, err := cli.ListFiles(ctx, ref.Raw()) + require.NoError(t, err) + fmt.Printf("List: %s\n", list) } { // DeleteFile - fmt.Printf("=== DeleteFile ===\n") - cli := client.NewClient(true) - ctx := context.Background() - ctx, _ = context.WithTimeout(ctx, 1*time.Second) + fmt.Printf("=== FileList ===\n") + timeout := 10 * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref, err := filecli.ParsePath(fileaddr) + require.NoError(t, err) + ref.SetUserinfo(user, pass) + //ref.PathType(filecli.PathTypePrefix) + fmt.Printf("Raw: %s\n", ref.Raw()) - err = cli.DeleteFile(ctx, srvaddr+"/foo.bin") - require.NoError(t, err) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + exist, err := cli.DeleteFile(ctx, ref.Raw()) + require.NoError(t, err) + fmt.Printf("Exist: %v\n", exist) + require.True(t, exist) } - { - // !FileInfo - fmt.Printf("=== FileInfo ===\n") - cli := client.NewClient(true) - ctx := context.Background() - ctx, _ = context.WithTimeout(ctx, 1*time.Second) - - exists, _, err := cli.FileInfo(ctx, srvaddr+"/foo.bin") - require.NoError(t, err) - require.False(t, exists) + { + // DeleteFile + fmt.Printf("=== FileList ===\n") + timeout := 10 * time.Second + ctx, _ := context.WithTimeout(context.Background(), timeout) + ref, err := filecli.ParsePath(fileaddr) + require.NoError(t, err) + ref.SetUserinfo(user, pass) + //ref.PathType(filecli.PathTypePrefix) + fmt.Printf("Raw: %s\n", ref.Raw()) + mw := filecli.NewBasicAuthMiddleware(ref.Userinfo()) + cli := filecli.NewClient(nil, mw) + exist, err := cli.DeleteFile(ctx, ref.Raw()) + require.NoError(t, err) + fmt.Printf("Exist: %v\n", exist) + require.False(t, exist) } - } diff --git a/test/test-oci.tar b/test/test-oci.tar deleted file mode 100644 index c0d5540..0000000 Binary files a/test/test-oci.tar and /dev/null differ diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go index 799c7ea..3258743 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go @@ -15,6 +15,7 @@ package transport import ( + "cmp" "context" "errors" "fmt" @@ -75,7 +76,10 @@ func pingSingle(ctx context.Context, reg name.Registry, t http.RoundTripper, sch resp.Body.Close() }() - insecure := scheme == "http" + // If resp.Request is set, we may have followed a redirect, + // so we want to prefer resp.Request.URL.Scheme (if it's set) + // falling back to the original request's scheme. + insecure := cmp.Or(resp.Request, req).URL.Scheme == "http" switch resp.StatusCode { case http.StatusOK: diff --git a/vendor/modules.txt b/vendor/modules.txt index fd48071..5a23a6e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -30,7 +30,7 @@ github.com/fsnotify/fsnotify/internal ## explicit; go 1.18 github.com/go-viper/mapstructure/v2 github.com/go-viper/mapstructure/v2/internal/errors -# github.com/google/go-containerregistry v0.21.0 +# github.com/google/go-containerregistry v0.21.2 ## explicit; go 1.25.6 github.com/google/go-containerregistry/internal/and github.com/google/go-containerregistry/internal/compression