ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

client rebuilding in progress

Browse Source
This commit is contained in:
Олег Бородин
2026-03-04 12:27:52 +02:00
parent 2d34ec5634
commit ae9c29de1e
31 changed files with 908 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/handler/aaafunc.go
+2
View File
@@ -52,7 +52,9 @@ func (hand *Handler) CheckAccess(rctx *router.Context) (bool, string, error) {
accountID = terms.AnonymousID
hand.logg.Debugf("URL: %s", rctx.URL().String())
authHeader := rctx.GetHeader("Authorization")
hand.logg.Debugf("Authorization: %s", authHeader)
if authHeader != "" {
username, password, err = auxhttp.ParseBasicAuth(authHeader)
if err != nil {
app/handler/blob.go
+3 -2
View File
@@ -57,6 +57,8 @@ func (hand *Handler) BlobExists(rctx *router.Context) {
}
// POST /v2/<name>/blobs/uploads/ 202 404
// POST /v2/<name>/blobs/uploads/?digest=<digest> 201/202 404/400
// POST /v2/<name>/blobs/uploads/?mount=<digest>&from=<other_name> 201 404
func (hand *Handler) PostUpload(rctx *router.Context) {
name, _ := rctx.GetSubpath("name")
@@ -102,8 +104,7 @@ func (hand *Handler) PostUpload(rctx *router.Context) {
rctx.SetStatus(code)
}
// POST /v2/<name>/blobs/uploads/?digest=<digest> 201/202 404/400
// POST /v2/<name>/blobs/uploads/?mount=<digest>&from=<other_name> 201 404
// PATCH /v2/<name>/blobs/uploads/<reference> 202 404/416
func (hand *Handler) PatchUpload(rctx *router.Context) {
app/operator/account.go
+1 -3
View File
@@ -256,9 +256,7 @@ type ListAccountsResult struct {
func (oper *Operator) ListAccounts(ctx context.Context, params *ListAccountsParams) (*ListAccountsResult, error) {
var err error
res := &ListAccountsResult{
Accounts: make([]descr.AccountShort, 0),
}
res := &ListAccountsResult{}
accountDescrs, err := oper.mdb.ReducedListAccounts(ctx)
if err != nil {
app/operator/file.go
+119 -123
View File
@@ -24,7 +24,7 @@ import (
"mstore/pkg/auxtool"
"mstore/pkg/auxuuid"
"mstore/pkg/descr"
"mstore/pkg/terms"
"mstore/pkg/filecli"
)
// FileInfo
@@ -39,7 +39,6 @@ type FileInfoResult struct {
ContentType string
ContentSize string
ContentDigest string
ContentCreatedAt string
ContentCreatedBy string
ContentUpdatedAt string
@@ -299,122 +298,10 @@ func (oper *Operator) DeleteFile(ctx context.Context, operatorID string, params
return code, res, err
}
// ListFiles
type ListFilesParams struct {
Filepath string
PathAs string `param:"pathAs"`
}
type ListFilesResult struct {
Files []descr.File `json:"files,omitempty"`
}
func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) {
var err error
res := &ListFilesResult{
Files: make([]descr.File, 0),
}
switch params.PathAs {
case terms.AsRegexp:
files, err := oper.listFilesWithRegex(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
case terms.AsPrefix:
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
files, err := oper.listFilesWithPrefix(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
default: // Fine
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
files, err := oper.listFilesInOneCollection(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
}
code := http.StatusOK
return code, res, err
}
func (oper *Operator) listFilesInOneCollection(ctx context.Context, collection string) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
files, err := oper.mdb.ListFilesByCollection(ctx, collection)
if err != nil {
return res, err
}
res = files
return res, err
}
func (oper *Operator) listFilesWithPrefix(ctx context.Context, prefix string) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
files, err := oper.mdb.ListAllFiles(ctx)
if err != nil {
return res, err
}
for _, file := range files {
fullpath := filepath.Join(file.Collection, file.Name)
if strings.HasPrefix(fullpath, prefix) {
res = append(res, file)
}
}
return res, err
}
func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
re, err := regexp.Compile(regex)
if err != nil {
return res, err
}
files, err := oper.mdb.ListAllFiles(ctx)
if err != nil {
return res, err
}
for _, file := range files {
fullpath := filepath.Join(file.Collection, file.Name)
if re.MatchString(fullpath) {
res = append(res, file)
}
}
return res, err
}
// ListCollections
type ListCollectionsParams struct {
Path string
PathAS string `param:"pathAs"`
PathType string `param:"pathType"`
}
type ListCollectionsResult struct {
Collections []string `json:"collection,omitempty"`
@@ -427,14 +314,14 @@ func (oper *Operator) ListCollections(ctx context.Context, operatorID string, pa
}
collectionList := make([]string, 0)
switch param.PathAS {
case terms.AsRegexp:
switch param.PathType {
case filecli.PathTypeRegexp:
collectionList, err = oper.listCollectionsWithRegexp(ctx, param.Path)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
case terms.AsPrefix:
case filecli.PathTypePrefix:
param.Path, err = cleanFilepath(param.Path)
if err != nil {
code := http.StatusInternalServerError
@@ -546,7 +433,7 @@ func (oper *Operator) listAllCollections(ctx context.Context) ([]string, error)
// DeleteColletion
type DeleteColletionParams struct {
Path string
PathAs string `param:"pathAs"`
PathType string `param:"pathType"`
DryRun bool `param:"dryRun"`
}
type DeleteColletionResult struct {
@@ -558,8 +445,8 @@ func (oper *Operator) DeleteColletion(ctx context.Context, operatorID string, pa
res := &DeleteColletionResult{
Files: make([]descr.File, 0),
}
switch param.PathAs {
case terms.AsRegexp:
switch param.PathType {
case filecli.PathTypeRegexp:
collections, err := oper.listCollectionsWithRegexp(ctx, param.Path)
if err != nil {
code := http.StatusInternalServerError
@@ -576,7 +463,7 @@ func (oper *Operator) DeleteColletion(ctx context.Context, operatorID string, pa
}
res.Files = allfiles
case terms.AsPrefix:
case filecli.PathTypePrefix:
param.Path, err = cleanFilepath(param.Path)
if err != nil {
code := http.StatusInternalServerError
@@ -598,7 +485,7 @@ func (oper *Operator) DeleteColletion(ctx context.Context, operatorID string, pa
allfiles = append(allfiles, files...)
}
res.Files = allfiles
default: // Fine
default:
param.Path, err = cleanFilepath(param.Path)
if err != nil {
code := http.StatusInternalServerError
@@ -641,3 +528,112 @@ func (oper *Operator) deleteFilesInCollection(ctx context.Context, collection st
}
return res, err
}
// ListFiles
type ListFilesParams struct {
Filepath string
PathType string `param:"pathType"`
}
type ListFilesResult struct {
Files []descr.File `json:"files,omitempty"`
}
func (oper *Operator) ListFiles(ctx context.Context, operatorID string, params *ListFilesParams) (int, *ListFilesResult, error) {
var err error
res := &ListFilesResult{}
switch params.PathType {
case filecli.PathTypeRegexp:
files, err := oper.listFilesWithRegex(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
case filecli.PathTypePrefix:
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
files, err := oper.listFilesWithPrefix(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
default:
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
params.Filepath, err = cleanFilepath(params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
files, err := oper.listFilesInCollection(ctx, params.Filepath)
if err != nil {
code := http.StatusInternalServerError
return code, res, err
}
res.Files = files
}
code := http.StatusOK
return code, res, err
}
func (oper *Operator) listFilesInCollection(ctx context.Context, collection string) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
files, err := oper.mdb.ListFilesByCollection(ctx, collection)
if err != nil {
return res, err
}
res = files
return res, err
}
func (oper *Operator) listFilesWithPrefix(ctx context.Context, prefix string) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
files, err := oper.mdb.ListAllFiles(ctx)
if err != nil {
return res, err
}
for _, file := range files {
fullpath := filepath.Join(file.Collection, file.Name)
if strings.HasPrefix(fullpath, prefix) {
res = append(res, file)
}
}
return res, err
}
func (oper *Operator) listFilesWithRegex(ctx context.Context, regex string) ([]descr.File, error) {
var err error
res := make([]descr.File, 0)
re, err := regexp.Compile(regex)
if err != nil {
return res, err
}
files, err := oper.mdb.ListAllFiles(ctx)
if err != nil {
return res, err
}
for _, file := range files {
fullpath := filepath.Join(file.Collection, file.Name)
if re.MatchString(fullpath) {
res = append(res, file)
}
}
return res, err
}
app/operator/manifest.go
+11 -1
View File
@@ -321,7 +321,7 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams
if !exists {
return res, http.StatusNotFound, err
}
/*
index, indexBytes, err := indexFromManigestDescrs(manifestDescrs)
if err != nil {
return res, http.StatusInternalServerError, err
@@ -332,6 +332,16 @@ func (oper *Operator) GetManifest(ctx context.Context, params *GetManifestParams
res.ContentLength = strconv.FormatInt(int64(len(indexBytes)), 10)
res.ContentType = index.MediaType
res.Payload = string(indexBytes)
*/
manifestDescr = manifestDescrs[0]
manifestDigest := auxoci.SHA256DigestFromString(manifestDescr.Payload)
res.DockerContentDigest = manifestDigest.String()
res.ContentLength = strconv.FormatInt(int64(len(manifestDescr.Payload)), 10)
res.ContentType = manifestDescr.ContentType
res.Payload = manifestDescr.Payload
}
return res, http.StatusOK, err
}
app/router/context.go
+6 -3
View File
@@ -15,6 +15,7 @@ import (
"encoding/json"
"io"
"net/http"
"net/url"
"strconv"
)
@@ -29,11 +30,10 @@ type Context struct {
}
func NewContext(writer http.ResponseWriter, request *http.Request) *Context {
ctx := context.Background()
rctx := &Context{
Writer: writer,
Request: request,
Ctx: ctx,
Ctx: request.Context(),
PathMap: make(map[string]string),
Bools: make(map[string]bool),
Strings: make(map[string]string),
@@ -66,6 +66,10 @@ func (rctx *Context) GetSubpath(key string) (string, bool) {
return value, exists
}
func (rctx *Context) URL() *url.URL {
return rctx.Request.URL
}
func (rctx *Context) GetQuery(key string) string {
return rctx.Request.URL.Query().Get(key)
}
@@ -83,7 +87,6 @@ func (rctx *Context) GetContext() context.Context {
}
// Binding
const emptyJSON = "{}"
func (rctx *Context) BindJSON(obj any) error {
cmd/mstorectl/command/filecmd.go
+129 -177
View File
@@ -11,6 +11,7 @@ package command
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
@@ -18,19 +19,17 @@ import (
"os"
"path"
"path/filepath"
"slices"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"mstore/pkg/client"
"mstore/pkg/filecli"
"mstore/pkg/descr"
"mstore/pkg/terms"
)
func (util *FileUtil) CreateFileCmds() *cobra.Command {
func (util *FileUtil) MakeFileCmds() *cobra.Command {
var subCmd = &cobra.Command{
Use: "files",
Short: "File operations",
@@ -127,7 +126,7 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command {
return subCmd
}
func (util *FileUtil) CreateCollectionCmds() *cobra.Command {
func (util *FileUtil) MakeCollectionCmds() *cobra.Command {
var subCmd = &cobra.Command{
Use: "collections",
Short: "Colletion operations",
@@ -176,8 +175,8 @@ type FileUtil struct {
listFilesParams ListFilesParams
importFilesParams ImportFilesParams
exportFilesParams ExportFilesParams
listCollectionsParams ListCollectionsParams
deleteCollectionParams DeleteCollectionParams
listCollectionsParams ListCollectionsParams
commonFileParams CommonFileParams
}
@@ -188,41 +187,6 @@ type CommonFileParams struct {
SkipTLSVerify bool
}
// FileInfo
type FileInfoParams struct {
Filepath string
}
type FileInfoResult struct {
File *descr.File `yaml:"file,omitempty"`
}
func (util *FileUtil) FileInfo(cmd *cobra.Command, args []string) {
util.fileInfoParams.Filepath = args[0]
res, err := util.fileInfo(&util.commonFileParams, &util.fileInfoParams)
printResponse(res, err)
}
func (util *FileUtil) fileInfo(common *CommonFileParams, params *FileInfoParams) (*FileInfoResult, error) {
var err error
res := &FileInfoResult{}
params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
if err != nil {
return res, err
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
exists, opres, err := client.NewClient(common.SkipTLSVerify).FileInfo(ctx, params.Filepath)
if err != nil {
return res, err
}
if !exists {
err = fmt.Errorf("File %s not exists", params.Filepath)
return res, err
}
res.File = opres
return res, err
}
// PutFile
type PutFileParams struct {
Source string
@@ -240,13 +204,27 @@ func (util *FileUtil) PutFile(cmd *cobra.Command, args []string) {
func (util *FileUtil) putFile(common *CommonFileParams, params *PutFileParams) (*PutFileResult, error) {
var err error
res := &PutFileResult{}
params.Dest, err = packUserinfo(params.Dest, common.Username, common.Password)
file, err := os.OpenFile(params.Dest, os.O_RDONLY, 0)
if err != nil {
return res, err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return res, err
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
err = client.NewClient(common.SkipTLSVerify).PutFile(ctx, params.Source, params.Dest)
ref, err := filecli.ParsePath(params.Dest)
if err != nil {
return res, err
}
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
err = cli.PutFile(ctx, ref.Raw(), file, stat.Size())
if err != nil {
return res, err
}
@@ -271,13 +249,27 @@ type GetFileResult struct{}
func (util *FileUtil) getFile(common *CommonFileParams, params *GetFileParams) (*GetFileResult, error) {
var err error
res := &GetFileResult{}
params.Dest, err = packUserinfo(params.Source, common.Username, common.Password)
err = os.MkdirAll(filepath.Dir(params.Dest), 0750)
if err != nil {
return res, err
}
file, err := os.OpenFile(params.Dest, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return res, err
}
defer file.Close()
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
_, err = client.NewClient(common.SkipTLSVerify).GetFile(ctx, params.Dest, params.Source)
ref, err := filecli.ParsePath(params.Source)
if err != nil {
return res, err
}
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
_, err = cli.GetFile(ctx, ref.Raw(), file)
if err != nil {
return res, err
}
@@ -299,71 +291,23 @@ func (util *FileUtil) DeleteFile(cmd *cobra.Command, args []string) {
func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFileParams) (*DeleteFileResult, error) {
var err error
res := &DeleteFileResult{}
params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
ref, err := filecli.ParsePath(params.Filepath)
if err != nil {
return res, err
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
err = client.NewClient(common.SkipTLSVerify).DeleteFile(ctx, params.Filepath)
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
_, err = cli.DeleteFile(ctx, ref.Raw())
if err != nil {
return res, err
}
return res, err
}
// ListFiles
type ListFilesParams struct {
Filepath string
Detail bool
AsPrefix bool
AsRegexp bool
}
type ListFilesResult struct {
Files []descr.File `yaml:"files,omitempty"`
Filenames []string `yaml:"filenames,omitempty"`
}
func (util *FileUtil) ListFiles(cmd *cobra.Command, args []string) {
util.listFilesParams.Filepath = args[0]
res, err := util.listFiles(&util.commonFileParams, &util.listFilesParams)
printResponse(res, err)
}
func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParams) (*ListFilesResult, error) {
var err error
res := &ListFilesResult{}
params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
if err != nil {
return res, err
}
if params.AsRegexp {
params.AsPrefix = false
}
var pathUsage string
switch {
case params.AsRegexp:
pathUsage = terms.AsRegexp
case params.AsPrefix:
pathUsage = terms.AsPrefix
default:
pathUsage = terms.AsFinePath
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
files, err := client.NewClient(common.SkipTLSVerify).ListFiles(ctx, params.Filepath, pathUsage)
if err != nil {
return res, err
}
if params.Detail {
res.Files = files
} else {
res.Filenames = makeFilelistFromFiles(files)
}
return res, err
}
// ImportFiles
type ImportFilesParams struct {
Source string
@@ -397,25 +341,39 @@ func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesP
if err != nil {
return err
}
if infoItem.Mode() == fs.ModeDevice {
return nil
}
if infoItem.Mode() == fs.ModeNamedPipe {
return nil
}
if infoItem.Mode() == fs.ModeCharDevice {
mode := infoItem.Mode()
skip := mode == fs.ModeDevice
skip = skip || mode == fs.ModeNamedPipe
skip = skip || mode == fs.ModeCharDevice
if skip {
return nil
}
if !infoItem.IsDir() {
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
relPath, _ := strings.CutPrefix(walkPath, params.Source)
dest, _ := url.JoinPath(params.Dest, relPath)
ref, err := filecli.ParsePath(dest)
if err != nil {
putErrors = append(putErrors, err)
return nil
}
err = client.NewClient(common.SkipTLSVerify).PutFile(ctx, walkPath, dest)
file, err := os.OpenFile(dest, os.O_RDONLY, 0)
if err != nil {
putErrors = append(putErrors, err)
return nil
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
putErrors = append(putErrors, err)
return nil
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref.SetUserinfo(common.Username, common.Password)
user, pass := ref.Userinfo()
mw := filecli.NewBasicAuthMiddleware(user, pass)
cli := filecli.NewClient(nil, mw)
err = cli.PutFile(ctx, ref.Raw(), file, stat.Size())
if err != nil {
putErrors = append(putErrors, err)
fmt.Printf("- %s: error: %v \n", walkPath, err)
@@ -459,57 +417,62 @@ func (util *FileUtil) ExportFiles(cmd *cobra.Command, args []string) {
func (util *FileUtil) exportFiles(common *CommonFileParams, params *ExportFilesParams) (*ExportFilesResult, error) {
var err error
res := &ExportFilesResult{}
params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := filecli.ParsePath(params.Filepath)
if err != nil {
return res, err
}
if params.AsRegexp {
params.AsPrefix = false
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
list, err := cli.ListFiles(ctx, ref.Raw())
if err != nil {
return res, err
}
var pathUsage string
switch {
case params.AsRegexp:
pathUsage = terms.AsRegexp
case params.AsPrefix:
pathUsage = terms.AsPrefix
default:
pathUsage = terms.AsFinePath
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
files, err := client.NewClient(common.SkipTLSVerify).ListFiles(ctx, params.Filepath, pathUsage)
files := make([]descr.File, 0)
err = json.Unmarshal(list, &files)
if err != nil {
return res, err
}
exportedFiles := make([]descr.File, 0)
for _, file := range files {
destdir := filepath.Join(params.Dest, file.Collection)
exported := make([]descr.File, 0)
for _, descr := range files {
destdir := filepath.Join(params.Dest, descr.Collection)
err = os.MkdirAll(destdir, 0750)
if err != nil {
return res, err
}
srcpath, err := makeFileURI(params.Filepath, file.Collection, file.Name)
destpath := filepath.Join(params.Dest, descr.Collection, descr.Name)
destfile, err := os.OpenFile(destpath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return res, err
}
destpath := filepath.Join(params.Dest, file.Collection, file.Name)
defer destfile.Close()
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
_, err = client.NewClient(common.SkipTLSVerify).GetFile(ctx, srcpath, destpath)
ref.SetResource(filepath.Join(descr.Collection, descr.Name))
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
_, err = cli.GetFile(ctx, ref.Raw(), destfile)
if err != nil {
fmt.Printf("- %s: error %v\n", srcpath, err)
//return res, err
fmt.Printf("- %s: error %v\n", ref.Raw(), err)
err = nil
} else {
fmt.Printf("- %s: ok\n", srcpath)
exportedFiles = append(exportedFiles, file)
fmt.Printf("- %s: ok\n", ref.Raw())
exported = append(exported, descr)
}
}
if params.Detail {
res.Files = exportedFiles
res.Files = exported
} else {
res.Filenames = makeFilelistFromFiles(exportedFiles)
files := descr.NewFiles()
files.Set(exported)
res.Filenames = files.List()
}
return res, err
}
@@ -548,29 +511,26 @@ func (util *FileUtil) listCollections(common *CommonFileParams, params *ListColl
res := &ListCollectionsResult{
Collections: make([]string, 0),
}
params.Path, err = packUserinfo(params.Path, common.Username, common.Password)
if err != nil {
return res, err
}
if params.AsRegexp {
params.AsPrefix = false
}
var pathUsage string
switch {
case params.AsRegexp:
pathUsage = terms.AsRegexp
case params.AsPrefix:
pathUsage = terms.AsPrefix
default:
pathUsage = terms.AsFinePath
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
collecions, err := client.NewClient(common.SkipTLSVerify).ListCollections(ctx, params.Path, pathUsage)
ref, err := filecli.ParsePath(params.Path)
if err != nil {
return res, err
}
res.Collections = collecions
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
list, err := cli.ListCollections(ctx, ref.Raw())
if err != nil {
return res, err
}
collections := make([]string, 0)
err = json.Unmarshal(list, collections)
if err != nil {
return res, err
}
res.Collections = collections
return res, err
}
@@ -599,36 +559,28 @@ func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteC
res := &DeleteCollectionResult{
Filenames: make([]string, 0),
}
params.Path, err = packUserinfo(params.Path, common.Username, common.Password)
if err != nil {
return res, err
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
var pathUsage string
switch {
case params.AsPrefix:
pathUsage = terms.AsPrefix
default:
pathUsage = terms.AsFinePath
}
files, err := client.NewClient(common.SkipTLSVerify).DeleteCollection(ctx, params.Path, pathUsage, params.DryRun)
ref, err := filecli.ParsePath(params.Path)
if err != nil {
return res, err
}
if params.Detail {
res.Files = files
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
list, err := cli.DeleteCollection(ctx, ref.Raw())
files := descr.NewFiles()
err = json.Unmarshal(list, files.ArrayPtr())
if err != nil {
return res, err
}
if !params.Detail {
res.Filenames = files.List()
} else {
res.Filenames = makeFilelistFromFiles(files)
res.Files = files.Array()
}
return res, err
}
func makeFilelistFromFiles(files []descr.File) []string {
res := make([]string, 0, len(files))
for _, file := range files {
res = append(res, filepath.Join(file.Collection, file.Name))
}
slices.Sort(res)
return res
}
cmd/mstorectl/command/imagecmd.go
+37 -12
View File
@@ -11,6 +11,8 @@ package command
import (
"context"
"encoding/json"
"fmt"
"net/url"
"os"
"path"
@@ -21,6 +23,7 @@ import (
"github.com/spf13/viper"
"mstore/pkg/client"
"mstore/pkg/repocli"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@@ -268,33 +271,55 @@ type ImageManifestParams struct {
}
type ImageManifestResult struct {
ImageManifest any `json:"imageManifest"`
Index *ocispec.Index `json:"index,omitempty"`
Manifest *ocispec.Manifest `json:"manifest,omitempty"`
}
func (util *ImageUtil) ImageManifest(cmd *cobra.Command, args []string) {
util.imageManifestParams.Imagepath = args[0]
res, err := util.imageInfo(&util.commonImageParams, &util.imageManifestParams)
res, err := util.imageManifest(&util.commonImageParams, &util.imageManifestParams)
printResponse(res, err)
}
func (util *ImageUtil) imageInfo(common *CommonImageParams, params *ImageManifestParams) (*ImageManifestResult, error) {
func (util *ImageUtil) imageManifest(common *CommonImageParams, params *ImageManifestParams) (*ImageManifestResult, error) {
var err error
res := &ImageManifestResult{}
ctx := context.Background()
cli := client.NewClient(common.SkipTLSVerify)
timeout := time.Duration(common.Timeout) * time.Second
res := &ImageManifestResult{
Index: &ocispec.Index{},
Manifest: &ocispec.Manifest{},
}
params.Imagepath, err = packUserinfo(params.Imagepath, common.Username, common.Password)
if err != nil {
return res, err
}
ctx, _ = context.WithTimeout(ctx, timeout)
opres, err := cli.ImageManifest(ctx, params.Imagepath)
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := repocli.ParseReference(params.Imagepath)
if err != nil {
return res, err
}
res.ImageManifest = opres
mw := repocli.NewBasicAuthMiddleware(ref.Userinfo())
cli := repocli.NewClientWithTransport(nil, mw)
exists, mime, man, err := cli.GetManifest(ctx, ref.Repo(), ref.Tag())
if !exists {
err = fmt.Errorf("Manifest not found")
return res, err
}
switch mime {
case repocli.MediaTypeDDMLv2, repocli.MediaTypeOIIv1:
err = json.Unmarshal(man, res.Index)
if err != nil {
return res, err
}
case repocli.MediaTypeDDMv2, repocli.MediaTypeOIMv1:
err = json.Unmarshal(man, res.Manifest)
if err != nil {
return res, err
}
default:
err = fmt.Errorf("Unknown content type: %s", mime)
return res, err
}
return res, err
}
go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mstore
go 1.25.6
require (
github.com/google/go-containerregistry v0.21.0
github.com/google/go-containerregistry v0.21.2
github.com/google/uuid v1.6.0
github.com/jmoiron/sqlx v1.4.0
github.com/mattn/go-sqlite3 v1.14.34
go.sum
+2 -2
View File
@@ -24,8 +24,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-containerregistry v0.21.0 h1:ocqxUOczFwAZQBMNE7kuzfqvDe0VWoZxQMOesXreCDI=
github.com/google/go-containerregistry v0.21.0/go.mod h1:ctO5aCaewH4AK1AumSF5DPW+0+R+d2FmylMJdp5G7p0=
github.com/google/go-containerregistry v0.21.2 h1:vYaMU4nU55JJGFC9JR/s8NZcTjbE9DBBbvusTW9NeS0=
github.com/google/go-containerregistry v0.21.2/go.mod h1:ctO5aCaewH4AK1AumSF5DPW+0+R+d2FmylMJdp5G7p0=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
pkg/descr/file.go
+45
View File
@@ -9,6 +9,10 @@
*/
package descr
import (
"path"
)
type File struct {
ID string `db:"id" json:"id,omitempty" yaml:"id,omitempty"`
Collection string `db:"collection" json:"collection,omitempty" yaml:"collection,omitempty"`
@@ -21,3 +25,44 @@ type File struct {
CreatedBy string `db:"created_by" json:"createdBy,omitempty" yaml:"createdBy,omitempty"`
UpdatedBy string `db:"updated_by" json:"updatedBy,omitempty" yaml:"updatedBy,omitempty"`
}
type Files struct {
files []File
}
func NewFiles() *Files {
return &Files{
files: make([]File, 0),
}
}
func xxxNewFiles(files []File) *Files {
return &Files{
files: files,
}
}
func (fi *Files) Set(files []File) {
fi.files = files
}
func (fi Files) ArrayPtr() *[]File {
return &fi.files
}
func (fi Files) Array() []File {
return fi.files
}
func (fi Files) List() []string {
list := make([]string, 0)
for _, file := range fi.files {
list = append(list, path.Join(file.Collection, file.Name))
}
return list
}
pkg/filecli/client.go
+29 -21
View File
@@ -11,13 +11,17 @@ type Client struct {
userAgent string
}
func NewClient(transport http.RoundTripper, mwFunc ...MiddlewareFunc) *Client {
func NewClient(transport http.RoundTripper, mwFuncs ...MiddlewareFunc) *Client {
if transport == nil {
transport = NewDefaultTransport()
}
for _, mwFunc := range mwFuncs {
transport = mwFunc(transport)
}
httpClient := &http.Client{
Transport: transport,
}
return &Client{
httpClient: httpClient,
userAgent: "ociClient/1.0",
@@ -34,26 +38,6 @@ func (cli *Client) UseMiddleware(mwFunc MiddlewareFunc) {
cli.httpClient.Transport = mwFunc(cli.httpClient.Transport)
}
// ExampleMiddleware
func NewExampleMiddleware() MiddlewareFunc {
return func(next http.RoundTripper) http.RoundTripper {
return newExampleTransport(next)
}
}
type ExampleTransport struct {
next http.RoundTripper
}
func newExampleTransport(next http.RoundTripper) *ExampleTransport {
return &ExampleTransport{
next: next,
}
}
func (tran ExampleTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return tran.next.RoundTrip(req)
}
// BasicAuthMiddleware
func NewBasicAuthMiddleware(user, pass string) MiddlewareFunc {
@@ -76,8 +60,10 @@ func newBasicAuthMW(next http.RoundTripper, user, pass string) *BasicAuthMW {
}
func (tran BasicAuthMW) RoundTrip(req *http.Request) (*http.Response, error) {
if tran.user != "" && tran.pass != "" {
pair := base64.StdEncoding.EncodeToString([]byte(tran.user + ":" + tran.pass))
req.Header.Set("Authorization", "Basic "+pair)
}
return tran.next.RoundTrip(req)
}
@@ -123,3 +109,25 @@ func NewDefaultTransport() *DefaultTransport {
func (wrap *DefaultTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return wrap.transport.RoundTrip(req)
}
// ExampleMiddleware
func NewExampleMiddleware() MiddlewareFunc {
return func(next http.RoundTripper) http.RoundTripper {
return newExampleTransport(next)
}
}
type ExampleTransport struct {
next http.RoundTripper
}
func newExampleTransport(next http.RoundTripper) *ExampleTransport {
return &ExampleTransport{
next: next,
}
}
func (tran ExampleTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return tran.next.RoundTrip(req)
}
pkg/filecli/delcoll.go
+51
View File
@@ -0,0 +1,51 @@
package filecli
import (
"bytes"
"context"
"fmt"
"net/http"
"strconv"
)
func (cli *Client) DeleteCollection(ctx context.Context, rawpath string) ([]byte, error) {
var err error
var list []byte
ref, err := ParsePath(rawpath)
if err != nil {
return list, err
}
uri := ref.Collection()
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, uri, nil)
if err != nil {
return list, err
}
req.Header.Set("User-Agent", cli.userAgent)
req.Header.Set("Accept", "*/*")
resp, err := cli.httpClient.Do(req)
if err != nil {
return list, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return list, err
}
if resp.StatusCode != http.StatusOK {
err := fmt.Errorf("Unxected response code %s", resp.Status)
return list, err
}
contentLength := resp.Header.Get("Content-Length")
blobSize, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return list, err
}
buffer := bytes.NewBuffer(nil)
recSize, err := Copy(ctx, buffer, resp.Body)
if blobSize != recSize {
err := fmt.Errorf("Mismatch declared and actual body size, %d and %d", blobSize, recSize)
return list, err
}
list = buffer.Bytes()
return list, err
}
pkg/filecli/fileexist.go → pkg/filecli/fileinfo.go
+11 -10
View File
@@ -7,44 +7,45 @@ import (
"strconv"
)
func (cli *Client) FileExists(ctx context.Context, rawpath string) (bool, int64, error) {
func (cli *Client) FileInfo(ctx context.Context, rawpath string) (bool, int64, string, error) {
var err error
var exist bool
var size int64
var digest string
ref, err := ParsePath(rawpath)
if err != nil {
return exist, size, err
return exist, size, digest, err
}
uri := ref.File()
fmt.Println(uri)
req, err := http.NewRequestWithContext(ctx, http.MethodHead, uri, nil)
if err != nil {
return exist, size, err
return exist, size, digest, err
}
req.Header.Set("User-Agent", cli.userAgent)
req.Header.Set("Accept", "*/*")
resp, err := cli.httpClient.Do(req)
if err != nil {
return exist, size, err
return exist, size, digest, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return exist, size, err
return exist, size, digest, err
}
if resp.StatusCode != http.StatusOK {
err := fmt.Errorf("Unxected response code %s", resp.Status)
return exist, size, err
return exist, size, digest, err
}
contentLength := resp.Header.Get("Content-Length")
contentLength := resp.Header.Get("Content-Size")
size, err = strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return exist, size, err
return exist, size, digest, err
}
digest = resp.Header.Get("Content-Digest")
exist = true
return exist, size, err
return exist, size, digest, err
}
pkg/filecli/listcoll.go
+53
View File
@@ -0,0 +1,53 @@
package filecli
import (
"bytes"
"context"
"fmt"
"net/http"
"strconv"
)
func (cli *Client) ListCollections(ctx context.Context, rawpath string) ([]byte, error) {
var err error
var list []byte
ref, err := ParsePath(rawpath)
if err != nil {
return list, err
}
uri := ref.Collections()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
if err != nil {
return list, err
}
req.Header.Set("User-Agent", cli.userAgent)
req.Header.Set("Accept", "*/*")
resp, err := cli.httpClient.Do(req)
if err != nil {
return list, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return list, err
}
if resp.StatusCode != http.StatusOK {
err := fmt.Errorf("Unexpected response code %s", resp.Status)
return list, err
}
contentLength := resp.Header.Get("Content-Length")
blobSize, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return list, err
}
buffer := bytes.NewBuffer(nil)
recSize, err := Copy(ctx, buffer, resp.Body)
if blobSize != recSize {
err := fmt.Errorf("Mismatch declared and actual body size, %d and %d", blobSize, recSize)
return list, err
}
list = buffer.Bytes()
return list, err
}
pkg/filecli/listfiles.go
+53
View File
@@ -0,0 +1,53 @@
package filecli
import (
"bytes"
"context"
"fmt"
"net/http"
"strconv"
)
func (cli *Client) ListFiles(ctx context.Context, rawpath string) ([]byte, error) {
var err error
var list []byte
ref, err := ParsePath(rawpath)
if err != nil {
return list, err
}
uri := ref.Files()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
if err != nil {
return list, err
}
req.Header.Set("User-Agent", cli.userAgent)
req.Header.Set("Accept", "*/*")
resp, err := cli.httpClient.Do(req)
if err != nil {
return list, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return list, err
}
if resp.StatusCode != http.StatusOK {
err := fmt.Errorf("Unexpected response code %s", resp.Status)
return list, err
}
contentLength := resp.Header.Get("Content-Length")
blobSize, err := strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return list, err
}
buffer := bytes.NewBuffer(nil)
recSize, err := Copy(ctx, buffer, resp.Body)
if blobSize != recSize {
err := fmt.Errorf("Mismatch declared and actual body size, %d and %d", blobSize, recSize)
return list, err
}
list = buffer.Bytes()
return list, err
}
pkg/filecli/putfile.go
+2 -2
View File
@@ -21,14 +21,14 @@ func (cli *Client) PutFile(ctx context.Context, rawpath string, src io.Reader, s
}
req.Header.Set("User-Agent", cli.userAgent)
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", strconv.FormatInt(size, 10))
req.Header.Set("Content-Size", strconv.FormatInt(size, 10))
resp, err := cli.httpClient.Do(req)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("File not accepted, code %d", resp.StatusCode)
err = fmt.Errorf("File not accepted, code %s", resp.Status)
return err
}
return err
pkg/filecli/repo.go
+46 -5
View File
@@ -2,17 +2,27 @@ package filecli
import (
"net/url"
"path"
"strings"
)
const (
PathTypeIdentic = "identic"
PathTypePrefix = "prefix"
PathTypeRegexp = "regexp"
)
type Repository struct {
urlobj *url.URL
user, pass string
resource string
values url.Values
}
func ParsePath(rawpath string) (*Repository, error) {
repo := &Repository{}
repo := &Repository{
values: url.Values{},
}
if !strings.Contains(rawpath, "://") {
rawpath = "https://" + rawpath
}
@@ -25,14 +35,30 @@ func ParsePath(rawpath string) (*Repository, error) {
repo.pass, _ = urlobj.User.Password()
urlobj.User = nil
}
repo.resource = repo.urlobj.Path
repo.resource = urlobj.Path
urlobj.Path = "/"
repo.urlobj = urlobj
repo.urlobj.Path = "/"
repo.urlobj = urlobj
repo.values = urlobj.Query()
return repo, err
}
func (repo *Repository) Raw() string {
res := path.Join(repo.urlobj.Host, repo.resource)
query := repo.values.Encode()
if query != "" {
return res + "?" + query
}
return res
}
func (repo *Repository) SetResource(resource string) {
repo.resource = resource
}
func (repo *Repository) PathType(typ string) {
repo.values.Set("pathType", typ)
}
func (repo *Repository) File() string {
curl := repo.urlobj.JoinPath("/v3/api/file", repo.resource)
return curl.String()
@@ -43,7 +69,22 @@ func (repo *Repository) Files() string {
return curl.String()
}
func (repo *Repository) Collection() string {
curl := repo.urlobj.JoinPath("/v3/api/collection", repo.resource)
return curl.String()
}
func (repo *Repository) Collections() string {
curl := repo.urlobj.JoinPath("/v3/api/collections", repo.resource)
return curl.String()
}
func (repo *Repository) Userinfo() (string, string) {
return repo.user, repo.pass
}
func (repo *Repository) SetUserinfo(user, pass string) {
if user != "" && pass != "" {
repo.user, repo.pass = user, pass
}
}
pkg/repocli/client.go
+2
View File
@@ -87,8 +87,10 @@ func newBasicAuthMW(next http.RoundTripper, user, pass string) *BasicAuthMW {
}
func (tran BasicAuthMW) RoundTrip(req *http.Request) (*http.Response, error) {
if tran.user != "" && tran.pass != "" {
pair := base64.StdEncoding.EncodeToString([]byte(tran.user + ":" + tran.pass))
req.Header.Set("Authorization", "Basic "+pair)
}
return tran.next.RoundTrip(req)
}
pkg/repocli/client_test.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"time"
)
func xxxTestClientGetManifest(t *testing.T) {
func TestClientGetManifest(t *testing.T) {
rawrepo := "mirror.gcr.io/alpine"
tags := []string{
"3.20.0",
pkg/repocli/mimetyp.go
+2 -2
View File
@@ -1,6 +1,6 @@
package repocli
const (
MediaTypeDDMLv2 = "application/vnd.docker.distribution.manifest.list.v2+json"
MediaTypeDDMv2 = "application/vnd.docker.distribution.manifest.v2+json"
//MediaTypeDDMLv2 = "application/vnd.docker.distribution.manifest.list.v2+json"
//MediaTypeDDMv2 = "application/vnd.docker.distribution.manifest.v2+json"
)
pkg/repocli/pullimage.go
+5 -5
View File
@@ -12,9 +12,9 @@ import (
const (
MediaTypeOIIv1 = "application/vnd.oci.image.index.v1+json"
MediatypeDDMLv2 = "application/vnd.docker.distribution.manifest.list.v2+json"
MediaTypeDDMLv2 = "application/vnd.docker.distribution.manifest.list.v2+json"
MediatypeDDMv2 = "application/vnd.docker.distribution.manifest.v2+json"
MediaTypeDDMv2 = "application/vnd.docker.distribution.manifest.v2+json"
MediaTypeOIMv1 = "application/vnd.oci.image.manifest.v1+json"
)
@@ -30,7 +30,7 @@ func NewDownloader(client *Client) *Downloader {
func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string) error {
var err error
ref, err := NewReference(rawref)
ref, err := ParseReference(rawref)
if err != nil {
return err
}
@@ -46,7 +46,7 @@ func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string)
return err
}
if mime == MediaTypeOIIv1 || mime == MediatypeDDMLv2 {
if mime == MediaTypeOIIv1 || mime == MediaTypeDDMLv2 {
var index ocispec.Index
err = json.Unmarshal(man, &index)
if err != nil {
@@ -72,7 +72,7 @@ func (down *Downloader) Pull(ctx context.Context, rawref, dir, os, arch string)
err = errors.New("Manifest not found")
return err
}
if mime != MediaTypeOIMv1 && mime != MediatypeDDMv2 {
if mime != MediaTypeOIMv1 && mime != MediaTypeDDMv2 {
err = errors.New("Unknown manifest media type")
return err
}
pkg/repocli/reference.go
+5 -1
View File
@@ -13,7 +13,7 @@ type Reference struct {
base, tag string
}
func NewReference(rawref string) (*Reference, error) {
func ParseReference(rawref string) (*Reference, error) {
ref := &Reference{}
if !strings.Contains(rawref, "://") {
rawref = "https://" + rawref
@@ -54,3 +54,7 @@ func (ref *Reference) Repo() string {
func (ref *Reference) Tag() string {
return ref.tag
}
func (ref *Reference) Userinfo() (string, string) {
return ref.user, ref.pass
}
test/account_test.go → test/attic/account_test.go
View File
test/attic/file_test.go
+163
View File
@@ -0,0 +1,163 @@
/*
* Copyright 2026 Oleg Borodin <onborodin@gmail.com>
*
* This work is published and licensed under a Creative Commons
* Attribution-NonCommercial-NoDerivatives 4.0 International License.
*
* Distribution of this work is permitted, but commercial use and
* modifications are strictly prohibited.
*/
package test
import (
"context"
"fmt"
"math/rand"
"os"
"path/filepath"
"sync"
"testing"
"time"
"mstore/app/server"
"mstore/pkg/client"
"mstore/pkg/terms"
"github.com/stretchr/testify/require"
)
func TestFileLife(t *testing.T) {
var srvport int64 = 10250
srvdir := t.TempDir()
srvaddr := fmt.Sprintf("mstore:mstore@127.0.0.1:%d", srvport)
srv, err := server.NewServer()
require.NoError(t, err)
{
err = srv.Configure()
require.NoError(t, err)
useTmpDir := true
if useTmpDir {
srv.SetDatadir(srvdir)
srv.SetLogdir(srvdir)
srv.SetRundir(srvdir)
}
srv.SetPort(srvport)
err = srv.Build()
require.NoError(t, err)
var svcWG sync.WaitGroup
errPipe := make(chan error, 5)
startFunc := func() {
err := srv.Service().Run()
errPipe <- err
svcWG.Done()
}
stopFunc := func() {
srv.Service().Stop()
svcWG.Wait()
err = <-errPipe
require.NoError(t, err)
}
defer stopFunc()
svcWG.Add(1)
go startFunc()
time.Sleep(1 * time.Second)
}
{
// ServiceHello
fmt.Printf("=== ServiceHello ===\n")
cli := client.NewClient(true)
ctx := context.Background()
helloRes, err := cli.ServiceHello(ctx, srvaddr+"/hello", 1*time.Second)
require.NoError(t, err)
require.True(t, helloRes)
}
filesize := 32
{
// PutFile
tmpdir := t.TempDir()
tmpfile := filepath.Join(tmpdir, "foo.bin")
filedata := make([]byte, filesize)
_, err = rand.Read(filedata)
require.NoError(t, err)
err := os.WriteFile(tmpfile, filedata, 0666)
require.NoError(t, err)
fmt.Printf("=== PutFile ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
err = cli.PutFile(ctx, tmpfile, srvaddr+"/foo.bin")
require.NoError(t, err)
}
{
// FileInfo
fmt.Printf("=== FileInfo ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
exists, file, err := cli.FileInfo(ctx, srvaddr+"/foo.bin")
require.NoError(t, err)
require.True(t, exists)
require.NotNil(t, file)
}
{
// GetFile
fmt.Printf("=== GetFile ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
tmpdir := t.TempDir()
tmpfile := filepath.Join(tmpdir, "foo.bin")
recsize, err := cli.GetFile(ctx, srvaddr+"/foo.bin", tmpfile)
require.NoError(t, err)
require.Equal(t, int64(filesize), recsize)
}
{
// ListFiles
fmt.Printf("=== ListFiles ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
files, err := cli.ListFiles(ctx, srvaddr+"/", terms.AsFinePath)
require.NoError(t, err)
require.NotZero(t, len(files))
}
{
// DeleteFile
fmt.Printf("=== DeleteFile ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
err = cli.DeleteFile(ctx, srvaddr+"/foo.bin")
require.NoError(t, err)
}
{
// !FileInfo
fmt.Printf("=== FileInfo ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
exists, _, err := cli.FileInfo(ctx, srvaddr+"/foo.bin")
require.NoError(t, err)
require.False(t, exists)
}
}
test/image_test.go → test/attic/image_test.go
View File
test/file_test.go
+93 -64
View File
@@ -12,16 +12,17 @@ package test
import (
"context"
"fmt"
"math/rand"
"os"
"path/filepath"
//"math/rand"
//"os"
//"path/filepath"
"sync"
"testing"
"time"
"bytes"
"mstore/app/server"
"mstore/pkg/client"
"mstore/pkg/terms"
"mstore/pkg/filecli"
"github.com/stretchr/testify/require"
)
@@ -29,7 +30,7 @@ import (
func TestFileLife(t *testing.T) {
var srvport int64 = 10250
srvdir := t.TempDir()
srvaddr := fmt.Sprintf("mstore:mstore@127.0.0.1:%d", srvport)
srvaddr := fmt.Sprintf("127.0.0.1:%d", srvport)
srv, err := server.NewServer()
require.NoError(t, err)
@@ -78,86 +79,114 @@ func TestFileLife(t *testing.T) {
require.NoError(t, err)
require.True(t, helloRes)
}
filesize := 32
fileaddr := srvaddr+"/foo2/bare.bin"
filedata := []byte("Hello, World")
filesize := int64(len(filedata))
user, pass := "mstore", "mstore"
diraddr := srvaddr+"/foo"
{
// PutFile
tmpdir := t.TempDir()
tmpfile := filepath.Join(tmpdir, "foo.bin")
filedata := make([]byte, filesize)
_, err = rand.Read(filedata)
require.NoError(t, err)
err := os.WriteFile(tmpfile, filedata, 0666)
require.NoError(t, err)
file := bytes.NewReader(filedata)
size := int64(len(filedata))
fmt.Printf("=== PutFile ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
err = cli.PutFile(ctx, tmpfile, srvaddr+"/foo.bin")
timeout := 10 * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := filecli.ParsePath(fileaddr)
require.NoError(t, err)
fmt.Printf("Raw: %s\n", ref.Raw())
ref.SetUserinfo(user, pass)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
err = cli.PutFile(ctx, ref.Raw(), file, size)
require.NoError(t, err)
}
{
// GetFile
file := bytes.NewBuffer(nil)
fmt.Printf("=== GetFile ===\n")
timeout := 10 * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := filecli.ParsePath(fileaddr)
require.NoError(t, err)
fmt.Printf("Raw: %s\n", ref.Raw())
ref.SetUserinfo(user, pass)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
exist, err := cli.GetFile(ctx, ref.Raw(), file)
require.NoError(t, err)
require.True(t, exist)
}
{
// FileInfo
fmt.Printf("=== FileInfo ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
exists, file, err := cli.FileInfo(ctx, srvaddr+"/foo.bin")
timeout := 10 * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := filecli.ParsePath(fileaddr)
require.NoError(t, err)
require.True(t, exists)
require.NotNil(t, file)
fmt.Printf("Raw: %s\n", ref.Raw())
ref.SetUserinfo(user, pass)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
exist, size, digest, err := cli.FileInfo(ctx, ref.Raw())
require.NoError(t, err)
require.True(t, exist)
require.Equal(t, size, filesize)
fmt.Printf("Digest: %s\n", digest)
}
{
// GetFile
fmt.Printf("=== GetFile ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
tmpdir := t.TempDir()
tmpfile := filepath.Join(tmpdir, "foo.bin")
recsize, err := cli.GetFile(ctx, srvaddr+"/foo.bin", tmpfile)
// FileList
fmt.Printf("=== FileList ===\n")
timeout := 10 * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := filecli.ParsePath(diraddr)
require.NoError(t, err)
require.Equal(t, int64(filesize), recsize)
}
{
// ListFiles
fmt.Printf("=== ListFiles ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
ref.SetUserinfo(user, pass)
ref.PathType(filecli.PathTypePrefix)
fmt.Printf("Raw: %s\n", ref.Raw())
files, err := cli.ListFiles(ctx, srvaddr+"/", terms.AsFinePath)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
list, err := cli.ListFiles(ctx, ref.Raw())
require.NoError(t, err)
require.NotZero(t, len(files))
fmt.Printf("List: %s\n", list)
}
{
// DeleteFile
fmt.Printf("=== DeleteFile ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
err = cli.DeleteFile(ctx, srvaddr+"/foo.bin")
fmt.Printf("=== FileList ===\n")
timeout := 10 * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := filecli.ParsePath(fileaddr)
require.NoError(t, err)
ref.SetUserinfo(user, pass)
//ref.PathType(filecli.PathTypePrefix)
fmt.Printf("Raw: %s\n", ref.Raw())
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
exist, err := cli.DeleteFile(ctx, ref.Raw())
require.NoError(t, err)
fmt.Printf("Exist: %v\n", exist)
require.True(t, exist)
}
{
// !FileInfo
fmt.Printf("=== FileInfo ===\n")
cli := client.NewClient(true)
ctx := context.Background()
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
exists, _, err := cli.FileInfo(ctx, srvaddr+"/foo.bin")
// DeleteFile
fmt.Printf("=== FileList ===\n")
timeout := 10 * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := filecli.ParsePath(fileaddr)
require.NoError(t, err)
require.False(t, exists)
ref.SetUserinfo(user, pass)
//ref.PathType(filecli.PathTypePrefix)
fmt.Printf("Raw: %s\n", ref.Raw())
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
exist, err := cli.DeleteFile(ctx, ref.Raw())
require.NoError(t, err)
fmt.Printf("Exist: %v\n", exist)
require.False(t, exist)
}
}
test/test-oci.tar
BIN
View File
Binary file not shown.
vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/ping.go
Generated Vendored
+5 -1
View File
@@ -15,6 +15,7 @@
package transport
import (
"cmp"
"context"
"errors"
"fmt"
@@ -75,7 +76,10 @@ func pingSingle(ctx context.Context, reg name.Registry, t http.RoundTripper, sch
resp.Body.Close()
}()
insecure := scheme == "http"
// If resp.Request is set, we may have followed a redirect,
// so we want to prefer resp.Request.URL.Scheme (if it's set)
// falling back to the original request's scheme.
insecure := cmp.Or(resp.Request, req).URL.Scheme == "http"
switch resp.StatusCode {
case http.StatusOK:
vendor/modules.txt
Vendored
+1 -1
View File
@@ -30,7 +30,7 @@ github.com/fsnotify/fsnotify/internal
## explicit; go 1.18
github.com/go-viper/mapstructure/v2
github.com/go-viper/mapstructure/v2/internal/errors
# github.com/google/go-containerregistry v0.21.0
# github.com/google/go-containerregistry v0.21.2
## explicit; go 1.25.6
github.com/google/go-containerregistry/internal/and
github.com/google/go-containerregistry/internal/compression