ziggi/mstore
1
0
Fork 0
Code Pull Requests Actions Packages Projects Releases Wiki Activity

client rebuilding in progress

Browse Source
This commit is contained in:
Олег Бородин
2026-03-04 12:27:52 +02:00
parent 2d34ec5634
commit ae9c29de1e
31 changed files with 908 additions and 467 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mstorectl/command/filecmd.go
+134 -182
View File
@@ -11,6 +11,7 @@ package command
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
@@ -18,19 +19,17 @@ import (
"os"
"path"
"path/filepath"
"slices"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"mstore/pkg/client"
"mstore/pkg/filecli"
"mstore/pkg/descr"
"mstore/pkg/terms"
)
func (util *FileUtil) CreateFileCmds() *cobra.Command {
func (util *FileUtil) MakeFileCmds() *cobra.Command {
var subCmd = &cobra.Command{
Use: "files",
Short: "File operations",
@@ -127,7 +126,7 @@ func (util *FileUtil) CreateFileCmds() *cobra.Command {
return subCmd
}
func (util *FileUtil) CreateCollectionCmds() *cobra.Command {
func (util *FileUtil) MakeCollectionCmds() *cobra.Command {
var subCmd = &cobra.Command{
Use: "collections",
Short: "Colletion operations",
@@ -176,8 +175,8 @@ type FileUtil struct {
listFilesParams ListFilesParams
importFilesParams ImportFilesParams
exportFilesParams ExportFilesParams
listCollectionsParams ListCollectionsParams
deleteCollectionParams DeleteCollectionParams
listCollectionsParams ListCollectionsParams
commonFileParams CommonFileParams
}
@@ -188,41 +187,6 @@ type CommonFileParams struct {
SkipTLSVerify bool
}
// FileInfo
type FileInfoParams struct {
Filepath string
}
type FileInfoResult struct {
File *descr.File `yaml:"file,omitempty"`
}
func (util *FileUtil) FileInfo(cmd *cobra.Command, args []string) {
util.fileInfoParams.Filepath = args[0]
res, err := util.fileInfo(&util.commonFileParams, &util.fileInfoParams)
printResponse(res, err)
}
func (util *FileUtil) fileInfo(common *CommonFileParams, params *FileInfoParams) (*FileInfoResult, error) {
var err error
res := &FileInfoResult{}
params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
if err != nil {
return res, err
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
exists, opres, err := client.NewClient(common.SkipTLSVerify).FileInfo(ctx, params.Filepath)
if err != nil {
return res, err
}
if !exists {
err = fmt.Errorf("File %s not exists", params.Filepath)
return res, err
}
res.File = opres
return res, err
}
// PutFile
type PutFileParams struct {
Source string
@@ -240,13 +204,27 @@ func (util *FileUtil) PutFile(cmd *cobra.Command, args []string) {
func (util *FileUtil) putFile(common *CommonFileParams, params *PutFileParams) (*PutFileResult, error) {
var err error
res := &PutFileResult{}
params.Dest, err = packUserinfo(params.Dest, common.Username, common.Password)
file, err := os.OpenFile(params.Dest, os.O_RDONLY, 0)
if err != nil {
return res, err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return res, err
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
err = client.NewClient(common.SkipTLSVerify).PutFile(ctx, params.Source, params.Dest)
ref, err := filecli.ParsePath(params.Dest)
if err != nil {
return res, err
}
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
err = cli.PutFile(ctx, ref.Raw(), file, stat.Size())
if err != nil {
return res, err
}
@@ -271,13 +249,27 @@ type GetFileResult struct{}
func (util *FileUtil) getFile(common *CommonFileParams, params *GetFileParams) (*GetFileResult, error) {
var err error
res := &GetFileResult{}
params.Dest, err = packUserinfo(params.Source, common.Username, common.Password)
err = os.MkdirAll(filepath.Dir(params.Dest), 0750)
if err != nil {
return res, err
}
file, err := os.OpenFile(params.Dest, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return res, err
}
defer file.Close()
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
_, err = client.NewClient(common.SkipTLSVerify).GetFile(ctx, params.Dest, params.Source)
ref, err := filecli.ParsePath(params.Source)
if err != nil {
return res, err
}
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
_, err = cli.GetFile(ctx, ref.Raw(), file)
if err != nil {
return res, err
}
@@ -299,71 +291,23 @@ func (util *FileUtil) DeleteFile(cmd *cobra.Command, args []string) {
func (util *FileUtil) deleteFile(common *CommonFileParams, params *DeleteFileParams) (*DeleteFileResult, error) {
var err error
res := &DeleteFileResult{}
params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
ref, err := filecli.ParsePath(params.Filepath)
if err != nil {
return res, err
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
err = client.NewClient(common.SkipTLSVerify).DeleteFile(ctx, params.Filepath)
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
_, err = cli.DeleteFile(ctx, ref.Raw())
if err != nil {
return res, err
}
return res, err
}
// ListFiles
type ListFilesParams struct {
Filepath string
Detail bool
AsPrefix bool
AsRegexp bool
}
type ListFilesResult struct {
Files []descr.File `yaml:"files,omitempty"`
Filenames []string `yaml:"filenames,omitempty"`
}
func (util *FileUtil) ListFiles(cmd *cobra.Command, args []string) {
util.listFilesParams.Filepath = args[0]
res, err := util.listFiles(&util.commonFileParams, &util.listFilesParams)
printResponse(res, err)
}
func (util *FileUtil) listFiles(common *CommonFileParams, params *ListFilesParams) (*ListFilesResult, error) {
var err error
res := &ListFilesResult{}
params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
if err != nil {
return res, err
}
if params.AsRegexp {
params.AsPrefix = false
}
var pathUsage string
switch {
case params.AsRegexp:
pathUsage = terms.AsRegexp
case params.AsPrefix:
pathUsage = terms.AsPrefix
default:
pathUsage = terms.AsFinePath
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
files, err := client.NewClient(common.SkipTLSVerify).ListFiles(ctx, params.Filepath, pathUsage)
if err != nil {
return res, err
}
if params.Detail {
res.Files = files
} else {
res.Filenames = makeFilelistFromFiles(files)
}
return res, err
}
// ImportFiles
type ImportFilesParams struct {
Source string
@@ -397,25 +341,39 @@ func (util *FileUtil) importFiles(common *CommonFileParams, params *ImportFilesP
if err != nil {
return err
}
if infoItem.Mode() == fs.ModeDevice {
return nil
}
if infoItem.Mode() == fs.ModeNamedPipe {
return nil
}
if infoItem.Mode() == fs.ModeCharDevice {
mode := infoItem.Mode()
skip := mode == fs.ModeDevice
skip = skip || mode == fs.ModeNamedPipe
skip = skip || mode == fs.ModeCharDevice
if skip {
return nil
}
if !infoItem.IsDir() {
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
relPath, _ := strings.CutPrefix(walkPath, params.Source)
dest, _ := url.JoinPath(params.Dest, relPath)
ref, err := filecli.ParsePath(dest)
if err != nil {
putErrors = append(putErrors, err)
return nil
}
err = client.NewClient(common.SkipTLSVerify).PutFile(ctx, walkPath, dest)
file, err := os.OpenFile(dest, os.O_RDONLY, 0)
if err != nil {
putErrors = append(putErrors, err)
return nil
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
putErrors = append(putErrors, err)
return nil
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref.SetUserinfo(common.Username, common.Password)
user, pass := ref.Userinfo()
mw := filecli.NewBasicAuthMiddleware(user, pass)
cli := filecli.NewClient(nil, mw)
err = cli.PutFile(ctx, ref.Raw(), file, stat.Size())
if err != nil {
putErrors = append(putErrors, err)
fmt.Printf("- %s: error: %v \n", walkPath, err)
@@ -459,57 +417,62 @@ func (util *FileUtil) ExportFiles(cmd *cobra.Command, args []string) {
func (util *FileUtil) exportFiles(common *CommonFileParams, params *ExportFilesParams) (*ExportFilesResult, error) {
var err error
res := &ExportFilesResult{}
params.Filepath, err = packUserinfo(params.Filepath, common.Username, common.Password)
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := filecli.ParsePath(params.Filepath)
if err != nil {
return res, err
}
if params.AsRegexp {
params.AsPrefix = false
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
list, err := cli.ListFiles(ctx, ref.Raw())
if err != nil {
return res, err
}
var pathUsage string
switch {
case params.AsRegexp:
pathUsage = terms.AsRegexp
case params.AsPrefix:
pathUsage = terms.AsPrefix
default:
pathUsage = terms.AsFinePath
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
files, err := client.NewClient(common.SkipTLSVerify).ListFiles(ctx, params.Filepath, pathUsage)
files := make([]descr.File, 0)
err = json.Unmarshal(list, &files)
if err != nil {
return res, err
}
exportedFiles := make([]descr.File, 0)
for _, file := range files {
destdir := filepath.Join(params.Dest, file.Collection)
exported := make([]descr.File, 0)
for _, descr := range files {
destdir := filepath.Join(params.Dest, descr.Collection)
err = os.MkdirAll(destdir, 0750)
if err != nil {
return res, err
}
srcpath, err := makeFileURI(params.Filepath, file.Collection, file.Name)
destpath := filepath.Join(params.Dest, descr.Collection, descr.Name)
destfile, err := os.OpenFile(destpath, os.O_WRONLY|os.O_CREATE, 0640)
if err != nil {
return res, err
}
defer destfile.Close()
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref.SetResource(filepath.Join(descr.Collection, descr.Name))
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
_, err = cli.GetFile(ctx, ref.Raw(), destfile)
if err != nil {
return res, err
}
destpath := filepath.Join(params.Dest, file.Collection, file.Name)
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
_, err = client.NewClient(common.SkipTLSVerify).GetFile(ctx, srcpath, destpath)
if err != nil {
fmt.Printf("- %s: error %v\n", srcpath, err)
//return res, err
fmt.Printf("- %s: error %v\n", ref.Raw(), err)
err = nil
} else {
fmt.Printf("- %s: ok\n", srcpath)
exportedFiles = append(exportedFiles, file)
fmt.Printf("- %s: ok\n", ref.Raw())
exported = append(exported, descr)
}
}
if params.Detail {
res.Files = exportedFiles
res.Files = exported
} else {
res.Filenames = makeFilelistFromFiles(exportedFiles)
files := descr.NewFiles()
files.Set(exported)
res.Filenames = files.List()
}
return res, err
}
@@ -548,29 +511,26 @@ func (util *FileUtil) listCollections(common *CommonFileParams, params *ListColl
res := &ListCollectionsResult{
Collections: make([]string, 0),
}
params.Path, err = packUserinfo(params.Path, common.Username, common.Password)
if err != nil {
return res, err
}
if params.AsRegexp {
params.AsPrefix = false
}
var pathUsage string
switch {
case params.AsRegexp:
pathUsage = terms.AsRegexp
case params.AsPrefix:
pathUsage = terms.AsPrefix
default:
pathUsage = terms.AsFinePath
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
collecions, err := client.NewClient(common.SkipTLSVerify).ListCollections(ctx, params.Path, pathUsage)
ref, err := filecli.ParsePath(params.Path)
if err != nil {
return res, err
}
res.Collections = collecions
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
list, err := cli.ListCollections(ctx, ref.Raw())
if err != nil {
return res, err
}
collections := make([]string, 0)
err = json.Unmarshal(list, collections)
if err != nil {
return res, err
}
res.Collections = collections
return res, err
}
@@ -599,36 +559,28 @@ func (util *FileUtil) deleteCollection(common *CommonFileParams, params *DeleteC
res := &DeleteCollectionResult{
Filenames: make([]string, 0),
}
params.Path, err = packUserinfo(params.Path, common.Username, common.Password)
if err != nil {
return res, err
}
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
var pathUsage string
switch {
case params.AsPrefix:
pathUsage = terms.AsPrefix
default:
pathUsage = terms.AsFinePath
}
files, err := client.NewClient(common.SkipTLSVerify).DeleteCollection(ctx, params.Path, pathUsage, params.DryRun)
ref, err := filecli.ParsePath(params.Path)
if err != nil {
return res, err
}
if params.Detail {
res.Files = files
ref.SetUserinfo(common.Username, common.Password)
mw := filecli.NewBasicAuthMiddleware(ref.Userinfo())
cli := filecli.NewClient(nil, mw)
list, err := cli.DeleteCollection(ctx, ref.Raw())
files := descr.NewFiles()
err = json.Unmarshal(list, files.ArrayPtr())
if err != nil {
return res, err
}
if !params.Detail {
res.Filenames = files.List()
} else {
res.Filenames = makeFilelistFromFiles(files)
res.Files = files.Array()
}
return res, err
}
func makeFilelistFromFiles(files []descr.File) []string {
res := make([]string, 0, len(files))
for _, file := range files {
res = append(res, filepath.Join(file.Collection, file.Name))
}
slices.Sort(res)
return res
}
cmd/mstorectl/command/imagecmd.go
+37 -12
View File
@@ -11,6 +11,8 @@ package command
import (
"context"
"encoding/json"
"fmt"
"net/url"
"os"
"path"
@@ -21,6 +23,7 @@ import (
"github.com/spf13/viper"
"mstore/pkg/client"
"mstore/pkg/repocli"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@@ -268,33 +271,55 @@ type ImageManifestParams struct {
}
type ImageManifestResult struct {
ImageManifest any `json:"imageManifest"`
Index *ocispec.Index `json:"index,omitempty"`
Manifest *ocispec.Manifest `json:"manifest,omitempty"`
}
func (util *ImageUtil) ImageManifest(cmd *cobra.Command, args []string) {
util.imageManifestParams.Imagepath = args[0]
res, err := util.imageInfo(&util.commonImageParams, &util.imageManifestParams)
res, err := util.imageManifest(&util.commonImageParams, &util.imageManifestParams)
printResponse(res, err)
}
func (util *ImageUtil) imageInfo(common *CommonImageParams, params *ImageManifestParams) (*ImageManifestResult, error) {
func (util *ImageUtil) imageManifest(common *CommonImageParams, params *ImageManifestParams) (*ImageManifestResult, error) {
var err error
res := &ImageManifestResult{}
ctx := context.Background()
cli := client.NewClient(common.SkipTLSVerify)
timeout := time.Duration(common.Timeout) * time.Second
res := &ImageManifestResult{
Index: &ocispec.Index{},
Manifest: &ocispec.Manifest{},
}
params.Imagepath, err = packUserinfo(params.Imagepath, common.Username, common.Password)
if err != nil {
return res, err
}
ctx, _ = context.WithTimeout(ctx, timeout)
opres, err := cli.ImageManifest(ctx, params.Imagepath)
timeout := time.Duration(common.Timeout) * time.Second
ctx, _ := context.WithTimeout(context.Background(), timeout)
ref, err := repocli.ParseReference(params.Imagepath)
if err != nil {
return res, err
}
res.ImageManifest = opres
mw := repocli.NewBasicAuthMiddleware(ref.Userinfo())
cli := repocli.NewClientWithTransport(nil, mw)
exists, mime, man, err := cli.GetManifest(ctx, ref.Repo(), ref.Tag())
if !exists {
err = fmt.Errorf("Manifest not found")
return res, err
}
switch mime {
case repocli.MediaTypeDDMLv2, repocli.MediaTypeOIIv1:
err = json.Unmarshal(man, res.Index)
if err != nil {
return res, err
}
case repocli.MediaTypeDDMv2, repocli.MediaTypeOIMv1:
err = json.Unmarshal(man, res.Manifest)
if err != nil {
return res, err
}
default:
err = fmt.Errorf("Unknown content type: %s", mime)
return res, err
}
return res, err
}