working commit
This commit is contained in:
+23
-4
@@ -22,7 +22,7 @@ func (hand *Handler) FileExists(rctx *router.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
rctx.SetHeader("Content-Type", res.ContentType)
|
rctx.SetHeader("Content-Type", res.ContentType)
|
||||||
rctx.SetHeader("Content-Length", res.ContentLength)
|
rctx.SetHeader("Content-Size", res.ContentSize)
|
||||||
rctx.SetHeader("Content-Digest", res.ContentDigest)
|
rctx.SetHeader("Content-Digest", res.ContentDigest)
|
||||||
rctx.SetStatus(code)
|
rctx.SetStatus(code)
|
||||||
|
|
||||||
@@ -31,13 +31,15 @@ func (hand *Handler) FileExists(rctx *router.Context) {
|
|||||||
func (hand *Handler) PutFile(rctx *router.Context) {
|
func (hand *Handler) PutFile(rctx *router.Context) {
|
||||||
hand.logg.Debugf("Handle PutFile")
|
hand.logg.Debugf("Handle PutFile")
|
||||||
|
|
||||||
contentLength := rctx.GetHeader("Content-Length")
|
contentSize := rctx.GetHeader("Content-Size")
|
||||||
contentType := rctx.GetHeader("Content-Type")
|
contentType := rctx.GetHeader("Content-Type")
|
||||||
filepath := rctx.PathMap["filepath"]
|
filepath := rctx.PathMap["filepath"]
|
||||||
|
|
||||||
|
hand.logg.Debugf("%s", contentSize)
|
||||||
|
|
||||||
params := &operator.PutFileParams{
|
params := &operator.PutFileParams{
|
||||||
Filepath: filepath,
|
Filepath: filepath,
|
||||||
ContentLength: contentLength,
|
ContentSize: contentSize,
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
Source: rctx.Request.Body,
|
Source: rctx.Request.Body,
|
||||||
}
|
}
|
||||||
@@ -67,7 +69,7 @@ func (hand *Handler) GetFile(rctx *router.Context) {
|
|||||||
}
|
}
|
||||||
rctx.SetStatus(code)
|
rctx.SetStatus(code)
|
||||||
rctx.SetHeader("Content-Type", res.ContentType)
|
rctx.SetHeader("Content-Type", res.ContentType)
|
||||||
rctx.SetHeader("Content-Length", res.ContentLength)
|
rctx.SetHeader("Content-Size", res.ContentSize)
|
||||||
rctx.SetHeader("Content-Digest", res.ContentDigest)
|
rctx.SetHeader("Content-Digest", res.ContentDigest)
|
||||||
|
|
||||||
if res.Source != nil {
|
if res.Source != nil {
|
||||||
@@ -93,3 +95,20 @@ func (hand *Handler) DeleteFile(rctx *router.Context) {
|
|||||||
}
|
}
|
||||||
rctx.SetStatus(code)
|
rctx.SetStatus(code)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (hand *Handler) ListFiles(rctx *router.Context) {
|
||||||
|
hand.logg.Debugf("Handle ListFiles")
|
||||||
|
|
||||||
|
filepath := rctx.PathMap["filepath"]
|
||||||
|
params := &operator.ListFilesParams{
|
||||||
|
Filepath: filepath,
|
||||||
|
}
|
||||||
|
code, res, err := hand.oper.ListFiles(params)
|
||||||
|
if err != nil {
|
||||||
|
hand.logg.Errorf("ListFiles error: %v", err)
|
||||||
|
rctx.SetStatus(code)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rctx.SetStatus(code)
|
||||||
|
rctx.SendJSON(res)
|
||||||
|
}
|
||||||
|
|||||||
+94
-22
@@ -1,9 +1,11 @@
|
|||||||
package operator
|
package operator
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"path"
|
"path"
|
||||||
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"mstore/app/descr"
|
"mstore/app/descr"
|
||||||
@@ -11,7 +13,7 @@ import (
|
|||||||
"mstore/pkg/auxuuid"
|
"mstore/pkg/auxuuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
// File exists
|
// FileExists
|
||||||
type FileExistsParams struct {
|
type FileExistsParams struct {
|
||||||
Filepath string
|
Filepath string
|
||||||
Source string
|
Source string
|
||||||
@@ -19,35 +21,50 @@ type FileExistsParams struct {
|
|||||||
}
|
}
|
||||||
type FileExistsResult struct {
|
type FileExistsResult struct {
|
||||||
ContentType string
|
ContentType string
|
||||||
ContentLength string
|
ContentSize string
|
||||||
ContentDigest string
|
ContentDigest string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cleanFilepath(filename string) (string, error) {
|
||||||
|
filename = "/" + filename
|
||||||
|
return filepath.Clean(filename), nil
|
||||||
|
}
|
||||||
|
|
||||||
func (oper *Operator) FileExists(param *FileExistsParams) (int, *FileExistsResult, error) {
|
func (oper *Operator) FileExists(param *FileExistsParams) (int, *FileExistsResult, error) {
|
||||||
var err error
|
var err error
|
||||||
code := http.StatusOK
|
code := http.StatusOK
|
||||||
res := &FileExistsResult{}
|
res := &FileExistsResult{}
|
||||||
|
|
||||||
filename := path.Base(param.Filepath)
|
xfilepath, err := cleanFilepath(param.Filepath)
|
||||||
collection := path.Dir(param.Filepath)
|
if err != nil {
|
||||||
|
code := http.StatusInternalServerError
|
||||||
|
return code, res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := path.Base(xfilepath)
|
||||||
|
collection := path.Dir(xfilepath)
|
||||||
|
|
||||||
exist, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
|
exist, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
|
||||||
|
if err != nil {
|
||||||
|
code := http.StatusInternalServerError
|
||||||
|
return code, res, err
|
||||||
|
}
|
||||||
if !exist {
|
if !exist {
|
||||||
code = http.StatusNotFound
|
code = http.StatusNotFound
|
||||||
return code, res, err
|
return code, res, err
|
||||||
}
|
}
|
||||||
res = &FileExistsResult{
|
res = &FileExistsResult{
|
||||||
ContentLength: strconv.FormatInt(fileDescr.Size, 10),
|
ContentSize: strconv.FormatInt(fileDescr.Size, 10),
|
||||||
ContentType: fileDescr.Type,
|
ContentType: fileDescr.Type,
|
||||||
ContentDigest: fileDescr.Checksum,
|
ContentDigest: fileDescr.Checksum,
|
||||||
}
|
}
|
||||||
return code, res, err
|
return code, res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Put file
|
// PutFile
|
||||||
type PutFileParams struct {
|
type PutFileParams struct {
|
||||||
ContentType string
|
ContentType string
|
||||||
ContentLength string
|
ContentSize string
|
||||||
Filepath string
|
Filepath string
|
||||||
Source io.ReadCloser
|
Source io.ReadCloser
|
||||||
}
|
}
|
||||||
@@ -59,7 +76,12 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
|
|||||||
var err error
|
var err error
|
||||||
res := &PutFileResult{}
|
res := &PutFileResult{}
|
||||||
|
|
||||||
size, err := strconv.ParseInt(param.ContentLength, 10, 64)
|
if param.ContentSize == "" {
|
||||||
|
code := http.StatusLengthRequired
|
||||||
|
err = fmt.Errorf("Content-Size is empty")
|
||||||
|
return code, res, err
|
||||||
|
}
|
||||||
|
size, err := strconv.ParseInt(param.ContentSize, 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
code := http.StatusLengthRequired
|
code := http.StatusLengthRequired
|
||||||
return code, res, err
|
return code, res, err
|
||||||
@@ -70,10 +92,15 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TODO: convert file path to a unified and secure state
|
// TODO: convert file path to a unified and secure state
|
||||||
|
xfilepath, err := cleanFilepath(param.Filepath)
|
||||||
|
if err != nil {
|
||||||
|
code := http.StatusInternalServerError
|
||||||
|
return code, res, err
|
||||||
|
}
|
||||||
|
filename := path.Base(xfilepath)
|
||||||
|
collection := path.Dir(xfilepath)
|
||||||
|
|
||||||
filename := path.Base(param.Filepath)
|
oper.logg.Debugf("Put file [%s] [%s]", collection, filename)
|
||||||
collection := path.Dir(param.Filepath)
|
|
||||||
oper.logg.Debugf("Put file %s %s", collection, filename)
|
|
||||||
|
|
||||||
tmpname, size, checksum, err := oper.store.WriteTempFile(param.Source)
|
tmpname, size, checksum, err := oper.store.WriteTempFile(param.Source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -125,13 +152,13 @@ func (oper *Operator) PutFile(param *PutFileParams) (int, *PutFileResult, error)
|
|||||||
return code, res, err
|
return code, res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get file
|
// GetFile
|
||||||
type GetFileParams struct {
|
type GetFileParams struct {
|
||||||
Filepath string
|
Filepath string
|
||||||
}
|
}
|
||||||
type GetFileResult struct {
|
type GetFileResult struct {
|
||||||
ContentType string
|
ContentType string
|
||||||
ContentLength string
|
ContentSize string
|
||||||
ContentDigest string
|
ContentDigest string
|
||||||
Source io.ReadCloser
|
Source io.ReadCloser
|
||||||
}
|
}
|
||||||
@@ -142,10 +169,15 @@ func (oper *Operator) GetFile(param *GetFileParams) (int, *GetFileResult, error)
|
|||||||
|
|
||||||
// TODO: convert file path to a unified and secure state
|
// TODO: convert file path to a unified and secure state
|
||||||
|
|
||||||
filename := path.Base(param.Filepath)
|
xfilepath, err := cleanFilepath(param.Filepath)
|
||||||
collection := path.Dir(param.Filepath)
|
if err != nil {
|
||||||
|
code := http.StatusInternalServerError
|
||||||
|
return code, res, err
|
||||||
|
}
|
||||||
|
filename := path.Base(xfilepath)
|
||||||
|
collection := path.Dir(xfilepath)
|
||||||
|
|
||||||
oper.logg.Debugf("Gut file %s %s", collection, filename)
|
oper.logg.Debugf("Get file [%s] [%s]", collection, filename)
|
||||||
|
|
||||||
descrExists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
|
descrExists, fileDescr, err := oper.mdb.GetFileByCollection(collection, filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -162,7 +194,7 @@ func (oper *Operator) GetFile(param *GetFileParams) (int, *GetFileResult, error)
|
|||||||
return code, res, err
|
return code, res, err
|
||||||
}
|
}
|
||||||
res = &GetFileResult{
|
res = &GetFileResult{
|
||||||
ContentLength: strconv.FormatInt(fileDescr.Size, 10),
|
ContentSize: strconv.FormatInt(fileDescr.Size, 10),
|
||||||
ContentType: fileDescr.Type,
|
ContentType: fileDescr.Type,
|
||||||
ContentDigest: fileDescr.Checksum,
|
ContentDigest: fileDescr.Checksum,
|
||||||
Source: reader,
|
Source: reader,
|
||||||
@@ -171,7 +203,7 @@ func (oper *Operator) GetFile(param *GetFileParams) (int, *GetFileResult, error)
|
|||||||
return code, res, err
|
return code, res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete file
|
// DeleteFile
|
||||||
type DeleteFileParams struct {
|
type DeleteFileParams struct {
|
||||||
Filepath string
|
Filepath string
|
||||||
}
|
}
|
||||||
@@ -182,8 +214,13 @@ func (oper *Operator) DeleteFile(param *DeleteFileParams) (int, *DeleteFileResul
|
|||||||
res := &DeleteFileResult{}
|
res := &DeleteFileResult{}
|
||||||
code := http.StatusOK
|
code := http.StatusOK
|
||||||
|
|
||||||
filename := path.Base(param.Filepath)
|
xfilepath, err := cleanFilepath(param.Filepath)
|
||||||
collection := path.Dir(param.Filepath)
|
if err != nil {
|
||||||
|
code := http.StatusInternalServerError
|
||||||
|
return code, res, err
|
||||||
|
}
|
||||||
|
filename := path.Base(xfilepath)
|
||||||
|
collection := path.Dir(xfilepath)
|
||||||
|
|
||||||
exist, _, err := oper.mdb.GetFileByCollection(collection, filename)
|
exist, _, err := oper.mdb.GetFileByCollection(collection, filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -202,7 +239,42 @@ func (oper *Operator) DeleteFile(param *DeleteFileParams) (int, *DeleteFileResul
|
|||||||
code = http.StatusInternalServerError
|
code = http.StatusInternalServerError
|
||||||
return code, res, err
|
return code, res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return code, res, err
|
return code, res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListFiles
|
||||||
|
type ListFilesParams struct {
|
||||||
|
Filepath string
|
||||||
|
}
|
||||||
|
type ListFilesResult struct {
|
||||||
|
FileDescrs []descr.File
|
||||||
|
}
|
||||||
|
|
||||||
|
func (oper *Operator) ListFiles(param *ListFilesParams) (int, *ListFilesResult, error) {
|
||||||
|
var err error
|
||||||
|
res := &ListFilesResult{
|
||||||
|
FileDescrs: make([]descr.File, 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: convert file path to a unified and secure state
|
||||||
|
|
||||||
|
xfilepath, err := cleanFilepath(param.Filepath)
|
||||||
|
if err != nil {
|
||||||
|
code := http.StatusInternalServerError
|
||||||
|
return code, res, err
|
||||||
|
}
|
||||||
|
collection := xfilepath
|
||||||
|
|
||||||
|
oper.logg.Debugf("List files by %s", collection)
|
||||||
|
|
||||||
|
fileDescrs, err := oper.mdb.ListAllFiles()
|
||||||
|
if err != nil {
|
||||||
|
code := http.StatusInternalServerError
|
||||||
|
return code, res, err
|
||||||
|
}
|
||||||
|
for _, item := range fileDescrs {
|
||||||
|
res.FileDescrs = append(res.FileDescrs, item)
|
||||||
|
}
|
||||||
|
code := http.StatusOK
|
||||||
|
return code, res, err
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -56,7 +56,7 @@ func pathCompiler(path string) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultRegexp = `[a-zA-Z0-9_][/\-\.a-zA-Z0-9_%%=:~]+`
|
defaultRegexp = `[a-zA-Z0-9_\.][/\-\.a-zA-Z0-9_%%=:~]+`
|
||||||
)
|
)
|
||||||
|
|
||||||
func convertRegexp(src []byte) []byte {
|
func convertRegexp(src []byte) []byte {
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ func (svc *Service) Build() error {
|
|||||||
svc.rout.Put("/v3/api/file/{filepath}", svc.hand.PutFile)
|
svc.rout.Put("/v3/api/file/{filepath}", svc.hand.PutFile)
|
||||||
svc.rout.Get("/v3/api/file/{filepath}", svc.hand.GetFile)
|
svc.rout.Get("/v3/api/file/{filepath}", svc.hand.GetFile)
|
||||||
svc.rout.Delete("/v3/api/file/{filepath}", svc.hand.DeleteFile)
|
svc.rout.Delete("/v3/api/file/{filepath}", svc.hand.DeleteFile)
|
||||||
|
svc.rout.Get("/v3/api/files/{filepath}", svc.hand.ListFiles)
|
||||||
|
|
||||||
selector := svc.rout.Selector()
|
selector := svc.rout.Selector()
|
||||||
for _, item := range selector.Routes {
|
for _, item := range selector.Routes {
|
||||||
|
|||||||
+41
-9
@@ -25,11 +25,34 @@ func NewStorage(basepath string) *Storage {
|
|||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const filesubdir = "files"
|
||||||
|
const tmpsubdir = "tmps"
|
||||||
|
|
||||||
|
func (store *Storage) makeCollecionpath(collection string) string {
|
||||||
|
return filepath.Join(store.basepath, filesubdir, collection)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeFilepath(collection, filename string) string {
|
||||||
|
return filepath.Join(store.basepath, filesubdir, collection, filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeTmppath(tmpname string) string {
|
||||||
|
return filepath.Join(store.basepath, tmpsubdir, tmpname)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeTmpsubdir() string {
|
||||||
|
return filepath.Join(store.basepath, tmpsubdir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (store *Storage) makeFilesubdir(collection, filename string) string {
|
||||||
|
return filepath.Join(store.basepath, filesubdir)
|
||||||
|
}
|
||||||
|
|
||||||
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
|
func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser, error) {
|
||||||
var err error
|
var err error
|
||||||
var res io.ReadCloser
|
var res io.ReadCloser
|
||||||
|
|
||||||
filename = filepath.Join(store.basepath, collection, filename)
|
filename = store.makeFilepath(collection, filename)
|
||||||
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
|
file, err := os.OpenFile(filename, os.O_RDONLY, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, err
|
return res, err
|
||||||
@@ -40,15 +63,16 @@ func (store *Storage) GetFileReader(collection, filename string) (io.ReadCloser,
|
|||||||
|
|
||||||
func (store *Storage) HardlinkFile(tmpname, collection, filename string) error {
|
func (store *Storage) HardlinkFile(tmpname, collection, filename string) error {
|
||||||
var err error
|
var err error
|
||||||
dirname := filepath.Join(store.basepath, collection)
|
dirname := store.makeCollecionpath(collection)
|
||||||
err = os.MkdirAll(dirname, 0750)
|
err = os.MkdirAll(dirname, 0750)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
filename = filepath.Join(store.basepath, collection, filename)
|
filename = store.makeFilepath(collection, filename)
|
||||||
os.Remove(filename) // TODO
|
os.Remove(filename) // TODO
|
||||||
|
|
||||||
|
tmpname = store.makeTmppath(tmpname)
|
||||||
err = os.Link(tmpname, filename)
|
err = os.Link(tmpname, filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -67,11 +91,17 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er
|
|||||||
|
|
||||||
tmpname := auxuuid.NewUUID()
|
tmpname := auxuuid.NewUUID()
|
||||||
tmpname = fmt.Sprintf("file-%s.tmp", tmpname)
|
tmpname = fmt.Sprintf("file-%s.tmp", tmpname)
|
||||||
tmppath := filepath.Join(store.basepath, tmpname)
|
tmppath := store.makeTmppath(tmpname)
|
||||||
|
|
||||||
|
tmpdirpath := store.makeTmpsubdir()
|
||||||
|
err = os.MkdirAll(tmpdirpath, 0750)
|
||||||
|
if err != nil {
|
||||||
|
return tmpname, size, digest, err
|
||||||
|
}
|
||||||
|
|
||||||
file, err := os.OpenFile(tmppath, os.O_WRONLY|os.O_CREATE, 0640)
|
file, err := os.OpenFile(tmppath, os.O_WRONLY|os.O_CREATE, 0640)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return tmppath, size, digest, err
|
return tmpname, size, digest, err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
@@ -80,23 +110,25 @@ func (store *Storage) WriteTempFile(source io.Reader) (string, int64, string, er
|
|||||||
|
|
||||||
size, err = io.Copy(writer, source)
|
size, err = io.Copy(writer, source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return tmppath, size, digest, err
|
return tmpname, size, digest, err
|
||||||
}
|
}
|
||||||
digest = hex.EncodeToString(hasher.Sum(nil))
|
digest = hex.EncodeToString(hasher.Sum(nil))
|
||||||
digest = fmt.Sprintf("sha256:%s", digest)
|
digest = fmt.Sprintf("sha256:%s", digest)
|
||||||
|
|
||||||
return tmppath, size, digest, err
|
return tmpname, size, digest, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (store *Storage) DeleteFile(collection, filename string) error {
|
func (store *Storage) DeleteFile(collection, filename string) error {
|
||||||
var err error
|
var err error
|
||||||
filename = filepath.Join(store.basepath, collection, filename)
|
filename = store.makeFilepath(collection, filename)
|
||||||
err = os.Remove(filename)
|
err = os.Remove(filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
dirname := filepath.Join(store.basepath, collection)
|
// TODO: more safe removing
|
||||||
|
dirname := store.makeCollecionpath(collection)
|
||||||
|
|
||||||
err = os.RemoveAll(dirname)
|
err = os.RemoveAll(dirname)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
package client
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const fileAPI = "/v3/api/file/"
|
||||||
|
const serviceAPI = "/v3/api/service/"
|
||||||
|
|
||||||
|
func createBasicAuthPair(username, password string) string {
|
||||||
|
auth := username + ":" + password
|
||||||
|
return "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertFileLink(ref string) (string, error) {
|
||||||
|
var err error
|
||||||
|
var res string
|
||||||
|
if !strings.Contains(ref, "://") {
|
||||||
|
ref = "https://" + ref
|
||||||
|
}
|
||||||
|
url, err := url.Parse(ref)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
url.Path = path.Join(fileAPI, url.Path)
|
||||||
|
url.User = nil
|
||||||
|
res = url.String()
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertServiceLink(ref string) (string, error) {
|
||||||
|
var err error
|
||||||
|
var res string
|
||||||
|
if !strings.Contains(ref, "://") {
|
||||||
|
ref = "https://" + ref
|
||||||
|
}
|
||||||
|
url, err := url.Parse(ref)
|
||||||
|
if err != nil {
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
url.Path = path.Join(serviceAPI, url.Path)
|
||||||
|
url.User = nil
|
||||||
|
res = url.String()
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
+82
-48
@@ -3,20 +3,14 @@ package client
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"encoding/base64"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"os"
|
"os"
|
||||||
"path"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strconv"
|
||||||
)
|
)
|
||||||
|
|
||||||
const fileAPI = "/v3/api/file/"
|
|
||||||
const serviceAPI = "/v3/api/service/"
|
|
||||||
|
|
||||||
type Client struct{}
|
type Client struct{}
|
||||||
|
|
||||||
func NewClient() *Client {
|
func NewClient() *Client {
|
||||||
@@ -31,7 +25,7 @@ func (cli *Client) ServiceHello(ctx context.Context, ref string) (bool, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", ref, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, ref, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
@@ -63,7 +57,7 @@ func (cli *Client) FileExists(ctx context.Context, ref string) (bool, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
req, err := http.NewRequestWithContext(ctx, "HEAD", ref, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodHead, ref, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
@@ -87,43 +81,6 @@ func (cli *Client) FileExists(ctx context.Context, ref string) (bool, error) {
|
|||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func createBasicAuthPair(username, password string) string {
|
|
||||||
auth := username + ":" + password
|
|
||||||
return "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
|
|
||||||
}
|
|
||||||
|
|
||||||
func convertFileLink(ref string) (string, error) {
|
|
||||||
var err error
|
|
||||||
var res string
|
|
||||||
if !strings.Contains(ref, "://") {
|
|
||||||
ref = "https://" + ref
|
|
||||||
}
|
|
||||||
url, err := url.Parse(ref)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
url.Path = path.Join(fileAPI, url.Path)
|
|
||||||
url.User = nil
|
|
||||||
res = url.String()
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func convertServiceLink(ref string) (string, error) {
|
|
||||||
var err error
|
|
||||||
var res string
|
|
||||||
if !strings.Contains(ref, "://") {
|
|
||||||
ref = "https://" + ref
|
|
||||||
}
|
|
||||||
url, err := url.Parse(ref)
|
|
||||||
if err != nil {
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
url.Path = path.Join(serviceAPI, url.Path)
|
|
||||||
url.User = nil
|
|
||||||
res = url.String()
|
|
||||||
return res, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (cli *Client) GetFile(ctx context.Context, ref, filename string) error {
|
func (cli *Client) GetFile(ctx context.Context, ref, filename string) error {
|
||||||
var err error
|
var err error
|
||||||
ref, err = convertFileLink(ref)
|
ref, err = convertFileLink(ref)
|
||||||
@@ -131,7 +88,7 @@ func (cli *Client) GetFile(ctx context.Context, ref, filename string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", ref, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, ref, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -150,7 +107,7 @@ func (cli *Client) GetFile(ctx context.Context, ref, filename string) error {
|
|||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
err := fmt.Errorf("Received wrong status code: %s", resp.StatusCode)
|
err := fmt.Errorf("Received wrong status code: %s", resp.Status)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,3 +126,80 @@ func (cli *Client) GetFile(ctx context.Context, ref, filename string) error {
|
|||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (cli *Client) PutFile(ctx context.Context, filename, ref string) error {
|
||||||
|
var err error
|
||||||
|
ref, err = convertFileLink(ref)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
file, err := os.Open(filename)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, ref, file)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
transport := &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{
|
||||||
|
InsecureSkipVerify: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
client := &http.Client{
|
||||||
|
Transport: transport,
|
||||||
|
}
|
||||||
|
fileinfo, err := os.Stat(filename)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
filesize := fileinfo.Size()
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/octet-stream")
|
||||||
|
req.Header.Set("Content-Size", strconv.FormatInt(filesize, 10))
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
err := fmt.Errorf("Received wrong status code: %s", resp.Status)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cli *Client) DeleteFile(ctx context.Context, ref, filename string) error {
|
||||||
|
var err error
|
||||||
|
ref, err = convertFileLink(ref)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, ref, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
transport := &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{
|
||||||
|
InsecureSkipVerify: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
client := &http.Client{
|
||||||
|
Transport: transport,
|
||||||
|
}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
err := fmt.Errorf("Received wrong status code: %s", resp.Status)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package client
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
//"mstore/pkg/client"
|
||||||
|
"mstore/app/server"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestService(t *testing.T) {
|
||||||
|
var srvport int64 = 10250
|
||||||
|
srvdir := t.TempDir()
|
||||||
|
srvaddr := fmt.Sprintf("127.0.0.1:%d", srvport)
|
||||||
|
|
||||||
|
srv, err := server.NewServer()
|
||||||
|
require.NoError(t, err)
|
||||||
|
{
|
||||||
|
err = srv.Configure()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
srv.SetDatadir(srvdir)
|
||||||
|
srv.SetLogdir(srvdir)
|
||||||
|
srv.SetRundir(srvdir)
|
||||||
|
srv.SetPort(srvport)
|
||||||
|
|
||||||
|
err = srv.Build()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
var svcWG sync.WaitGroup
|
||||||
|
errPipe := make(chan error, 5)
|
||||||
|
|
||||||
|
startFunc := func() {
|
||||||
|
err := srv.Service().Run()
|
||||||
|
errPipe <- err
|
||||||
|
svcWG.Done()
|
||||||
|
}
|
||||||
|
|
||||||
|
stopFunc := func() {
|
||||||
|
srv.Service().Stop()
|
||||||
|
svcWG.Wait()
|
||||||
|
err = <-errPipe
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
defer stopFunc()
|
||||||
|
|
||||||
|
svcWG.Add(1)
|
||||||
|
go startFunc()
|
||||||
|
time.Sleep(1 * time.Second)
|
||||||
|
}
|
||||||
|
{
|
||||||
|
fmt.Printf("=== ServiceHello ===\n")
|
||||||
|
cli := NewClient()
|
||||||
|
ctx := context.Background()
|
||||||
|
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
|
||||||
|
|
||||||
|
helloRes, err := cli.ServiceHello(ctx, srvaddr+"/hello")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.True(t, helloRes)
|
||||||
|
}
|
||||||
|
{
|
||||||
|
tmpdir := t.TempDir()
|
||||||
|
tmpfile := filepath.Join(tmpdir, "foo.bin")
|
||||||
|
|
||||||
|
filedata := make([]byte, 32)
|
||||||
|
_, err = rand.Read(filedata)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
err := os.WriteFile(tmpfile, filedata, 0666)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
fmt.Printf("=== PutFile ===\n")
|
||||||
|
cli := NewClient()
|
||||||
|
ctx := context.Background()
|
||||||
|
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
|
||||||
|
|
||||||
|
err = cli.PutFile(ctx, tmpfile, srvaddr+"/foo.bin")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
+17
-12
@@ -10,26 +10,29 @@ import (
|
|||||||
"math/rand"
|
"math/rand"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"mstore/app/router"
|
"mstore/app/router"
|
||||||
"mstore/app/server"
|
"mstore/app/server"
|
||||||
)
|
)
|
||||||
|
|
||||||
func xxxTestFileOperations(t *testing.T) {
|
func TestFileOperations(t *testing.T) {
|
||||||
var err error
|
var err error
|
||||||
fmt.Printf("=== MakeServer ===\n")
|
fmt.Printf("=== MakeServer ===\n")
|
||||||
srv, err := server.NewServer()
|
srv, err := server.NewServer()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
filename := `bare.bin`
|
||||||
|
|
||||||
{
|
{
|
||||||
err = srv.Configure()
|
err = srv.Configure()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
tmpdir := t.TempDir()
|
//tmpdir := t.TempDir()
|
||||||
srv.SetDatadir(tmpdir)
|
//srv.SetDatadir(tmpdir)
|
||||||
srv.SetLogdir(tmpdir)
|
//srv.SetLogdir(tmpdir)
|
||||||
srv.SetRundir(tmpdir)
|
//srv.SetRundir(tmpdir)
|
||||||
|
|
||||||
err = srv.Build()
|
err = srv.Build()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -37,7 +40,7 @@ func xxxTestFileOperations(t *testing.T) {
|
|||||||
|
|
||||||
{
|
{
|
||||||
fmt.Printf("=== PutFile ===\n")
|
fmt.Printf("=== PutFile ===\n")
|
||||||
reqPath := `/v3/api/file/foo/bare`
|
reqPath := `/v3/api/file/` + filename
|
||||||
routePath := `/v3/api/file/{filepath}`
|
routePath := `/v3/api/file/{filepath}`
|
||||||
|
|
||||||
rout := router.NewRouter()
|
rout := router.NewRouter()
|
||||||
@@ -46,19 +49,20 @@ func xxxTestFileOperations(t *testing.T) {
|
|||||||
|
|
||||||
rout.Put(routePath, hand.PutFile)
|
rout.Put(routePath, hand.PutFile)
|
||||||
|
|
||||||
datasize := 16
|
datasize := 64
|
||||||
filedata := make([]byte, datasize)
|
filedata := make([]byte, datasize)
|
||||||
_, err = rand.Read(filedata)
|
_, err = rand.Read(filedata)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
filedata = []byte(hex.EncodeToString(filedata))
|
filedata = []byte(hex.EncodeToString(filedata))
|
||||||
|
datasize *= 2
|
||||||
|
|
||||||
source := bytes.NewReader(filedata)
|
source := bytes.NewReader(filedata)
|
||||||
|
|
||||||
request, err := http.NewRequest("PUT", reqPath, source)
|
request, err := http.NewRequest("PUT", reqPath, source)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
request.Header.Set("Content-Length", fmt.Sprintf("%d", datasize))
|
request.Header.Set("Content-Size", fmt.Sprintf("%d", datasize))
|
||||||
request.Header.Set("Content-Type", "application/octet-stream")
|
request.Header.Set("Content-Type", "application/octet-stream")
|
||||||
|
|
||||||
recorder := httptest.NewRecorder()
|
recorder := httptest.NewRecorder()
|
||||||
@@ -75,7 +79,7 @@ func xxxTestFileOperations(t *testing.T) {
|
|||||||
{
|
{
|
||||||
fmt.Printf("=== FileExists ===\n")
|
fmt.Printf("=== FileExists ===\n")
|
||||||
|
|
||||||
reqPath := `/v3/api/file/foo/bare`
|
reqPath := filepath.Join(`/v3/api/file`, filename)
|
||||||
routePath := `/v3/api/file/{filepath}`
|
routePath := `/v3/api/file/{filepath}`
|
||||||
|
|
||||||
rout := router.NewRouter()
|
rout := router.NewRouter()
|
||||||
@@ -100,7 +104,7 @@ func xxxTestFileOperations(t *testing.T) {
|
|||||||
}
|
}
|
||||||
{
|
{
|
||||||
fmt.Printf("=== GetFile ===\n")
|
fmt.Printf("=== GetFile ===\n")
|
||||||
reqPath := `/v3/api/file/foo/bare`
|
reqPath := filepath.Join(`/v3/api/file`, filename)
|
||||||
routePath := `/v3/api/file/{filepath}`
|
routePath := `/v3/api/file/{filepath}`
|
||||||
|
|
||||||
rout := router.NewRouter()
|
rout := router.NewRouter()
|
||||||
@@ -123,9 +127,10 @@ func xxxTestFileOperations(t *testing.T) {
|
|||||||
|
|
||||||
fmt.Printf("Response body: %s\n", string(bodyBytes))
|
fmt.Printf("Response body: %s\n", string(bodyBytes))
|
||||||
}
|
}
|
||||||
|
return
|
||||||
{
|
{
|
||||||
fmt.Printf("=== DeleteFile ===\n")
|
fmt.Printf("=== DeleteFile ===\n")
|
||||||
reqPath := `/v3/api/file/foo/bare`
|
reqPath := filepath.Join(`/v3/api/file`, filename)
|
||||||
routePath := `/v3/api/file/{filepath}`
|
routePath := `/v3/api/file/{filepath}`
|
||||||
|
|
||||||
rout := router.NewRouter()
|
rout := router.NewRouter()
|
||||||
@@ -152,7 +157,7 @@ func xxxTestFileOperations(t *testing.T) {
|
|||||||
{
|
{
|
||||||
fmt.Printf("=== FileNotExists ===\n")
|
fmt.Printf("=== FileNotExists ===\n")
|
||||||
|
|
||||||
reqPath := `/v3/api/file/foo/bare`
|
reqPath := filepath.Join(`/v3/api/file`, filename)
|
||||||
routePath := `/v3/api/file/{filepath}`
|
routePath := `/v3/api/file/{filepath}`
|
||||||
|
|
||||||
rout := router.NewRouter()
|
rout := router.NewRouter()
|
||||||
|
|||||||
+8
-9
@@ -9,14 +9,13 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"mstore/pkg/client"
|
|
||||||
"mstore/app/server"
|
"mstore/app/server"
|
||||||
|
"mstore/pkg/client"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestService(t *testing.T) {
|
func xxxTestService(t *testing.T) {
|
||||||
var srvport int64 = 10250
|
var srvport int64 = 10250
|
||||||
srvdir := t.TempDir()
|
srvdir := t.TempDir()
|
||||||
srvaddr := fmt.Sprintf("127.0.0.1:%d", srvport)
|
srvaddr := fmt.Sprintf("127.0.0.1:%d", srvport)
|
||||||
@@ -74,13 +73,13 @@ func TestService(t *testing.T) {
|
|||||||
err := os.WriteFile(tmpfile, filedata, 0666)
|
err := os.WriteFile(tmpfile, filedata, 0666)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
//fmt.Printf("=== PutFile ===\n")
|
fmt.Printf("=== PutFile ===\n")
|
||||||
//cli := client.NewClient()
|
cli := client.NewClient()
|
||||||
//ctx := context.Background()
|
ctx := context.Background()
|
||||||
//ctx, _ = context.WithTimeout(ctx, 1*time.Second)
|
ctx, _ = context.WithTimeout(ctx, 1*time.Second)
|
||||||
|
|
||||||
//err = cli.PutFile(ctx, tmpfile, srvaddr+"/foo.bin")
|
err = cli.PutFile(ctx, tmpfile, srvaddr+"/foo.bin")
|
||||||
//require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user