alittle splitted mstorectl code; etc

2026-03-05 12:26:14 +02:00
parent 80d6a244cf
commit 223ae2e96e
24 changed files with 631 additions and 104 deletions
@@ -0,0 +1,417 @@
+/*
+ * Copyright 2026 Oleg Borodin <onborodin@gmail.com>
+ *
+ * This work is published and licensed under a Creative Commons
+ * Attribution-NonCommercial-NoDerivatives 4.0 International License.
+ *
+ * Distribution of this work is permitted, but commercial use and
+ * modifications are strictly prohibited.
+ */
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"mstore/pkg/auxhttp"
+	"mstore/pkg/descr"
+)
+
+func (cli *Client) FileInfo(ctx context.Context, fileuri string) (bool, *descr.File, error) {
+	var exists bool
+	var err error
+	file := &descr.File{}
+	fileuri, username, password, err := repackServiceURI(fileuri)
+	if err != nil {
+		return exists, file, err
+	}
+	fileuri, err = convertFileURI(fileuri)
+	if err != nil {
+		return exists, file, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, fileuri, nil)
+	if err != nil {
+		return exists, file, err
+	}
+	if username != "" && password != "" {
+		basic := auxhttp.EncodeBasicAuth(username, password)
+		req.Header.Add("Authorization", basic)
+	}
+	client := makeHTTPClient(cli.skipTLSVerify)
+	resp, err := client.Do(req)
+	if err != nil {
+		return exists, file, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		file.Collection = resp.Header.Get("Content-Collection")
+		file.Name = resp.Header.Get("Content-Name")
+		contentSize := resp.Header.Get("Content-Size")
+		size, err := strconv.ParseInt(contentSize, 10, 64)
+		if err != nil {
+			return exists, file, err
+		}
+		file.Size = size
+		file.Type = resp.Header.Get("Content-Type")
+		file.Checksum = resp.Header.Get("Content-Digest")
+		exists = true
+	}
+	return exists, file, err
+}
+
+func (cli *Client) PutFile(ctx context.Context, filename, fileuri string) error {
+	var err error
+	fileuri, username, password, err := repackServiceURI(fileuri)
+	if err != nil {
+		return err
+	}
+	fileuri, err = convertFileURI(fileuri)
+	if err != nil {
+		return err
+	}
+	file, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, fileuri, file)
+	if err != nil {
+		return err
+	}
+	fileinfo, err := os.Stat(filename)
+	if err != nil {
+		return err
+	}
+	filesize := fileinfo.Size()
+
+	req.ContentLength = filesize
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("Content-Size", strconv.FormatInt(filesize, 10))
+	if username != "" && password != "" {
+		basic := auxhttp.EncodeBasicAuth(username, password)
+		req.Header.Add("Authorization", basic)
+	}
+	client := makeHTTPClient(cli.skipTLSVerify)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("Received wrong status code: %s", resp.Status)
+		return err
+	}
+	return err
+}
+
+func (cli *Client) GetFile(ctx context.Context, fileuri, filename string) (int64, error) {
+	var err error
+	var size int64
+
+	fileuri, username, password, err := repackServiceURI(fileuri)
+	if err != nil {
+		return size, err
+	}
+	fileuri, err = convertFileURI(fileuri)
+	if err != nil {
+		return size, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileuri, nil)
+	if err != nil {
+		return size, err
+	}
+	if username != "" && password != "" {
+		basic := auxhttp.EncodeBasicAuth(username, password)
+		req.Header.Add("Authorization", basic)
+	}
+	client := makeHTTPClient(cli.skipTLSVerify)
+	resp, err := client.Do(req)
+	if err != nil {
+		return size, err
+	}
+	defer resp.Body.Close()
+
+	contentLength := resp.Header.Get("Content-Length")
+	if contentLength == "" {
+		err = fmt.Errorf("Empty Content-Length received")
+		return size, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("Received wrong status code: %s", resp.Status)
+		return size, err
+	}
+	declSize, err := strconv.ParseInt(contentLength, 10, 64)
+	if err != nil {
+		err = fmt.Errorf("Wrong Content-Length value: %v", err)
+		return size, err
+	}
+	dirname := filepath.Dir(filename)
+	err = os.MkdirAll(dirname, 0750)
+	if err != nil {
+		return size, err
+	}
+	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0640)
+	if err != nil {
+		return size, err
+	}
+	size, err = io.Copy(file, resp.Body)
+	if err != nil {
+		return size, err
+	}
+	if size != declSize {
+		err := fmt.Errorf("Mismatch Content-Length and recorded filesize")
+		return size, err
+	}
+	return size, err
+}
+
+func (cli *Client) DeleteFile(ctx context.Context, fileuri string) error {
+	var err error
+
+	fileuri, username, password, err := repackServiceURI(fileuri)
+	if err != nil {
+		return err
+	}
+	fileuri, err = convertFileURI(fileuri)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodDelete, fileuri, nil)
+	if err != nil {
+		return err
+	}
+	if username != "" && password != "" {
+		basic := auxhttp.EncodeBasicAuth(username, password)
+		req.Header.Add("Authorization", basic)
+	}
+	client := makeHTTPClient(cli.skipTLSVerify)
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("Received wrong status code: %s", resp.Status)
+		return err
+	}
+	return err
+}
+
+func (cli *Client) ListFiles(ctx context.Context, catalogURI, usePathAs string) ([]descr.File, error) {
+	var err error
+	res := make([]descr.File, 0)
+
+	catalogURI, username, password, err := repackServiceURI(catalogURI)
+	if err != nil {
+		return res, err
+	}
+	catalogURI, err = convertFilesURI(catalogURI)
+	if err != nil {
+		return res, err
+	}
+	// Add values
+	values := url.Values{}
+	if usePathAs != "" {
+		values.Add("pathAs", string(usePathAs))
+	}
+	encodedValues := values.Encode()
+	if encodedValues != "" {
+		catalogURI = catalogURI + "?" + encodedValues
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, catalogURI, nil)
+	if err != nil {
+		return res, err
+	}
+	if username != "" && password != "" {
+		basic := auxhttp.EncodeBasicAuth(username, password)
+		req.Header.Add("Authorization", basic)
+	}
+	client := makeHTTPClient(cli.skipTLSVerify)
+	resp, err := client.Do(req)
+	if err != nil {
+		return res, err
+	}
+	defer resp.Body.Close()
+
+	contentLength := resp.Header.Get("Content-Length")
+	if contentLength == "" {
+		err = fmt.Errorf("Empty Content-Length received")
+		return res, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("Received wrong status code: %s", resp.Status)
+		return res, err
+	}
+	declSize, err := strconv.ParseInt(contentLength, 10, 64)
+	if err != nil {
+		err = fmt.Errorf("Wrong Content-Length value: %v", err)
+		return res, err
+	}
+	respBuffer := bytes.NewBuffer(nil)
+	size, err := io.Copy(respBuffer, resp.Body)
+	if err != nil {
+		return res, err
+	}
+	respBytes := respBuffer.Bytes()
+	if size != declSize {
+		err := fmt.Errorf("Mismatch Content-Length and recorded filesize")
+		return res, err
+	}
+	err = json.Unmarshal(respBytes, &res)
+	if err != nil {
+		return res, err
+	}
+	return res, err
+}
+
+func (cli *Client) ListCollections(ctx context.Context, catalogURI, usePathAs string) ([]string, error) {
+	var err error
+	res := make([]string, 0)
+
+	catalogURI, username, password, err := repackServiceURI(catalogURI)
+	if err != nil {
+		return res, err
+	}
+	catalogURI, err = convertCollectionsURI(catalogURI)
+	if err != nil {
+		return res, err
+	}
+
+	// Add values
+	values := url.Values{}
+	if usePathAs != "" {
+		values.Add("pathAs", string(usePathAs))
+	}
+	encodedValues := values.Encode()
+	if encodedValues != "" {
+		catalogURI = catalogURI + "?" + encodedValues
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, catalogURI, nil)
+	if err != nil {
+		return res, err
+	}
+	if username != "" && password != "" {
+		basic := auxhttp.EncodeBasicAuth(username, password)
+		req.Header.Add("Authorization", basic)
+	}
+	client := makeHTTPClient(cli.skipTLSVerify)
+	resp, err := client.Do(req)
+	if err != nil {
+		return res, err
+	}
+	defer resp.Body.Close()
+
+	contentLength := resp.Header.Get("Content-Length")
+	if contentLength == "" {
+		err = fmt.Errorf("Empty Content-Length received")
+		return res, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("Received wrong status code: %s", resp.Status)
+		return res, err
+	}
+	declSize, err := strconv.ParseInt(contentLength, 10, 64)
+	if err != nil {
+		err = fmt.Errorf("Wrong Content-Length value: %v", err)
+		return res, err
+	}
+	respBuffer := bytes.NewBuffer(nil)
+	size, err := io.Copy(respBuffer, resp.Body)
+	if err != nil {
+		return res, err
+	}
+	respBytes := respBuffer.Bytes()
+	if size != declSize {
+		err := fmt.Errorf("Mismatch Content-Length and recorded filesize")
+		return res, err
+	}
+	err = json.Unmarshal(respBytes, &res)
+	if err != nil {
+		return res, err
+	}
+	return res, err
+}
+
+func (cli *Client) DeleteCollection(ctx context.Context, catalogURI, usePathAs string, dryRun bool) ([]descr.File, error) {
+	var err error
+	res := make([]descr.File, 0)
+
+	catalogURI, username, password, err := repackServiceURI(catalogURI)
+	if err != nil {
+		return res, err
+	}
+	catalogURI, err = convertCollectionURI(catalogURI)
+	if err != nil {
+		return res, err
+	}
+	// Add values
+	values := url.Values{}
+	if usePathAs != "" {
+		values.Add("pathAs", string(usePathAs))
+	}
+	if dryRun {
+		values.Add("dryRun", "true")
+	}
+	encodedValues := values.Encode()
+	if encodedValues != "" {
+		catalogURI = catalogURI + "?" + encodedValues
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodDelete, catalogURI, nil)
+	if err != nil {
+		return res, err
+	}
+	if username != "" && password != "" {
+		basic := auxhttp.EncodeBasicAuth(username, password)
+		req.Header.Add("Authorization", basic)
+	}
+	client := makeHTTPClient(cli.skipTLSVerify)
+	resp, err := client.Do(req)
+	if err != nil {
+		return res, err
+	}
+	defer resp.Body.Close()
+
+	contentLength := resp.Header.Get("Content-Length")
+	if contentLength == "" {
+		err = fmt.Errorf("Empty Content-Length received")
+		return res, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("Received wrong status code: %s", resp.Status)
+		return res, err
+	}
+	declSize, err := strconv.ParseInt(contentLength, 10, 64)
+	if err != nil {
+		err = fmt.Errorf("Wrong Content-Length value: %v", err)
+		return res, err
+	}
+	respBuffer := bytes.NewBuffer(nil)
+	size, err := io.Copy(respBuffer, resp.Body)
+	if err != nil {
+		return res, err
+	}
+	respBytes := respBuffer.Bytes()
+	if size != declSize {
+		err := fmt.Errorf("Mismatch Content-Length and recorded filesize")
+		return res, err
+	}
+	err = json.Unmarshal(respBytes, &res)
+	if err != nil {
+		return res, err
+	}
+	return res, err
+}
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2026 Oleg Borodin <onborodin@gmail.com>
+ *
+ * This work is published and licensed under a Creative Commons
+ * Attribution-NonCommercial-NoDerivatives 4.0 International License.
+ *
+ * Distribution of this work is permitted, but commercial use and
+ * modifications are strictly prohibited.
+ */
+package client
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+)
+
+func makeHTTPClient(skipTLSVerify bool) *http.Client {
+	transport := &http.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: skipTLSVerify,
+		},
+	}
+	client := &http.Client{
+		Transport: transport,
+	}
+	return client
+}
+
+func convertFileURI(fileuri string) (string, error) {
+	var err error
+	var res string
+	uri, err := url.Parse(fileuri)
+	if err != nil {
+		return res, err
+	}
+	const fileAPI = "/v3/api/file/"
+	uri.Path, err = url.JoinPath(fileAPI, uri.Path)
+	if err != nil {
+		return res, err
+	}
+
+	res = uri.String()
+	return res, err
+}
+
+func convertFilesURI(fileuri string) (string, error) {
+	var err error
+	var res string
+	uri, err := url.Parse(fileuri)
+	const filesAPI = "/v3/api/files/"
+	uri.Path, err = url.JoinPath(filesAPI, uri.Path)
+	if err != nil {
+		return res, err
+	}
+	res = uri.String()
+	return res, err
+}
+
+func convertCollectionsURI(fileuri string) (string, error) {
+	var err error
+	var res string
+	uri, err := url.Parse(fileuri)
+	const prefixAPI = "/v3/api/collections/"
+	uri.Path, err = url.JoinPath(prefixAPI, uri.Path)
+	if err != nil {
+		return res, err
+	}
+	res = uri.String()
+	return res, err
+}
+
+func convertCollectionURI(fileuri string) (string, error) {
+	var err error
+	var res string
+	uri, err := url.Parse(fileuri)
+	const prefixAPI = "/v3/api/collection/"
+	uri.Path, err = url.JoinPath(prefixAPI, uri.Path)
+	if err != nil {
+		return res, err
+	}
+	res = uri.String()
+	return res, err
+}
+
+func repackServiceURI(fileuri string) (string, string, string, error) {
+	var err error
+	var res, username, password string
+	if !strings.Contains(fileuri, "://") {
+		fileuri = "https://" + fileuri
+	}
+	uri, err := url.Parse(fileuri)
+	if err != nil {
+		return res, username, password, err
+	}
+	uri.Path = path.Clean(uri.Path)
+	if uri.User != nil {
+		username = uri.User.Username()
+		password, _ = uri.User.Password()
+	}
+	uri.User = nil
+	//uri.Scheme = "https"
+	res = uri.String()
+	return res, username, password, err
+}