feat(storage): add copy command

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Features
+
+- storage: add `copy` command for copying objects within and across buckets (#887)
+
 ### Bug fixes
 
 - instance create: apply delete protection in the instance's zone, fixing a wrong-zone "Not Found" error when creating protected instances outside the default zone (#879)
@@ -11,6 +15,7 @@
 
 - AUR package: reset pkgrel in PKGBUILD (#878)
 - utils: remove unused ICMP formatters (#884)
+- deps: update golang.org/x dependencies (#885)
 
 ## 1.97.0
 

cmd/storage/storage_copy.go

@@ -0,0 +1,43 @@
+package storage
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var storageCopyCmd = &cobra.Command{
+	Use:     "copy sos://BUCKET/[OBJECT|PREFIX/] sos://BUCKET/[OBJECT|PREFIX/]",
+	Aliases: []string{"cp"},
+	Short:   "Copy objects within a bucket or across buckets",
+	Long: `Copy objects within a bucket or across buckets.
+
+This command copies objects server-side without downloading them. Object
+metadata, headers, and ACLs are preserved. Existing destination objects are
+overwritten.
+
+Multi-object prefix copies are processed serially. A trailing slash on the
+source selects prefix mode; -r controls recursion into subdirectories.
+
+Examples:
+
+    exo storage copy sos://my-bucket/file-a sos://my-bucket/folder/file-a
+
+    exo storage copy sos://my-bucket/file-a sos://other-bucket/file-a
+
+    exo storage copy -r sos://my-bucket/prefix/ sos://other-bucket/prefix/
+
+    exo storage copy -n sos://my-bucket/file-a sos://other-bucket/file-a
+`,
+	PreRunE: validateStorageTransferCommand,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runStorageTransfer(cmd, args, false)
+	},
+}
+
+func init() {
+	storageCopyCmd.Flags().BoolP("dry-run", "n", false, "simulate the copy operation")
+	storageCopyCmd.Flags().BoolP("force", "f", false, "skip confirmation prompt")
+	storageCopyCmd.Flags().BoolP("recursive", "r", false, "copy objects recursively")
+	storageCopyCmd.Flags().BoolP("verbose", "v", false, "output copied objects")
+	storageCopyCmd.Flags().Int("multipart-concurrency", 4, "number of concurrent parts for multipart copies")
+	storageCmd.AddCommand(storageCopyCmd)
+}

cmd/storage/storage_move.go

@@ -15,8 +15,9 @@ import (
 )
 
 var storageMoveCmd = &cobra.Command{
-	Use:   "move sos://BUCKET/[OBJECT|PREFIX/] sos://BUCKET/[OBJECT|PREFIX/]",
-	Short: "Move objects within a bucket or across buckets",
+	Use:     "move sos://BUCKET/[OBJECT|PREFIX/] sos://BUCKET/[OBJECT|PREFIX/]",
+	Aliases: []string{"mv"},
+	Short:   "Move objects within a bucket or across buckets",
 	Long: `Move objects within a bucket or across buckets.
 
 This command moves objects by performing a server-side copy followed by
@@ -32,73 +33,17 @@ source selects prefix mode; -r controls recursion into subdirectories.
 
 Examples:
 
-    exo storage move sos://my-bucket/file-a sos://my-bucket/folder/
+    exo storage move sos://my-bucket/file-a sos://my-bucket/folder/file-a
 
     exo storage move sos://my-bucket/file-a sos://other-bucket/file-a
 
     exo storage move -r sos://my-bucket/prefix/ sos://other-bucket/prefix/
 
-    exo storage move -n sos://my-bucket/file-a sos://other-bucket/
+    exo storage move -n sos://my-bucket/file-a sos://other-bucket/file-a
 `,
-	PreRunE: func(cmd *cobra.Command, args []string) error {
-		if len(args) != 2 {
-			exocmd.CmdExitOnUsageError(cmd, "invalid arguments")
-		}
-		return validateMoveArgs(args)
-	},
-
+	PreRunE: validateStorageTransferCommand,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		recursive, err := cmd.Flags().GetBool("recursive")
-		if err != nil {
-			return err
-		}
-		force, err := cmd.Flags().GetBool("force")
-		if err != nil {
-			return err
-		}
-		multipartConcurrency, err := cmd.Flags().GetInt("multipart-concurrency")
-		if err != nil {
-			return err
-		}
-		verbose, err := cmd.Flags().GetBool("verbose")
-		if err != nil {
-			return err
-		}
-		dryRun, err := cmd.Flags().GetBool("dry-run")
-		if err != nil {
-			return err
-		}
-
-		srcBucket, srcKey := parseBucketKey(args[0])
-		dstBucket, dstKey := parseBucketKey(args[1])
-
-		storage, err := sos.NewStorageClient(
-			exocmd.GContext,
-			sos.ClientOptZoneFromBucket(exocmd.GContext, srcBucket),
-		)
-		if err != nil {
-			return fmt.Errorf("unable to initialize storage client: %w", err)
-		}
-
-		isPrefix := strings.HasSuffix(srcKey, "/") || recursive
-
-		if !force && !dryRun && isPrefix {
-			if !utils.AskQuestion(exocmd.GContext, fmt.Sprintf(
-				"Are you sure you want to move all objects from %s%s/%s to %s%s/%s?",
-				sos.BucketPrefix, srcBucket, srcKey, sos.BucketPrefix, dstBucket, dstKey)) {
-				return nil
-			}
-		}
-
-		if dryRun {
-			fmt.Println("[DRY-RUN]")
-		}
-
-		if !isPrefix {
-			return runSingleObjectMove(storage, srcBucket, srcKey, dstBucket, dstKey, multipartConcurrency, verbose, dryRun)
-		}
-
-		return runPrefixMove(storage, srcBucket, srcKey, dstBucket, dstKey, multipartConcurrency, recursive, verbose, dryRun)
+		return runStorageTransfer(cmd, args, true)
 	},
 }
 
@@ -111,9 +56,23 @@ func init() {
 	storageCmd.AddCommand(storageMoveCmd)
 }
 
-func validateMoveArgs(args []string) error {
+func validateStorageTransferCommand(cmd *cobra.Command, args []string) error {
+	if len(args) != 2 {
+		exocmd.CmdExitOnUsageError(cmd, "invalid arguments")
+	}
+
+	recursive, err := cmd.Flags().GetBool("recursive")
+	if err != nil {
+		return err
+	}
+
+	return validateStorageTransferArgs(args, recursive)
+}
+
+func validateStorageTransferArgs(args []string, recursive bool) error {
 	srcBucket, srcKey := parseBucketKey(args[0])
 	dstBucket, dstKey := parseBucketKey(args[1])
+	isPrefix := strings.HasSuffix(srcKey, "/") || recursive
 
 	if srcBucket == "" {
 		return fmt.Errorf("source must include a bucket name: %s", args[0])
@@ -124,9 +83,15 @@ func validateMoveArgs(args []string) error {
 	if srcKey == "" && dstKey == "" {
 		return fmt.Errorf("at least one of source/destination must include an object key or prefix")
 	}
-	if srcKey != "" && dstKey == "" {
+	if !isPrefix && srcKey != "" && dstKey == "" {
 		return fmt.Errorf("destination must include an object key when source is a single object: %s", args[1])
 	}
+	if srcBucket == dstBucket && srcKey == dstKey {
+		return fmt.Errorf("source and destination must differ")
+	}
+	if isPrefix && srcBucket == dstBucket && (strings.HasPrefix(dstKey, srcKey) || strings.HasPrefix(srcKey, dstKey)) {
+		return fmt.Errorf("source and destination prefixes must not overlap")
+	}
 
 	return nil
 }
@@ -141,32 +106,91 @@ func parseBucketKey(url string) (bucket, key string) {
 	return
 }
 
-func runSingleObjectMove(storage *sos.Client, srcBucket, srcKey, dstBucket, dstKey string, multipartConcurrency int, verbose, dryRun bool) error {
+func runStorageTransfer(cmd *cobra.Command, args []string, move bool) error {
+	recursive, err := cmd.Flags().GetBool("recursive")
+	if err != nil {
+		return err
+	}
+	force, err := cmd.Flags().GetBool("force")
+	if err != nil {
+		return err
+	}
+	multipartConcurrency, err := cmd.Flags().GetInt("multipart-concurrency")
+	if err != nil {
+		return err
+	}
+	verbose, err := cmd.Flags().GetBool("verbose")
+	if err != nil {
+		return err
+	}
+	dryRun, err := cmd.Flags().GetBool("dry-run")
+	if err != nil {
+		return err
+	}
+
+	srcBucket, srcKey := parseBucketKey(args[0])
+	dstBucket, dstKey := parseBucketKey(args[1])
+
+	storage, err := sos.NewStorageClient(
+		exocmd.GContext,
+		sos.ClientOptZoneFromBucket(exocmd.GContext, srcBucket),
+	)
+	if err != nil {
+		return fmt.Errorf("unable to initialize storage client: %w", err)
+	}
+
+	action := cmd.Name()
+	pastTense := "copied"
+	if move {
+		pastTense = "moved"
+	}
+	isPrefix := strings.HasSuffix(srcKey, "/") || recursive
+
+	if !force && !dryRun && isPrefix {
+		if !utils.AskQuestion(exocmd.GContext, fmt.Sprintf(
+			"Are you sure you want to %s all objects from %s%s/%s to %s%s/%s?",
+			action, sos.BucketPrefix, srcBucket, srcKey, sos.BucketPrefix, dstBucket, dstKey)) {
+			return nil
+		}
+	}
+
+	if dryRun {
+		fmt.Println("[DRY-RUN]")
+	}
+
+	if !isPrefix {
+		return runSingleObjectTransfer(storage, srcBucket, srcKey, dstBucket, dstKey, action, pastTense, multipartConcurrency, verbose, dryRun, move)
+	}
+
+	return runPrefixTransfer(storage, srcBucket, srcKey, dstBucket, dstKey, action, pastTense, multipartConcurrency, recursive, verbose, dryRun, move)
+}
+
+func runSingleObjectTransfer(storage *sos.Client, srcBucket, srcKey, dstBucket, dstKey, action, pastTense string, multipartConcurrency int, verbose, dryRun, move bool) error {
 	if srcKey == "" {
-		return fmt.Errorf("source must be an object key, not just a bucket: use a trailing slash for prefix moves")
+		return fmt.Errorf("source must be an object key, not just a bucket: use --recursive for bucket prefixes")
 	}
 
 	if dryRun {
-		fmt.Printf("move %s%s/%s -> %s%s/%s\n", sos.BucketPrefix, srcBucket, srcKey, sos.BucketPrefix, dstBucket, dstKey)
+		fmt.Printf("%s %s%s/%s -> %s%s/%s\n", action, sos.BucketPrefix, srcBucket, srcKey, sos.BucketPrefix, dstBucket, dstKey)
 		return nil
 	}
 
-	if err := storage.MoveObject(exocmd.GContext, srcBucket, srcKey, dstBucket, dstKey, multipartConcurrency, verbose); err != nil {
-		return fmt.Errorf("move failed: %w", err)
+	if err := transferObject(storage, srcBucket, srcKey, dstBucket, dstKey, multipartConcurrency, verbose, move); err != nil {
+		return fmt.Errorf("%s failed: %w", action, err)
 	}
 
 	if verbose {
 		showObj, err := storage.ShowObject(exocmd.GContext, dstBucket, dstKey)
 		if err == nil {
-			fmt.Printf("moved: %s -> %s (%d bytes, %s)\n", srcKey, showObj.URL, showObj.Size, showObj.LastModified)
+			fmt.Printf("%s: %s -> %s (%d bytes, %s)\n", pastTense, srcKey, showObj.URL, showObj.Size, showObj.LastModified)
 		}
 	}
 
 	return nil
 }
 
-func runPrefixMove(storage *sos.Client, srcBucket, srcKey, dstBucket, dstKey string, multipartConcurrency int, recursive, verbose, dryRun bool) error {
-	var moved, failed int
+func runPrefixTransfer(storage *sos.Client, srcBucket, srcKey, dstBucket, dstKey, action, pastTense string, multipartConcurrency int, recursive, verbose, dryRun, move bool) error {
+	var transferred, failed int
 	err := storage.ForEachObject(exocmd.GContext, srcBucket, srcKey, recursive, func(o *types.Object) error {
 		if o.Key == nil {
 			return nil
@@ -177,39 +201,47 @@ func runPrefixMove(storage *sos.Client, srcBucket, srcKey, dstBucket, dstKey str
 		dstObjectKey := dstKey + srcObjectKeyTrimmed
 
 		if dryRun {
-			fmt.Printf("move %s%s/%s -> %s%s/%s\n", sos.BucketPrefix, srcBucket, srcObjectKey, sos.BucketPrefix, dstBucket, dstObjectKey)
+			fmt.Printf("%s %s%s/%s -> %s%s/%s\n", action, sos.BucketPrefix, srcBucket, srcObjectKey, sos.BucketPrefix, dstBucket, dstObjectKey)
 			return nil
 		}
 
-		if err := storage.MoveObject(exocmd.GContext, srcBucket, srcObjectKey, dstBucket, dstObjectKey, multipartConcurrency, verbose); err != nil {
-			fmt.Fprintf(os.Stderr, "move failed for %s: %v\n", srcObjectKey, err)
+		if err := transferObject(storage, srcBucket, srcObjectKey, dstBucket, dstObjectKey, multipartConcurrency, verbose, move); err != nil {
+			fmt.Fprintf(os.Stderr, "%s failed for %s: %v\n", action, srcObjectKey, err)
 			failed++
 			return nil
 		}
 
-		moved++
+		transferred++
 		if verbose && !globalstate.Quiet {
-			fmt.Printf("moved: %s%s/%s -> %s%s/%s\n", sos.BucketPrefix, srcBucket, srcObjectKey, sos.BucketPrefix, dstBucket, dstObjectKey)
+			fmt.Printf("%s: %s%s/%s -> %s%s/%s\n", pastTense, sos.BucketPrefix, srcBucket, srcObjectKey, sos.BucketPrefix, dstBucket, dstObjectKey)
 		}
 
 		return nil
 	})
 
 	if err != nil {
-		return fmt.Errorf("move failed: %w", err)
+		return fmt.Errorf("%s failed: %w", action, err)
 	}
 
 	if failed > 0 {
-		return fmt.Errorf("%d object(s) failed to move", failed)
+		return fmt.Errorf("%d object(s) failed to %s", failed, action)
 	}
 
-	if moved == 0 && !dryRun && !globalstate.Quiet {
+	if transferred == 0 && !dryRun && !globalstate.Quiet {
 		fmt.Printf("no objects exist at %q\n", srcKey)
 	}
 
-	if verbose && !globalstate.Quiet && moved > 0 {
-		fmt.Printf("moved %d objects\n", moved)
+	if verbose && !globalstate.Quiet && transferred > 0 {
+		fmt.Printf("%s %d objects\n", pastTense, transferred)
 	}
 
 	return nil
 }
+
+func transferObject(storage *sos.Client, srcBucket, srcKey, dstBucket, dstKey string, multipartConcurrency int, verbose, move bool) error {
+	if move {
+		return storage.MoveObject(exocmd.GContext, srcBucket, srcKey, dstBucket, dstKey, multipartConcurrency, verbose)
+	}
+
+	return storage.CopyObjectTo(exocmd.GContext, srcBucket, srcKey, dstBucket, dstKey, multipartConcurrency, verbose)
+}