fix: kaniko dir env unused (GoogleContainerTools#2067)

* update kaniko-dir flag description

* add dir check anonymous func for check to config or env

* add kaniko dir constant in default options

* Update cmd/executor/cmd/root.go

Co-authored-by: Jason Hall <jasonhall@redhat.com>

* add CopyDir util usage for checkKanikoDir func

* add removal of kaniko dir on successful copy of new files

* add comment for CopyDir usage

Co-authored-by: Jason Hall <jasonhall@redhat.com>

cmd/executor/cmd/root.go

@@ -68,7 +68,13 @@ var RootCmd = &cobra.Command{
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 		if cmd.Use == "executor" {
 
-			if err := checkKanikoDir(config.KanikoDir); err != nil {
+			// Command line flag takes precedence over the KANIKO_DIR environment variable.
+			dir := config.KanikoDir
+			if opts.KanikoDir != constants.DefaultKanikoPath {
+				dir = opts.KanikoDir
+			}
+
+			if err := checkKanikoDir(dir); err != nil {
 				return err
 			}
 
@@ -187,7 +193,7 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().BoolVarP(&opts.SkipTLSVerifyPull, "skip-tls-verify-pull", "", false, "Pull from insecure registry ignoring TLS verify")
 	RootCmd.PersistentFlags().IntVar(&opts.PushRetry, "push-retry", 0, "Number of retries for the push operation")
 	RootCmd.PersistentFlags().IntVar(&opts.ImageFSExtractRetry, "image-fs-extract-retry", 0, "Number of retries for image FS extraction")
-	RootCmd.PersistentFlags().StringVarP(&opts.KanikoDir, "kaniko-dir", "", "/kaniko", "Path to the kaniko directory")
+	RootCmd.PersistentFlags().StringVarP(&opts.KanikoDir, "kaniko-dir", "", constants.DefaultKanikoPath, "Path to the kaniko directory, this takes precedence over the KANIKO_DIR environment variable.")
 	RootCmd.PersistentFlags().StringVarP(&opts.TarPath, "tarPath", "", "", "Path to save the image in as a tarball instead of pushing")
 	RootCmd.PersistentFlags().BoolVarP(&opts.SingleSnapshot, "single-snapshot", "", false, "Take a single snapshot at the end of the build.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible")
@@ -244,11 +250,12 @@ func addHiddenFlags(cmd *cobra.Command) {
 func checkKanikoDir(dir string) error {
 	if dir != constants.DefaultKanikoPath {
 
-		if err := os.MkdirAll(dir, os.ModeDir); err != nil {
+		// The destination directory may be across a different partition, so we cannot simply rename/move the directory in this case.
+		if _, err := util.CopyDir(constants.DefaultKanikoPath, dir, util.FileContext{}, util.DoNotChangeUID, util.DoNotChangeGID); err != nil {
 			return err
 		}
 
-		if err := os.Rename(constants.DefaultKanikoPath, dir); err != nil {
+		if err := os.RemoveAll(constants.DefaultKanikoPath); err != nil {
 			return err
 		}
 	}