Merge pull request seaweedfs#5 from chrislusf/master

sync

README.md

@@ -4,7 +4,7 @@
 [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
-[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=86400)](https://hub.docker.com/r/chrislusf/seaweedfs/)
+[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
 
@@ -112,17 +112,19 @@ On top of the object store, optional [Filer] can support directories and POSIX a
 [Back to TOC](#table-of-contents)
 
 ## Filer Features ##
-* [filer server][Filer] provide "normal" directories and files via http.
-* [mount filer][Mount] to read and write files directly as a local directory via FUSE.
+* [Filer server][Filer] provide "normal" directories and files via http.
+* [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
+* [Mount filer][Mount] to read and write files directly as a local directory via FUSE.
 * [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
 * [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
 * [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
 * [WebDAV] access as a mapped drive on Mac and Windows, or from mobile devices.
 * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
 * [File TTL][FilerTTL] automatically purge file metadata and actual file data.
-* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=604800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
+* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
 
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
+[SuperLargeFiles]: https://github.com/chrislusf/seaweedfs/wiki/Data-Structure-for-Large-Files
 [Mount]: https://github.com/chrislusf/seaweedfs/wiki/FUSE-Mount
 [AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
 [BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/golang/protobuf v1.4.2
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
-	github.com/gorilla/mux v1.7.3
+	github.com/gorilla/mux v1.7.4
 	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
@@ -90,7 +90,4 @@ require (
 	gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
 )
 
-replace (
-	github.com/satori/go.uuid v1.2.0 => github.com/satori/go.uuid v0.0.0-20181028125025-b2ce2384e17b
-	go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547
-)
+replace go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547

go.sum

@@ -236,6 +236,8 @@ github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=

test/s3/basic/basic_test.go

@@ -2,14 +2,14 @@ package basic
 
 import (
 	"fmt"
-	"os"
-	"strings"
-	"testing"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
 )
 
 var (
@@ -109,3 +109,101 @@ func exitErrorf(msg string, args ...interface{}) {
 	fmt.Fprintf(os.Stderr, msg+"\n", args...)
 	os.Exit(1)
 }
+
+const (
+	Bucket = "theBucket"
+	object = "foo/bar"
+	Data   = "<data>"
+)
+
+func TestObjectOp(t *testing.T) {
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(Bucket),
+	})
+	if err != nil {
+		exitErrorf("Unable to create bucket, %v", err)
+	}
+
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(object),
+		Body:   strings.NewReader(Data),
+	})
+	if err != nil {
+		exitErrorf("Unable to put object, %v", err)
+	}
+
+	dest := fmt.Sprintf("%s_bak", object)
+	copyObj, err := svc.CopyObject(&s3.CopyObjectInput{
+		Bucket:     aws.String(Bucket),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", Bucket, object)),
+		Key:        aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to copy object, %v", err)
+	}
+	t.Log("copy object result -> ", copyObj.CopyObjectResult)
+
+	getObj, err := svc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to get copy object, %v", err)
+	}
+
+	data, err := ioutil.ReadAll(getObj.Body)
+	if err != nil {
+		exitErrorf("Unable to read object data, %v", err)
+	}
+	if string(data) != Data {
+		t.Error("object data -> ", string(data))
+	}
+
+	listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket: aws.String(Bucket),
+		Prefix: aws.String("foo/"),
+	})
+	if err != nil {
+		exitErrorf("Unable to list objects, %v", err)
+	}
+	count := 0
+	for _, content := range listObj.Contents {
+		key := aws.StringValue(content.Key)
+		if key == dest {
+			count++
+		} else if key == object {
+			count++
+		}
+		if count == 2 {
+			break
+		}
+	}
+	if count != 2 {
+		exitErrorf("Unable to find two objects, %v", listObj.Contents)
+	}
+
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(object),
+	})
+	if err != nil {
+		exitErrorf("Unable to delete source object, %v", err)
+	}
+
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to delete object, %v", err)
+	}
+
+	_, err = svc.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(Bucket),
+	})
+
+	if err != nil {
+		exitErrorf("Unable to delete bucket, %v", err)
+	}
+}

weed/filesys/dir.go

@@ -27,6 +27,7 @@ type Dir struct {
 var _ = fs.Node(&Dir{})
 var _ = fs.NodeCreater(&Dir{})
 var _ = fs.NodeMkdirer(&Dir{})
+var _ = fs.NodeFsyncer(&Dir{})
 var _ = fs.NodeRequestLookuper(&Dir{})
 var _ = fs.HandleReadDirAller(&Dir{})
 var _ = fs.NodeRemover(&Dir{})
@@ -90,23 +91,27 @@ func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
 	attr.BlockSize = 1024 * 1024
 }
 
+func (dir *Dir) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
+	// fsync works at OS level
+	// write the file chunks to the filerGrpcAddress
+	glog.V(3).Infof("dir %s fsync %+v", dir.FullPath(), req)
+
+	return nil
+}
+
 func (dir *Dir) newFile(name string, entry *filer_pb.Entry) fs.Node {
-	return dir.wfs.fsNodeCache.EnsureFsNode(util.NewFullPath(dir.FullPath(), name), func() fs.Node {
-		return &File{
-			Name:           name,
-			dir:            dir,
-			wfs:            dir.wfs,
-			entry:          entry,
-			entryViewCache: nil,
-		}
-	})
+	return &File{
+		Name:           name,
+		dir:            dir,
+		wfs:            dir.wfs,
+		entry:          entry,
+		entryViewCache: nil,
+	}
 }
 
 func (dir *Dir) newDirectory(fullpath util.FullPath, entry *filer_pb.Entry) fs.Node {
 
-	return dir.wfs.fsNodeCache.EnsureFsNode(fullpath, func() fs.Node {
-		return &Dir{name: entry.Name, wfs: dir.wfs, entry: entry, parent: dir}
-	})
+	return &Dir{name: entry.Name, wfs: dir.wfs, entry: entry, parent: dir}
 
 }
 
@@ -306,8 +311,6 @@ func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error {
 
 	dir.wfs.deleteFileChunks(entry.Chunks)
 
-	dir.wfs.fsNodeCache.DeleteFsNode(filePath)
-
 	dir.wfs.metaCache.DeleteEntry(context.Background(), filePath)
 
 	glog.V(3).Infof("remove file: %v", req)
@@ -324,7 +327,6 @@ func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error {
 func (dir *Dir) removeFolder(req *fuse.RemoveRequest) error {
 
 	t := util.NewFullPath(dir.FullPath(), req.Name)
-	dir.wfs.fsNodeCache.DeleteFsNode(t)
 
 	dir.wfs.metaCache.DeleteEntry(context.Background(), t)
 
@@ -417,8 +419,6 @@ func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp
 
 func (dir *Dir) Forget() {
 	glog.V(3).Infof("Forget dir %s", dir.FullPath())
-
-	dir.wfs.fsNodeCache.DeleteFsNode(util.FullPath(dir.FullPath()))
 }
 
 func (dir *Dir) maybeLoadEntry() error {

weed/filesys/dir_rename.go

@@ -3,11 +3,12 @@ package filesys
 import (
 	"context"
 
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/seaweedfs/fuse"
-	"github.com/seaweedfs/fuse/fs"
 )
 
 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
@@ -19,7 +20,15 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 
 	glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath)
 
-	err := dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	// find local old entry
+	oldEntry, err := dir.wfs.metaCache.FindEntry(context.Background(), oldPath)
+	if err != nil {
+		glog.V(0).Infof("dir Rename can not find source %s : %v", oldPath, err)
+		return fuse.ENOENT
+	}
+
+	// update remote filer
+	err = dir.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.AtomicRenameEntryRequest{
 			OldDirectory: dir.FullPath(),
@@ -30,21 +39,30 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 
 		_, err := client.AtomicRenameEntry(context.Background(), request)
 		if err != nil {
-			glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err)
 			return fuse.EIO
 		}
 
 		return nil
 
 	})
+	if err != nil {
+		glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err)
+		return fuse.EIO
+	}
 
-	if err == nil {
-
-		// fmt.Printf("rename path: %v => %v\n", oldPath, newPath)
-		dir.wfs.fsNodeCache.Move(oldPath, newPath)
-		delete(dir.wfs.handles, oldPath.AsInode())
-
+	// TODO: replicate renaming logic on filer
+	if err := dir.wfs.metaCache.DeleteEntry(context.Background(), oldPath); err != nil {
+		glog.V(0).Infof("dir Rename delete local %s => %s : %v", oldPath, newPath, err)
+		return fuse.EIO
+	}
+	oldEntry.FullPath = newPath
+	if err := dir.wfs.metaCache.InsertEntry(context.Background(), oldEntry); err != nil {
+		glog.V(0).Infof("dir Rename insert local %s => %s : %v", oldPath, newPath, err)
+		return fuse.EIO
 	}
 
+	// fmt.Printf("rename path: %v => %v\n", oldPath, newPath)
+	delete(dir.wfs.handles, oldPath.AsInode())
+
 	return err
 }

weed/filesys/file.go

@@ -213,7 +213,6 @@ func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
 func (file *File) Forget() {
 	t := util.NewFullPath(file.dir.FullPath(), file.Name)
 	glog.V(3).Infof("Forget file %s", t)
-	file.wfs.fsNodeCache.DeleteFsNode(t)
 }
 
 func (file *File) maybeLoadEntry(ctx context.Context) error {

weed/filesys/filehandle.go

@@ -191,10 +191,16 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 
 		if fh.f.entry.Attributes != nil {
 			fh.f.entry.Attributes.Mime = fh.contentType
-			fh.f.entry.Attributes.Uid = req.Uid
-			fh.f.entry.Attributes.Gid = req.Gid
+			if fh.f.entry.Attributes.Uid == 0 {
+				fh.f.entry.Attributes.Uid = req.Uid
+			}
+			if fh.f.entry.Attributes.Gid == 0 {
+				fh.f.entry.Attributes.Gid = req.Gid
+			}
+			if fh.f.entry.Attributes.Crtime == 0 {
+				fh.f.entry.Attributes.Crtime = time.Now().Unix()
+			}
 			fh.f.entry.Attributes.Mtime = time.Now().Unix()
-			fh.f.entry.Attributes.Crtime = time.Now().Unix()
 			fh.f.entry.Attributes.FileMode = uint32(os.FileMode(fh.f.entry.Attributes.FileMode) &^ fh.f.wfs.option.Umask)
 			fh.f.entry.Attributes.Collection = fh.dirtyPages.collection
 			fh.f.entry.Attributes.Replication = fh.dirtyPages.replication