Merge pull request Terry-Mao#69 from zhapuyu/master

add proxy cache and fix hbase del bug
AlexanderChiuluvB · Jun 4, 2017 · ed63965 · ed63965
2 parents 77c52eb + 98bda64
commit ed63965
Show file tree

Hide file tree

Showing 44 changed files with 1,679 additions and 32,835 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,7 +24,7 @@ _testmain.go
 *.prof
 *.gitattributes
 
-*.iml
-*.ipr
-*.iws
-.idea/
+directory/directory
+pitchfork/pitchfork
+proxy/proxy
+store/store
diff --git a/README.md b/README.md
@@ -1,107 +1,8 @@
-bfs
-==============
-`bfs` 是基于facebook haystack 用golang实现的小文件存储系统。
+##### Version 2.0.1
+> proxy 访问store加上限流日志  
 
----------------------------------------
-  * [特性](#特性)
-  * [安装](#安装)
-  * [集群](#集群)
-  * [API](#API)
-  * [更多](#更多)
+##### Version 2.0.0
+> 1.bfs 缓存限制在1M以内  
 
----------------------------------------
-
-## 特性
- * 高吞吐量和低延迟
- * 容错性
- * 高效
- * 维护简单
-
-## 安装
-
-### 一、安装hbase、zookeeper
-
- * 参考hbase官网. 安装、启动请查看[这里](https://hbase.apache.org/).
- * 参考zookeeper官网. 安装、启动请查看[这里](http://zookeeper.apache.org/).
-
-### 二、搭建golang、python环境
-
- * 参考golang官网. 安装请查看[这里](https://golang.org/doc/install).
- * 参考python官网. 安装请查看[这里]
-(https://www.python.org/)
-
-### 三、安装gosnowflake
-
- * 参考[这里](https://github.com/Terry-Mao/gosnowflake)
-
-### 四、部署
-1.下载bfs及依赖包
-```sh
-$ go get -u github.com/Terry-Mao/bfs
-$ cd /data/apps/go/src/github.com/Terry-Mao/bfs
-$ go get ./...
-```
-
-2.安装directory、store、pitchfork、proxy模块(配置文件请依据实际机器环境配置)
-```sh
-$ cd $GOPATH/src/github.com/Terry-Mao/bfs/directory
-$ go install
-$ cp directory.toml $GOPATH/bin/directory.toml
-$ cd ../store/
-$ go install
-$ cp store.toml $GOPATH/bin/store.toml
-$ cd ../pitchfork/
-$ go install
-$ cp pitchfork.toml $GOPATH/bin/pitchfork.toml
-$ cd ../proxy
-$ go install
-$ cp proxy.toml $GOPATH/bin/proxy.toml
-
-```
-到此所有的环境都搭建完成！
-
-### 五、启动
-```sh
-$ cd /$GOPATH/bin
-$ nohup $GOPATH/bin/directory -c $GOPATH/bin/directory.toml &
-$ nohup $GOPATH/bin/store -c $GOPATH/bin/store.toml &
-$ nohup $GOPATH/bin/pitchfork -c $GOPATH/bin/pitchfork.toml &
-$ nohup $GOPATH/bin/proxy -c $GOPATH/bin/proxy.toml &
-$ cd $GOPATH/github.com/Terry-Mao/bfs/ops
-$ nohup python runserver.py &
-```
-
-### 六、测试
- * bfs初始化，分配存储空间，请查看[这里](https://github.com/Terry-Mao/bfs/doc/ops.md)
- * 请求bfs，请查看[这里](https://github.com/Terry-Mao/bfs/doc/proxy.md)
-
-## 集群
-
-![Aaron Swartz](http://i0.hdslb.com/bfs/active/bfs_server.png)
-
-### directory
-
- * directory主要负责请求的均匀调度和元数据管理，元数据存放在hbase，由gosnowflake产生文件key
-
-### store
-
- * store主要负责文件的物理存储
-
-### pitchfork
-
- * pitchfork负责监控store的服务状态、可用性和磁盘状态
-
-### proxy
-
- * proxy作为bfs存储的代理以及维护bucket相关
-
-### ops
-
- * ops作为bfs的后台管理界面，负责分配存储、扩容、压缩等维护工作
-
-## API
-[api文档](https://github.com/Terry-Mao/bfs/blob/master/doc/api.md)
-
-## 更多
-
- * [bfs-image-server](https://github.com/YonkaFang/bfs-image-server) 
+# bfs
+distributed file system(small file storage) writen by golang according to facebook haystack.
diff --git a/directory/directory.go b/directory/directory.go
@@ -8,10 +8,11 @@ import (
 	"bfs/libs/errors"
 	"bfs/libs/meta"
 	"encoding/json"
-	log "github.com/golang/glog"
-	"github.com/samuel/go-zookeeper/zk"
 	"strconv"
 	"time"
+
+	log "github.com/golang/glog"
+	"github.com/samuel/go-zookeeper/zk"
 )
 
 const (
@@ -306,6 +307,7 @@ func (d *Directory) UploadStores(bucket string, f *meta.File) (n *meta.Needle, s
 	n.Key = key
 	n.Vid = vid
 	n.Cookie = d.cookie()
+	n.MTime = f.MTime
 	f.Key = key
 	if err = d.hBase.Put(bucket, f, n); err != nil {
 		if err != errors.ErrNeedleExist {

diff --git a/directory/hbase/hbase.go b/directory/hbase/hbase.go
@@ -7,8 +7,9 @@ import (
 	"bytes"
 	"crypto/sha1"
 	"encoding/binary"
-	log "github.com/golang/glog"
 	"time"
+
+	log "github.com/golang/glog"
 )
 
 const (
@@ -144,7 +145,7 @@ func (h *HBaseClient) putNeedle(n *meta.Needle) (err error) {
 	}
 	binary.BigEndian.PutUint32(vbuf, uint32(n.Vid))
 	binary.BigEndian.PutUint32(cbuf, uint32(n.Cookie))
-	binary.BigEndian.PutUint64(ubuf, uint64(time.Now().UnixNano()))
+	binary.BigEndian.PutUint64(ubuf, uint64(n.MTime))
 	if err = c.Put(_table, &hbasethrift.TPut{
 		Row: ks,
 		ColumnValues: []*hbasethrift.TColumnValue{
@@ -185,16 +186,6 @@ func (h *HBaseClient) delNeedle(key int64) (err error) {
 	ks = h.key(key)
 	if err = c.DeleteSingle(_table, &hbasethrift.TDelete{
 		Row: ks,
-		Columns: []*hbasethrift.TColumn{
-			&hbasethrift.TColumn{
-				Family:    _familyBasic,
-				Qualifier: _columnVid,
-			},
-			&hbasethrift.TColumn{
-				Family:    _familyBasic,
-				Qualifier: _columnCookie,
-			},
-		},
 	}); err != nil {
 		hbasePool.Put(c, true)
 		return
@@ -274,7 +265,7 @@ func (h *HBaseClient) putFile(bucket string, f *meta.File) (err error) {
 	}
 	binary.BigEndian.PutUint64(kbuf, uint64(f.Key))
 	binary.BigEndian.PutUint32(stbuf, uint32(f.Status))
-	binary.BigEndian.PutUint64(ubuf, uint64(time.Now().UnixNano()))
+	binary.BigEndian.PutUint64(ubuf, uint64(f.MTime))
 	if err = c.Put(h.tableName(bucket), &hbasethrift.TPut{
 		Row: ks,
 		ColumnValues: []*hbasethrift.TColumnValue{
@@ -351,28 +342,6 @@ func (h *HBaseClient) delFile(bucket, filename string) (err error) {
 	ks = []byte(filename)
 	if err = c.DeleteSingle(h.tableName(bucket), &hbasethrift.TDelete{
 		Row: ks,
-		Columns: []*hbasethrift.TColumn{
-			&hbasethrift.TColumn{
-				Family:    _familyFile,
-				Qualifier: _columnKey,
-			},
-			&hbasethrift.TColumn{
-				Family:    _familyFile,
-				Qualifier: _columnSha1,
-			},
-			&hbasethrift.TColumn{
-				Family:    _familyFile,
-				Qualifier: _columnMine,
-			},
-			&hbasethrift.TColumn{
-				Family:    _familyFile,
-				Qualifier: _columnStatus,
-			},
-			&hbasethrift.TColumn{
-				Family:    _familyFile,
-				Qualifier: _columnUpdateTime,
-			},
-		},
 	}); err != nil {
 		hbasePool.Put(c, true)
 		return

diff --git a/directory/http_api.go b/directory/http_api.go
@@ -4,9 +4,12 @@ import (
 	"bfs/libs/errors"
 	"bfs/libs/meta"
 	"encoding/json"
-	log "github.com/golang/glog"
 	"net/http"
 	"time"
+
+	"strconv"
+
+	log "github.com/golang/glog"
 )
 
 const (
@@ -86,13 +89,14 @@ func (s *server) get(wr http.ResponseWriter, r *http.Request) {
 
 func (s *server) upload(wr http.ResponseWriter, r *http.Request) {
 	var (
-		err    error
-		n      *meta.Needle
-		f      *meta.File
-		bucket string
-		res    meta.Response
-		ok     bool
-		uerr   errors.Error
+		err      error
+		n        *meta.Needle
+		f        *meta.File
+		bucket   string
+		res      meta.Response
+		ok       bool
+		uerr     errors.Error
+		mtimeStr string
 	)
 	if r.Method != "POST" {
 		http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
@@ -115,6 +119,14 @@ func (s *server) upload(wr http.ResponseWriter, r *http.Request) {
 		http.Error(wr, "bad request", http.StatusBadRequest)
 		return
 	}
+	if mtimeStr = r.FormValue("mtime"); mtimeStr == "" {
+		http.Error(wr, "bad request", http.StatusBadRequest)
+		return
+	}
+	if f.MTime, err = strconv.ParseInt(mtimeStr, 10, 64); err != nil {
+		http.Error(wr, "bad request", http.StatusBadRequest)
+		return
+	}
 	defer HttpUploadWriter(r, wr, time.Now(), &res)
 
 	res.Ret = errors.RetOK
@@ -144,6 +156,10 @@ func (s *server) upload(wr http.ResponseWriter, r *http.Request) {
 	res.Key = n.Key
 	res.Cookie = n.Cookie
 	res.Vid = n.Vid
+	res.MTime = n.MTime
+	if f.MTime > 0 {
+		res.MTime = f.MTime
+	}
 	return
 }
 

diff --git a/directory/snowflake/gosnowflake.go b/directory/snowflake/gosnowflake.go
@@ -17,7 +17,7 @@
 package snowflake
 
 import (
-	log "github.com/golang/glog"
+	log "code.google.com/p/log4go"
 	"encoding/json"
 	"errors"
 	myrpc "github.com/Terry-Mao/gosnowflake/rpc"
@@ -174,7 +174,7 @@ func (c *Client) client() (*rpc.Client, error) {
 // watchWorkerId watch the zk node change.
 func (c *Client) watchWorkerId(workerId int64, workerIdStr string) {
 	workerIdPath := path.Join(zkPath, workerIdStr)
-	//log.Debug("workerIdPath: %s", workerIdPath)
+	log.Debug("workerIdPath: %s", workerIdPath)
 	for {
 		rpcs, _, watch, err := zkConn.ChildrenW(workerIdPath)
 		if err != nil {

diff --git a/doc/api.md b/doc/api.md
diff --git a/doc/bfs.graffle b/doc/bfs.graffle