Merge pull request Terry-Mao#72 from hurkgu/feature/bfs_upgrade

upgrade

README.md

@@ -1,8 +1,104 @@
-##### Version 2.0.1
-> proxy 访问store加上限流日志  
+bfs
+==============
+`bfs` 是基于facebook haystack 用golang实现的小文件存储系统。
 
-##### Version 2.0.0
-> 1.bfs 缓存限制在1M以内  
+---------------------------------------
+  * [特性](#特性)
+  * [安装](#安装)
+  * [集群](#集群)
+  * [API](#API)
+  * [更多](#更多)
 
-# bfs
-distributed file system(small file storage) writen by golang according to facebook haystack.
+---------------------------------------
+
+## 特性
+ * 高吞吐量和低延迟
+ * 容错性
+ * 高效
+ * 维护简单
+
+## 安装
+
+### 一、安装hbase、zookeeper
+
+ * 参考hbase官网. 安装、启动请查看[这里](https://hbase.apache.org/).
+ * 参考zookeeper官网. 安装、启动请查看[这里](http://zookeeper.apache.org/).
+
+### 二、搭建golang、python环境
+
+ * 参考golang官网. 安装请查看[这里](https://golang.org/doc/install).
+ * 参考python官网. 安装请查看[这里]
+(https://www.python.org/)
+
+### 三、安装gosnowflake
+
+ * 参考[这里](https://github.com/Terry-Mao/gosnowflake)
+
+### 四、部署
+1.下载bfs及依赖包
+```sh
+$ go get -u github.com/Terry-Mao/bfs
+$ cd /data/apps/go/src/github.com/Terry-Mao/bfs
+$ go get ./...
+```
+
+2.安装directory、store、pitchfork、proxy模块(配置文件请依据实际机器环境配置)
+```sh
+$ cd $GOPATH/src/github.com/Terry-Mao/bfs/directory
+$ go install
+$ cp directory.toml $GOPATH/bin/directory.toml
+$ cd ../store/
+$ go install
+$ cp store.toml $GOPATH/bin/store.toml
+$ cd ../pitchfork/
+$ go install
+$ cp pitchfork.toml $GOPATH/bin/pitchfork.toml
+$ cd ../proxy
+$ go install
+$ cp proxy.toml $GOPATH/bin/proxy.toml
+
+```
+到此所有的环境都搭建完成！
+
+### 五、启动
+```sh
+$ cd /$GOPATH/bin
+$ nohup $GOPATH/bin/directory -c $GOPATH/bin/directory.toml &
+$ nohup $GOPATH/bin/store -c $GOPATH/bin/store.toml &
+$ nohup $GOPATH/bin/pitchfork -c $GOPATH/bin/pitchfork.toml &
+$ nohup $GOPATH/bin/proxy -c $GOPATH/bin/proxy.toml &
+$ cd $GOPATH/github.com/Terry-Mao/bfs/ops
+$ nohup python runserver.py &
+```
+
+### 六、测试
+ * bfs初始化，分配存储空间，请查看[这里](https://github.com/Terry-Mao/bfs/doc/ops.md)
+ * 请求bfs，请查看[这里](https://github.com/Terry-Mao/bfs/doc/proxy.md)
+
+## 集群
+
+![Aaron Swartz](http://i0.hdslb.com/bfs/active/bfs_server.png)
+
+### directory
+
+ * directory主要负责请求的均匀调度和元数据管理，元数据存放在hbase，由gosnowflake产生文件key
+
+### store
+
+ * store主要负责文件的物理存储
+
+### pitchfork
+
+ * pitchfork负责监控store的服务状态、可用性和磁盘状态
+
+### proxy
+
+ * proxy作为bfs存储的代理以及维护bucket相关
+
+### ops
+
+ * ops作为bfs的后台管理界面，负责分配存储、扩容、压缩等维护工作
+
+## API
+[api文档](https://github.com/Terry-Mao/bfs/blob/master/doc/api.md)
+## 更多

directory/conf/config.go

@@ -1,10 +1,13 @@
 package conf
 
 import (
-	"github.com/BurntSushi/toml"
 	"io/ioutil"
 	"os"
 	"time"
+
+	xtime "bfs/libs/time"
+
+	"github.com/BurntSushi/toml"
 )
 
 type Config struct {
@@ -34,12 +37,24 @@ type Zookeeper struct {
 	GroupRoot    string
 }
 
+// HBase config.
 type HBase struct {
-	Addr       string
-	MaxActive  int
-	MaxIdle    int
-	Timeout    duration
-	LvsTimeout duration
+	ZookeeperHbase *ZookeeperHbase
+	// default "" means use default hbase zk path. It should correspond to server config
+	Master        string
+	Meta          string
+	TestRowKey    string
+	DialTimeout   xtime.Duration // 0 means no dial timeout
+	ReadTimeout   xtime.Duration
+	ReadsTimeout  xtime.Duration
+	WriteTimeout  xtime.Duration
+	WritesTimeout xtime.Duration
+}
+
+type ZookeeperHbase struct {
+	Root    string
+	Addrs   []string
+	Timeout xtime.Duration
 }
 
 // Code to implement the TextUnmarshaler interface for `duration`:

directory/directory.go

@@ -19,7 +19,7 @@ const (
 	retrySleep = time.Second * 1
 )
 
-// Directory
+// Directory .
 // id means store serverid; vid means volume id; gid means group id
 type Directory struct {
 	// STORE
@@ -34,15 +34,15 @@ type Directory struct {
 	volume      map[int32]*meta.VolumeState // volume_id:volume_state
 	volumeStore map[int32][]string          // volume_id:store_server_id
 
-	genkey     *snowflake.Genkey  // snowflake client for gen key
-	hBase      *hbase.HBaseClient // hBase client
-	dispatcher *Dispatcher        // dispatch for write or read reqs
+	genkey     *snowflake.Genkey // snowflake client for gen key
+	hBase      *hbase.Client     // hBase client
+	dispatcher *Dispatcher       // dispatch for write or read reqs
 
 	config *conf.Config
 	zk     *myzk.Zookeeper
 }
 
-// NewDirectory
+// NewDirectory .
 func NewDirectory(config *conf.Config) (d *Directory, err error) {
 	d = &Directory{}
 	d.config = config
@@ -52,10 +52,8 @@ func NewDirectory(config *conf.Config) (d *Directory, err error) {
 	if d.genkey, err = snowflake.NewGenkey(config.Snowflake.ZkAddrs, config.Snowflake.ZkPath, config.Snowflake.ZkTimeout.Duration, config.Snowflake.WorkId); err != nil {
 		return
 	}
-	if err = hbase.Init(config); err != nil {
-		return
-	}
-	d.hBase = hbase.NewHBaseClient()
+
+	d.hBase = hbase.NewClient(config.HBase)
 	d.dispatcher = NewDispatcher()
 	go d.SyncZookeeper()
 	return
@@ -259,7 +257,7 @@ func (d *Directory) GetStores(bucket, filename string) (n *meta.Needle, f *meta.
 	stores = make([]string, 0, len(svrs))
 	for _, store = range svrs {
 		if storeMeta, ok = d.store[store]; !ok {
-			log.Errorf("store cannot match store:", store)
+			log.Errorf("store cannot match store:%s", store)
 			continue
 		}
 		if !storeMeta.CanRead() {
@@ -283,8 +281,8 @@ func (d *Directory) UploadStores(bucket string, f *meta.File) (n *meta.Needle, s
 		storeMeta *meta.Store
 		ok        bool
 	)
-	if vid, err = d.dispatcher.VolumeId(d.group, d.storeVolume); err != nil {
-		log.Errorf("dispatcher.VolumeId error(%v)", err)
+	if vid, err = d.dispatcher.VolumeID(d.group, d.storeVolume); err != nil {
+		log.Errorf("dispatcher.VolumeID error(%v)", err)
 		err = errors.ErrStoreNotAvailable
 		return
 	}

directory/directory.toml

@@ -23,7 +23,7 @@ ZkPath = "/gosnowflake-servers"
 ZkTimeout = "15s"
 
 # workid
-WorkId = 0
+WorkId = 1
 
 [zookeeper]
 # zookeeper cluster addrs, multiple addrs split by ",".
@@ -47,14 +47,14 @@ GroupRoot = "/group"
 PullInterval = "10s"
 
 [hbase]
-# addr 172.16.13.90:9090
-Addr = "172.16.13.90:9090"
-
-# Note that you must specify a number here.
-MaxActive = 100
-
-# Note that you must specify a number here.
-MaxIdle = 100
-
-# Note that you must specify a number here.
-Timeout = "1s"
+master = ""
+meta = ""
+dialTimeout = "1s"
+readTimeout = "10s"
+readsTimeout = "10s"
+writeTimeout = "10s"
+writesTimeout = "10s"
+    [hbase.zookeeperHbase]
+    root = ""
+    addrs = ["localhost:2181"]
+    timeout = "30s"

directory/directory_test.go

@@ -2,34 +2,38 @@ package main
 
 import (
 	"testing"
-	"time"
+	// "time"
+
+	"bfs/directory/conf"
+	dzk "bfs/directory/zk"
 )
 
 func TestDirectory(t *testing.T) {
 	var (
 		err    error
-		config *Config
-		zk     *Zookeeper
+		config *conf.Config
+		zk     *dzk.Zookeeper
 		d      *Directory
 	)
-	if config, err = NewConfig("./directory.conf"); err != nil {
+	if config, err = conf.NewConfig("./directory.toml"); err != nil {
 		t.Errorf("NewConfig() error(%v)", err)
 		t.FailNow()
 	}
 
-	if zk, err = NewZookeeper([]string{"localhost:2181"}, time.Second*1, "/rack", "/volume", "/group"); err != nil {
+	if zk, err = dzk.NewZookeeper(config); err != nil {
 		t.Errorf("NewZookeeper() error(%v)", err)
 		t.FailNow()
 	}
-	if d, err = NewDirectory(config, zk); err != nil {
+	defer zk.Close()
+	if d, err = NewDirectory(config); err != nil {
 		t.Errorf("NewDirectory() error(%v)", err)
 		t.FailNow()
 	}
 	if _, err = d.syncStores(); err != nil {
 		t.Errorf("syncStores() error(%v)", err)
 		t.FailNow()
 	}
-	if _, err = d.syncGroups(); err != nil {
+	if err = d.syncGroups(); err != nil {
 		t.Errorf("syncGroups() error(%v)", err)
 		t.FailNow()
 	}