Merge branch 'master' of github.com:Terry-Mao/bfs

ops/README.md

@@ -1,7 +1,19 @@
-<<<<<<< HEAD
-# bfsops
-The background management interface of BFS.
-=======
 # bfsops
 The background management interface of BFS.
->>>>>>> 7d8b931a9e68fda757ca26536ef0a50189407e43
+
+## 初始化流程(请参照test/ops_initialization.py)：
+
+###step 1
+启动store、directory、proxy、pitchfork
+store启动后，zookeeper看到/rack 有store节点
+
+###step 2:
+调用space()函数，初始化store，调用完成后，磁盘空间会被分配；bfs存储目录看到生成的volume文件。
+
+###step 3:
+调用groups()函数，store分组，调用完成后，zookeeper看到/group/ 有组节点
+
+###step 4:
+调用volumes()函数， 生效volume，调用完成后，zookeeper看到/volume/有volume节点
+
+### Done

ops/bfsOps/commons/config.py → ops/commons/config.py

@@ -1,9 +1,12 @@
 # -*- encoding:utf8 -*-
 
+# store.
 store_admin_port = 6063
 store_stat_port = 6061
 store_block_size = 32 # GB
 
-zk_hosts = '127.0.0.1:2181'
+# zookeeper  port:2181.
+zk_hosts = '127.0.0.1'
 
+# log dir.
 log_dir = '/tmp'

ops/bfsOps/commons/store_client.py → ops/commons/store_client.py

@@ -22,6 +22,7 @@ def __init__(self, host, port):
     def request(self, method, url, body=None, headers={}):
         '''return [True|False], status, data'''
         try:
+            print url,body
             if self.http_conn is None:
                 self.http_conn = httplib.HTTPConnection(self.host, self.port, timeout=HttpConnection.DEFAULT_TIMEOUT)
 
@@ -60,7 +61,7 @@ def genStoreConn(store_ip, store_port):
 def storeAddFreeVolume(store_ip, base_dir, num_volumes):
     if num_volumes <= 0:
         return None
-
+    print store_ip    
     store_conn = genStoreConn(store_ip, config.store_admin_port)
     url = "/add_free_volume"
     value = {}

ops/bfsOps/commons/zk_client.py → ops/commons/zk_client.py

@@ -84,6 +84,8 @@ def getAllVolume():
 				if not STORE_VOLUME.has_key(store_id):
 					STORE_VOLUME[store_id] = []
 				STORE_VOLUME[store_id].append(volume_id)
+                        print "max:",MAX_VOLUME_ID
+                        logger.error("你好")
 		return True
 	except Exception as ex:
 		logger.error("getAllVolume() called   error: %s", str(ex))
@@ -124,6 +126,7 @@ def getAllGroup():
 				if not GROUP_STORE.has_key(group_id):
 					GROUP_STORE[group_id] = []
 				GROUP_STORE[group_id].append(store_id)
+                print GROUP_STORE
 		return True
 	except Exception as ex:
 		logger.error("getAllGroup() called   error: %s", str(ex))	

ops/setup.py

@@ -1,16 +1,10 @@
 from setuptools import setup
 
 setup(
-    name="IMDaDa",
+    name="bfs",
     version="1.0",
     install_requires=[
         "requests",
         "flask",
-        "flask-wtf",
-        "Flask-Login",
-        "Flask-OAuth",
-        "Flask-OAuthlib",
-        "Flask-migrate",
-        "flask-debugtoolbar",
     ],
 )

ops/test/ops_initialization.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import urllib2
+import json
+
+# initialize bfs space
+def space():
+	url = 'http://127.0.0.1:9000/bfsops/initialization'
+	# ips 机器的ip列表
+	# dirs 磁盘目录
+	# size 每块磁盘的空间大小
+	value = {"ips":"xx.xx.xx.xx, yy.yy.yy.yy","dirs":"/data1/bfsdata/,/data2/bfsdata/","size":"10T"}
+
+	jdata = json.dumps(value)
+	req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"})
+	response = urllib2.urlopen(req)
+	return response.read()
+# initialize groups
+def groups():
+	url = 'http://127.0.0.1:9000/bfsops/groups'
+	# ips 要分组的ip列表
+	# copys 副本数（包括本身）
+	# racks 跨机架 默认1即可，具体请参考ops代码
+	value = {"ips":"xx.xx.xx.xx, yy.yy.yy.yy", "copys":2, "rack":1}
+
+	jdata = json.dumps(value)
+	req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"})
+	response = urllib2.urlopen(req)
+	return response.read()
+
+# initialize volumes
+def volumes():
+	url = 'http://127.0.0.1:9000/bfsops/volumes'
+	# groups 生效某个group
+	value = {"groups":"2"}
+
+	jdata = json.dumps(value)
+	req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"})
+	response = urllib2.urlopen(req)
+	return response.read()
+
+# store启动后，zookeeper看到/rack 有store节点
+
+# 初始化流程：
+# 分别依次调用3个函数，分别完成：初始化store；store分组；生效volume.
+#step 1:
+#space()
+#调用完成后，磁盘空间会被分配；bfs存储目录看到生成的volume文件。
+
+#step 2:
+#groups()
+#调用完成后，zookeeper看到/group/ 有组节点
+
+#step 3:
+#volumes()
+#调用完成后，zookeeper看到/volume/有volume节点
+
+

ops/bfsOps/views/ops.py → ops/views/ops.py

@@ -6,15 +6,7 @@
 from commons import *
 from commons.global_var import *
 
-
 from flask import request,render_template,jsonify,session,redirect,url_for,abort
-#from bfsOps.decorates import login_required
-
-@app.route('/')
-@app.route('/index.html')
-#@login_required
-def home():
-	return render_template('index.html')
 
 
 @app.route('/bfsops/initialization', methods = ["POST"])
@@ -178,7 +170,7 @@ def bfsopsGroupsGet():
 def bfsopsVolumesPost():
 	if not request.json:
 		abort(400)
-	groups = list(set(request.json['groups']))
+	groups = list(set(request.json['groups'].split(',')))
 	for group_id in groups:
 		if not GROUP_STORE.has_key(group_id.encode('utf-8')):
 			abort(400)
@@ -188,7 +180,7 @@ def bfsopsVolumesPost():
 	resp['errorMsg'] = ""
 
 	need_break = False
-	global MAX_VOLUME_ID
+    global MAX_VOLUME_ID
 	for group_id_u in groups:
 		group_id = group_id_u.encode('utf-8')
 		stores = GROUP_STORE[group_id]
@@ -225,8 +217,3 @@ def bfsopsVolumesPost():
 	resp_str = json.dumps(resp)
 	logger.info("bfsopsVolumesPost() called, success, resp: %s", resp_str)
 	return resp_str
-
-@app.route('/bfsops/volumes', methods = ["GET"])
-#@login_required
-def bfsopsVolumesGet():
-	pass