diff --git a/ops/README.md b/ops/README.md index c48e246..5c5771a 100644 --- a/ops/README.md +++ b/ops/README.md @@ -1,7 +1,19 @@ -<<<<<<< HEAD -# bfsops -The background management interface of BFS. -======= # bfsops The background management interface of BFS. ->>>>>>> 7d8b931a9e68fda757ca26536ef0a50189407e43 + +## 初始化流程(请参照test/ops_initialization.py)： + +###step 1 +启动store、directory、proxy、pitchfork +store启动后，zookeeper看到/rack 有store节点 + +###step 2: +调用space()函数，初始化store，调用完成后，磁盘空间会被分配；bfs存储目录看到生成的volume文件。 + +###step 3: +调用groups()函数，store分组，调用完成后，zookeeper看到/group/ 有组节点 + +###step 4: +调用volumes()函数， 生效volume，调用完成后，zookeeper看到/volume/有volume节点 + +### Done \ No newline at end of file diff --git a/ops/bfsOps/test/test.py b/ops/bfsOps/test/test.py deleted file mode 100644 index 773e77e..0000000 --- a/ops/bfsOps/test/test.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import urllib2 -import json - -def http_post(): - url = 'http://127.0.0.1:9000/bfsops/initialization' - #url = 'http://127.0.0.1:9000/groups' - #url = 'http://127.0.0.1:9000/volumes' - value = {"ips":"123.56.108.22","dirs":"/tmp","size":"3G"} - values = [] - values.append(value) - values_d = {} - values_d['initlist'] = values - - jdata = json.dumps(values_d) - req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"}) - response = urllib2.urlopen(req) - return response.read() - -resp = http_post() - -print resp \ No newline at end of file diff --git a/ops/bfsOps/commons/__init__.py b/ops/commons/__init__.py similarity index 100% rename from ops/bfsOps/commons/__init__.py rename to ops/commons/__init__.py diff --git a/ops/commons/__init__.pyc b/ops/commons/__init__.pyc new file mode 100644 index 0000000..d392edf Binary files /dev/null and b/ops/commons/__init__.pyc differ diff --git a/ops/bfsOps/commons/blogging.py b/ops/commons/blogging.py similarity index 100% rename from ops/bfsOps/commons/blogging.py rename to ops/commons/blogging.py diff --git a/ops/commons/blogging.pyc b/ops/commons/blogging.pyc new file mode 100644 index 0000000..6fdb4c6 Binary files /dev/null and b/ops/commons/blogging.pyc differ diff --git a/ops/bfsOps/commons/config.py b/ops/commons/config.py similarity index 59% rename from ops/bfsOps/commons/config.py rename to ops/commons/config.py index 7e413e6..cfcdbfd 100644 --- a/ops/bfsOps/commons/config.py +++ b/ops/commons/config.py @@ -1,9 +1,12 @@ # -*- encoding:utf8 -*- +# store. store_admin_port = 6063 store_stat_port = 6061 store_block_size = 32 # GB -zk_hosts = '127.0.0.1:2181' +# zookeeper port:2181. +zk_hosts = '127.0.0.1' +# log dir. log_dir = '/tmp' diff --git a/ops/commons/config.pyc b/ops/commons/config.pyc new file mode 100644 index 0000000..a9979ff Binary files /dev/null and b/ops/commons/config.pyc differ diff --git a/ops/bfsOps/commons/global_var.py b/ops/commons/global_var.py similarity index 100% rename from ops/bfsOps/commons/global_var.py rename to ops/commons/global_var.py diff --git a/ops/commons/global_var.pyc b/ops/commons/global_var.pyc new file mode 100644 index 0000000..844df44 Binary files /dev/null and b/ops/commons/global_var.pyc differ diff --git a/ops/bfsOps/commons/store_client.py b/ops/commons/store_client.py similarity index 98% rename from ops/bfsOps/commons/store_client.py rename to ops/commons/store_client.py index e26ad41..4f57a71 100644 --- a/ops/bfsOps/commons/store_client.py +++ b/ops/commons/store_client.py @@ -22,6 +22,7 @@ def __init__(self, host, port): def request(self, method, url, body=None, headers={}): '''return [True|False], status, data''' try: + print url,body if self.http_conn is None: self.http_conn = httplib.HTTPConnection(self.host, self.port, timeout=HttpConnection.DEFAULT_TIMEOUT) @@ -60,7 +61,7 @@ def genStoreConn(store_ip, store_port): def storeAddFreeVolume(store_ip, base_dir, num_volumes): if num_volumes <= 0: return None - + print store_ip store_conn = genStoreConn(store_ip, config.store_admin_port) url = "/add_free_volume" value = {} diff --git a/ops/commons/store_client.pyc b/ops/commons/store_client.pyc new file mode 100644 index 0000000..a607206 Binary files /dev/null and b/ops/commons/store_client.pyc differ diff --git a/ops/bfsOps/commons/zk_client.py b/ops/commons/zk_client.py similarity index 93% rename from ops/bfsOps/commons/zk_client.py rename to ops/commons/zk_client.py index 0ffddac..e2b158a 100644 --- a/ops/bfsOps/commons/zk_client.py +++ b/ops/commons/zk_client.py @@ -84,6 +84,8 @@ def getAllVolume(): if not STORE_VOLUME.has_key(store_id): STORE_VOLUME[store_id] = [] STORE_VOLUME[store_id].append(volume_id) + print "max:",MAX_VOLUME_ID + logger.error("你好") return True except Exception as ex: logger.error("getAllVolume() called error: %s", str(ex)) @@ -124,6 +126,7 @@ def getAllGroup(): if not GROUP_STORE.has_key(group_id): GROUP_STORE[group_id] = [] GROUP_STORE[group_id].append(store_id) + print GROUP_STORE return True except Exception as ex: logger.error("getAllGroup() called error: %s", str(ex)) diff --git a/ops/commons/zk_client.pyc b/ops/commons/zk_client.pyc new file mode 100644 index 0000000..99100ba Binary files /dev/null and b/ops/commons/zk_client.pyc differ diff --git a/ops/setup.py b/ops/setup.py index dbaeb04..e405e20 100644 --- a/ops/setup.py +++ b/ops/setup.py @@ -1,16 +1,10 @@ from setuptools import setup setup( - name="IMDaDa", + name="bfs", version="1.0", install_requires=[ "requests", "flask", - "flask-wtf", - "Flask-Login", - "Flask-OAuth", - "Flask-OAuthlib", - "Flask-migrate", - "flask-debugtoolbar", ], ) diff --git a/ops/test/ops_initialization.py b/ops/test/ops_initialization.py new file mode 100644 index 0000000..119e4e1 --- /dev/null +++ b/ops/test/ops_initialization.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import urllib2 +import json + +# initialize bfs space +def space(): + url = 'http://127.0.0.1:9000/bfsops/initialization' + # ips 机器的ip列表 + # dirs 磁盘目录 + # size 每块磁盘的空间大小 + value = {"ips":"xx.xx.xx.xx, yy.yy.yy.yy","dirs":"/data1/bfsdata/,/data2/bfsdata/","size":"10T"} + + jdata = json.dumps(value) + req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"}) + response = urllib2.urlopen(req) + return response.read() +# initialize groups +def groups(): + url = 'http://127.0.0.1:9000/bfsops/groups' + # ips 要分组的ip列表 + # copys 副本数（包括本身） + # racks 跨机架 默认1即可，具体请参考ops代码 + value = {"ips":"xx.xx.xx.xx, yy.yy.yy.yy", "copys":2, "rack":1} + + jdata = json.dumps(value) + req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"}) + response = urllib2.urlopen(req) + return response.read() + +# initialize volumes +def volumes(): + url = 'http://127.0.0.1:9000/bfsops/volumes' + # groups 生效某个group + value = {"groups":"2"} + + jdata = json.dumps(value) + req = urllib2.Request(url, jdata, headers = {"Content-type":"application/json"}) + response = urllib2.urlopen(req) + return response.read() + +# store启动后，zookeeper看到/rack 有store节点 + +# 初始化流程： +# 分别依次调用3个函数，分别完成：初始化store；store分组；生效volume. +#step 1: +#space() +#调用完成后，磁盘空间会被分配；bfs存储目录看到生成的volume文件。 + +#step 2: +#groups() +#调用完成后，zookeeper看到/group/ 有组节点 + +#step 3: +#volumes() +#调用完成后，zookeeper看到/volume/有volume节点 + + diff --git a/ops/bfsOps/views/__init__.py b/ops/views/__init__.py similarity index 100% rename from ops/bfsOps/views/__init__.py rename to ops/views/__init__.py diff --git a/ops/views/__init__.pyc b/ops/views/__init__.pyc new file mode 100644 index 0000000..60410e5 Binary files /dev/null and b/ops/views/__init__.pyc differ diff --git a/ops/bfsOps/views/ops.py b/ops/views/ops.py similarity index 92% rename from ops/bfsOps/views/ops.py rename to ops/views/ops.py index 8163f4b..0f37a4c 100644 --- a/ops/bfsOps/views/ops.py +++ b/ops/views/ops.py @@ -6,15 +6,7 @@ from commons import * from commons.global_var import * - from flask import request,render_template,jsonify,session,redirect,url_for,abort -#from bfsOps.decorates import login_required - -@app.route('/') -@app.route('/index.html') -#@login_required -def home(): - return render_template('index.html') @app.route('/bfsops/initialization', methods = ["POST"]) @@ -178,7 +170,7 @@ def bfsopsGroupsGet(): def bfsopsVolumesPost(): if not request.json: abort(400) - groups = list(set(request.json['groups'])) + groups = list(set(request.json['groups'].split(','))) for group_id in groups: if not GROUP_STORE.has_key(group_id.encode('utf-8')): abort(400) @@ -188,7 +180,7 @@ def bfsopsVolumesPost(): resp['errorMsg'] = "" need_break = False - global MAX_VOLUME_ID + global MAX_VOLUME_ID for group_id_u in groups: group_id = group_id_u.encode('utf-8') stores = GROUP_STORE[group_id] @@ -225,8 +217,3 @@ def bfsopsVolumesPost(): resp_str = json.dumps(resp) logger.info("bfsopsVolumesPost() called, success, resp: %s", resp_str) return resp_str - -@app.route('/bfsops/volumes', methods = ["GET"]) -#@login_required -def bfsopsVolumesGet(): - pass diff --git a/ops/views/ops.pyc b/ops/views/ops.pyc new file mode 100644 index 0000000..52113af Binary files /dev/null and b/ops/views/ops.pyc differ