Merge branch 'master' of https://github.com/FederatedAI/KubeFATE

.gitignore

@@ -1,5 +1,6 @@
 *.cfg
 *.idea/
+*.vscode/
 *bin/
 *.todo
 *.exe

Makefile

@@ -1,23 +1,30 @@
 BUILD-PATH = ${shell pwd}
-K8S-DEPLOY="./k8s-deploy"
+K8S-DEPLOY = ./k8s-deploy
+CHART = ./helm-charts
+FML = ./fml_manager
+RELEASE_VERSION ?= v1.5.0
 
 define sub_make
  cd $1 && make $2 && cd ${BUILD-PATH}
 endef
 
-all: build-linux-binary zip build-docker-image
+all: release
 
-build-linux-binary:
-	$(call sub_make, ${K8S-DEPLOY}, build-linux-binary)
+release: k8s-release docker-compose-release helm-chart-release
+	mkdir -p ${BUILD-PATH}/release
+	mv ${K8S-DEPLOY}/release/* release/
+	mv ${BUILD-PATH}/kubefate-docker-compose-${RELEASE_VERSION}.tar.gz release/
+	mv ${CHART}/fate-*.tgz release/
 
-build-docker-image:
-	$(call sub_make, ${K8S-DEPLOY}, build-docker-image)
+clean:
+	rm -r release
 
-zip:
+k8s-release:
+	${call sub_make, ${K8S-DEPLOY}, release RELEASE_VERSION=${RELEASE_VERSION}}
+docker-compose-release:
 	tar -czvf kubefate-docker-compose-${RELEASE_VERSION}.tar.gz ./docker-deploy/* ./docker-deploy/.env
+helm-chart-release:
+	${call sub_make, ${CHART}, release  RELEASE_VERSION=${RELEASE_VERSION}}
 
-release: zip
-	${call sub_make, ${K8S-DEPLOY}, release RELEASE_VERSION=${RELEASE_VERSION}} && mv ${K8S-DEPLOY}/kubefate-k8s-${RELEASE_VERSION}.tar.gz ./ && curl -LJO https://federatedai.github.io/KubeFATE/package/fate-${RELEASE_VERSION}.tgz 
-
-clean:
-	rm kubefate-docker-compose-*.tar.gz kubefate-k8s-*.tar.gz fate-*.tgz && $(call sub_make, ${K8S-DEPLOY}, clean)
+fml-release:
+	${call sub_make, ${FML}, docker-save VERSION=${RELEASE_VERSION}}

docker-deploy/generate_config.sh

@@ -86,8 +86,6 @@ GenerateConfig() {
 			sed -i "s#<nodemanager.port>#${nodemanager_port}#g" ./confs-$party_id/confs/eggroll/conf/eggroll.properties
 			sed -i "s#<party.id>#${party_id}#g" ./confs-$party_id/confs/eggroll/conf/eggroll.properties
 
-			#python env
-			sed -i "s#<venv>#${venv_dir}#g" ./confs-$party_id/confs/eggroll/conf/eggroll.properties
 			#pythonpath, very import, do not modify."
 			sed -i "s#<python.path>#/data/projects/fate/python:/data/projects/fate/eggroll/python#g" ./confs-$party_id/confs/eggroll/conf/eggroll.properties
 
@@ -108,6 +106,16 @@ GenerateConfig() {
 			#sed -i 's#image: "redis:5"#image: "${RegistryURI}/redis:5"#g' ./confs-$party_id/docker-compose.yml
 		fi
 
+        # check if use python-nn
+        if [ "$enabled_nn" = "true" ]; then
+            sed -i 's#image: "federatedai/python:${TAG}"#image: "federatedai/python-nn:${TAG}"#g' ./confs-$party_id/docker-compose.yml
+        fi
+
+        # replace namenode in training_template/public/fate_flow/conf/service_conf.yaml
+        if [ "$name_node" != "" ]; then
+            sed -i "s#name_node: hdfs://namenode:9000#name_node: ${name_node}#g" ./confs-$party_id/confs/fate_flow/conf/service_conf.yaml
+        fi
+
 		# update serving ip
 		sed -i "s/fate-serving/${serving_ip}/g" ./confs-$party_id/docker-compose.yml
 

docker-deploy/parties.conf

@@ -9,6 +9,9 @@ serving_ip_list=(192.168.1.1 192.168.1.2)
 # computing_backend could be eggroll or spark.
 computing_backend=eggroll
 
+# true if you need python-nn else false, the default value will be false
+enabled_nn=false
+
 # default
 exchangeip=
 
@@ -22,3 +25,5 @@ mysql_db=fate_flow
 redis_ip=redis
 redis_port=6379
 redis_password=fate_dev
+
+name_node=hdfs://namenode:9000

docker-deploy/training_template/backends/eggroll/conf/eggroll.properties

@@ -28,7 +28,7 @@ eggroll.resourcemanager.process.tag=<party.id>
 eggroll.bootstrap.root.script=bin/eggroll_boot.sh
 
 eggroll.resourcemanager.bootstrap.egg_pair.exepath=bin/roll_pair/egg_pair_bootstrap.sh
-eggroll.resourcemanager.bootstrap.egg_pair.venv=<venv>
+eggroll.resourcemanager.bootstrap.egg_pair.venv=
 eggroll.resourcemanager.bootstrap.egg_pair.pythonpath=<python.path>
 eggroll.resourcemanager.bootstrap.egg_pair.filepath=python/eggroll/roll_pair/egg_pair.py
 eggroll.resourcemanager.bootstrap.egg_pair.ld_library_path=

docker-deploy/training_template/docker-compose-eggroll.yml

@@ -109,6 +109,7 @@ services:
     networks:
       fate-network:
         ipv4_address: 192.167.0.100
+    command: ["/bin/bash", "-c", "sleep 5 && python ./fate_flow/fate_flow_server.py"]
 
   client:
     image: "federatedai/client:${TAG}"

docs/Use_image_pull_secrets.md

@@ -0,0 +1,32 @@
+## 如何配置使用私有镜像仓库部署FATE
+
+### 1. 生成`imagePullSecrets`
+
+生成docker hub的Secrets
+
+```bash
+DOCKER_REGISTRY_SERVER=<registry的URL>
+DOCKER_USER=<你的docker用户名>
+DOCKER_EMAIL=<你的docker邮箱>
+DOCKER_PASSWORD=<你的docker密码>
+
+kubectl create secret docker-registry myregistrykey \
+  --docker-server=$DOCKER_REGISTRY_SERVER \
+  --docker-username=$DOCKER_USER \
+  --docker-password=$DOCKER_PASSWORD \
+  --docker-email=$DOCKER_EMAIL
+```
+
+> docker hub的registry的URL：https://index.docker.io/v1/
+
+### 2. 配置
+
+在要部署FATE的对应namespace下生成上述secret，然后将secret的name写入`cluster.yaml`的`imagePullSecrets`
+
+```bash
+# 例如这样
+imagePullSecrets: 
+- name: myregistrykey
+```
+
+配合`registry`可以使用私有的仓库镜像，也可以应对docker hub限流。

fml_manager/Makefile

@@ -0,0 +1,12 @@
+NAME ?= federatedai/client
+VERSION ?= v1.5.0
+IMG ?= ${NAME}:${VERSION:v%=%-release}
+
+docker-build:
+	docker build . -t ${IMG}
+
+docker-push:
+	docker push ${IMG}
+
+docker-save: docker-build
+	docker save -o client-${VERSION}.docker ${IMG} 

helm-charts/FATE-Serving/templates/ingress.yaml

@@ -10,7 +10,7 @@
 # limitations under the License.
 
 {{ if .Values.servingProxy.include }}
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: serving-proxy

helm-charts/FATE/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: v1.5.0
 description: A Helm chart for fate-training
 name: fate
-version: v1.5.0
+version: v1.5.0-a
 home: https://fate.fedai.org
 icon: https://aisp-1251170195.cos.ap-hongkong.myqcloud.com/wp-content/uploads/sites/12/2019/09/logo.png
 sources:

helm-charts/FATE/templates/NOTES.txt

@@ -3,6 +3,16 @@ Then you should be able to visit the FateBoard portal at {{ .Values.host.fateboa
 For more details, please visit https://github.com/FederatedAI/KubeFATE.
 
 # optional
-If you have enabled the istio, please visit:
-- FateBoard at http://${istio-gateway}/fateboard-{{ .Values.partyId }}/
-- NoteBook at http://${istio-gateway}/notebook-{{ .Values.partyId }}/ 
+If you have configured ingress or isto, you may need to add "<host_ip/istio_ip> [ <FateBoard>, <NoteBook>...]"to the hosts.
+{{- if .Values.modules.fateboard.include }}
+- FateBoard at http://{{ .Values.host.fateboard }}
+{{- end }}
+{{- if .Values.modules.client.include  }}
+- NoteBook at http://{{ .Values.host.client }}
+{{- end }}
+{{- if .Values.modules.spark.include  }}
+- SparkUI at http://{{ .Values.host.sparkUI }}
+{{- end }}
+{{- if .Values.modules.rabbitmq.include  }}
+- RabbitmqUI at http://{{ .Values.host.rabbitmqUI }}
+{{- end }}

helm-charts/FATE/templates/clustermanager-module.yaml

@@ -41,7 +41,16 @@ spec:
           command:
           - bash
           - -c
-          - "java -Dlog4j.configurationFile=$${EGGROLL_HOME}/conf/log4j2.properties -cp $${EGGROLL_HOME}/lib/*: com.webank.eggroll.core.Bootstrap --bootstraps com.webank.eggroll.core.resourcemanager.ClusterManagerBootstrap -c $${EGGROLL_HOME}/conf/eggroll.properties -p 4670 -s 'EGGROLL_DEAMON'"
+          - |
+            set -x
+            mkdir -p /data/projects/fate/eggroll/logs/eggroll/
+            touch /data/projects/fate/eggroll/logs/eggroll/eggroll-audit.log
+            ln -sf /dev/stdout /data/projects/fate/eggroll/logs/eggroll/eggroll-audit.log
+            touch /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.log
+            ln -sf /dev/stdout /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.log
+            touch /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.err.log 
+            ln -sf /dev/stderr /data/projects/fate/eggroll/logs/eggroll/eggroll.jvm.err.log
+            java -Dlog4j.configurationFile=$${EGGROLL_HOME}/conf/log4j2.properties -cp $${EGGROLL_HOME}/lib/*: com.webank.eggroll.core.Bootstrap --bootstraps com.webank.eggroll.core.resourcemanager.ClusterManagerBootstrap -c $${EGGROLL_HOME}/conf/eggroll.properties -p 4670 -s 'EGGROLL_DEAMON'
           ports:
             - containerPort: 4670
           volumeMounts:
@@ -51,6 +60,10 @@ spec:
       {{- with .Values.modules.mysql.nodeSelector }}
       nodeSelector: 
 {{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with .Values.image.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml . | indent 6 }}
       {{- end }}
       restartPolicy: Always
       volumes:
@@ -67,7 +80,7 @@ metadata:
 {{ include "fate.labels" . | indent 4 }}
 spec:
   ports:
-    - name: "4670"
+    - name: "tcp-clustermanager"
       port: 4670
       targetPort: 4670
       protocol: TCP

helm-charts/FATE/templates/hdfs-dn.yaml

@@ -100,6 +100,10 @@ spec:
       {{- with .Values.modules.hdfs.datanode.nodeSelector }}
       nodeSelector: 
 {{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with .Values.image.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml . | indent 6 }}
       {{- end }}
       restartPolicy: Always
       volumes:
@@ -115,15 +119,15 @@ metadata:
 {{ include "fate.labels" . | indent 4 }}
 spec:
   ports:
-    - name: "9000"
+    - name: "tcp-9000"
       port: 9000
       targetPort: 9000
       protocol: TCP
-    - name: "9870"
+    - name: "tcp-9870"
       port: 9870
       targetPort: 9870
       protocol: TCP
-    - name: "50070"
+    - name: "tcp-50070"
       port: 50070
       targetPort: 50070
       protocol: TCP

helm-charts/FATE/templates/hdfs-nn.yaml

@@ -119,6 +119,10 @@ spec:
       {{- with .Values.modules.hdfs.namenode.nodeSelector }}
       nodeSelector: 
 {{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with .Values.image.imagePullSecrets }}
+      imagePullSecrets:
+{{ toYaml . | indent 6 }}
       {{- end }}
       restartPolicy: Always
       volumes:
@@ -137,15 +141,15 @@ metadata:
 {{ include "fate.labels" . | indent 4 }}
 spec:
   ports:
-    - name: "9000"
+    - name: "tcp-9000"
       port: 9000
       targetPort: 9000
       protocol: TCP
-    - name: "9870"
+    - name: "tcp-9870"
       port: 9870
       targetPort: 9870
       protocol: TCP
-    - name: "50070"
+    - name: "tcp-50070"
       port: 50070
       targetPort: 50070
       protocol: TCP

helm-charts/FATE/templates/ingress.yaml

@@ -8,8 +8,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+{{ if .Values.istio.enabled }}
+{{ else }}
 {{ if .Values.modules.fateboard.include }}
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: fateboard
@@ -29,7 +31,7 @@ spec:
 {{ end }}
 
 {{ if .Values.modules.client.include }}
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: client
@@ -49,7 +51,7 @@ spec:
 {{ end }}
 
 {{ if .Values.modules.spark.include }}
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: spark
@@ -69,7 +71,7 @@ spec:
 {{ end }}
 
 {{ if .Values.modules.rabbitmq.include }}
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: rabbitmq
@@ -86,4 +88,5 @@ spec:
           serviceName: rabbitmq
           servicePort: 15672
 ---
+{{ end }}
 {{ end }}