canal

bigdata/flink/NOTE.txt

@@ -9,3 +9,26 @@ https://github.com/apache/flink-docker
 Flink job cluster on Kubernetes
 
 Native Kubernetes Setup
+
+1.下载解压安装包
+https://flink.apache.org/downloads.html
+
+2.创建serviceaccount
+kubectl create ns flink
+kubectl apply -f flink-serviceaccount.yml
+
+3.创建集群
+./bin/kubernetes-session.sh \
+  -Dkubernetes.cluster-id=my-flink-cluster \
+  -Dkubernetes.container.image=flink:1.12.1-scala_2.12 \
+  -Dkubernetes.container.image.pull-policy=Always \
+  -Dkubernetes.namespace=flink \
+  -Dkubernetes.service-account=flink \
+  -Dtaskmanager.memory.process.size=2048m \
+  -Dkubernetes.taskmanager.cpu=-1 \
+  -Dtaskmanager.numberOfTaskSlots=4 \
+  -Dresourcemanager.taskmanager-timeout=360000 \
+  -Dkubernetes.service.create-timeout=100min \
+  -Dkubernetes.jobmanager.cpu=1.0 \
+  -Dkubernetes.rest-service.exposed.type=ClusterIP
+

bigdata/flink/flink-native-kubernetes/flink-serviceaccount.yml

@@ -0,0 +1,63 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flink
+  namespace: flink
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: flink-role
+  namespace: flink
+rules:
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get","list","watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["get","list","watch"]
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["list"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get","list","watch"]
+- apiGroups: ["apps"]
+  resources: ["deployments"] # KubernetesPipelineTest#cascadingDelete
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: ["extensions"]
+  resources: ["deployments"] # ditto
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: ["apps"]
+  resources: ["replicasets"] # ditto
+  verbs: ["create","delete","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["persistentvolumeclaims"] # KubernetesPipelineTest#dynamicPVC
+  verbs: ["create","delete","get","list","patch","update","watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: flink-role-binding
+  namespace: flink
+subjects:
+- kind: ServiceAccount
+  name: flink
+  namespace: flink
+roleRef:
+  kind: Role
+  name: flink-role
+  apiGroup: rbac.authorization.k8s.io

bigdata/flink/flink-native-kubernetes/kubernetes-session.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+# get Flink config
+. "$bin"/config.sh
+
+if [ "$FLINK_IDENT_STRING" = "" ]; then
+        FLINK_IDENT_STRING="$USER"
+fi
+
+JVM_ARGS="$JVM_ARGS -Xmx512m"
+
+CC_CLASSPATH=`manglePathList $(constructFlinkClassPath):$INTERNAL_HADOOP_CLASSPATHS`
+
+log=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-k8s-session-$HOSTNAME.log
+log_setting="-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4j-session.properties -Dlog4j.configurationFile=file:"$FLINK_CONF_DIR"/log4j-session.properties -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/logback-session.xml"
+
+export FLINK_CONF_DIR
+
+$JAVA_RUN $JVM_ARGS -classpath "$CC_CLASSPATH" $log_setting org.apache.flink.kubernetes.cli.KubernetesSessionCli "$@"

bigdata/hbase/NOTE.txt

@@ -0,0 +1,11 @@
+https://github.com/HariSekhon/Dockerfiles
+
+hbase-single单机版
+
+docker run -d -h hbase01 \
+-p 2181:2181 -p 8080:8080 -p 8085:8085 -p 9090:9090 -p 9095:9095    \
+-p 16000:16000 -p 16010:16010 -p 16201:16201 -p 16301:16301   \
+--name hbase01     \
+harisekhon/hbase:1.3
+
+harisekhon/hbase:2.1

bigdata/hbase/pinpoint-hbase-all-in-one.yml

@@ -0,0 +1,106 @@
+---
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+      app: pinpoint-hbase
+  name: pinpoint-hbase-svc
+  namespace: tracing
+spec:
+  ports:
+  - port: 2181
+    targetPort: 2181
+    name: zookeeper
+  - port: 60000
+    targetPort: 60000
+    name: hmaster-api
+  - port: 60020
+    targetPort: 60020
+    name: hregion-api
+  clusterIP: None
+  selector:
+    app: pinpoint-hbase
+---
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+      app: pinpoint-hbase
+  name: pinpoint-hbase-ui
+  namespace: tracing
+spec:
+  ports:
+  - port: 16010
+    targetPort: 16010
+    name: hmaster-ui
+  - port: 16030
+    targetPort: 16030
+    name: hregion-ui
+  selector:
+    app: pinpoint-hbase
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app: pinpoint-hbase
+  name: pinpoint-hbase
+  namespace: tracing
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pinpoint-hbase
+  serviceName: pinpoint-hbase-svc
+  template:
+    metadata:
+      labels:
+        app: pinpoint-hbase
+        name: pinpoint-hbase
+      annotations:
+        #disable istio sidecar injection
+        #sidecar.istio.io/inject: "false"
+        #scheduler.alpha.kubernetes.io/critical-pod=""
+    spec:
+      containers:
+        #image: pinpointdocker/pinpoint-hbase:2.2.0
+      - image: harisekhon/hbase:2.1
+        imagePullPolicy: Always
+        name: pinpoint-hbase
+        ports:
+        - containerPort: 2181
+          name: zookeeper
+          protocol: TCP
+        - containerPort: 60000
+          name: hmaster-api
+          protocol: TCP
+        - containerPort: 16010
+          name: hmaster-ui
+          protocol: TCP
+        - containerPort: 60020
+          name: hregion-api
+          protocol: TCP
+        - containerPort: 16030
+          name: hregion-ui
+          protocol: TCP
+        resources:
+          limits:
+            cpu: "2"
+            memory: 4Gi
+          requests:
+            cpu: 500m
+            memory: 1Gi
+        volumeMounts:
+        #- mountPath: /home/pinpoint
+        - mountPath: /hbase-data
+          name: pinpoint-hbase-store
+  volumeClaimTemplates:
+  - metadata:
+      name: pinpoint-hbase-store
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 80Gi

bigdata/maxwell/NOTE.txt

@@ -0,0 +1,30 @@
+http://maxwells-daemon.io/quickstart/
+
+docker run -it --rm zendesk/maxwell bin/maxwell --user=$MYSQL_USERNAME \
+    --password=$MYSQL_PASSWORD --host=$MYSQL_HOST --producer=kafka \
+    --kafka.bootstrap.servers=$KAFKA_HOST:$KAFKA_PORT --kafka_topic=maxwell
+
+针对数据初始化的问题，Maxwell 提供了一个命令工具 maxwell-bootstrap 帮助我们完成数据初始化，maxwell-bootstrap 是基于 SELECT * FROM table 的方式进行全量数据初始化，不会产生多余的binlog！
+
+这个工具有下面这些参数：
+
+参数	说明
+--log_level LOG_LEVEL	日志级别（DEBUG, INFO, WARN or ERROR）
+--user USER	mysql 用户名
+--password PASSWORD	mysql 密码
+--host HOST	mysql 地址
+--port PORT	mysql 端口
+--database DATABASE	要bootstrap的表所在的数据库
+--table TABLE	要引导的表
+--where WHERE_CLAUSE	设置过滤条件
+--client_id CLIENT_ID	指定执行引导操作的Maxwell实例
+
+docker run -it --rm zendesk/maxwell bin/maxwell-bootstrap --user maxwell  \
+    --password 123456 --host 10.100.97.246  --database test --table test --client_id maxwell
+
+部署以及同步过程
+1.首次全量同步
+job maxwell-bootstrap 
+2.实时增量同步
+deploy maxwell
+

bigdata/maxwell/kubernetes/maxwell-bootstrap-id01-job.yml

@@ -0,0 +1,54 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: maxwell-bootstrap-id01
+  namespace: default
+spec:
+  template:
+    metadata:
+      name: maxwell-bootstrap
+    spec:
+      containers:
+      - name: maxwell-bootstrap
+        image: zendesk/maxwell
+        env:
+          - name: LOG_LEVEL
+            value: "info"
+          - name: HOST
+            value: "mysql-server.dbaas"
+          - name: PORT
+            value: "3306"
+          - name: USER
+            value: "root"
+          - name: PASSWORD
+            value: "root_password"
+          #- name: DATABASE
+          #  value: "mydb"
+          #- name: TABLE
+          #  value: "table01"
+          #- name: WHERE_CLAUSE
+          #  value: "where"
+          #- name: PRODUCER
+          #  value: "stdout"
+          - name: PRODUCER
+            value: "kafka"
+          - name: CLIENT_ID
+            value: "maxwell"
+          - name: KAFKA_BROKERS
+            value: "kafka-svc:9092"
+          - name: KAFKA_TOPIC
+            value: "maxwell-topic"
+        command: [ "/bin/sh", "-c", "bin/maxwell-bootstrap --log_level $LOG_LEVEL \
+                                    --producer=$PRODUCER --host $HOST --user $USER \
+                                    --password $PASSWORD --port $PORT \
+                                    #--database $DATABASE --table $TABLE \
+                                    --kafka.bootstrap.servers=${KAFKA_BROKERS} \
+                                    --kafka_topic=$KAFKA_TOPIC \
+                                    --kafka_partition_hash=murmur3 \
+                                    --kafka_key_format=hash \
+                                    --kafka.compression.type=snappy \
+                                    --kafka.retries=5 \
+                                    --kafka.acks=all \
+                                    --producer_partition_by=primary_key \
+                                    --bootstrapper=async " ]
+      restartPolicy: Never