Merge pull request #147 from linshenkx/dev

添加amoro、hbase、hive、kafka、kyuubi、spark组件
dromara · May 6, 2024 · 097b9ef · 097b9ef
2 parents 7823874 + a920870
commit 097b9ef
Show file tree

Hide file tree

Showing 121 changed files with 17,389 additions and 761 deletions.
diff --git a/cloudeon-server/src/main/java/org/dromara/cloudeon/controller/AlertController.java b/cloudeon-server/src/main/java/org/dromara/cloudeon/controller/AlertController.java
diff --git a/cloudeon-server/src/main/java/org/dromara/cloudeon/controller/ClusterServiceController.java b/cloudeon-server/src/main/java/org/dromara/cloudeon/controller/ClusterServiceController.java
@@ -472,7 +472,9 @@ public ResultDTO<Void> stopRoles(Integer roleInstanceId) {
 
         ServiceRoleInstanceEntity roleInstanceEntity = roleInstanceRepository.findById(roleInstanceId).get();
         ServiceInstanceEntity serviceInstanceEntity = serviceInstanceRepository.findById(roleInstanceEntity.getServiceInstanceId()).get();
-
+        if (roleInstanceEntity.getServiceRoleState() != ServiceRoleState.ROLE_STARTED) {
+            throw new RuntimeException("角色未启动,无法执行停止!");
+        }
         // 更新角色实例状态
         roleInstanceEntity.setServiceRoleState(ServiceRoleState.STOPPING_ROLE);
         roleInstanceRepository.save(roleInstanceEntity);
@@ -492,6 +494,9 @@ public ResultDTO<Void> startRole(Integer roleInstanceId) {
 
         ServiceRoleInstanceEntity roleInstanceEntity = roleInstanceRepository.findById(roleInstanceId).get();
         ServiceInstanceEntity serviceInstanceEntity = serviceInstanceRepository.findById(roleInstanceEntity.getServiceInstanceId()).get();
+        if (roleInstanceEntity.getServiceRoleState() != ServiceRoleState.ROLE_STOPPED) {
+            throw new RuntimeException("角色未停止,无法执行启动!");
+        }
         // 更新角色实例状态
         roleInstanceEntity.setServiceRoleState(ServiceRoleState.STARTING_ROLE);
         roleInstanceRepository.save(roleInstanceEntity);

diff --git a/cloudeon-server/src/main/java/org/dromara/cloudeon/dao/AlertMessageRepository.java b/cloudeon-server/src/main/java/org/dromara/cloudeon/dao/AlertMessageRepository.java
@@ -24,11 +24,13 @@
 import java.util.List;
 
 public interface AlertMessageRepository extends JpaRepository<AlertMessageEntity, Integer> {
-    AlertMessageEntity findByFireTimeAndAlertNameAndHostname(String fireTime, String alertName, String hostname);
+    List<AlertMessageEntity> findByFireTimeAndAlertNameAndHostname(String fireTime, String alertName, String hostname);
 
     @Query(value = "select a  from AlertMessageEntity a  where  a.resolved =:resolved and a.clusterId=:clusterId")
     List<AlertMessageEntity> findByIsResolve(@Param("resolved")Boolean resolved,@Param("clusterId")Integer clusterId);
 
+    List<AlertMessageEntity> findByResolved(@Param("resolved") Boolean resolved);
+
     List<AlertMessageEntity> findByServiceInstanceIdAndResolved(Integer serviceInstanceId, boolean resolved);
     List<AlertMessageEntity> findByServiceRoleInstanceIdAndResolved(Integer roleInstanceId, boolean resolved);
 

diff --git a/cloudeon-server/src/main/java/org/dromara/cloudeon/dao/ClusterNodeRepository.java b/cloudeon-server/src/main/java/org/dromara/cloudeon/dao/ClusterNodeRepository.java
@@ -32,5 +32,7 @@ public interface ClusterNodeRepository extends JpaRepository<ClusterNodeEntity,
 
     ClusterNodeEntity findByHostname(String hostname);
 
-    public List<ClusterNodeEntity> findByClusterId(Integer clusterId);
+    ClusterNodeEntity findByHostnameOrIp(String hostname, String ip);
+
+    List<ClusterNodeEntity> findByClusterId(Integer clusterId);
 }
diff --git a/cloudeon-server/src/main/java/org/dromara/cloudeon/processor/ScaleK8sServiceTask.java b/cloudeon-server/src/main/java/org/dromara/cloudeon/processor/ScaleK8sServiceTask.java
@@ -48,7 +48,7 @@ public void internalExecute() {
 
         kubeService.executeWithKubeClient(serviceInstanceEntity.getClusterId(), client -> {
             RollableScalableResource<Deployment> resource = client.apps().deployments().inNamespace(namespace).withName(deploymentName);
-            Integer replicas = resource.get().getStatus().getReplicas();
+            Integer replicas = resource.get().getSpec().getReplicas();
             log.info("当前deployment: {} Replicas: {}", deploymentName, replicas);
             if (!isApplyTask()) {
                 int count = replicas - 1;

diff --git a/cloudeon-server/src/main/java/org/dromara/cloudeon/service/ServiceService.java b/cloudeon-server/src/main/java/org/dromara/cloudeon/service/ServiceService.java
@@ -8,6 +8,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.dromara.cloudeon.config.CloudeonConfigProp;
 import org.dromara.cloudeon.dao.*;
+import org.dromara.cloudeon.dto.NodeInfo;
 import org.dromara.cloudeon.dto.RoleNodeInfo;
 import org.dromara.cloudeon.entity.*;
 import org.dromara.cloudeon.enums.RoleType;
@@ -48,6 +49,8 @@ public class ServiceService {
     private ClusterAlertRuleRepository clusterAlertRuleRepository;
     @Resource
     private Environment environment;
+    @Resource
+    private ClusterNodeRepository nodeRepository;
 
     public String getNamespace(Integer clusterId) {
         String namespace = clusterInfoRepository.findById(clusterId).get().getNamespace();
@@ -103,6 +106,10 @@ public Map<String, Object> getDataModel(Integer serviceInstanceId, String roleNa
         Map<String, Object> dataModel = new HashMap<>();
         Integer clusterId = serviceInstanceEntity.getClusterId();
         dataModel.put("clusterId", clusterId);
+        List<ClusterNodeEntity> nodeEntityList = nodeRepository.findByClusterId(clusterId);
+        Map<String, NodeInfo> nodeInfoMap = nodeEntityList.stream().map(nodeEntity -> NodeInfo.builder().ip(nodeEntity.getIp()).hostName(nodeEntity.getHostname()).build())
+                .collect(Collectors.toMap(NodeInfo::getHostName, nodeInfo -> nodeInfo));
+        dataModel.put("nodeInfo", nodeInfoMap);
         Integer stackId = stackServiceEntity.getStackId();
         dataModel.put("stackId", stackId);
         // 查询服务实例所有配置项,包括全局

diff --git a/cloudeon-stack/EDP-2.0.0/amoro/alert-rule.yaml b/cloudeon-stack/EDP-2.0.0/amoro/alert-rule.yaml
@@ -0,0 +1,13 @@
+rules:
+    - alert: AmoroAms存活数检测
+      promql: sum(up{job="metrics-amoro-ams"})<#{{${serviceRoles['AMORO_AMS']?size}}}#
+      alertLevel: exception
+      serviceRoleName: AMORO_AMS
+      alertAdvice: AmoroAms发生宕机，请检查日志或执行重启
+      alertInfo: "Flink历史服务器发生宕机，当前存活实例数为：{{$value}}"
+    - alert: AmoroAms进程存活
+      promql: up{job="metrics-amoro-ams"} != 1
+      alertLevel: exception
+      serviceRoleName: AMORO_AMS
+      alertAdvice: AmoroAms发生宕机，请检查日志或执行重启
+      alertInfo: "{{ $labels.job }}的{{ $labels.instance }}实例发生宕机"
diff --git a/cloudeon-stack/EDP-2.0.0/amoro/docker/Dockerfile b/cloudeon-stack/EDP-2.0.0/amoro/docker/Dockerfile
@@ -0,0 +1,22 @@
+FROM registry.cn-guangzhou.aliyuncs.com/bigdata200/flink:1.15.4
+
+ENV AMORO_HOME=/opt/amoro
+ENV PATH=$PATH:$AMORO_HOME/bin
+
+RUN yum install -y unzip
+
+RUN wget https://github.com/apache/amoro/releases/download/v0.6.1/amoro-flink-runtime-1.15-0.6.1.jar \
+    && mv amoro-flink-runtime-1.15-0.6.1.jar ${FLINK_HOME}/lib/
+
+RUN wget https://github.com/apache/amoro/releases/download/v0.6.1/amoro-0.6.1-bin.zip
+
+RUN unzip  amoro-*-bin.zip -d /opt/ \
+    && rm -f amoro-*-bin.zip && mv /opt/amoro-*  $AMORO_HOME
+
+RUN yum install -y mariadb.x86_64 mariadb-libs.x86_64
+
+ENV MYSQL_JDBC_DRIVER_VERSION=8.0.30
+RUN wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_JDBC_DRIVER_VERSION}/mysql-connector-java-${MYSQL_JDBC_DRIVER_VERSION}.jar \
+    && mv mysql-connector-java-${MYSQL_JDBC_DRIVER_VERSION}.jar $AMORO_HOME/lib/
+
+WORKDIR $AMORO_HOME
diff --git a/cloudeon-stack/EDP-2.0.0/amoro/docker/build.sh b/cloudeon-stack/EDP-2.0.0/amoro/docker/build.sh
@@ -0,0 +1,3 @@
+docker build --build-arg "https_proxy=http://my.win:7890" -t registry.cn-guangzhou.aliyuncs.com/bigdata200/amoro:0.6.1 .
+docker push  registry.cn-guangzhou.aliyuncs.com/bigdata200/amoro:0.6.1
+
diff --git a/cloudeon-stack/EDP-2.0.0/amoro/icons/app.png b/cloudeon-stack/EDP-2.0.0/amoro/icons/app.png
diff --git a/cloudeon-stack/EDP-2.0.0/amoro/k8s-render/metrics-service.yaml.ftl b/cloudeon-stack/EDP-2.0.0/amoro/k8s-render/metrics-service.yaml.ftl
@@ -0,0 +1,13 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: metrics-amoro-ams
+  labels:
+    sname: ${serviceFullName}
+    enable-default-service-monitor: true
+spec:
+  selector:
+    sname: ${serviceFullName}
+  ports:
+  - name: metrics
+    port: 5559
diff --git a/cloudeon-stack/EDP-2.0.0/amoro/k8s/amoro.yaml.ftl b/cloudeon-stack/EDP-2.0.0/amoro/k8s/amoro.yaml.ftl
@@ -0,0 +1,166 @@
+<#if conf["data.path.list"]??&& conf["data.path.list"]?trim?has_content>
+  <#assign primeDataPathList=conf["data.path.list"]?trim?split(",")>
+<#else >
+  <#assign primeDataPathList=[conf['global.persistence.basePath']]>
+</#if>
+<#assign dataPathList = []>
+<#list primeDataPathList as dataPath>
+  <#if dataPath?ends_with("/")>
+    <#assign dataPathList = dataPathList + [dataPath+ roleFullName]>
+  <#else>
+    <#assign dataPathList = dataPathList + [dataPath+"/"+ roleFullName]>
+  </#if>
+</#list>
+---
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  labels:
+    name: "${roleServiceFullName}"
+    sname: "${serviceFullName}"
+    roleFullName: "${roleFullName}"
+  name: "${roleServiceFullName}"
+spec:
+  replicas: ${roleNodeCnt}
+  selector:
+    matchLabels:
+      app: "${roleServiceFullName}"
+      sname: "${serviceFullName}"
+      roleFullName: "${roleFullName}"
+  strategy:
+    type: "RollingUpdate"
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  minReadySeconds: 5
+  revisionHistoryLimit: 10
+  template:
+    metadata:
+      labels:
+        name: "${roleServiceFullName}"
+        sname: "${serviceFullName}"
+        roleFullName: "${roleFullName}"
+        app: "${roleServiceFullName}"
+        podConflictName: "${roleServiceFullName}"
+        inject-filebeat: "true"
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchLabels:
+                name: "${roleServiceFullName}"
+                podConflictName: "${roleServiceFullName}"
+            topologyKey: "kubernetes.io/hostname"
+      hostPID: false
+      hostNetwork: true
+      nodeSelector:
+        ${roleServiceFullName}: "true"
+      terminationGracePeriodSeconds: 30
+      containers:
+      - image: "${conf['serverImage']}"
+        imagePullPolicy: "${conf['global.imagePullPolicy']}"
+        name: "${roleServiceFullName}"
+        command: ["/bin/bash","-c"]
+        args:
+          - |
+            /bin/bash /opt/global/bootstrap.sh && \
+            /bin/bash /opt/service-common/bootstrap.sh;
+        readinessProbe:
+          exec:
+            command:
+            - "/bin/bash"
+            - "-c"
+            - "curl --fail --connect-timeout 15 --max-time 15 \"http://`hostname`:${conf['http-server.port']}/\"\
+            \n"
+          failureThreshold: 3
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          requests:
+            memory: "${conf['amoro.ams.container.request.memory']}Mi"
+            cpu: "${conf['amoro.ams.container.request.cpu']}"
+          limits:
+            memory: "${conf['amoro.ams.container.limit.memory']}Mi"
+            cpu: "${conf['amoro.ams.container.limit.cpu']}"
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: MEM_LIMIT
+            valueFrom:
+              resourceFieldRef:
+                resource: limits.memory
+          - name: RENDER_TPL_DIR
+            value: "/opt/service-render"
+          - name: RENDER_MODEL
+            value: "/opt/service-common/values.json"
+          - name: ROLE_FULL_NAME
+            value: "${roleFullName}"
+        volumeMounts:
+        - mountPath: "/etc/localtime"
+          name: "timezone"
+        - name: global-service-common
+          mountPath: /opt/global/bootstrap.sh
+          subPath: bootstrap.sh
+        - name: global-render-config
+          mountPath: /opt/global/10.render
+        - name: global-usersync-config
+          mountPath: /opt/global/20.usersync
+        - name: global-copy-filebeat-config
+          mountPath: /opt/global/30.copy-filebeat-config
+        - name: service-render
+          mountPath: /opt/service-render
+        - name: service-common
+          mountPath: /opt/service-common
+        - name: hdfs-config
+          mountPath: /etc/hdfs-config
+        - name: yarn-config
+          mountPath: /etc/yarn-config
+        - mountPath: "/workspace"
+          name: "workspace"
+<#list dataPathList as dataPath>
+        - name: local-data-${dataPath?index+1}
+          mountPath: /data/${dataPath?index+1}
+</#list>
+      volumes:
+      - hostPath:
+          path: "/etc/localtime"
+        name: "timezone"
+      - name: global-service-common
+        configMap:
+          name: global-service-common
+      - name: global-render-config
+        configMap:
+          name: global-render-config
+      - name: global-usersync-config
+        configMap:
+          name: global-usersync-config
+      - name: global-copy-filebeat-config
+        configMap:
+          name: global-copy-filebeat-config
+      - name: service-render
+        configMap:
+          name: amoro-service-render
+      - name: service-common
+        configMap:
+          name: amoro-service-common
+      - name: hdfs-config
+        configMap:
+          name: hdfs-config
+      - name: yarn-config
+        configMap:
+          name: yarn-config
+      - name: "workspace"
+        hostPath:
+          type: DirectoryOrCreate
+          path: "${dataPathList[0]}/workspace"
+<#list dataPathList as dataPath>
+      - name: local-data-${dataPath?index+1}
+        hostPath:
+          path: ${dataPath}/data
+          type: DirectoryOrCreate
+</#list>
diff --git a/cloudeon-stack/EDP-2.0.0/amoro/kube-prometheus-render/alert-rule.yaml.ftl b/cloudeon-stack/EDP-2.0.0/amoro/kube-prometheus-render/alert-rule.yaml.ftl
@@ -0,0 +1,33 @@
+<#macro executeDynamicCode userInput>
+    <#assign startIdx = userInput?index_of('#{{')>
+    <#assign endIdx = userInput?last_index_of('}}#')>
+    <#if startIdx gt 0 && endIdx gt 0>
+        <#assign dynamicCode = userInput?substring(startIdx + 3, endIdx)>
+        ${userInput?substring(0,startIdx)}<@dynamicCode?interpret />${userInput?substring(endIdx+3)}<#t>
+    <#else >
+        ${userInput}
+    </#if><#t>
+</#macro>
+<#if alertRules ?? && alertRules?size gt 0>
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ${serviceFullName}
+spec:
+  groups:
+    - name: ${serviceFullName}
+      rules:
+      <#list alertRules as rule>
+      -  alert: ${rule.ruleName}
+         expr: <@executeDynamicCode rule.promql />
+         labels:
+             receiver: "webhook"
+             alertLevel: "${rule.alertLevel}"
+             clusterId: "${rule.clusterId}"
+             serviceRoleName: "${rule.stackRoleName}"
+             serviceName: "${rule.stackServiceName}"
+         annotations:
+             alertAdvice: "${rule.alertAdvice}"
+             alertInfo:  "${rule.alertInfo}"
+      </#list>
+</#if>
diff --git a/cloudeon-stack/EDP-2.0.0/amoro/service-common/bootstrap.sh b/cloudeon-stack/EDP-2.0.0/amoro/service-common/bootstrap.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -e
+
+\cp -f /opt/service-render-output/* $AMORO_HOME/conf/
+
+mv $AMORO_HOME/conf/flink-conf.yaml $FLINK_HOME/conf/
+
+\cp -f /etc/hdfs-config/* $HADOOP_CONF_DIR
+\cp -f /etc/yarn-config/* $HADOOP_CONF_DIR
+
+mkdir -p /workspace/logs
+rm -rf $AMORO_HOME/logs
+ln -s /workspace/logs $AMORO_HOME/logs
+
+ams.sh start
+
+until find /workspace/logs -mmin -1 -type f -name '*.log' ! -name '*gc*' | grep -q .
+do
+  echo "`date`: Waiting for logs..."
+  sleep 2
+done
+find /workspace/logs -mmin -1 -type f -name '*.log' ! -name '*gc*' -exec tail -F {} +
+
+echo "---------------------------------------------开始----------------------------------------------"
+tail -f /dev/null
+
+
+
diff --git a/cloudeon-stack/EDP-2.0.0/amoro/service-common/filebeat.input.yml b/cloudeon-stack/EDP-2.0.0/amoro/service-common/filebeat.input.yml
@@ -0,0 +1,11 @@
+- type: log
+  enabled: true
+  fields:
+    service: AMORO
+    role: AMORO_AMS
+  paths:
+    - /workspace/logs/ams*.log
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+  close_timeout: 5m