Support disable statement in OAL (apache#2402)

* Try to make disable work.

* Make disable works.

* Add disable document.

docs/en/concepts-and-designs/oal.md

@@ -10,10 +10,13 @@ The OAL scripts will be compiled to normal Java codes in package stage.
 ## Grammar
 Scripts should be named as `*.oal`
 ```
-
+// Declare the metric.
 METRIC_NAME = from(SCOPE.(* | [FIELD][,FIELD ...]))
 [.filter(FIELD OP [INT | STRING])]
 .FUNCTION([PARAM][, PARAM ...])
+
+// Disable hard code 
+disable(METRIC_NAME);
 ```
 
 ## Scope
@@ -67,6 +70,13 @@ All metric data will be grouped by Scope.ID and min-level TimeBucket.
 
 - In `Endpoint` scope, the Scope.ID = Endpoint id (the unique id based on service and its Endpoint)
 
+## Disable
+`Disable` is an advanced statement in OAL, which is only used in certain case.
+Some of the aggregation and metric are defined through core hard codes,
+this `disable` statement is designed for make them de-active,
+such as `segment`, `top_n_database_statement`.
+In default, no one is being disable.
+
 ## Examples
 ```
 // Caculate p99 of both Endpoint1 and Endpoint2
@@ -93,4 +103,8 @@ Endpoint_500 = from(Endpoint.*).filter(responseCode like "5%").percent()
 
 // Caculate the sum of calls for each service.
 EndpointCalls = from(Endpoint.*).sum()
+
+disable(segment);
+disable(endpoint_relation_server_side);
+disable(top_n_database_statement);
 ```

oap-server/generate-tool-grammar/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALLexer.g4

@@ -26,6 +26,7 @@ lexer grammar OALLexer;
 
 FROM: 'from';
 FILTER: 'filter';
+DISABLE: 'disable';
 SRC_ALL: 'All';
 SRC_SERVICE: 'Service';
 SRC_SERVICE_INSTANCE: 'ServiceInstance';
@@ -43,6 +44,14 @@ SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC';
 SRC_SERVICE_INSTANCE_CLR_THREAD: 'ServiceInstanceCLRThread';
 SRC_ENVOY_INSTANCE_METRIC: 'EnvoyInstanceMetric';
 
+//hard code sources, only used when need to be deactived.
+SRC_SEGMENT: 'segment';
+SRC_TOP_N_DB_STATEMENT: 'top_n_database_statement';
+SRC_ENDPOINT_RELATION_SERVER_SIDE: 'endpoint_relation_server_side';
+SRC_SERVICE_RELATION_SERVER_SIDE: 'service_relation_server_side';
+SRC_SERVICE_RELATION_CLIENT_SIDE: 'service_relation_client_side';
+SRC_ALARM_RECORD: 'alarm_record';
+
 // Literals
 
 BOOL_LITERAL:       'true'

oap-server/generate-tool-grammar/src/main/antlr4/org/apache/skywalking/oal/tool/grammar/OALParser.g4

@@ -26,13 +26,17 @@ options { tokenVocab=OALLexer; }
 // Top Level Description
 
 root
-    : (aggregationStatement)*
+    : (aggregationStatement | disableStatement)*
     ;
 
 aggregationStatement
     : variable (SPACE)? EQUAL (SPACE)? metricStatement DelimitedComment? LineComment? (SEMI|EOF)
     ;
 
+disableStatement
+    : DISABLE LR_BRACKET disableSource RR_BRACKET DelimitedComment? LineComment? (SEMI|EOF)
+    ;
+
 metricStatement
     : FROM LR_BRACKET source  DOT sourceAttribute RR_BRACKET (filterStatement+)? DOT aggregateFunction
     ;
@@ -53,6 +57,11 @@ source
       SRC_ENVOY_INSTANCE_METRIC
     ;
 
+disableSource
+    : SRC_SEGMENT | SRC_TOP_N_DB_STATEMENT | SRC_ENDPOINT_RELATION_SERVER_SIDE | SRC_SERVICE_RELATION_SERVER_SIDE |
+      SRC_SERVICE_RELATION_CLIENT_SIDE | SRC_ALARM_RECORD
+    ;
+
 sourceAttribute
     : IDENTIFIER | ALL
     ;

oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/Main.java

@@ -20,7 +20,6 @@
 
 import freemarker.template.TemplateException;
 import java.io.*;
-import java.util.List;
 import org.apache.skywalking.apm.util.StringUtil;
 import org.apache.skywalking.oal.tool.meta.*;
 import org.apache.skywalking.oal.tool.output.FileGenerator;
@@ -59,9 +58,9 @@ public static void main(String[] args) throws IOException, TemplateException {
         SourceColumnsFactory.setSettings(metaSettings);
 
         ScriptParser scriptParser = ScriptParser.createFromFile(scriptFilePath);
-        List<AnalysisResult> analysisResults = scriptParser.parse();
+        OALScripts oalScripts = scriptParser.parse();
 
-        FileGenerator generator = new FileGenerator(analysisResults, outputPath);
+        FileGenerator generator = new FileGenerator(oalScripts, outputPath);
         generator.generate();
     }
 }

oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/output/FileGenerator.java

@@ -21,16 +21,18 @@
 import freemarker.template.*;
 import java.io.*;
 import java.util.*;
-import org.apache.skywalking.oal.tool.parser.AnalysisResult;
+import org.apache.skywalking.oal.tool.parser.*;
 
 public class FileGenerator {
     private List<AnalysisResult> results;
+    private DisableCollection collection;
     private String outputPath;
     private Configuration configuration;
     private AllDispatcherContext allDispatcherContext;
 
-    public FileGenerator(List<AnalysisResult> results, String outputPath) {
-        this.results = results;
+    public FileGenerator(OALScripts oalScripts, String outputPath) {
+        this.results = oalScripts.getIndicatorStmts();
+        this.collection = oalScripts.getDisableCollection();
         this.outputPath = outputPath;
         configuration = new Configuration(new Version("2.3.28"));
         configuration.setEncoding(Locale.ENGLISH, "UTF-8");
@@ -48,6 +50,7 @@ public void generate() throws IOException, TemplateException {
             createFile(file);
             generateDispatcher(result, new FileWriter(file));
         }
+        generateDisable();
     }
 
     private void generate(AnalysisResult result, String fileSuffix,
@@ -103,4 +106,10 @@ private void buildDispatcherContext() {
             context.getIndicators().add(result);
         }
     }
+
+    private void generateDisable() throws IOException, TemplateException {
+        File file = new File(outputPath, "generated/DisableSourceDefinition.java");
+        createFile(file);
+        configuration.getTemplate("DisableSourceDefinition.ftl").process(collection, new FileWriter(file));
+    }
 }

oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/DisableCollection.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oal.tool.parser;
+
+import java.util.*;
+import lombok.Getter;
+
+/**
+ * @author wusheng
+ */
+@Getter
+public class DisableCollection {
+    private List<String> allDisableSources = new ArrayList<>();
+
+    public void add(String source) {
+        allDisableSources.add(source);
+    }
+}

oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/OALListener.java

@@ -26,11 +26,13 @@
 public class OALListener extends OALParserBaseListener {
     private List<AnalysisResult> results;
     private AnalysisResult current;
+    private DisableCollection collection;
 
     private ConditionExpression conditionExpression;
 
-    public OALListener(List<AnalysisResult> results) {
-        this.results = results;
+    public OALListener(OALScripts scripts) {
+        this.results = scripts.getIndicatorStmts();
+        this.collection = scripts.getDisableCollection();
     }
 
     @Override
@@ -150,6 +152,15 @@ private String metricNameFormat(String source) {
         return source;
     }
 
+    /**
+     * Disable source
+     *
+     * @param ctx
+     */
+    @Override public void enterDisableSource(OALParser.DisableSourceContext ctx) {
+        collection.add(ctx.getText());
+    }
+
     private String firstLetterUpper(String source) {
         return source.substring(0, 1).toUpperCase() + source.substring(1);
     }

oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/OALScripts.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oal.tool.parser;
+
+import java.util.*;
+import lombok.*;
+
+@Getter
+public class OALScripts {
+    private List<AnalysisResult> indicatorStmts;
+    private DisableCollection disableCollection;
+
+    public OALScripts() {
+        indicatorStmts = new LinkedList<>();
+        disableCollection = new DisableCollection();
+    }
+}

oap-server/generate-tool/src/main/java/org/apache/skywalking/oal/tool/parser/ScriptParser.java

@@ -43,18 +43,19 @@ public static ScriptParser createFromScriptText(String script) throws IOExceptio
         return parser;
     }
 
-    public List<AnalysisResult> parse() throws IOException {
-        List<AnalysisResult> results = new LinkedList<>();
+    public OALScripts parse() throws IOException {
+        OALScripts scripts = new OALScripts();
+
         CommonTokenStream tokens = new CommonTokenStream(lexer);
 
         OALParser parser = new OALParser(tokens);
 
         ParseTree tree = parser.root();
         ParseTreeWalker walker = new ParseTreeWalker();
 
-        walker.walk(new OALListener(results), tree);
+        walker.walk(new OALListener(scripts), tree);
 
-        return results;
+        return scripts;
     }
 
     public void close() {

oap-server/generate-tool/src/main/resources/code-templates/DisableSourceDefinition.ftl

@@ -0,0 +1,33 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*/
+
+package org.apache.skywalking.oap.server.core.analysis.generated;
+<#if (allDisableSources?size>0)>
+import org.apache.skywalking.oap.server.core.analysis.Disable;
+</#if>
+
+/**
+* This class is auto generated. Please don't change this class manually.
+*
+* @author Observability Analysis Language code generator
+*/
+<#list allDisableSources as disableSource>
+@Disable("${disableSource}")
+</#list>
+public class DisableSourceDefinition {
+}

oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/output/FileGeneratorTest.java

@@ -79,10 +79,10 @@ private AnalysisResult buildResult() {
     public void testGenerateIndicatorImplementor() throws IOException, TemplateException {
         AnalysisResult result = buildResult();
 
-        List<AnalysisResult> results = new LinkedList<>();
-        results.add(result);
+        OALScripts oalScripts = new OALScripts();
+        oalScripts.getIndicatorStmts().add(result);
 
-        FileGenerator fileGenerator = new FileGenerator(results, ".");
+        FileGenerator fileGenerator = new FileGenerator(oalScripts, ".");
         StringWriter writer = new StringWriter();
         fileGenerator.generateIndicatorImplementor(result, writer);
         Assert.assertEquals(readExpectedFile("IndicatorImplementorExpected.java"), writer.toString());
@@ -94,10 +94,10 @@ public void testGenerateIndicatorImplementor() throws IOException, TemplateExcep
     public void testServiceDispatcher() throws IOException, TemplateException {
         AnalysisResult result = buildResult();
 
-        List<AnalysisResult> results = new LinkedList<>();
-        results.add(result);
+        OALScripts oalScripts = new OALScripts();
+        oalScripts.getIndicatorStmts().add(result);
 
-        FileGenerator fileGenerator = new FileGenerator(results, ".");
+        FileGenerator fileGenerator = new FileGenerator(oalScripts, ".");
         StringWriter writer = new StringWriter();
         fileGenerator.generateDispatcher(result, writer);
         Assert.assertEquals(readExpectedFile("ServiceDispatcherExpected.java"), writer.toString());

oap-server/generate-tool/src/test/java/org/apache/skywalking/oal/tool/parser/ScriptParserTest.java

@@ -50,7 +50,7 @@ public void testParse() throws IOException {
             "Endpoint_avg = from(Endpoint.latency).longAvg(); //comment test" + "\n" +
                 "Service_avg = from(Service.latency).longAvg()"
         );
-        List<AnalysisResult> results = parser.parse();
+        List<AnalysisResult> results = parser.parse().getIndicatorStmts();
 
         Assert.assertEquals(2, results.size());
 
@@ -72,7 +72,7 @@ public void testParse2() throws IOException {
         ScriptParser parser = ScriptParser.createFromScriptText(
             "Endpoint_percent = from(Endpoint.*).percent(status == true);"
         );
-        List<AnalysisResult> results = parser.parse();
+        List<AnalysisResult> results = parser.parse().getIndicatorStmts();
 
         AnalysisResult endpointPercent = results.get(0);
         Assert.assertEquals("EndpointPercent", endpointPercent.getMetricName());
@@ -91,7 +91,7 @@ public void testParse3() throws IOException {
         ScriptParser parser = ScriptParser.createFromScriptText(
             "Endpoint_percent = from(Endpoint.*).filter(status == true).filter(name == \"/product/abc\").longAvg();"
         );
-        List<AnalysisResult> results = parser.parse();
+        List<AnalysisResult> results = parser.parse().getIndicatorStmts();
 
         AnalysisResult endpointPercent = results.get(0);
         Assert.assertEquals("EndpointPercent", endpointPercent.getMetricName());
@@ -121,7 +121,7 @@ public void testParse4() throws IOException {
                 "service_response_s3_summary = from(Service.latency).filter(latency >= 3000).sum();" + "\n" +
                 "service_response_s4_summary = from(Service.latency).filter(latency <= 4000).sum();"
         );
-        List<AnalysisResult> results = parser.parse();
+        List<AnalysisResult> results = parser.parse().getIndicatorStmts();
 
         AnalysisResult responseSummary = results.get(0);
         Assert.assertEquals("ServiceResponseS1Summary", responseSummary.getMetricName());
@@ -167,4 +167,14 @@ public void testParse4() throws IOException {
         Assert.assertEquals("4000", booleanMatchExp.getValue());
         Assert.assertEquals("lessEqualMatch", booleanMatchExp.getExpressionType());
     }
+
+    @Test
+    public void testDisable() throws IOException {
+        ScriptParser parser = ScriptParser.createFromScriptText(
+            "disable(segment);");
+        DisableCollection collection = parser.parse().getDisableCollection();
+        List<String> sources = collection.getAllDisableSources();
+        Assert.assertEquals(1, sources.size());
+        Assert.assertEquals("segment", sources.get(0));
+    }
 }