Merge pull request #2 from geoffhigginbottom/Dev

Added MS SQL Server
geoffhigginbottom · Oct 4, 2021 · 8b69e2e · 8b69e2e
2 parents 5a3678e + 5eda8ec
commit 8b69e2e
Show file tree

Hide file tree

Showing 8 changed files with 383 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -48,7 +48,6 @@ The quantities of each EC2 Instance deployed as part of the 'Instances' Module a
 
 ### Enable / Disable Modules
 eks_cluster_enabled         = false
-ecs_cluster_enabled         = false
 instances_enabled           = false
 phone_shop_enabled          = false
 lambda_sqs_dynamodb_enabled = false
@@ -77,6 +76,13 @@ mysql_ids = [
   "mysql3"
   ]
 
+ms_sql_count = "1" # min 0 : max = subnet_count
+ms_sql_ids = [
+  "ms_sql1",
+  "ms_sql2",
+  "ms_sql3"
+  ]
+
 apache_count = "1" # min 0 : max = subnet_count
 apache_ids = [
   "apache1",
@@ -181,7 +187,8 @@ The following EC2 Instances can be deployed:
 - Collectors
 - HAProxy
 - MySQL
-- apache (just a basic Apache server in reality)
+- MS SQL
+- Apache (just a basic Apache server in reality)
 - Splunk Enterprise
 
 Each Instance has Infrastructure Monitoring 'monitors' configured to match the services running on them.  The configuration for each monitor is deployed into /etc/signalfx/monitors/xxx.yaml, this means the /etc/signalfx/agent.yaml file is the same regardless of role.

diff --git a/main.tf b/main.tf
@@ -149,13 +149,17 @@ module "instances" {
   private_key_path        = var.private_key_path
   instance_type           = var.instance_type
   collector_instance_type = var.collector_instance_type
+  ms_sql_instance_type    = var.ms_sql_instance_type
   ami                     = data.aws_ami.latest-ubuntu.id
+  ms_sql_ami              = data.aws_ami.ms-sql-server.id
   collector_count         = var.collector_count
   collector_ids           = var.collector_ids
   haproxy_count           = var.haproxy_count
   haproxy_ids             = var.haproxy_ids
   mysql_count             = var.mysql_count
   mysql_ids               = var.mysql_ids
+  ms_sql_count            = var.ms_sql_count
+  ms_sql_ids              = var.ms_sql_ids
   apache_web_count        = var.apache_web_count
   apache_web_ids          = var.apache_web_ids
   splunk_ent_count        = var.splunk_ent_count
@@ -175,6 +179,9 @@ output "HAProxy_Servers" {
 output "MySQL_Servers" {
   value = var.instances_enabled ? module.instances.*.mysql_details : null
 }
+output "MS_SQL_Servers" {
+  value = var.instances_enabled ? module.instances.*.ms_sql_details : null
+}
 output "Apache_Web_Servers" {
   value = var.instances_enabled ? module.instances.*.apache_web_details : null
 }
@@ -185,6 +192,11 @@ output "SQS_Test_Server" {
   value = var.lambda_sqs_dynamodb_enabled ? module.lambda_sqs_dynamodb.*.sqs_test_server_details : null
 }
 
+output "MS_SQL_Administrator_Password"{
+  value = var.instances_enabled ? module.instances.*.Administrator_Password : null
+}
+
+
 ### Phone Shop Outputs ###
 output "Phone_Shop_Server" {
   value = var.phone_shop_enabled ? module.phone_shop.*.phone_shop_server_details : null

diff --git a/modules/instances/config_files/ms_sql_agent_config.yaml b/modules/instances/config_files/ms_sql_agent_config.yaml
@@ -0,0 +1,197 @@
+# Default configuration file for the Linux (deb/rpm) and Windows MSI collector packages
+
+# If the collector is installed without the Linux/Windows installer script, the following
+# environment variables are required to be manually defined or configured below:
+# - SPLUNK_ACCESS_TOKEN: The Splunk access token to authenticate requests
+# - SPLUNK_API_URL: The Splunk API URL, e.g. https://api.us0.signalfx.com
+# - SPLUNK_BUNDLE_DIR: The path to the Smart Agent bundle, e.g. /usr/lib/splunk-otel-collector/agent-bundle
+# - SPLUNK_COLLECTD_DIR: The path to the collectd config directory for the Smart Agent, e.g. /usr/lib/splunk-otel-collector/agent-bundle/run/collectd
+# - SPLUNK_HEC_TOKEN: The Splunk HEC authentication token
+# - SPLUNK_HEC_URL: The Splunk HEC endpoint URL, e.g. https://ingest.us0.signalfx.com/v1/log
+# - SPLUNK_INGEST_URL: The Splunk ingest URL, e.g. https://ingest.us0.signalfx.com
+# - SPLUNK_TRACE_URL: The Splunk trace endpoint URL, e.g. https://ingest.us0.signalfx.com/v2/trace
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+  http_forwarder:
+    ingress:
+      endpoint: 0.0.0.0:6060
+    egress:
+      endpoint: "${SPLUNK_API_URL}"
+      # Use instead when sending to gateway
+      #endpoint: "${SPLUNK_GATEWAY_URL}"
+  smartagent:
+    bundleDir: "${SPLUNK_BUNDLE_DIR}"
+    collectd:
+      configDir: "${SPLUNK_COLLECTD_DIR}"
+  zpages:
+    #endpoint: 0.0.0.0:55679
+  memory_ballast:
+    # In general, the ballast should be set to 1/3 of the collector's memory, the limit
+    # should be 90% of the collector's memory.
+    # The simplest way to specify the ballast size is set the value of SPLUNK_BALLAST_SIZE_MIB env variable.
+    size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
+
+receivers:
+  fluentforward:
+    endpoint: 127.0.0.1:8006
+  hostmetrics:
+    collection_interval: 10s
+    scrapers:
+      cpu:
+      disk:
+      filesystem:
+      memory:
+      network:
+      # System load average metrics https://en.wikipedia.org/wiki/Load_(computing)
+      load:
+      # Paging/Swap space utilization and I/O metrics
+      paging:
+      # Aggregated system process count metrics
+      processes:
+      # System processes metrics, disabled by default
+      # process:
+  jaeger:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:14250
+      thrift_binary:
+        endpoint: 0.0.0.0:6832
+      thrift_compact:
+        endpoint: 0.0.0.0:6831
+      thrift_http:
+        endpoint: 0.0.0.0:14268
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:55681
+  # This section is used to collect the OpenTelemetry Collector metrics
+  # Even if just a Splunk APM customer, these metrics are included
+  prometheus/internal:
+    config:
+      scrape_configs:
+      - job_name: 'otel-collector'
+        scrape_interval: 10s
+        static_configs:
+        - targets: ['0.0.0.0:8888']
+        metric_relabel_configs:
+          - source_labels: [ __name__ ]
+            regex: '.*grpc_io.*'
+            action: drop
+  smartagent/signalfx-forwarder:
+    type: signalfx-forwarder
+    listenAddress: 0.0.0.0:9080
+  signalfx:
+    endpoint: 0.0.0.0:9943
+  zipkin:
+    endpoint: 0.0.0.0:9411
+  smartagent/sqlserver:
+    type: telegraf/sqlserver
+    host: localhost
+    port: 1433
+    userID: ${SPLUNK_SQL_USER}
+    password: ${SPLUNK_SQL_USER_PWD}
+    appName: signalfxagent
+
+processors:
+  batch:
+  # Enabling the memory_limiter is strongly recommended for every pipeline.
+  # Configuration is based on the amount of memory allocated to the collector.
+  # For more information about memory limiter, see
+  # https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiter/README.md
+  memory_limiter:
+    check_interval: 2s
+    limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
+  # detect if the collector is running on a cloud system
+  # important for creating unique cloud provider dimensions
+  resourcedetection:
+    detectors: [system, gce, ecs, ec2, azure]
+    override: false
+
+  # Same as above but overrides resource attributes set by receivers
+  resourcedetection/internal:
+    detectors: [system, gce, ecs, ec2, azure]
+    override: true
+
+  # Optional: The following processor can be used to add a default "deployment.environment" attribute to the logs and 
+  # traces when it's not populated by instrumentation libraries.
+  # If enabled, make sure to enable this processor in the pipeline below.
+  #resource/add_environment:
+    #attributes:
+      #- action: insert
+        #value: staging/production/...
+        #key: deployment.environment
+
+exporters:
+  # Traces
+  sapm:
+    access_token: "${SPLUNK_ACCESS_TOKEN}"
+    endpoint: "${SPLUNK_TRACE_URL}"
+  # Metrics + Events
+  signalfx:
+    access_token: "${SPLUNK_ACCESS_TOKEN}"
+    api_url: "${SPLUNK_API_URL}"
+    ingest_url: "${SPLUNK_INGEST_URL}"
+    # Use instead when sending to gateway
+    #api_url: http://${SPLUNK_GATEWAY_URL}:6060
+    #ingest_url: http://${SPLUNK_GATEWAY_URL}:9943
+    sync_host_metadata: true
+    correlation:
+  # Logs
+  splunk_hec:
+    token: "${SPLUNK_HEC_TOKEN}"
+    endpoint: "${SPLUNK_HEC_URL}"
+    source: "otel"
+    sourcetype: "otel"
+  # Send to gateway
+  otlp:
+    endpoint: "${SPLUNK_GATEWAY_URL}:4317"
+    insecure: true
+  # Debug
+  logging:
+    loglevel: debug
+
+service:
+  extensions: [health_check, http_forwarder, zpages, memory_ballast]
+  pipelines:
+    traces:
+      receivers: [jaeger, otlp, smartagent/signalfx-forwarder, zipkin]
+      processors:
+      - memory_limiter
+      - batch
+      - resourcedetection
+      #- resource/add_environment
+      exporters: [sapm, signalfx]
+      # Use instead when sending to gateway
+      #exporters: [otlp, signalfx]
+    metrics:
+      receivers: [hostmetrics, otlp, signalfx, smartagent/signalfx-forwarder, smartagent/sqlserver]
+      processors: [memory_limiter, batch, resourcedetection]
+      exporters: [signalfx]
+      # Use instead when sending to gateway
+      #exporters: [otlp]
+    metrics/internal:
+      receivers: [prometheus/internal]
+      processors: [memory_limiter, batch, resourcedetection/internal]
+      exporters: [signalfx]
+      # Use instead when sending to gateway
+      #exporters: [otlp]
+    logs/signalfx:
+      receivers: [signalfx]
+      processors: [memory_limiter, batch]
+      exporters: [signalfx]
+      # Use instead when sending to gateway
+      #exporters: [otlp]
+    logs:
+      receivers: [fluentforward, otlp]
+      processors:
+      - memory_limiter
+      - batch
+      - resourcedetection
+      #- resource/add_environment
+      exporters: [splunk_hec]
+      # Use instead when sending to gateway
+      #exporters: [otlp]
diff --git a/modules/instances/security_groups.tf b/modules/instances/security_groups.tf
@@ -19,6 +19,22 @@ resource "aws_security_group" "instances_sg" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 
+  ## Allow RDP - Enable Windows Remote Desktop
+  ingress {
+    from_port   = 3389
+    to_port     = 3389
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ## Allow WinRM - Enable Windows Remote Desktop
+  ingress {
+    from_port   = 5985
+    to_port     = 5985
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
   ## Allow HTTP
   ingress {
     from_port   = 80

diff --git a/modules/instances/variables.tf b/modules/instances/variables.tf
@@ -23,9 +23,15 @@ variable "instance_type" {
 variable "collector_instance_type" {
   default = []
 }
+variable "ms_sql_instance_type" {
+  default = []
+}
 variable "ami" {
   default = {}
 }
+variable "ms_sql_ami" {
+  default = {}
+}
 
 ### SignalFX Variables ###
 variable "access_token" {
@@ -49,7 +55,6 @@ variable "ballast" {
 variable "environment" {
   default = []
 }
-
 variable "collector_count" {
   default = {}
 }
@@ -68,13 +73,21 @@ variable "mysql_count" {
 variable "mysql_ids" {
   default = []
 }
+variable "ms_sql_count" {
+  default = {}
+}
+variable "ms_sql_ids" {
+  default = []
+}
 variable "apache_web_count" {
   default = {}
 }
 variable "apache_web_ids" {
   default = []
 }
-
+variable "branch" {
+  default = []
+}
 
 
 ### Splunk Enterprise Variables ###