diff --git a/deploy/helm/sumologic/conf/setup/fields.tf b/deploy/helm/sumologic/conf/setup/fields.tf new file mode 100644 index 0000000000..c2f782ba23 --- /dev/null +++ b/deploy/helm/sumologic/conf/setup/fields.tf @@ -0,0 +1,9 @@ +{{- range $value := .Values.sumologic.logs.fields }} +resource "sumologic_field" {{ $value | quote }} { + count = var.create_fields ? 1 : 0 + + field_name = {{ $value | quote }} + data_type = "String" + state = "Enabled" +} +{{- end }} diff --git a/deploy/helm/sumologic/conf/setup/setup.sh b/deploy/helm/sumologic/conf/setup/setup.sh old mode 100644 new mode 100755 index 7b7aba05b3..a113aeb645 --- a/deploy/helm/sumologic/conf/setup/setup.sh +++ b/deploy/helm/sumologic/conf/setup/setup.sh @@ -1,21 +1,67 @@ #!/bin/bash -cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform -cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} +function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' +} + +# Check if we'd have at least 10 fields remaining after additional fields +# would be created for the collection +function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - {{ len .Values.sumologic.logs.fields }} )) -ge 10 ]] ; then + return 0 + else + return 1 + fi +} + +cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ +cd /terraform + COLLECTOR_NAME="{{- if .Values.sumologic.collectorName }}{{ .Values.sumologic.collectorName }}{{- else}}{{ .Values.sumologic.clusterName }}{{- end}}" terraform init -# Sumo Collector and HTTP sources +# Sumo Logic fields +if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=({{ include "helm-toolkit.utils.joinListWithSpaces" .Values.sumologic.logs.fields }}) + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done +else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." +fi + +# Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" + {{- $ctx := .Values -}} {{- range $type, $sources := .Values.sumologic.sources }} {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }} @@ -30,7 +76,9 @@ terraform import sumologic_http_source.{{ template "terraform.sources.name" (dic # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret {{ .Release.Namespace }}/sumologic -terraform apply -auto-approve +# Apply planned changes +terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= diff --git a/deploy/helm/sumologic/conf/setup/variables.tf b/deploy/helm/sumologic/conf/setup/variables.tf index 7ed9274c43..60f648cb93 100644 --- a/deploy/helm/sumologic/conf/setup/variables.tf +++ b/deploy/helm/sumologic/conf/setup/variables.tf @@ -15,4 +15,10 @@ variable "collector_name" { variable "namespace_name" { type = string default = "{{ .Release.Namespace }}" -} \ No newline at end of file +} + +variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true +} diff --git a/deploy/helm/sumologic/templates/NOTES.txt b/deploy/helm/sumologic/templates/NOTES.txt index 6fb7489259..922752a9d5 100644 --- a/deploy/helm/sumologic/templates/NOTES.txt +++ b/deploy/helm/sumologic/templates/NOTES.txt @@ -11,4 +11,8 @@ Check the release status by running: {{- if eq .Values.fluentd.persistence.enabled false }} WARNING: File persistence for fluentd is disabled. This might lead to loss of data in case of memory buffer overflow. We recommend turning this property on for production environments by setting fluentd.persistence.enabled=true -{{- end }} \ No newline at end of file +{{- end }} + +We've tried to automatically create fields. In an unlikely scenario that this +fails please refer to the following to create them manually: +https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/2b3ca63/deploy/docs/Installation_with_Helm.md#prerequisite diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 7d1857e6c3..cf8e45e175 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -778,3 +778,19 @@ Example: key: {{ template "terraform.sources.config-map-variable" (dict "Type" $type "Context" $ctx "Name" $key) }} {{- end }} {{- end -}} + +{{/* +Generate a space separated list of quoted values: + +Example: + +{{ include "helm-toolkit.utils.joinListWithSpaces" .Values.sumologic.logs.fields }} +*/}} +{{- define "helm-toolkit.utils.joinListWithSpaces" -}} +{{- $local := dict "first" true -}} +{{- range $k, $v := . -}} +{{- if not $local.first }} {{ end -}} +{{- $v | quote -}} +{{- $_ := set $local "first" false -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 38633a574d..11b3c08145 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -101,6 +101,19 @@ sumologic: logs: enabled: true + ## Fields to be created at Sumo Logic to ensure logs are tagged with + ## relevant metadata. + ## https://help.sumologic.com/Manage/Fields#Manage_fields + fields: + - cluster + - container + - deployment + - host + - namespace + - node + - pod + - service + ### Metrics configuration ## Set the enabled flag to false for disabling metrics ingestion altogether. metrics: diff --git a/deploy/kubernetes/setup-sumologic.yaml.tmpl b/deploy/kubernetes/setup-sumologic.yaml.tmpl index d72de07f9b..909ef8e7cb 100644 --- a/deploy/kubernetes/setup-sumologic.yaml.tmpl +++ b/deploy/kubernetes/setup-sumologic.yaml.tmpl @@ -81,6 +81,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { default_events_source = "events" @@ -193,22 +250,67 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="$COLLECTOR_NAME" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events" terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs" @@ -224,7 +326,9 @@ data: # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret $NAMESPACE/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -232,7 +336,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "$CLUSTER_NAME" @@ -247,6 +351,12 @@ data: type = string default = "$NAMESPACE" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + } --- # Source: sumologic/templates/setup/setup-custom-configmap.yaml apiVersion: v1 diff --git a/tests/terraform/static/all_fields.output.yaml b/tests/terraform/static/all_fields.output.yaml index 6850283ec6..517334f677 100644 --- a/tests/terraform/static/all_fields.output.yaml +++ b/tests/terraform/static/all_fields.output.yaml @@ -49,6 +49,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { default_events_source = "events" @@ -208,22 +265,67 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="kubernetes" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events" terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs" @@ -240,7 +342,9 @@ data: # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -248,7 +352,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "kubernetes" @@ -263,3 +367,9 @@ data: type = string default = "sumologic" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + } diff --git a/tests/terraform/static/collector_fields.output.yaml b/tests/terraform/static/collector_fields.output.yaml index dfa820d56d..62a802cebb 100644 --- a/tests/terraform/static/collector_fields.output.yaml +++ b/tests/terraform/static/collector_fields.output.yaml @@ -49,6 +49,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { default_events_source = "events" @@ -163,22 +220,67 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="kubernetes" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events" terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs" @@ -194,7 +296,9 @@ data: # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -202,7 +306,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "kubernetes" @@ -217,3 +321,9 @@ data: type = string default = "sumologic" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + } diff --git a/tests/terraform/static/conditional_sources.output.yaml b/tests/terraform/static/conditional_sources.output.yaml index 56352b43ab..5310c33f72 100644 --- a/tests/terraform/static/conditional_sources.output.yaml +++ b/tests/terraform/static/conditional_sources.output.yaml @@ -49,6 +49,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { } @@ -90,28 +147,75 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="kubernetes" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -119,7 +223,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "kubernetes" @@ -134,3 +238,9 @@ data: type = string default = "sumologic" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + } diff --git a/tests/terraform/static/custom.output.yaml b/tests/terraform/static/custom.output.yaml index 56352b43ab..5310c33f72 100644 --- a/tests/terraform/static/custom.output.yaml +++ b/tests/terraform/static/custom.output.yaml @@ -49,6 +49,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { } @@ -90,28 +147,75 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="kubernetes" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -119,7 +223,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "kubernetes" @@ -134,3 +238,9 @@ data: type = string default = "sumologic" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + } diff --git a/tests/terraform/static/default.output.yaml b/tests/terraform/static/default.output.yaml index 445a6a9896..56c58308c1 100644 --- a/tests/terraform/static/default.output.yaml +++ b/tests/terraform/static/default.output.yaml @@ -49,6 +49,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { default_events_source = "events" @@ -161,22 +218,67 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="kubernetes" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events" terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs" @@ -192,7 +294,9 @@ data: # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -200,7 +304,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "kubernetes" @@ -215,3 +319,9 @@ data: type = string default = "sumologic" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + } diff --git a/tests/terraform/static/disable_default_metrics.output.yaml b/tests/terraform/static/disable_default_metrics.output.yaml index 5b69cd32ef..64d56e0188 100644 --- a/tests/terraform/static/disable_default_metrics.output.yaml +++ b/tests/terraform/static/disable_default_metrics.output.yaml @@ -49,6 +49,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { default_events_source = "events" @@ -154,22 +211,67 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="kubernetes" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events" terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs" @@ -184,7 +286,9 @@ data: # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -192,7 +296,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "kubernetes" @@ -207,3 +311,9 @@ data: type = string default = "sumologic" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + } diff --git a/tests/terraform/static/strip_extrapolation.output.yaml b/tests/terraform/static/strip_extrapolation.output.yaml index 52d9520ded..dc08a55fc6 100644 --- a/tests/terraform/static/strip_extrapolation.output.yaml +++ b/tests/terraform/static/strip_extrapolation.output.yaml @@ -49,6 +49,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { default_events_source = "events" @@ -162,22 +219,67 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="kubernetes" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events" terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs" @@ -193,7 +295,9 @@ data: # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -201,7 +305,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "kubernetes" @@ -216,3 +320,9 @@ data: type = string default = "sumologic" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + } diff --git a/tests/terraform/static/traces.output.yaml b/tests/terraform/static/traces.output.yaml index 278ff09236..4dd65115bf 100644 --- a/tests/terraform/static/traces.output.yaml +++ b/tests/terraform/static/traces.output.yaml @@ -49,6 +49,63 @@ data: done cd "${target}" && bash setup.sh done + fields.tf: | + resource "sumologic_field" "cluster" { + count = var.create_fields ? 1 : 0 + + field_name = "cluster" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "container" { + count = var.create_fields ? 1 : 0 + + field_name = "container" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "deployment" { + count = var.create_fields ? 1 : 0 + + field_name = "deployment" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "host" { + count = var.create_fields ? 1 : 0 + + field_name = "host" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "namespace" { + count = var.create_fields ? 1 : 0 + + field_name = "namespace" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "node" { + count = var.create_fields ? 1 : 0 + + field_name = "node" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "pod" { + count = var.create_fields ? 1 : 0 + + field_name = "pod" + data_type = "String" + state = "Enabled" + } + resource "sumologic_field" "service" { + count = var.create_fields ? 1 : 0 + + field_name = "service" + data_type = "String" + state = "Enabled" + } locals.tf: | locals { default_traces_source = "traces" @@ -98,29 +155,76 @@ data: } setup.sh: | #!/bin/bash - cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform - cd /terraform # Fix URL to remove "v1" or "v1/" export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*} - # Support proxy for terraform export HTTP_PROXY=${HTTP_PROXY:=""} export HTTPS_PROXY=${HTTPS_PROXY:=""} export NO_PROXY=${NO_PROXY:=""} + function remaining_fields() { + local RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields/quota)" + + echo "${RESPONSE}" | jq '.remaining' + } + + # Check if we'd have at least 10 fields remaining after additional fields + # would be created for the collection + function should_create_fields() { + local REMAINING=$(remaining_fields) + if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then + return 0 + else + return 1 + fi + } + + cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/ + cd /terraform + COLLECTOR_NAME="kubernetes" terraform init - # Sumo Collector and HTTP sources + # Sumo Logic fields + if should_create_fields ; then + readonly CREATE_FIELDS=1 + readonly FIELDS_RESPONSE="$(curl -XGET -s \ + -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \ + "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )" + + declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service") + for FIELD in "${FIELDS[@]}" ; do + FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" ) + # Don't try to import non existing fields + if [[ -z "${FIELD_ID}" ]]; then + continue + fi + + terraform import \ + -var="create_fields=1" \ + sumologic_field."${FIELD}" "${FIELD_ID}" + done + else + readonly CREATE_FIELDS=0 + echo "Couldn't automatically create fields" + echo "You do not have enough field capacity to create the required fields automatically." + echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity." + fi + + # Sumo Logic Collector and HTTP sources terraform import sumologic_collector.collector "$COLLECTOR_NAME" terraform import sumologic_http_source.default_traces_source "$COLLECTOR_NAME/traces" # Kubernetes Secret terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic - terraform apply -auto-approve + # Apply planned changes + terraform apply -auto-approve \ + -var="create_fields=${CREATE_FIELDS}" # Cleanup env variables export SUMOLOGIC_BASE_URL= @@ -128,7 +232,7 @@ data: export SUMOLOGIC_ACCESSID= bash /etc/terraform/custom.sh - variables.tf: |- + variables.tf: | variable "cluster_name" { type = string default = "kubernetes" @@ -143,3 +247,9 @@ data: type = string default = "sumologic" } + + variable "create_fields" { + description = "If set, terraform will attempt to create fields at Sumo Logic" + type = bool + default = true + }