From 8807ee197c38ccf8db08f9848de7a7092d74cf02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 9 Jun 2020 10:23:21 +0200 Subject: [PATCH 1/6] Fixed label naming --- pkg/mentix/exporters/promfilesd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index c2906ac1e4..5dcc90a1ee 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -115,7 +115,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site Targets: []string{path.Join(host, endpoint.Path)}, Labels: map[string]string{ "site": site.Name, - "service-type": endpoint.Type.Name, + "service_type": endpoint.Type.Name, }, } } From 7e0ece2c7e70ffb6aaaf4223005ae20a4673d3f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 9 Jun 2020 13:09:02 +0200 Subject: [PATCH 2/6] Added support for service ports --- pkg/mentix/connectors/gocdb.go | 14 +++++++++++++- pkg/mentix/exporters/promfilesd.go | 3 +-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index d10690f15b..f4975eae06 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -21,6 +21,7 @@ package connectors import ( "encoding/xml" "fmt" + "net/url" "strings" "github.com/rs/zerolog" @@ -146,6 +147,17 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site // Copy retrieved data into the mesh data site.Services = nil for _, service := range services.Services { + host := service.Host + + // If a URL is provided, extract the port from it and append it to the host + if len(service.URL) > 0 { + if hostURL, err := url.Parse(service.URL); err == nil { + if port := hostURL.Port(); len(port) > 0 { + host += ":" + port + } + } + } + // Assemble additional endpoints var endpoints []*meshdata.ServiceEndpoint for _, endpoint := range service.Endpoints.Endpoints { @@ -167,7 +179,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site IsMonitored: strings.EqualFold(service.IsMonitored, "Y"), Properties: connector.extensionsToMap(&service.Extensions), }, - Host: service.Host, + Host: host, AdditionalEndpoints: endpoints, }) } diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index 5dcc90a1ee..0479c333ff 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -23,7 +23,6 @@ import ( "fmt" "io/ioutil" "os" - "path" "path/filepath" "github.com/rs/zerolog" @@ -112,7 +111,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.Sc func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { return &prometheus.ScrapeConfig{ - Targets: []string{path.Join(host, endpoint.Path)}, + Targets: []string{host}, Labels: map[string]string{ "site": site.Name, "service_type": endpoint.Type.Name, From 2e1ab438a039afbb4ced4b6708ccf3c767246088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 24 Jun 2020 10:50:40 +0200 Subject: [PATCH 3/6] Added support for differing metrics paths when exporting scrapes to Prometheus --- pkg/mentix/config/ids.go | 4 ++++ pkg/mentix/exporters/promfilesd.go | 15 +++++++++++---- pkg/mentix/meshdata/service.go | 13 +++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index f8aed9c0dd..e3e837feb9 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -26,3 +26,7 @@ const ( ExporterIDWebAPI = "webapi" ExporterIDPrometheusFileSD = "prom_filesd" ) + +const ( + PropertyMetricsPath = "metrics_path" +) diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index 0479c333ff..39c2772fc3 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -110,12 +110,19 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.Sc } func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { + labels := map[string]string{ + "site": site.Name, + "service_type": endpoint.Type.Name, + } + + // If a metrics path was specified as a property, use that one by setting the corresponding label + if metricsPath := endpoint.GetPropertyValue(config.PropertyMetricsPath, ""); len(metricsPath) > 0 { + labels["__metrics_path__"] = metricsPath + } + return &prometheus.ScrapeConfig{ Targets: []string{host}, - Labels: map[string]string{ - "site": site.Name, - "service_type": endpoint.Type.Name, - }, + Labels: labels, } } diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index 4ff18ec63e..240c8b25bb 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -18,6 +18,8 @@ package meshdata +import "strings" + // Service represents a service managed by Mentix. type Service struct { ServiceEndpoint @@ -40,3 +42,14 @@ type ServiceEndpoint struct { IsMonitored bool Properties map[string]string } + +// GetPropertyValue performs a case-insensitive search for the given property. +func (endpoint *ServiceEndpoint) GetPropertyValue(id string, defValue string) string { + for key := range endpoint.Properties { + if strings.EqualFold(key, id) { + return endpoint.Properties[key] + } + } + + return defValue +} From 34413c2f6f7701f76a03f80c5e7c01babe7c39e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Jun 2020 11:47:51 +0200 Subject: [PATCH 4/6] Added support for organization names --- pkg/mentix/config/ids.go | 4 ---- pkg/mentix/connectors/gocdb.go | 9 ++++++-- pkg/mentix/exporters/promfilesd.go | 2 +- pkg/mentix/meshdata/properties.go | 37 ++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 pkg/mentix/meshdata/properties.go diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index e3e837feb9..f8aed9c0dd 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -26,7 +26,3 @@ const ( ExporterIDWebAPI = "webapi" ExporterIDPrometheusFileSD = "prom_filesd" ) - -const ( - PropertyMetricsPath = "metrics_path" -) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index f4975eae06..390463e76d 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -121,16 +121,21 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { // Copy retrieved data into the mesh data meshData.Sites = nil for _, site := range sites.Sites { + properties := connector.extensionsToMap(&site.Extensions) + + // See if an organization has been defined using properties; otherwise, use the official name + organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName) + meshsite := &meshdata.Site{ Name: site.ShortName, FullName: site.OfficialName, - Organization: "", + Organization: organization, Domain: site.Domain, Homepage: site.Homepage, Email: site.Email, Description: site.Description, Services: nil, - Properties: connector.extensionsToMap(&site.Extensions), + Properties: properties, } meshData.Sites = append(meshData.Sites, meshsite) } diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index 39c2772fc3..7c1fe91e40 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -116,7 +116,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site } // If a metrics path was specified as a property, use that one by setting the corresponding label - if metricsPath := endpoint.GetPropertyValue(config.PropertyMetricsPath, ""); len(metricsPath) > 0 { + if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 { labels["__metrics_path__"] = metricsPath } diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go new file mode 100644 index 0000000000..2d6f1cf62f --- /dev/null +++ b/pkg/mentix/meshdata/properties.go @@ -0,0 +1,37 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package meshdata + +import "strings" + +const ( + PropertyOrganization = "organization" + PropertyMetricsPath = "metrics_path" +) + +// GetPropertyValue performs a case-insensitive search for the given property. +func GetPropertyValue(props map[string]string, id string, defValue string) string { + for key := range props { + if strings.EqualFold(key, id) { + return props[key] + } + } + + return defValue +} From 7fc02aa4c7a7c581f3e907e81ebb4140cd6454b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 25 Jun 2020 13:09:35 +0200 Subject: [PATCH 5/6] Service URLs are now properly combined and exported --- pkg/mentix/connectors/gocdb.go | 43 ++++++++++++++++++++++++++++++++-- pkg/mentix/meshdata/service.go | 15 +----------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 390463e76d..22c5989620 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -22,6 +22,7 @@ import ( "encoding/xml" "fmt" "net/url" + "path" "strings" "github.com/rs/zerolog" @@ -149,6 +150,14 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site return err } + getServiceURLString := func(service *gocdb.Service, endpoint *gocdb.ServiceEndpoint, host string) string { + urlstr := "https://" + host // Fall back to the provided hostname + if svcURL, err := connector.getServiceURL(service, endpoint); err == nil { + urlstr = svcURL.String() + } + return urlstr + } + // Copy retrieved data into the mesh data site.Services = nil for _, service := range services.Services { @@ -169,7 +178,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site endpoints = append(endpoints, &meshdata.ServiceEndpoint{ Type: connector.findServiceType(meshData, endpoint.Type), Name: endpoint.Name, - Path: endpoint.URL, + URL: getServiceURLString(service, endpoint, host), IsMonitored: strings.EqualFold(endpoint.IsMonitored, "Y"), Properties: connector.extensionsToMap(&endpoint.Extensions), }) @@ -180,7 +189,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site ServiceEndpoint: meshdata.ServiceEndpoint{ Type: connector.findServiceType(meshData, service.Type), Name: fmt.Sprintf("%v - %v", service.Host, service.Type), - Path: "", + URL: getServiceURLString(service, nil, host), IsMonitored: strings.EqualFold(service.IsMonitored, "Y"), Properties: connector.extensionsToMap(&service.Extensions), }, @@ -211,6 +220,36 @@ func (connector *GOCDBConnector) extensionsToMap(extensions *gocdb.Extensions) m return properties } +func (connector *GOCDBConnector) getServiceURL(service *gocdb.Service, endpoint *gocdb.ServiceEndpoint) (*url.URL, error) { + urlstr := service.URL + if len(urlstr) == 0 { + // The URL defaults to the hostname using the HTTPS protocol + urlstr = "https://" + service.Host + } + + svcURL, err := url.ParseRequestURI(urlstr) + if err != nil { + return nil, fmt.Errorf("unable to parse URL '%v': %v", urlstr, err) + } + + // If an endpoint was provided, use its path + if endpoint != nil { + // If the endpoint URL is an absolute one, just use that; otherwise, make an absolute one out of it + if endpointURL, err := url.ParseRequestURI(endpoint.URL); err == nil && len(endpointURL.Scheme) > 0 { + svcURL = endpointURL + } else { + // Replace entire URL path if the relative path starts with a slash; otherwise, just append + if strings.HasPrefix(endpoint.URL, "/") { + svcURL.Path = endpoint.URL + } else { + svcURL.Path = path.Join(svcURL.Path, endpoint.URL) + } + } + } + + return svcURL, nil +} + func (connector *GOCDBConnector) GetName() string { return "GOCDB" } diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index 240c8b25bb..e0f240653d 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -18,8 +18,6 @@ package meshdata -import "strings" - // Service represents a service managed by Mentix. type Service struct { ServiceEndpoint @@ -38,18 +36,7 @@ type ServiceType struct { type ServiceEndpoint struct { Type *ServiceType Name string - Path string + URL string IsMonitored bool Properties map[string]string } - -// GetPropertyValue performs a case-insensitive search for the given property. -func (endpoint *ServiceEndpoint) GetPropertyValue(id string, defValue string) string { - for key := range endpoint.Properties { - if strings.EqualFold(key, id) { - return endpoint.Properties[key] - } - } - - return defValue -} From 2efc8765921f9d9213591d122579f29d55217ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 1 Jul 2020 16:47:37 +0200 Subject: [PATCH 6/6] Added country information to Mentix --- pkg/mentix/connectors/gocdb.go | 2 ++ pkg/mentix/connectors/gocdb/types.go | 2 ++ pkg/mentix/exporters/promfilesd.go | 1 + pkg/mentix/meshdata/site.go | 3 +++ 4 files changed, 8 insertions(+) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 22c5989620..cd6206bc80 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -135,6 +135,8 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { Homepage: site.Homepage, Email: site.Email, Description: site.Description, + Country: site.Country, + CountryCode: site.CountryCode, Services: nil, Properties: properties, } diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go index cc09e6e107..9448b47f23 100755 --- a/pkg/mentix/connectors/gocdb/types.go +++ b/pkg/mentix/connectors/gocdb/types.go @@ -48,6 +48,8 @@ type Site struct { Homepage string `xml:"HOME_URL"` Email string `xml:"CONTACT_EMAIL"` Domain string `xml:"DOMAIN>DOMAIN_NAME"` + Country string `xml:"COUNTRY"` + CountryCode string `xml:"COUNTRY_CODE"` Extensions Extensions `xml:"EXTENSIONS"` } diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index 7c1fe91e40..ec49dd484f 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -112,6 +112,7 @@ func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.Sc func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { labels := map[string]string{ "site": site.Name, + "country": site.CountryCode, "service_type": endpoint.Type.Name, } diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 6fba7eced0..8c90ace8e9 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -27,6 +27,9 @@ type Site struct { Homepage string Email string Description string + Country string + CountryCode string + Location string Services []*Service Properties map[string]string