From 30038cefdfa61614f79f11b3bb9a157f7b95cb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 14 May 2020 16:15:50 +0200 Subject: [PATCH 01/13] Added Mentix (Mesh Entity Exporter) as a Reva service --- .gitignore | 4 + CONTRIBUTORS.md | 1 + examples/mentix/mentix.toml | 24 +++ go.sum | 12 ++ internal/http/services/loader/loader.go | 1 + internal/http/services/mentix/mentix.go | 118 ++++++++++++ pkg/mentix/config/config.go | 40 +++++ pkg/mentix/config/ids.go | 28 +++ pkg/mentix/connectors/connector.go | 65 +++++++ pkg/mentix/connectors/gocdb.go | 201 +++++++++++++++++++++ pkg/mentix/connectors/gocdb/query.go | 51 ++++++ pkg/mentix/connectors/gocdb/types.go | 76 ++++++++ pkg/mentix/engine/engine.go | 219 +++++++++++++++++++++++ pkg/mentix/exporters/exporter.go | 118 ++++++++++++ pkg/mentix/exporters/prometheus/types.go | 24 +++ pkg/mentix/exporters/promfilesd.go | 137 ++++++++++++++ pkg/mentix/exporters/reqexporter.go | 47 +++++ pkg/mentix/exporters/webapi.go | 66 +++++++ pkg/mentix/exporters/webapi/query.go | 53 ++++++ pkg/mentix/mentix.go | 72 ++++++++ pkg/mentix/meshdata/meshdata.go | 75 ++++++++ pkg/mentix/meshdata/service.go | 39 ++++ pkg/mentix/meshdata/site.go | 32 ++++ pkg/mentix/network/utils.go | 68 +++++++ 24 files changed, 1571 insertions(+) create mode 100644 examples/mentix/mentix.toml create mode 100644 internal/http/services/mentix/mentix.go create mode 100644 pkg/mentix/config/config.go create mode 100644 pkg/mentix/config/ids.go create mode 100755 pkg/mentix/connectors/connector.go create mode 100755 pkg/mentix/connectors/gocdb.go create mode 100755 pkg/mentix/connectors/gocdb/query.go create mode 100755 pkg/mentix/connectors/gocdb/types.go create mode 100644 pkg/mentix/engine/engine.go create mode 100755 pkg/mentix/exporters/exporter.go create mode 100755 pkg/mentix/exporters/prometheus/types.go create mode 100755 pkg/mentix/exporters/promfilesd.go create mode 100644 pkg/mentix/exporters/reqexporter.go create mode 100755 pkg/mentix/exporters/webapi.go create mode 100755 pkg/mentix/exporters/webapi/query.go create mode 100644 pkg/mentix/mentix.go create mode 100644 pkg/mentix/meshdata/meshdata.go create mode 100644 pkg/mentix/meshdata/service.go create mode 100644 pkg/mentix/meshdata/site.go create mode 100644 pkg/mentix/network/utils.go diff --git a/.gitignore b/.gitignore index c5a436c23f..60a37c4459 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ # Visual Studio Code .vscode +# GoLand +.idea + # Output of the go coverage tool, specifically when used with LiteIDE *.out @@ -35,3 +38,4 @@ docs/tech-doc-hugo docs/themes/ dist/ +bin/ diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 227defe057..e98bf07e46 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -17,3 +17,4 @@ - Mohitty - Thomas Boerger - Miroslav Bauer +- Daniel Mueller diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml new file mode 100644 index 0000000000..4870e1403e --- /dev/null +++ b/examples/mentix/mentix.toml @@ -0,0 +1,24 @@ +[shared] +jwt_secret = "Ment1x-T0pS3cr3t" + +[http] +address = "0.0.0.0:9600" +enabled_services = ["mentix"] + +[http.services.mentix] +connector = "gocdb" +exporters = ["webapi"] +# Enable the Prometheus File Service Discovery: +# exporters = ["webapi", "prom-filesd"] +update-interval = "15m" + +[http.services.mentix.gocdb] +address = "http://sciencemesh-test.uni-muenster.de" + +[http.services.mentix.webapi] +endpoint = "/" + +# Configure the Prometheus File Service Discovery: +# [http.services.mentix.prom-filesd] +# Prometheus must be configured to read the following file: +# output-file = "/usr/share/prom/sciencemesh_services.json" diff --git a/go.sum b/go.sum index 9b33b804b5..45746ab446 100644 --- a/go.sum +++ b/go.sum @@ -43,7 +43,9 @@ github.com/aws/aws-sdk-go v1.30.26 h1:wP0N6DBb/3EyHTtWNz4jzgGVi1l290zoFGfu4HFGeM github.com/aws/aws-sdk-go v1.30.26/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98= @@ -123,6 +125,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -212,6 +215,7 @@ github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prY github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -219,10 +223,12 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCb github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= @@ -294,6 +300,7 @@ golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced h1:4oqSq7eft7MdPKBGQK11X9 golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -335,6 +342,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.0 h1:2tJEkRfnZL5g1GeBUlITh/rqT5HG3sFcoVCUUxmgJ2g= google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -373,16 +381,20 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/Acconut/lockfile.v1 v1.1.0/go.mod h1:6UCz3wJ8tSFUsPR6uP/j8uegEtDuEEqFxlpi0JI4Umw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU= gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/square/go-jose.v2 v2.1.9/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index 3e66815aeb..9d35075393 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -23,6 +23,7 @@ import ( _ "github.com/cs3org/reva/internal/http/services/datagateway" _ "github.com/cs3org/reva/internal/http/services/dataprovider" _ "github.com/cs3org/reva/internal/http/services/helloworld" + _ "github.com/cs3org/reva/internal/http/services/mentix" _ "github.com/cs3org/reva/internal/http/services/meshdirectory" _ "github.com/cs3org/reva/internal/http/services/ocmd" _ "github.com/cs3org/reva/internal/http/services/oidcprovider" diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go new file mode 100644 index 0000000000..d5fc3290e9 --- /dev/null +++ b/internal/http/services/mentix/mentix.go @@ -0,0 +1,118 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package mentix + +import ( + "net/http" + + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + + "github.com/cs3org/reva/pkg/mentix" + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exporters" + "github.com/cs3org/reva/pkg/rhttp/global" +) + +type svc struct { + conf *config.Configuration + ment *mentix.Mentix + + stop chan struct{} +} + +func (s *svc) Close() error { + // Trigger and close the stop signal channel to stop Mentix + s.stop <- struct{}{} + close(s.stop) + + return nil +} + +func (s *svc) Prefix() string { + return s.conf.Prefix +} + +func (s *svc) Unprotected() []string { + return []string{"/"} +} + +func (s *svc) Handler() http.Handler { + // Forward requests to the engine + return http.HandlerFunc(s.ment.Engine().RequestHandler) +} + +func (s *svc) startBackgroundService() { + // Just run Mentix in the background + go s.ment.Run(s.stop) +} + +func parseConfig(m map[string]interface{}) (*config.Configuration, error) { + cfg := defaultConfig() + if err := mapstructure.Decode(m, &cfg); err != nil { + return nil, errors.Wrap(err, "mentix: error decoding configuration") + } + return cfg, nil +} + +func defaultConfig() *config.Configuration { + conf := &config.Configuration{} + + // General settings + conf.Prefix = "mentix" + conf.Connector = config.ConnectorID_GOCDB + conf.Exporters = exporters.RegisteredExporterIDs() + conf.UpdateInterval = "1h" + + // GOCDB settings + conf.GOCDB.Scope = "SM" + + // WebAPI settings + conf.WebAPI.Endpoint = "/" + + return conf +} + +// New returns a new Mentix service +func New(m map[string]interface{}) (global.Service, error) { + // Prepare the configuration + conf, err := parseConfig(m) + if err != nil { + return nil, err + } + + // Create the Mentix instance + ment, err := mentix.New(conf) + if err != nil { + return nil, errors.Wrap(err, "mentix: error creating instance") + } + + // Create the service and start its background activity + s := &svc{ + conf: conf, + ment: ment, + stop: make(chan struct{}), + } + s.startBackgroundService() + return s, nil +} + +func init() { + global.Register("mentix", New) +} diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go new file mode 100644 index 0000000000..595cc59a93 --- /dev/null +++ b/pkg/mentix/config/config.go @@ -0,0 +1,40 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +type Configuration struct { + Prefix string `mapstructure:"prefix"` + + Connector string `mapstructure:"connector"` + Exporters []string `mapstructure:"exporters"` + UpdateInterval string `mapstructure:"update-interval"` + + GOCDB struct { + Address string `mapstructure:"address"` + Scope string `mapstructure:"scope"` + } `mapstructure:"gocdb"` + + WebAPI struct { + Endpoint string `mapstructure:"endpoint"` + } `yaml:"webapi"` + + PrometheusFileSD struct { + OutputFile string `mapstructure:"output-file"` + } `mapstructure:"prom-filesd"` +} diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go new file mode 100644 index 0000000000..ec3224fbd1 --- /dev/null +++ b/pkg/mentix/config/ids.go @@ -0,0 +1,28 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +const ( + ConnectorID_GOCDB = "gocdb" +) + +const ( + ExporterID_WebAPI = "webapi" + ExporterID_PrometheusFileSD = "prom-filesd" +) diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go new file mode 100755 index 0000000000..5d8ce1bf8c --- /dev/null +++ b/pkg/mentix/connectors/connector.go @@ -0,0 +1,65 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package connectors + +import ( + "fmt" + "strings" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +var ( + registeredConnectors = map[string]Connector{} +) + +type Connector interface { + Activate(conf *config.Configuration) error + RetrieveMeshData() (*meshdata.MeshData, error) + + GetName() string +} + +type BaseConnector struct { + conf *config.Configuration +} + +func (connector *BaseConnector) Activate(conf *config.Configuration) error { + if conf == nil { + return fmt.Errorf("no configuration provided") + } + connector.conf = conf + + return nil +} + +func FindConnector(connectorID string) (Connector, error) { + for id, connector := range registeredConnectors { + if strings.EqualFold(id, connectorID) { + return connector, nil + } + } + + return nil, fmt.Errorf("no connector with ID '%v' registered", connectorID) +} + +func registerConnector(id string, connector Connector) { + registeredConnectors[id] = connector +} diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go new file mode 100755 index 0000000000..5c598d5461 --- /dev/null +++ b/pkg/mentix/connectors/gocdb.go @@ -0,0 +1,201 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package connectors + +import ( + "encoding/xml" + "fmt" + "strings" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/connectors/gocdb" + "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/network" +) + +type GOCDBConnector struct { + BaseConnector + + gocdbAddress string +} + +func (connector *GOCDBConnector) Activate(conf *config.Configuration) error { + if err := connector.BaseConnector.Activate(conf); err != nil { + return err + } + + // Check and store GOCDB specific settings + connector.gocdbAddress = conf.GOCDB.Address + if len(connector.gocdbAddress) == 0 { + return fmt.Errorf("no GOCDB address configured") + } + + return nil +} + +func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) { + meshData := new(meshdata.MeshData) + + // Query all data from GOCDB + if err := connector.queryServiceTypes(meshData); err != nil { + return nil, fmt.Errorf("could not query service types: %v", err) + } + + if err := connector.querySites(meshData); err != nil { + return nil, fmt.Errorf("could not query sites: %v", err) + } + + for _, site := range meshData.Sites { + // Get services associated with the current site + if err := connector.queryServices(meshData, site); err != nil { + return nil, fmt.Errorf("could not query services of site '%v': %v", site.Name, err) + } + } + + return meshData, nil +} + +func (connector *GOCDBConnector) query(v interface{}, method string, isPrivate bool, hasScope bool, params network.URLParams) error { + var scope string + if hasScope { + scope = connector.conf.GOCDB.Scope + } + + // Get the data from GOCDB + data, err := gocdb.QueryGOCDB(connector.conf.GOCDB.Address, method, isPrivate, scope, params) + if err != nil { + return err + } + + // Unmarshal it + if err := xml.Unmarshal(data, v); err != nil { + return fmt.Errorf("unable to unmarshal data: %v", err) + } + + return nil +} + +func (connector *GOCDBConnector) queryServiceTypes(meshData *meshdata.MeshData) error { + var serviceTypes gocdb.ServiceTypes + if err := connector.query(&serviceTypes, "get_service_types", false, false, network.URLParams{}); err != nil { + return err + } + + // Copy retrieved data into the mesh data + meshData.ServiceTypes = nil + for _, serviceType := range serviceTypes.Types { + meshData.ServiceTypes = append(meshData.ServiceTypes, &meshdata.ServiceType{ + Name: serviceType.Name, + Description: serviceType.Description, + }) + } + + return nil +} + +func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { + var sites gocdb.Sites + if err := connector.query(&sites, "get_site", false, true, network.URLParams{}); err != nil { + return err + } + + // Copy retrieved data into the mesh data + meshData.Sites = nil + for _, site := range sites.Sites { + meshsite := &meshdata.Site{ + Name: site.ShortName, + FullName: site.OfficialName, + Organization: "", + Domain: site.Domain, + Homepage: site.Homepage, + Email: site.Email, + Description: site.Description, + Services: nil, + Properties: connector.extensionsToMap(&site.Extensions), + } + meshData.Sites = append(meshData.Sites, meshsite) + } + + return nil +} + +func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site *meshdata.Site) error { + var services gocdb.Services + if err := connector.query(&services, "get_service", false, true, network.URLParams{"sitename": site.Name}); err != nil { + return err + } + + // Copy retrieved data into the mesh data + site.Services = nil + for _, service := range services.Services { + // Assemble additional endpoints + var endpoints []*meshdata.ServiceEndpoint + for _, endpoint := range service.Endpoints.Endpoints { + endpoints = append(endpoints, &meshdata.ServiceEndpoint{ + Type: connector.findServiceType(meshData, endpoint.Type), + Name: endpoint.Name, + Path: endpoint.URL, + IsMonitored: strings.EqualFold(endpoint.IsMonitored, "Y"), + Properties: connector.extensionsToMap(&endpoint.Extensions), + }) + } + + // Add the service to the site + site.Services = append(site.Services, &meshdata.Service{ + ServiceEndpoint: meshdata.ServiceEndpoint{ + Type: connector.findServiceType(meshData, service.Type), + Name: fmt.Sprintf("%v - %v", service.Host, service.Type), + Path: "", + IsMonitored: strings.EqualFold(service.IsMonitored, "Y"), + Properties: connector.extensionsToMap(&service.Extensions), + }, + Host: service.Host, + AdditionalEndpoints: endpoints, + }) + } + + return nil +} + +func (connector *GOCDBConnector) findServiceType(meshData *meshdata.MeshData, name string) *meshdata.ServiceType { + for _, serviceType := range meshData.ServiceTypes { + if strings.EqualFold(serviceType.Name, name) { + return serviceType + } + } + + // If the service type doesn't exist, create a default one + return &meshdata.ServiceType{Name: name, Description: ""} +} + +func (connector *GOCDBConnector) extensionsToMap(extensions *gocdb.Extensions) map[string]string { + properties := make(map[string]string) + for _, ext := range extensions.Extensions { + properties[ext.Key] = ext.Value + } + return properties +} + +func (connector *GOCDBConnector) GetName() string { + return "GOCDB" +} + +func init() { + registerConnector(config.ConnectorID_GOCDB, &GOCDBConnector{}) +} diff --git a/pkg/mentix/connectors/gocdb/query.go b/pkg/mentix/connectors/gocdb/query.go new file mode 100755 index 0000000000..f1bb720254 --- /dev/null +++ b/pkg/mentix/connectors/gocdb/query.go @@ -0,0 +1,51 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package gocdb + +import ( + "fmt" + + "github.com/cs3org/reva/pkg/mentix/network" +) + +func QueryGOCDB(address string, method string, isPrivate bool, scope string, params network.URLParams) ([]byte, error) { + // The method must always be specified + params["method"] = method + + // If a scope was specified, pass it to the endpoint as well + if len(scope) > 0 { + params["scope"] = scope + } + + // GOCDB's public API is located at /gocdbpi/public, the private one at /gocdbpi/private + var path string + if isPrivate { + path = "/gocdbpi/private" + } else { + path = "/gocdbpi/public" + } + + // Query the data from GOCDB + data, err := network.ReadEndpoint(address, path, params) + if err != nil { + return nil, fmt.Errorf("unable to read GOCDB endpoint: %v", err) + } + + return data, nil +} diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go new file mode 100755 index 0000000000..0ead7df2bb --- /dev/null +++ b/pkg/mentix/connectors/gocdb/types.go @@ -0,0 +1,76 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package gocdb + +type Extension struct { + Key string `xml:"KEY"` + Value string `xml:"VALUE"` +} + +type Extensions struct { + Extensions []*Extension `xml:"EXTENSION"` +} + +type ServiceType struct { + Name string `xml:"SERVICE_TYPE_NAME"` + Description string `xml:"SERVICE_TYPE_DESC"` +} + +type ServiceTypes struct { + Types []*ServiceType `xml:"SERVICE_TYPE"` +} + +type Site struct { + ShortName string `xml:"SHORT_NAME"` + OfficialName string `xml:"OFFICIAL_NAME"` + Description string `xml:"SITE_DESCRIPTION"` + Homepage string `xml:"HOME_URL"` + Email string `xml:"CONTACT_EMAIL"` + Domain string `xml:"DOMAIN>DOMAIN_NAME"` + Extensions Extensions `xml:"EXTENSIONS"` +} + +type Sites struct { + Sites []*Site `xml:"SITE"` +} + +type ServiceEndpoint struct { + Name string `xml:"NAME"` + URL string `xml:"URL"` + Type string `xml:"INTERFACENAME"` + IsMonitored string `xml:"ENDPOINT_MONITORED"` + Extensions Extensions `xml:"EXTENSIONS"` +} + +type ServiceEndpoints struct { + Endpoints []*ServiceEndpoint `xml:"ENDPOINT"` +} + +type Service struct { + Host string `xml:"HOSTNAME"` + Type string `xml:"SERVICE_TYPE"` + IsMonitored string `xml:"NODE_MONITORED"` + URL string `xml:"URL"` + Endpoints ServiceEndpoints `xml:"ENDPOINTS"` + Extensions Extensions `xml:"EXTENSIONS"` +} + +type Services struct { + Services []*Service `xml:"SERVICE_ENDPOINT"` +} diff --git a/pkg/mentix/engine/engine.go b/pkg/mentix/engine/engine.go new file mode 100644 index 0000000000..585ecf52b6 --- /dev/null +++ b/pkg/mentix/engine/engine.go @@ -0,0 +1,219 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package engine + +import ( + "fmt" + "net/http" + "time" + + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/connectors" + "github.com/cs3org/reva/pkg/mentix/exporters" + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +const ( + runLoopSleeptime = time.Millisecond * 500 +) + +type Engine struct { + conf *config.Configuration + + meshData *meshdata.MeshData + connector connectors.Connector + exporters []exporters.Exporter + + updateInterval time.Duration +} + +func (engine *Engine) initialize(conf *config.Configuration) error { + if conf == nil { + return fmt.Errorf("no configuration provided") + } + engine.conf = conf + + // Initialize the connector that will be used to gather the mesh data + if err := engine.initConnector(); err != nil { + return fmt.Errorf("unable to initialize connector: %v", err) + } + + // Initialize the exporters + if err := engine.initExporters(); err != nil { + return fmt.Errorf("unable to initialize exporters: %v", err) + } + + // Get the update interval + duration, err := time.ParseDuration(engine.conf.UpdateInterval) + if err != nil { + // If the duration can't be parsed, default to one hour + duration = time.Hour + } + engine.updateInterval = duration + + // Create empty mesh data + engine.meshData = meshdata.New() + + return nil +} + +func (engine *Engine) initConnector() error { + // Try to get a connector with the configured ID + connector, err := connectors.FindConnector(engine.conf.Connector) + if err != nil { + return fmt.Errorf("the desired connector could be found: %v", err) + } + engine.connector = connector + + // Activate the selected connector + if err := engine.connector.Activate(engine.conf); err != nil { + return fmt.Errorf("unable to activate connector: %v", err) + } + + return nil +} + +func (engine *Engine) initExporters() error { + // Use all exporters exposed by the exporters package + exporters, err := exporters.AvailableExporters(engine.conf) + if err != nil { + return fmt.Errorf("unable to get registered exporters: %v", err) + } + var names []string + for _, exporter := range exporters { + names = append(names, exporter.GetName()) + } + engine.exporters = exporters + + // Activate all exporters + for _, exporter := range engine.exporters { + if err := exporter.Activate(engine.conf); err != nil { + return fmt.Errorf("unable to activate exporter '%v': %v", exporter.GetName(), err) + } + } + + return nil +} + +func (engine *Engine) startExporters() error { + // Start all exporters + for _, exporter := range engine.exporters { + if err := exporter.Start(); err != nil { + return fmt.Errorf("unable to start exporter '%v': %v", exporter.GetName(), err) + } + } + + return nil +} + +func (engine *Engine) stopExporters() { + // Stop all exporters + for _, exporter := range engine.exporters { + exporter.Stop() + } +} + +func (engine *Engine) destroy() { + // Stop all exporters + engine.stopExporters() +} + +func (engine *Engine) Run(stopSignal <-chan struct{}) error { + // Destroy the engine automatically after Run() has finished + defer engine.destroy() + + // Start all exporters; they will be stopped in Engine.destroy + if err := engine.startExporters(); err != nil { + return fmt.Errorf("unable to start exporters: %v", err) + } + + updateTimestamp := time.Time{} +loop: + for { + // Poll the stopSignal channel; if a signal was received, break the loop, terminating Mentix gracefully + select { + case <-stopSignal: + break loop + + default: + } + + // If enough time has passed, retrieve the latest mesh data and update it + if time.Since(updateTimestamp) >= engine.updateInterval { + meshData, err := engine.retrieveMeshData() + if err == nil { + if err := engine.applyMeshData(meshData); err != nil { + } + } else { + } + + updateTimestamp = time.Now() + } + + time.Sleep(runLoopSleeptime) + } + + return nil +} + +func (engine *Engine) retrieveMeshData() (*meshdata.MeshData, error) { + meshData, err := engine.connector.RetrieveMeshData() + if err != nil { + return nil, fmt.Errorf("retrieving mesh data failed: %v", err) + } + return meshData, nil +} + +func (engine *Engine) applyMeshData(meshData *meshdata.MeshData) error { + if !meshData.Compare(engine.meshData) { + engine.meshData = meshData + + for _, exporter := range engine.exporters { + if err := exporter.UpdateMeshData(meshData); err != nil { + return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) + } + } + } + + return nil +} + +func (engine *Engine) RequestHandler(w http.ResponseWriter, r *http.Request) { + log := appctx.GetLogger(r.Context()) + + // Ask each RequestExporter if it wants to handle the request + for _, exporter := range engine.exporters { + if reqExporter, ok := exporter.(exporters.RequestExporter); ok { + if reqExporter.WantsRequest(r) { + if err := reqExporter.HandleRequest(w, r); err != nil { + log.Err(err).Msg("error handling request") + } + } + } + } +} + +func New(conf *config.Configuration) (*Engine, error) { + engine := new(Engine) + if err := engine.initialize(conf); err != nil { + return nil, fmt.Errorf("unable to initialize engine: %v", err) + } + return engine, nil +} diff --git a/pkg/mentix/exporters/exporter.go b/pkg/mentix/exporters/exporter.go new file mode 100755 index 0000000000..b7a449a66d --- /dev/null +++ b/pkg/mentix/exporters/exporter.go @@ -0,0 +1,118 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "fmt" + "sync" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +var ( + registeredExporters = map[string]Exporter{} +) + +type Exporter interface { + Activate(conf *config.Configuration) error + Start() error + Stop() + + UpdateMeshData(*meshdata.MeshData) error + + GetName() string +} + +type BaseExporter struct { + conf *config.Configuration + + meshData *meshdata.MeshData + locker sync.RWMutex +} + +func (exporter *BaseExporter) Activate(conf *config.Configuration) error { + if conf == nil { + return fmt.Errorf("no configuration provided") + } + exporter.conf = conf + + return nil +} + +func (exporter *BaseExporter) Start() error { + return nil +} + +func (exporter *BaseExporter) Stop() { + +} + +func (exporter *BaseExporter) UpdateMeshData(meshData *meshdata.MeshData) error { + // Update the stored mesh data + if err := exporter.storeMeshData(meshData); err != nil { + return fmt.Errorf("unable to store the mesh data: %v", err) + } + + return nil +} + +func (exporter *BaseExporter) storeMeshData(meshData *meshdata.MeshData) error { + exporter.locker.Lock() + defer exporter.locker.Unlock() + + // Store the new mesh data by cloning it + exporter.meshData = meshData.Clone() + if exporter.meshData == nil { + return fmt.Errorf("unable to clone the mesh data") + } + + return nil +} + +func registerExporter(id string, exporter Exporter) { + registeredExporters[id] = exporter +} + +func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { + // Try to add all exporters configured in the environment + var exporters []Exporter + for _, exporterID := range conf.Exporters { + if exporter, ok := registeredExporters[exporterID]; ok { + exporters = append(exporters, exporter) + } else { + return nil, fmt.Errorf("no exporter with ID '%v' registered", exporterID) + } + } + + // At least one exporter must be configured + if len(exporters) == 0 { + return nil, fmt.Errorf("no exporters available") + } + + return exporters, nil +} + +func RegisteredExporterIDs() []string { + var keys []string + for k := range registeredExporters { + keys = append(keys, k) + } + return keys +} diff --git a/pkg/mentix/exporters/prometheus/types.go b/pkg/mentix/exporters/prometheus/types.go new file mode 100755 index 0000000000..d55ab47d41 --- /dev/null +++ b/pkg/mentix/exporters/prometheus/types.go @@ -0,0 +1,24 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package prometheus + +type ScrapeConfig struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels"` +} diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go new file mode 100755 index 0000000000..d2b1de2502 --- /dev/null +++ b/pkg/mentix/exporters/promfilesd.go @@ -0,0 +1,137 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exporters/prometheus" + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +type PrometheusFileSDExporter struct { + BaseExporter + + outputFilename string +} + +func (exporter *PrometheusFileSDExporter) Activate(conf *config.Configuration) error { + if err := exporter.BaseExporter.Activate(conf); err != nil { + return err + } + + // Check and store Prometheus File SD specific settings + exporter.outputFilename = conf.PrometheusFileSD.OutputFile + if len(exporter.outputFilename) == 0 { + return fmt.Errorf("no output filename configured") + } + + // Create the output directory + os.MkdirAll(filepath.Dir(exporter.outputFilename), os.ModePerm) + + return nil +} + +func (exporter *PrometheusFileSDExporter) UpdateMeshData(meshData *meshdata.MeshData) error { + if err := exporter.BaseExporter.UpdateMeshData(meshData); err != nil { + return err + } + + // Perform exporting the data asynchronously + go exporter.exportMeshData() + return nil +} + +func (exporter *PrometheusFileSDExporter) exportMeshData() { + // Data is read, so acquire a read lock + exporter.locker.RLock() + defer exporter.locker.RUnlock() + + scrapes := exporter.createScrapeConfigs() + if err := exporter.exportScrapeConfig(scrapes); err != nil { + } +} + +func (exporter *PrometheusFileSDExporter) createScrapeConfigs() []*prometheus.ScrapeConfig { + var scrapes []*prometheus.ScrapeConfig + var addScrape = func(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) { + if scrape := exporter.createScrapeConfig(site, host, endpoint); scrape != nil { + scrapes = append(scrapes, scrape) + } + } + + // Create a scrape config for each service alongside any additional endpoints + for _, site := range exporter.meshData.Sites { + for _, service := range site.Services { + if !service.IsMonitored { + continue + } + + // Add the "main" service to the scrapes + addScrape(site, service.Host, &service.ServiceEndpoint) + + for _, endpoint := range service.AdditionalEndpoints { + if endpoint.IsMonitored { + addScrape(site, service.Host, endpoint) + } + } + } + } + + return scrapes +} + +func (exporter *PrometheusFileSDExporter) createScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { + return &prometheus.ScrapeConfig{ + Targets: []string{path.Join(host, endpoint.Path)}, + Labels: map[string]string{ + "site": site.Name, + "service-type": endpoint.Type.Name, + }, + } +} + +func (exporter *PrometheusFileSDExporter) exportScrapeConfig(v interface{}) error { + // Encode scrape config as JSON + data, err := json.MarshalIndent(v, "", "\t") + if err != nil { + return fmt.Errorf("unable to marshal scrape config: %v", err) + } + + // Write the data to disk + if err := ioutil.WriteFile(exporter.outputFilename, data, os.ModePerm); err != nil { + return fmt.Errorf("unable to write scrape config '%v': %v", exporter.outputFilename, err) + } + + return nil +} + +func (exporter *PrometheusFileSDExporter) GetName() string { + return "Prometheus File SD" +} + +func init() { + registerExporter(config.ExporterID_PrometheusFileSD, &PrometheusFileSDExporter{}) +} diff --git a/pkg/mentix/exporters/reqexporter.go b/pkg/mentix/exporters/reqexporter.go new file mode 100644 index 0000000000..77800d933e --- /dev/null +++ b/pkg/mentix/exporters/reqexporter.go @@ -0,0 +1,47 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "net/http" + "strings" +) + +type RequestExporter interface { + Exporter + + WantsRequest(r *http.Request) bool + HandleRequest(resp http.ResponseWriter, req *http.Request) error +} + +type BaseRequestExporter struct { + BaseExporter + + endpoint string +} + +func (exporter *BaseRequestExporter) WantsRequest(r *http.Request) bool { + // Make sure that the endpoint starts with a / + endpoint := exporter.endpoint + if !strings.HasPrefix(endpoint, "/") { + endpoint = "/" + endpoint + } + + return r.URL.Path == endpoint +} diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exporters/webapi.go new file mode 100755 index 0000000000..df4f8a62de --- /dev/null +++ b/pkg/mentix/exporters/webapi.go @@ -0,0 +1,66 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package exporters + +import ( + "fmt" + "net/http" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exporters/webapi" +) + +type WebAPIExporter struct { + BaseRequestExporter +} + +func (exporter *WebAPIExporter) Activate(conf *config.Configuration) error { + if err := exporter.BaseExporter.Activate(conf); err != nil { + return err + } + + // Store WebAPI specific settings + exporter.endpoint = conf.WebAPI.Endpoint + + return nil +} + +func (exporter *WebAPIExporter) HandleRequest(resp http.ResponseWriter, req *http.Request) error { + // Data is read, so acquire a read lock + exporter.locker.RLock() + defer exporter.locker.RUnlock() + + data, err := webapi.HandleQuery(exporter.meshData, req.URL.Query()) + if err == nil { + resp.Write(data) + } else { + resp.Write([]byte(fmt.Sprintf("Error while serving API request: %v", err))) + return fmt.Errorf("error while serving API request: %v", err) + } + + return nil +} + +func (exporter *WebAPIExporter) GetName() string { + return "WebAPI" +} + +func init() { + registerExporter(config.ExporterID_WebAPI, &WebAPIExporter{}) +} diff --git a/pkg/mentix/exporters/webapi/query.go b/pkg/mentix/exporters/webapi/query.go new file mode 100755 index 0000000000..03799ce825 --- /dev/null +++ b/pkg/mentix/exporters/webapi/query.go @@ -0,0 +1,53 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package webapi + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + + "github.com/cs3org/reva/pkg/mentix/meshdata" +) + +const ( + queryMethodDefault = "" +) + +func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + method := params.Get("method") + switch strings.ToLower(method) { + case queryMethodDefault: + return handleDefaultQuery(meshData, params) + + default: + return []byte{}, fmt.Errorf("unknown API method '%v'", method) + } +} + +func handleDefaultQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { + // Just return the plain, unfiltered data as JSON + data, err := json.MarshalIndent(meshData, "", "\t") + if err != nil { + return []byte{}, fmt.Errorf("unable to marshal the mesh data: %v", err) + } + + return data, nil +} diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go new file mode 100644 index 0000000000..745fb5d0c4 --- /dev/null +++ b/pkg/mentix/mentix.go @@ -0,0 +1,72 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by mentlicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In mentlying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package mentix + +import ( + "fmt" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/engine" +) + +type Mentix struct { + conf *config.Configuration + + engine *engine.Engine +} + +func (ment *Mentix) initialize(conf *config.Configuration) error { + if conf == nil { + return fmt.Errorf("no configuration provided") + } + ment.conf = conf + + // Initialize the engine + engine, err := engine.New(conf) + if err != nil { + return fmt.Errorf("unable to create engine: %v", err) + } + ment.engine = engine + + return nil +} + +func (ment *Mentix) destroy() { + +} + +func (ment *Mentix) Run(stopSignal <-chan struct{}) error { + // Shut down the ment automatically after Run() has finished + defer ment.destroy() + + // The engine will do the actual work + return ment.engine.Run(stopSignal) +} + +func (ment *Mentix) Engine() *engine.Engine { + return ment.engine +} + +func New(conf *config.Configuration) (*Mentix, error) { + ment := new(Mentix) + if err := ment.initialize(conf); err != nil { + return nil, fmt.Errorf("unable to initialize Mentix: %v", err) + } + return ment, nil +} diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go new file mode 100644 index 0000000000..c53d658ca4 --- /dev/null +++ b/pkg/mentix/meshdata/meshdata.go @@ -0,0 +1,75 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package meshdata + +import ( + "encoding/json" + "fmt" +) + +type MeshData struct { + Sites []*Site + ServiceTypes []*ServiceType +} + +func (meshData *MeshData) Clear() { + meshData.Sites = nil + meshData.ServiceTypes = nil +} + +func (meshData *MeshData) ToJSON() (string, error) { + data, err := json.MarshalIndent(meshData, "", "\t") + if err != nil { + return "", fmt.Errorf("unable to marshal the mesh data: %v", err) + } + return string(data), nil +} + +func (meshData *MeshData) FromJSON(data string) error { + meshData.Clear() + if err := json.Unmarshal([]byte(data), meshData); err != nil { + return fmt.Errorf("unable to unmarshal the mesh data: %v", err) + } + return nil +} + +func (meshData *MeshData) Clone() *MeshData { + clone := &MeshData{} + + // To avoid any "deep copy" packages, use JSON en- and decoding instead + data, err := meshData.ToJSON() + if err == nil { + clone.FromJSON(data) + } + + return clone +} + +func (meshData *MeshData) Compare(other *MeshData) bool { + // To avoid cumbersome comparisons, just compare the JSON-encoded data + json1, _ := meshData.ToJSON() + json2, _ := other.ToJSON() + return json1 == json2 +} + +func New() *MeshData { + meshData := &MeshData{} + meshData.Clear() + return meshData +} diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go new file mode 100644 index 0000000000..6bdfaa0633 --- /dev/null +++ b/pkg/mentix/meshdata/service.go @@ -0,0 +1,39 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package meshdata + +type Service struct { + ServiceEndpoint + + Host string + AdditionalEndpoints []*ServiceEndpoint +} + +type ServiceType struct { + Name string + Description string +} + +type ServiceEndpoint struct { + Type *ServiceType + Name string + Path string + IsMonitored bool + Properties map[string]string +} diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go new file mode 100644 index 0000000000..bdb61f0cab --- /dev/null +++ b/pkg/mentix/meshdata/site.go @@ -0,0 +1,32 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package meshdata + +type Site struct { + Name string + FullName string + Organization string + Domain string + Homepage string + Email string + Description string + + Services []*Service + Properties map[string]string +} diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/network/utils.go new file mode 100644 index 0000000000..0d303bc57d --- /dev/null +++ b/pkg/mentix/network/utils.go @@ -0,0 +1,68 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package network + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "path" +) + +type URLParams map[string]string + +func GenerateURL(baseURL string, basePath string, params URLParams) (*url.URL, error) { + fullURL, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("unable to generate URL: base=%v, path=%v, params=%v", baseURL, basePath, params) + } + + fullURL.Path = path.Join(fullURL.Path, basePath) + + query := make(url.Values) + for key, value := range params { + query.Set(key, value) + } + fullURL.RawQuery = query.Encode() + + return fullURL, nil +} + +func AllowInsecureConnections() { + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} +} + +func ReadEndpoint(baseURL string, path string, params URLParams) ([]byte, error) { + endpointURL, err := GenerateURL(baseURL, path, params) + if err != nil { + return nil, fmt.Errorf("unable to generate endpoint URL: %v", err) + } + + // Fetch the data and read the body + resp, err := http.Get(endpointURL.String()) + if err != nil { + return nil, fmt.Errorf("unable to get data from endpoint: %v", err) + } + + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + return body, nil +} From 12d60e3cf4c335eacea8bf8481d1ec21e5231e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 May 2020 10:13:40 +0200 Subject: [PATCH 02/13] Merged the engine into the main Mentix type --- internal/http/services/mentix/mentix.go | 2 +- pkg/mentix/engine/engine.go | 219 ------------------------ pkg/mentix/mentix.go | 170 ++++++++++++++++-- 3 files changed, 159 insertions(+), 232 deletions(-) delete mode 100644 pkg/mentix/engine/engine.go diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index d5fc3290e9..65e9bdd0a2 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -55,7 +55,7 @@ func (s *svc) Unprotected() []string { func (s *svc) Handler() http.Handler { // Forward requests to the engine - return http.HandlerFunc(s.ment.Engine().RequestHandler) + return http.HandlerFunc(s.ment.RequestHandler) } func (s *svc) startBackgroundService() { diff --git a/pkg/mentix/engine/engine.go b/pkg/mentix/engine/engine.go deleted file mode 100644 index 585ecf52b6..0000000000 --- a/pkg/mentix/engine/engine.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package engine - -import ( - "fmt" - "net/http" - "time" - - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/connectors" - "github.com/cs3org/reva/pkg/mentix/exporters" - "github.com/cs3org/reva/pkg/mentix/meshdata" -) - -const ( - runLoopSleeptime = time.Millisecond * 500 -) - -type Engine struct { - conf *config.Configuration - - meshData *meshdata.MeshData - connector connectors.Connector - exporters []exporters.Exporter - - updateInterval time.Duration -} - -func (engine *Engine) initialize(conf *config.Configuration) error { - if conf == nil { - return fmt.Errorf("no configuration provided") - } - engine.conf = conf - - // Initialize the connector that will be used to gather the mesh data - if err := engine.initConnector(); err != nil { - return fmt.Errorf("unable to initialize connector: %v", err) - } - - // Initialize the exporters - if err := engine.initExporters(); err != nil { - return fmt.Errorf("unable to initialize exporters: %v", err) - } - - // Get the update interval - duration, err := time.ParseDuration(engine.conf.UpdateInterval) - if err != nil { - // If the duration can't be parsed, default to one hour - duration = time.Hour - } - engine.updateInterval = duration - - // Create empty mesh data - engine.meshData = meshdata.New() - - return nil -} - -func (engine *Engine) initConnector() error { - // Try to get a connector with the configured ID - connector, err := connectors.FindConnector(engine.conf.Connector) - if err != nil { - return fmt.Errorf("the desired connector could be found: %v", err) - } - engine.connector = connector - - // Activate the selected connector - if err := engine.connector.Activate(engine.conf); err != nil { - return fmt.Errorf("unable to activate connector: %v", err) - } - - return nil -} - -func (engine *Engine) initExporters() error { - // Use all exporters exposed by the exporters package - exporters, err := exporters.AvailableExporters(engine.conf) - if err != nil { - return fmt.Errorf("unable to get registered exporters: %v", err) - } - var names []string - for _, exporter := range exporters { - names = append(names, exporter.GetName()) - } - engine.exporters = exporters - - // Activate all exporters - for _, exporter := range engine.exporters { - if err := exporter.Activate(engine.conf); err != nil { - return fmt.Errorf("unable to activate exporter '%v': %v", exporter.GetName(), err) - } - } - - return nil -} - -func (engine *Engine) startExporters() error { - // Start all exporters - for _, exporter := range engine.exporters { - if err := exporter.Start(); err != nil { - return fmt.Errorf("unable to start exporter '%v': %v", exporter.GetName(), err) - } - } - - return nil -} - -func (engine *Engine) stopExporters() { - // Stop all exporters - for _, exporter := range engine.exporters { - exporter.Stop() - } -} - -func (engine *Engine) destroy() { - // Stop all exporters - engine.stopExporters() -} - -func (engine *Engine) Run(stopSignal <-chan struct{}) error { - // Destroy the engine automatically after Run() has finished - defer engine.destroy() - - // Start all exporters; they will be stopped in Engine.destroy - if err := engine.startExporters(); err != nil { - return fmt.Errorf("unable to start exporters: %v", err) - } - - updateTimestamp := time.Time{} -loop: - for { - // Poll the stopSignal channel; if a signal was received, break the loop, terminating Mentix gracefully - select { - case <-stopSignal: - break loop - - default: - } - - // If enough time has passed, retrieve the latest mesh data and update it - if time.Since(updateTimestamp) >= engine.updateInterval { - meshData, err := engine.retrieveMeshData() - if err == nil { - if err := engine.applyMeshData(meshData); err != nil { - } - } else { - } - - updateTimestamp = time.Now() - } - - time.Sleep(runLoopSleeptime) - } - - return nil -} - -func (engine *Engine) retrieveMeshData() (*meshdata.MeshData, error) { - meshData, err := engine.connector.RetrieveMeshData() - if err != nil { - return nil, fmt.Errorf("retrieving mesh data failed: %v", err) - } - return meshData, nil -} - -func (engine *Engine) applyMeshData(meshData *meshdata.MeshData) error { - if !meshData.Compare(engine.meshData) { - engine.meshData = meshData - - for _, exporter := range engine.exporters { - if err := exporter.UpdateMeshData(meshData); err != nil { - return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) - } - } - } - - return nil -} - -func (engine *Engine) RequestHandler(w http.ResponseWriter, r *http.Request) { - log := appctx.GetLogger(r.Context()) - - // Ask each RequestExporter if it wants to handle the request - for _, exporter := range engine.exporters { - if reqExporter, ok := exporter.(exporters.RequestExporter); ok { - if reqExporter.WantsRequest(r) { - if err := reqExporter.HandleRequest(w, r); err != nil { - log.Err(err).Msg("error handling request") - } - } - } - } -} - -func New(conf *config.Configuration) (*Engine, error) { - engine := new(Engine) - if err := engine.initialize(conf); err != nil { - return nil, fmt.Errorf("unable to initialize engine: %v", err) - } - return engine, nil -} diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 745fb5d0c4..9540099748 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -20,47 +20,193 @@ package mentix import ( "fmt" + "net/http" + "time" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/engine" + "github.com/cs3org/reva/pkg/mentix/connectors" + "github.com/cs3org/reva/pkg/mentix/exporters" + "github.com/cs3org/reva/pkg/mentix/meshdata" ) type Mentix struct { conf *config.Configuration - engine *engine.Engine + meshData *meshdata.MeshData + connector connectors.Connector + exporters []exporters.Exporter + + updateInterval time.Duration } +const ( + runLoopSleeptime = time.Millisecond * 500 +) + func (ment *Mentix) initialize(conf *config.Configuration) error { if conf == nil { return fmt.Errorf("no configuration provided") } ment.conf = conf - // Initialize the engine - engine, err := engine.New(conf) + // Initialize the connector that will be used to gather the mesh data + if err := ment.initConnector(); err != nil { + return fmt.Errorf("unable to initialize connector: %v", err) + } + + // Initialize the exporters + if err := ment.initExporters(); err != nil { + return fmt.Errorf("unable to initialize exporters: %v", err) + } + + // Get the update interval + duration, err := time.ParseDuration(ment.conf.UpdateInterval) if err != nil { - return fmt.Errorf("unable to create engine: %v", err) + // If the duration can't be parsed, default to one hour + duration = time.Hour } - ment.engine = engine + ment.updateInterval = duration + + // Create empty mesh data + ment.meshData = meshdata.New() return nil } -func (ment *Mentix) destroy() { +func (ment *Mentix) initConnector() error { + // Try to get a connector with the configured ID + connector, err := connectors.FindConnector(ment.conf.Connector) + if err != nil { + return fmt.Errorf("the desired connector could be found: %v", err) + } + ment.connector = connector + + // Activate the selected connector + if err := ment.connector.Activate(ment.conf); err != nil { + return fmt.Errorf("unable to activate connector: %v", err) + } + + return nil +} + +func (ment *Mentix) initExporters() error { + // Use all exporters exposed by the exporters package + exporters, err := exporters.AvailableExporters(ment.conf) + if err != nil { + return fmt.Errorf("unable to get registered exporters: %v", err) + } + var names []string + for _, exporter := range exporters { + names = append(names, exporter.GetName()) + } + ment.exporters = exporters + + // Activate all exporters + for _, exporter := range ment.exporters { + if err := exporter.Activate(ment.conf); err != nil { + return fmt.Errorf("unable to activate exporter '%v': %v", exporter.GetName(), err) + } + } + + return nil +} +func (ment *Mentix) startExporters() error { + // Start all exporters + for _, exporter := range ment.exporters { + if err := exporter.Start(); err != nil { + return fmt.Errorf("unable to start exporter '%v': %v", exporter.GetName(), err) + } + } + + return nil +} + +func (ment *Mentix) stopExporters() { + // Stop all exporters + for _, exporter := range ment.exporters { + exporter.Stop() + } +} + +func (ment *Mentix) destroy() { + // Stop all exporters + ment.stopExporters() } func (ment *Mentix) Run(stopSignal <-chan struct{}) error { - // Shut down the ment automatically after Run() has finished defer ment.destroy() - // The engine will do the actual work - return ment.engine.Run(stopSignal) + // Start all exporters; they will be stopped in ment.destroy + if err := ment.startExporters(); err != nil { + return fmt.Errorf("unable to start exporters: %v", err) + } + + updateTimestamp := time.Time{} +loop: + for { + // Poll the stopSignal channel; if a signal was received, break the loop, terminating Mentix gracefully + select { + case <-stopSignal: + break loop + + default: + } + + // If enough time has passed, retrieve the latest mesh data and update it + if time.Since(updateTimestamp) >= ment.updateInterval { + meshData, err := ment.retrieveMeshData() + if err == nil { + if err := ment.applyMeshData(meshData); err != nil { + } + } else { + } + + updateTimestamp = time.Now() + } + + time.Sleep(runLoopSleeptime) + } + + return nil } -func (ment *Mentix) Engine() *engine.Engine { - return ment.engine +func (ment *Mentix) retrieveMeshData() (*meshdata.MeshData, error) { + meshData, err := ment.connector.RetrieveMeshData() + if err != nil { + return nil, fmt.Errorf("retrieving mesh data failed: %v", err) + } + return meshData, nil +} + +func (ment *Mentix) applyMeshData(meshData *meshdata.MeshData) error { + if !meshData.Compare(ment.meshData) { + ment.meshData = meshData + + for _, exporter := range ment.exporters { + if err := exporter.UpdateMeshData(meshData); err != nil { + return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) + } + } + } + + return nil +} + +func (ment *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { + log := appctx.GetLogger(r.Context()) + + // Ask each RequestExporter if it wants to handle the request + for _, exporter := range ment.exporters { + if reqExporter, ok := exporter.(exporters.RequestExporter); ok { + if reqExporter.WantsRequest(r) { + if err := reqExporter.HandleRequest(w, r); err != nil { + log.Err(err).Msg("error handling request") + } + } + } + } } func New(conf *config.Configuration) (*Mentix, error) { From a6f68ba51e95ad039b5ad6cc26d99d050bccd75c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 May 2020 10:39:56 +0200 Subject: [PATCH 03/13] Cleanup --- internal/http/services/mentix/mentix.go | 50 ++++++----- pkg/mentix/exporters/reqexporter.go | 10 ++- pkg/mentix/mentix.go | 111 +++++++++++++----------- pkg/mentix/network/utils.go | 19 ++-- 4 files changed, 103 insertions(+), 87 deletions(-) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 65e9bdd0a2..97e1d8fdbe 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -32,15 +32,19 @@ import ( type svc struct { conf *config.Configuration - ment *mentix.Mentix + mntx *mentix.Mentix - stop chan struct{} + stopSignal chan struct{} } +const ( + serviceName = "mentix" +) + func (s *svc) Close() error { - // Trigger and close the stop signal channel to stop Mentix - s.stop <- struct{}{} - close(s.stop) + // Trigger and close the stopSignal signal channel to stop Mentix + s.stopSignal <- struct{}{} + close(s.stopSignal) return nil } @@ -50,17 +54,22 @@ func (s *svc) Prefix() string { } func (s *svc) Unprotected() []string { - return []string{"/"} + // Get all endpoints exposed by the RequestExporters + var endpoints []string + for _, exporter := range s.mntx.GetRequestExporters() { + endpoints = append(endpoints, exporter.Endpoint()) + } + return endpoints } func (s *svc) Handler() http.Handler { - // Forward requests to the engine - return http.HandlerFunc(s.ment.RequestHandler) + // Forward requests to Mentix + return http.HandlerFunc(s.mntx.RequestHandler) } func (s *svc) startBackgroundService() { // Just run Mentix in the background - go s.ment.Run(s.stop) + go s.mntx.Run(s.stopSignal) } func parseConfig(m map[string]interface{}) (*config.Configuration, error) { @@ -74,16 +83,13 @@ func parseConfig(m map[string]interface{}) (*config.Configuration, error) { func defaultConfig() *config.Configuration { conf := &config.Configuration{} - // General settings - conf.Prefix = "mentix" - conf.Connector = config.ConnectorID_GOCDB - conf.Exporters = exporters.RegisteredExporterIDs() - conf.UpdateInterval = "1h" + conf.Prefix = serviceName - // GOCDB settings - conf.GOCDB.Scope = "SM" + conf.Connector = config.ConnectorID_GOCDB // Use GOCDB + conf.Exporters = exporters.RegisteredExporterIDs() // Enable all exporters + conf.UpdateInterval = "1h" // Update once per hour - // WebAPI settings + conf.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future conf.WebAPI.Endpoint = "/" return conf @@ -98,21 +104,21 @@ func New(m map[string]interface{}) (global.Service, error) { } // Create the Mentix instance - ment, err := mentix.New(conf) + mntx, err := mentix.New(conf) if err != nil { return nil, errors.Wrap(err, "mentix: error creating instance") } // Create the service and start its background activity s := &svc{ - conf: conf, - ment: ment, - stop: make(chan struct{}), + conf: conf, + mntx: mntx, + stopSignal: make(chan struct{}), } s.startBackgroundService() return s, nil } func init() { - global.Register("mentix", New) + global.Register(serviceName, New) } diff --git a/pkg/mentix/exporters/reqexporter.go b/pkg/mentix/exporters/reqexporter.go index 77800d933e..69d0d54fb9 100644 --- a/pkg/mentix/exporters/reqexporter.go +++ b/pkg/mentix/exporters/reqexporter.go @@ -26,6 +26,7 @@ import ( type RequestExporter interface { Exporter + Endpoint() string WantsRequest(r *http.Request) bool HandleRequest(resp http.ResponseWriter, req *http.Request) error } @@ -36,12 +37,15 @@ type BaseRequestExporter struct { endpoint string } -func (exporter *BaseRequestExporter) WantsRequest(r *http.Request) bool { - // Make sure that the endpoint starts with a / +func (exporter *BaseRequestExporter) Endpoint() string { + // Ensure that the endpoint starts with a / endpoint := exporter.endpoint if !strings.HasPrefix(endpoint, "/") { endpoint = "/" + endpoint } + return strings.TrimSpace(endpoint) +} - return r.URL.Path == endpoint +func (exporter *BaseRequestExporter) WantsRequest(r *http.Request) bool { + return r.URL.Path == exporter.Endpoint() } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 9540099748..79b75e4adf 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -44,55 +44,55 @@ const ( runLoopSleeptime = time.Millisecond * 500 ) -func (ment *Mentix) initialize(conf *config.Configuration) error { +func (mntx *Mentix) initialize(conf *config.Configuration) error { if conf == nil { return fmt.Errorf("no configuration provided") } - ment.conf = conf + mntx.conf = conf // Initialize the connector that will be used to gather the mesh data - if err := ment.initConnector(); err != nil { + if err := mntx.initConnector(); err != nil { return fmt.Errorf("unable to initialize connector: %v", err) } // Initialize the exporters - if err := ment.initExporters(); err != nil { + if err := mntx.initExporters(); err != nil { return fmt.Errorf("unable to initialize exporters: %v", err) } // Get the update interval - duration, err := time.ParseDuration(ment.conf.UpdateInterval) + duration, err := time.ParseDuration(mntx.conf.UpdateInterval) if err != nil { // If the duration can't be parsed, default to one hour duration = time.Hour } - ment.updateInterval = duration + mntx.updateInterval = duration // Create empty mesh data - ment.meshData = meshdata.New() + mntx.meshData = meshdata.New() return nil } -func (ment *Mentix) initConnector() error { +func (mntx *Mentix) initConnector() error { // Try to get a connector with the configured ID - connector, err := connectors.FindConnector(ment.conf.Connector) + connector, err := connectors.FindConnector(mntx.conf.Connector) if err != nil { return fmt.Errorf("the desired connector could be found: %v", err) } - ment.connector = connector + mntx.connector = connector // Activate the selected connector - if err := ment.connector.Activate(ment.conf); err != nil { + if err := mntx.connector.Activate(mntx.conf); err != nil { return fmt.Errorf("unable to activate connector: %v", err) } return nil } -func (ment *Mentix) initExporters() error { +func (mntx *Mentix) initExporters() error { // Use all exporters exposed by the exporters package - exporters, err := exporters.AvailableExporters(ment.conf) + exporters, err := exporters.AvailableExporters(mntx.conf) if err != nil { return fmt.Errorf("unable to get registered exporters: %v", err) } @@ -100,11 +100,11 @@ func (ment *Mentix) initExporters() error { for _, exporter := range exporters { names = append(names, exporter.GetName()) } - ment.exporters = exporters + mntx.exporters = exporters // Activate all exporters - for _, exporter := range ment.exporters { - if err := exporter.Activate(ment.conf); err != nil { + for _, exporter := range mntx.exporters { + if err := exporter.Activate(mntx.conf); err != nil { return fmt.Errorf("unable to activate exporter '%v': %v", exporter.GetName(), err) } } @@ -112,9 +112,9 @@ func (ment *Mentix) initExporters() error { return nil } -func (ment *Mentix) startExporters() error { +func (mntx *Mentix) startExporters() error { // Start all exporters - for _, exporter := range ment.exporters { + for _, exporter := range mntx.exporters { if err := exporter.Start(); err != nil { return fmt.Errorf("unable to start exporter '%v': %v", exporter.GetName(), err) } @@ -123,42 +123,44 @@ func (ment *Mentix) startExporters() error { return nil } -func (ment *Mentix) stopExporters() { +func (mntx *Mentix) stopExporters() { // Stop all exporters - for _, exporter := range ment.exporters { + for _, exporter := range mntx.exporters { exporter.Stop() } } -func (ment *Mentix) destroy() { +func (mntx *Mentix) destroy() { // Stop all exporters - ment.stopExporters() + mntx.stopExporters() } -func (ment *Mentix) Run(stopSignal <-chan struct{}) error { - defer ment.destroy() +func (mntx *Mentix) Run(stopSignal <-chan struct{}) error { + defer mntx.destroy() - // Start all exporters; they will be stopped in ment.destroy - if err := ment.startExporters(); err != nil { + // Start all exporters; they will be stopped in mntx.destroy + if err := mntx.startExporters(); err != nil { return fmt.Errorf("unable to start exporters: %v", err) } updateTimestamp := time.Time{} loop: for { - // Poll the stopSignal channel; if a signal was received, break the loop, terminating Mentix gracefully - select { - case <-stopSignal: - break loop + if stopSignal != nil { + // Poll the stopSignal channel; if a signal was received, break the loop, terminating Mentix gracefully + select { + case <-stopSignal: + break loop - default: + default: + } } // If enough time has passed, retrieve the latest mesh data and update it - if time.Since(updateTimestamp) >= ment.updateInterval { - meshData, err := ment.retrieveMeshData() + if time.Since(updateTimestamp) >= mntx.updateInterval { + meshData, err := mntx.retrieveMeshData() if err == nil { - if err := ment.applyMeshData(meshData); err != nil { + if err := mntx.applyMeshData(meshData); err != nil { } } else { } @@ -172,19 +174,19 @@ loop: return nil } -func (ment *Mentix) retrieveMeshData() (*meshdata.MeshData, error) { - meshData, err := ment.connector.RetrieveMeshData() +func (mntx *Mentix) retrieveMeshData() (*meshdata.MeshData, error) { + meshData, err := mntx.connector.RetrieveMeshData() if err != nil { return nil, fmt.Errorf("retrieving mesh data failed: %v", err) } return meshData, nil } -func (ment *Mentix) applyMeshData(meshData *meshdata.MeshData) error { - if !meshData.Compare(ment.meshData) { - ment.meshData = meshData +func (mntx *Mentix) applyMeshData(meshData *meshdata.MeshData) error { + if !meshData.Compare(mntx.meshData) { + mntx.meshData = meshData - for _, exporter := range ment.exporters { + for _, exporter := range mntx.exporters { if err := exporter.UpdateMeshData(meshData); err != nil { return fmt.Errorf("unable to update mesh data on exporter '%v': %v", exporter.GetName(), err) } @@ -194,25 +196,34 @@ func (ment *Mentix) applyMeshData(meshData *meshdata.MeshData) error { return nil } -func (ment *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { +func (mntx *Mentix) GetRequestExporters() []exporters.RequestExporter { + // Return all exporters that implement the RequestExporter interface + var reqExporters []exporters.RequestExporter + for _, exporter := range mntx.exporters { + if reqExporter, ok := exporter.(exporters.RequestExporter); ok { + reqExporters = append(reqExporters, reqExporter) + } + } + return reqExporters +} + +func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { log := appctx.GetLogger(r.Context()) // Ask each RequestExporter if it wants to handle the request - for _, exporter := range ment.exporters { - if reqExporter, ok := exporter.(exporters.RequestExporter); ok { - if reqExporter.WantsRequest(r) { - if err := reqExporter.HandleRequest(w, r); err != nil { - log.Err(err).Msg("error handling request") - } + for _, exporter := range mntx.GetRequestExporters() { + if exporter.WantsRequest(r) { + if err := exporter.HandleRequest(w, r); err != nil { + log.Err(err).Msg("error handling request") } } } } func New(conf *config.Configuration) (*Mentix, error) { - ment := new(Mentix) - if err := ment.initialize(conf); err != nil { + mntx := new(Mentix) + if err := mntx.initialize(conf); err != nil { return nil, fmt.Errorf("unable to initialize Mentix: %v", err) } - return ment, nil + return mntx, nil } diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/network/utils.go index 0d303bc57d..75f8e4240c 100644 --- a/pkg/mentix/network/utils.go +++ b/pkg/mentix/network/utils.go @@ -19,23 +19,22 @@ package network import ( - "crypto/tls" "fmt" "io/ioutil" "net/http" "net/url" - "path" + p "path" ) type URLParams map[string]string -func GenerateURL(baseURL string, basePath string, params URLParams) (*url.URL, error) { - fullURL, err := url.Parse(baseURL) +func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { + fullURL, err := url.Parse(host) if err != nil { - return nil, fmt.Errorf("unable to generate URL: base=%v, path=%v, params=%v", baseURL, basePath, params) + return nil, fmt.Errorf("unable to generate URL: base=%v, path=%v, params=%v", host, path, params) } - fullURL.Path = path.Join(fullURL.Path, basePath) + fullURL.Path = p.Join(fullURL.Path, path) query := make(url.Values) for key, value := range params { @@ -46,12 +45,8 @@ func GenerateURL(baseURL string, basePath string, params URLParams) (*url.URL, e return fullURL, nil } -func AllowInsecureConnections() { - http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} -} - -func ReadEndpoint(baseURL string, path string, params URLParams) ([]byte, error) { - endpointURL, err := GenerateURL(baseURL, path, params) +func ReadEndpoint(host string, path string, params URLParams) ([]byte, error) { + endpointURL, err := GenerateURL(host, path, params) if err != nil { return nil, fmt.Errorf("unable to generate endpoint URL: %v", err) } From 41e7f01272717e7fde8ab6f776406ebf0cdb37e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 15 May 2020 10:47:47 +0200 Subject: [PATCH 04/13] Configuration names adjusted to Reva naming style --- examples/mentix/mentix.toml | 6 +++--- internal/http/services/mentix/mentix.go | 6 +++--- pkg/mentix/config/config.go | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 4870e1403e..68e0681cd5 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -10,7 +10,7 @@ connector = "gocdb" exporters = ["webapi"] # Enable the Prometheus File Service Discovery: # exporters = ["webapi", "prom-filesd"] -update-interval = "15m" +update_interval = "15m" [http.services.mentix.gocdb] address = "http://sciencemesh-test.uni-muenster.de" @@ -19,6 +19,6 @@ address = "http://sciencemesh-test.uni-muenster.de" endpoint = "/" # Configure the Prometheus File Service Discovery: -# [http.services.mentix.prom-filesd] +# [http.services.mentix.prom_filesd] # Prometheus must be configured to read the following file: -# output-file = "/usr/share/prom/sciencemesh_services.json" +# output_file = "/usr/share/prom/sciencemesh_services.json" diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 97e1d8fdbe..9c6875dcdd 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -19,10 +19,10 @@ package mentix import ( + "fmt" "net/http" "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" "github.com/cs3org/reva/pkg/mentix" "github.com/cs3org/reva/pkg/mentix/config" @@ -75,7 +75,7 @@ func (s *svc) startBackgroundService() { func parseConfig(m map[string]interface{}) (*config.Configuration, error) { cfg := defaultConfig() if err := mapstructure.Decode(m, &cfg); err != nil { - return nil, errors.Wrap(err, "mentix: error decoding configuration") + return nil, fmt.Errorf("error decoding configuration: %v", err) } return cfg, nil } @@ -106,7 +106,7 @@ func New(m map[string]interface{}) (global.Service, error) { // Create the Mentix instance mntx, err := mentix.New(conf) if err != nil { - return nil, errors.Wrap(err, "mentix: error creating instance") + return nil, fmt.Errorf("error creating Mentix: %v", err) } // Create the service and start its background activity diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 595cc59a93..d24f9470c6 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -23,7 +23,7 @@ type Configuration struct { Connector string `mapstructure:"connector"` Exporters []string `mapstructure:"exporters"` - UpdateInterval string `mapstructure:"update-interval"` + UpdateInterval string `mapstructure:"update_interval"` GOCDB struct { Address string `mapstructure:"address"` @@ -35,6 +35,6 @@ type Configuration struct { } `yaml:"webapi"` PrometheusFileSD struct { - OutputFile string `mapstructure:"output-file"` - } `mapstructure:"prom-filesd"` + OutputFile string `mapstructure:"output_file"` + } `mapstructure:"prom_filesd"` } From 2d4a96ef17d9db366e7a96908deb50a35896021c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 18 May 2020 13:03:25 +0200 Subject: [PATCH 05/13] The HTTP service creator function now receives a handle to the global message log --- internal/http/services/datagateway/datagateway.go | 3 ++- internal/http/services/dataprovider/dataprovider.go | 3 ++- internal/http/services/helloworld/helloworld.go | 3 ++- internal/http/services/meshdirectory/meshdirectory.go | 3 ++- internal/http/services/ocmd/ocmd.go | 3 ++- internal/http/services/oidcprovider/oidcprovider.go | 3 ++- internal/http/services/owncloud/ocdav/ocdav.go | 3 ++- internal/http/services/owncloud/ocs/ocs.go | 3 ++- internal/http/services/prometheus/prometheus.go | 3 ++- internal/http/services/wellknown/wellknown.go | 3 ++- pkg/rhttp/global/global.go | 8 ++++++-- pkg/rhttp/rhttp.go | 2 +- 12 files changed, 27 insertions(+), 13 deletions(-) diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 27c6937f97..0a9e119465 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -31,6 +31,7 @@ import ( "github.com/dgrijalva/jwt-go" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" + "github.com/rs/zerolog" ) const ( @@ -57,7 +58,7 @@ type svc struct { } // New returns a new datagateway -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 519755283e..6bfde5e1f5 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -27,6 +27,7 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" tusd "github.com/tus/tusd/pkg/handler" ) @@ -47,7 +48,7 @@ type svc struct { } // New returns a new datasvc -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/helloworld/helloworld.go b/internal/http/services/helloworld/helloworld.go index 0d8dc6872e..ba12a2d2ad 100644 --- a/internal/http/services/helloworld/helloworld.go +++ b/internal/http/services/helloworld/helloworld.go @@ -24,6 +24,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" ) func init() { @@ -31,7 +32,7 @@ func init() { } // New returns a new helloworld service -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/meshdirectory/meshdirectory.go b/internal/http/services/meshdirectory/meshdirectory.go index b7e4eeda36..3e7df19d45 100644 --- a/internal/http/services/meshdirectory/meshdirectory.go +++ b/internal/http/services/meshdirectory/meshdirectory.go @@ -28,6 +28,7 @@ import ( "github.com/cs3org/reva/pkg/meshdirectory" "github.com/cs3org/reva/pkg/meshdirectory/manager/registry" "github.com/pkg/errors" + "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" @@ -67,7 +68,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a new Mesh Directory HTTP service -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/http/services/ocmd/ocmd.go b/internal/http/services/ocmd/ocmd.go index 6275f07bde..e819cefa3c 100644 --- a/internal/http/services/ocmd/ocmd.go +++ b/internal/http/services/ocmd/ocmd.go @@ -26,6 +26,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/sharedconf" "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" ) // Config holds the config options that need to be passed down to all ocdav handlers @@ -49,7 +50,7 @@ func init() { } // New returns a new ocmd object -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &Config{} diff --git a/internal/http/services/oidcprovider/oidcprovider.go b/internal/http/services/oidcprovider/oidcprovider.go index 624e0e2e3f..d2be4d47c4 100644 --- a/internal/http/services/oidcprovider/oidcprovider.go +++ b/internal/http/services/oidcprovider/oidcprovider.go @@ -36,6 +36,7 @@ import ( "github.com/ory/fosite/storage" "github.com/ory/fosite/token/jwt" "github.com/pkg/errors" + "github.com/rs/zerolog" ) func init() { @@ -70,7 +71,7 @@ type svc struct { } // New returns a new oidcprovidersvc -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { c := &config{} if err := mapstructure.Decode(m, c); err != nil { return nil, err diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 192bf8ef76..4656b5a5b7 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -37,6 +37,7 @@ import ( "github.com/cs3org/reva/pkg/storage/templates" ctxuser "github.com/cs3org/reva/pkg/user" "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" ) type ctxKey int @@ -73,7 +74,7 @@ type svc struct { } // New returns a new ocdav -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &Config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/owncloud/ocs/ocs.go b/internal/http/services/owncloud/ocs/ocs.go index 0481da58ec..ffdb20e76a 100644 --- a/internal/http/services/owncloud/ocs/ocs.go +++ b/internal/http/services/owncloud/ocs/ocs.go @@ -28,6 +28,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/sharedconf" "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" ) const ( @@ -45,7 +46,7 @@ type svc struct { } // New returns a new capabilitiessvc -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &config.Config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index 738c14541c..456c8163fd 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -25,6 +25,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" + "github.com/rs/zerolog" "go.opencensus.io/stats/view" ) @@ -33,7 +34,7 @@ func init() { } // New returns a new prometheus service -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/wellknown/wellknown.go b/internal/http/services/wellknown/wellknown.go index 7751e11d71..a2f3432ff5 100644 --- a/internal/http/services/wellknown/wellknown.go +++ b/internal/http/services/wellknown/wellknown.go @@ -25,6 +25,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" ) func init() { @@ -49,7 +50,7 @@ type svc struct { } // New returns a new webuisvc -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/pkg/rhttp/global/global.go b/pkg/rhttp/global/global.go index 2f90128984..6404eab2f0 100644 --- a/pkg/rhttp/global/global.go +++ b/pkg/rhttp/global/global.go @@ -18,7 +18,11 @@ package global -import "net/http" +import ( + "net/http" + + "github.com/rs/zerolog" +) // NewMiddlewares contains all the registered new middleware functions. var NewMiddlewares = map[string]NewMiddleware{} @@ -43,7 +47,7 @@ func Register(name string, newFunc NewService) { } // NewService is the function that HTTP services need to register at init time. -type NewService func(conf map[string]interface{}) (Service, error) +type NewService func(conf map[string]interface{}, log *zerolog.Logger) (Service, error) // Service represents a HTTP service. type Service interface { diff --git a/pkg/rhttp/rhttp.go b/pkg/rhttp/rhttp.go index adb64001ee..628b5f0c76 100644 --- a/pkg/rhttp/rhttp.go +++ b/pkg/rhttp/rhttp.go @@ -189,7 +189,7 @@ func (s *Server) registerServices() error { for svcName := range s.conf.Services { if s.isServiceEnabled(svcName) { newFunc := global.Services[svcName] - svc, err := newFunc(s.conf.Services[svcName]) + svc, err := newFunc(s.conf.Services[svcName], &s.log) if err != nil { err = errors.Wrapf(err, "http service %s could not be started,", svcName) return err From 174dbb3b6da0dae7cf598f974da5dbff4856c18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 18 May 2020 13:03:49 +0200 Subject: [PATCH 06/13] Added logging --- examples/mentix/mentix.toml | 2 +- internal/http/services/mentix/mentix.go | 53 ++++++++++++++++--------- pkg/mentix/config/ids.go | 2 +- pkg/mentix/connectors/connector.go | 12 +++++- pkg/mentix/connectors/gocdb.go | 6 ++- pkg/mentix/exporters/exporter.go | 12 +++++- pkg/mentix/exporters/promfilesd.go | 9 ++++- pkg/mentix/exporters/webapi.go | 6 ++- pkg/mentix/mentix.go | 25 +++++++++--- pkg/mentix/network/utils.go | 3 ++ 10 files changed, 95 insertions(+), 35 deletions(-) diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 68e0681cd5..760b33d7e6 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -9,7 +9,7 @@ enabled_services = ["mentix"] connector = "gocdb" exporters = ["webapi"] # Enable the Prometheus File Service Discovery: -# exporters = ["webapi", "prom-filesd"] +# exporters = ["webapi", "prom_filesd"] update_interval = "15m" [http.services.mentix.gocdb] diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 9c6875dcdd..75049dab8c 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -19,10 +19,11 @@ package mentix import ( - "fmt" "net/http" "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/rs/zerolog" "github.com/cs3org/reva/pkg/mentix" "github.com/cs3org/reva/pkg/mentix/config" @@ -30,9 +31,14 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" ) +func init() { + global.Register(serviceName, New) +} + type svc struct { conf *config.Configuration mntx *mentix.Mentix + log *zerolog.Logger stopSignal chan struct{} } @@ -73,30 +79,44 @@ func (s *svc) startBackgroundService() { } func parseConfig(m map[string]interface{}) (*config.Configuration, error) { - cfg := defaultConfig() + cfg := &config.Configuration{} if err := mapstructure.Decode(m, &cfg); err != nil { - return nil, fmt.Errorf("error decoding configuration: %v", err) + return nil, errors.Wrap(err, "mentix: error decoding configuration") } + applyDefaultConfig(cfg) return cfg, nil } -func defaultConfig() *config.Configuration { +func applyDefaultConfig(*config.Configuration) { conf := &config.Configuration{} - conf.Prefix = serviceName + if conf.Prefix == "" { + conf.Prefix = serviceName + } + + if conf.Connector == "" { + conf.Connector = config.ConnectorID_GOCDB // Use GOCDB + } + + if len(conf.Exporters) == 0 { + conf.Exporters = exporters.RegisteredExporterIDs() // Enable all exporters + } - conf.Connector = config.ConnectorID_GOCDB // Use GOCDB - conf.Exporters = exporters.RegisteredExporterIDs() // Enable all exporters - conf.UpdateInterval = "1h" // Update once per hour + if conf.UpdateInterval == "" { + conf.UpdateInterval = "1h" // Update once per hour + } - conf.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future - conf.WebAPI.Endpoint = "/" + if conf.GOCDB.Scope == "" { + conf.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future + } - return conf + if conf.WebAPI.Endpoint == "" { + conf.WebAPI.Endpoint = "/" + } } // New returns a new Mentix service -func New(m map[string]interface{}) (global.Service, error) { +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { // Prepare the configuration conf, err := parseConfig(m) if err != nil { @@ -104,21 +124,18 @@ func New(m map[string]interface{}) (global.Service, error) { } // Create the Mentix instance - mntx, err := mentix.New(conf) + mntx, err := mentix.New(conf, log) if err != nil { - return nil, fmt.Errorf("error creating Mentix: %v", err) + return nil, errors.Wrap(err, "mentix: error creating Mentix") } // Create the service and start its background activity s := &svc{ conf: conf, mntx: mntx, + log: log, stopSignal: make(chan struct{}), } s.startBackgroundService() return s, nil } - -func init() { - global.Register(serviceName, New) -} diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index ec3224fbd1..f9ba7b8f23 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -24,5 +24,5 @@ const ( const ( ExporterID_WebAPI = "webapi" - ExporterID_PrometheusFileSD = "prom-filesd" + ExporterID_PrometheusFileSD = "prom_filesd" ) diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index 5d8ce1bf8c..3aa56dce4e 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -22,6 +22,8 @@ import ( "fmt" "strings" + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -31,7 +33,7 @@ var ( ) type Connector interface { - Activate(conf *config.Configuration) error + Activate(conf *config.Configuration, log *zerolog.Logger) error RetrieveMeshData() (*meshdata.MeshData, error) GetName() string @@ -39,14 +41,20 @@ type Connector interface { type BaseConnector struct { conf *config.Configuration + log *zerolog.Logger } -func (connector *BaseConnector) Activate(conf *config.Configuration) error { +func (connector *BaseConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { if conf == nil { return fmt.Errorf("no configuration provided") } connector.conf = conf + if log == nil { + return fmt.Errorf("no logger provided") + } + connector.log = log + return nil } diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 5c598d5461..320bfbb119 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -23,6 +23,8 @@ import ( "fmt" "strings" + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/connectors/gocdb" "github.com/cs3org/reva/pkg/mentix/meshdata" @@ -35,8 +37,8 @@ type GOCDBConnector struct { gocdbAddress string } -func (connector *GOCDBConnector) Activate(conf *config.Configuration) error { - if err := connector.BaseConnector.Activate(conf); err != nil { +func (connector *GOCDBConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := connector.BaseConnector.Activate(conf, log); err != nil { return err } diff --git a/pkg/mentix/exporters/exporter.go b/pkg/mentix/exporters/exporter.go index b7a449a66d..e1de66229e 100755 --- a/pkg/mentix/exporters/exporter.go +++ b/pkg/mentix/exporters/exporter.go @@ -22,6 +22,8 @@ import ( "fmt" "sync" + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/meshdata" ) @@ -31,7 +33,7 @@ var ( ) type Exporter interface { - Activate(conf *config.Configuration) error + Activate(conf *config.Configuration, log *zerolog.Logger) error Start() error Stop() @@ -42,17 +44,23 @@ type Exporter interface { type BaseExporter struct { conf *config.Configuration + log *zerolog.Logger meshData *meshdata.MeshData locker sync.RWMutex } -func (exporter *BaseExporter) Activate(conf *config.Configuration) error { +func (exporter *BaseExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { if conf == nil { return fmt.Errorf("no configuration provided") } exporter.conf = conf + if log == nil { + return fmt.Errorf("no logger provided") + } + exporter.log = log + return nil } diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index d2b1de2502..c60142645c 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -26,6 +26,8 @@ import ( "path" "path/filepath" + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exporters/prometheus" "github.com/cs3org/reva/pkg/mentix/meshdata" @@ -37,8 +39,8 @@ type PrometheusFileSDExporter struct { outputFilename string } -func (exporter *PrometheusFileSDExporter) Activate(conf *config.Configuration) error { - if err := exporter.BaseExporter.Activate(conf); err != nil { +func (exporter *PrometheusFileSDExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := exporter.BaseExporter.Activate(conf, log); err != nil { return err } @@ -71,6 +73,9 @@ func (exporter *PrometheusFileSDExporter) exportMeshData() { scrapes := exporter.createScrapeConfigs() if err := exporter.exportScrapeConfig(scrapes); err != nil { + exporter.log.Err(err).Str("file", exporter.outputFilename).Msg("error exporting Prometheus File SD") + } else { + exporter.log.Debug().Str("file", exporter.outputFilename).Msg("exported Prometheus File SD") } } diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exporters/webapi.go index df4f8a62de..e1a2ff2f5d 100755 --- a/pkg/mentix/exporters/webapi.go +++ b/pkg/mentix/exporters/webapi.go @@ -22,6 +22,8 @@ import ( "fmt" "net/http" + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exporters/webapi" ) @@ -30,8 +32,8 @@ type WebAPIExporter struct { BaseRequestExporter } -func (exporter *WebAPIExporter) Activate(conf *config.Configuration) error { - if err := exporter.BaseExporter.Activate(conf); err != nil { +func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := exporter.BaseExporter.Activate(conf, log); err != nil { return err } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 79b75e4adf..f941ed5b29 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -21,8 +21,11 @@ package mentix import ( "fmt" "net/http" + "strings" "time" + "github.com/rs/zerolog" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/connectors" @@ -32,6 +35,7 @@ import ( type Mentix struct { conf *config.Configuration + log *zerolog.Logger meshData *meshdata.MeshData connector connectors.Connector @@ -44,12 +48,17 @@ const ( runLoopSleeptime = time.Millisecond * 500 ) -func (mntx *Mentix) initialize(conf *config.Configuration) error { +func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) error { if conf == nil { return fmt.Errorf("no configuration provided") } mntx.conf = conf + if log == nil { + return fmt.Errorf("no logger provided") + } + mntx.log = log + // Initialize the connector that will be used to gather the mesh data if err := mntx.initConnector(); err != nil { return fmt.Errorf("unable to initialize connector: %v", err) @@ -81,9 +90,10 @@ func (mntx *Mentix) initConnector() error { return fmt.Errorf("the desired connector could be found: %v", err) } mntx.connector = connector + mntx.log.Info().Msgf("mentix connector: %v", connector.GetName()) // Activate the selected connector - if err := mntx.connector.Activate(mntx.conf); err != nil { + if err := mntx.connector.Activate(mntx.conf, mntx.log); err != nil { return fmt.Errorf("unable to activate connector: %v", err) } @@ -101,10 +111,11 @@ func (mntx *Mentix) initExporters() error { names = append(names, exporter.GetName()) } mntx.exporters = exporters + mntx.log.Info().Msgf("mentix exporters: %v", strings.Join(names, "; ")) // Activate all exporters for _, exporter := range mntx.exporters { - if err := exporter.Activate(mntx.conf); err != nil { + if err := exporter.Activate(mntx.conf, mntx.log); err != nil { return fmt.Errorf("unable to activate exporter '%v': %v", exporter.GetName(), err) } } @@ -161,8 +172,10 @@ loop: meshData, err := mntx.retrieveMeshData() if err == nil { if err := mntx.applyMeshData(meshData); err != nil { + mntx.log.Err(err).Msg("failed to apply mesh data") } } else { + mntx.log.Err(err).Msg("failed to retrieve mesh data") } updateTimestamp = time.Now() @@ -184,6 +197,8 @@ func (mntx *Mentix) retrieveMeshData() (*meshdata.MeshData, error) { func (mntx *Mentix) applyMeshData(meshData *meshdata.MeshData) error { if !meshData.Compare(mntx.meshData) { + mntx.log.Debug().Msg("mesh data changed, applying") + mntx.meshData = meshData for _, exporter := range mntx.exporters { @@ -220,9 +235,9 @@ func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { } } -func New(conf *config.Configuration) (*Mentix, error) { +func New(conf *config.Configuration, log *zerolog.Logger) (*Mentix, error) { mntx := new(Mentix) - if err := mntx.initialize(conf); err != nil { + if err := mntx.initialize(conf, log); err != nil { return nil, fmt.Errorf("unable to initialize Mentix: %v", err) } return mntx, nil diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/network/utils.go index 75f8e4240c..1d968d405d 100644 --- a/pkg/mentix/network/utils.go +++ b/pkg/mentix/network/utils.go @@ -56,6 +56,9 @@ func ReadEndpoint(host string, path string, params URLParams) ([]byte, error) { if err != nil { return nil, fmt.Errorf("unable to get data from endpoint: %v", err) } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("invalid response received: %v", resp.Status) + } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) From 47669d4f59de0979566f855ab1a1488efd7a76ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 18 May 2020 13:18:14 +0200 Subject: [PATCH 07/13] Improved logging slightly --- pkg/mentix/mentix.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index f941ed5b29..0eb472c748 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -80,6 +80,13 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) // Create empty mesh data mntx.meshData = meshdata.New() + // Log some infos + var exporterNames []string + for _, exporter := range mntx.exporters { + exporterNames = append(exporterNames, exporter.GetName()) + } + log.Info().Msgf("mentix started with connector: %v; exporters: %v; update interval: %v", mntx.connector.GetName(), strings.Join(exporterNames, ","), duration) + return nil } @@ -90,7 +97,6 @@ func (mntx *Mentix) initConnector() error { return fmt.Errorf("the desired connector could be found: %v", err) } mntx.connector = connector - mntx.log.Info().Msgf("mentix connector: %v", connector.GetName()) // Activate the selected connector if err := mntx.connector.Activate(mntx.conf, mntx.log); err != nil { @@ -111,7 +117,6 @@ func (mntx *Mentix) initExporters() error { names = append(names, exporter.GetName()) } mntx.exporters = exporters - mntx.log.Info().Msgf("mentix exporters: %v", strings.Join(names, "; ")) // Activate all exporters for _, exporter := range mntx.exporters { From 615cbd7f9e92a28ddcab669e1572dc77028270b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Mon, 18 May 2020 14:20:02 +0200 Subject: [PATCH 08/13] Added Mentix configuration to the docs --- .../HTTP/Services/Mentix/GOCDB/_index.md | 27 ++++++++ .../HTTP/Services/Mentix/PromFileSD/_index.md | 19 ++++++ .../HTTP/Services/Mentix/WebAPI/_index.md | 19 ++++++ .../Config/HTTP/Services/Mentix/_index.md | 63 +++++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 docs/content/en/docs/Config/HTTP/Services/Mentix/GOCDB/_index.md create mode 100644 docs/content/en/docs/Config/HTTP/Services/Mentix/PromFileSD/_index.md create mode 100644 docs/content/en/docs/Config/HTTP/Services/Mentix/WebAPI/_index.md create mode 100644 docs/content/en/docs/Config/HTTP/Services/Mentix/_index.md diff --git a/docs/content/en/docs/Config/HTTP/Services/Mentix/GOCDB/_index.md b/docs/content/en/docs/Config/HTTP/Services/Mentix/GOCDB/_index.md new file mode 100644 index 0000000000..4ae7cdf81f --- /dev/null +++ b/docs/content/en/docs/Config/HTTP/Services/Mentix/GOCDB/_index.md @@ -0,0 +1,27 @@ +--- +title: "gocdb" +linkTitle: "gocdb" +weight: 10 +description: > + Configuration for the GOCDB connector of the Mentix service +--- + +{{% pageinfo %}} +When using the [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) connector, at least its address has to be configured. +{{% /pageinfo %}} + +{{% dir name="address" type="string" default="" %}} +The address of the GOCDB instance; must be a valid URL (e.g., http://gocdb.uni-muenster.de). **Note:** The public API must be reachable under `
/gocdbpi/public`. +{{< highlight toml >}} +[http.services.mentix.gocdb] +address = "http://gocdb.example.com" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="scope" type="string" default="SM" %}} +The scope to use for filtering sites and services. +{{< highlight toml >}} +[http.services.mentix.gocdb] +scope = "SM" +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/Config/HTTP/Services/Mentix/PromFileSD/_index.md b/docs/content/en/docs/Config/HTTP/Services/Mentix/PromFileSD/_index.md new file mode 100644 index 0000000000..c3fb13cded --- /dev/null +++ b/docs/content/en/docs/Config/HTTP/Services/Mentix/PromFileSD/_index.md @@ -0,0 +1,19 @@ +--- +title: "prom_filesd" +linkTitle: "prom_filesd" +weight: 10 +description: > + Configuration for the Prometheus File SD exporter of the Mentix service +--- + +{{% pageinfo %}} +When using the Prometheus File SD exporter, the output filename has to be configured first. +{{% /pageinfo %}} + +{{% dir name="output_file" type="string" default="" %}} +The target filename of the generated Prometheus File SD scrape config. +{{< highlight toml >}} +[http.services.mentix.prom_filesd] +output_file = "/var/shared/prometheus/sciencemesh.json" +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/Config/HTTP/Services/Mentix/WebAPI/_index.md b/docs/content/en/docs/Config/HTTP/Services/Mentix/WebAPI/_index.md new file mode 100644 index 0000000000..6f937aa60c --- /dev/null +++ b/docs/content/en/docs/Config/HTTP/Services/Mentix/WebAPI/_index.md @@ -0,0 +1,19 @@ +--- +title: "webapi" +linkTitle: "webapi" +weight: 10 +description: > + Configuration for the WebAPI of the Mentix service +--- + +{{% pageinfo %}} +The WebAPI exporter supports multiple endpoints for exporting data. As there currently is only one such endpoint, the WebAPI settings should not be modified. +{{% /pageinfo %}} + +{{% dir name="endpoint" type="string" default="/" %}} +The endpoint where the mesh data can be queried. +{{< highlight toml >}} +[http.services.mentix.webapi] +endpoint = "data" +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/Config/HTTP/Services/Mentix/_index.md b/docs/content/en/docs/Config/HTTP/Services/Mentix/_index.md new file mode 100644 index 0000000000..6f8b2a8fc6 --- /dev/null +++ b/docs/content/en/docs/Config/HTTP/Services/Mentix/_index.md @@ -0,0 +1,63 @@ +--- +title: "mentix" +linkTitle: "mentix" +weight: 10 +description: > + Configuration for the Mentix service +--- + +{{% pageinfo %}} +Mentix (_**Me**sh E**nti**ty E**x**porter_) is a service to read mesh topology data from a source (e.g., a GOCDB instance) and export it to various targets like an HTTP endpoint or Prometheus. +{{% /pageinfo %}} + +{{% dir name="prefix" type="string" default="mentix" %}} +The relative root path of all exposed HTTP endpoints of Mentix. +{{< highlight toml >}} +[http.services.mentix] +prefix = "/mentix" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="connector" type="string" default="gocdb" %}} +Mentix is decoupled from the actual source of the mesh data by using a so-called _connector_. A connector is used to gather the data from a certain source, which are then converted into Mentix' own internal format. + +Supported values are: + +- **gocdb** +The [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) is a database specifically designed to organize the topology of a mesh of distributed sites and services. In order to use GOCDB with Mentix, its instance address has to be configured (see [here](gocdb)). + +{{< highlight toml >}} +[http.services.mentix] +connector = "gocdb" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="exporters" type="[]string" default="[webapi,prom_filesd]" %}} +Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint. + +Supported values are: + +- **webapi** +Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data can be retrieved at the configured relative endpoint (see [here](webapi)). The web API currently doesn't support any parameters but will most likely be extended in the future. +- **prom_filesd** +[Prometheus](https://prometheus.io/) supports discovering new services it should monitor via external configuration files (hence, this is called _file-based service discovery_). Mentix can create such files using the `prom_filesd` exporter. To use this exporter, you have to specify the target output file in the configuration (see [here](prom_filesd)). You also have to set up the discovery service in Prometheus by adding a scrape configuration like the following example to the Prometheus configuration (for more information, visit the official [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)): + ``` scrape_configs: + - job_name: 'sciencemesh' + file_sd_configs: + - files: + - '/usr/share/prom/sciencemesh_services.json' + ``` + +{{< highlight toml >}} +[http.services.mentix] +exporters = ["webapi", "prom_filesd"] +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="update_interval" type="string" default="1h" %}} +How frequently Mentix should pull and update the mesh data. Supports common time duration strings, like "1h30m", "1d" etc. +{{< highlight toml >}} +[http.services.mentix] +update_interval = "15m" +{{< /highlight >}} +{{% /dir %}} From 295d0d4f47a275c67c028a6071082f71341b5ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 19 May 2020 10:15:23 +0200 Subject: [PATCH 09/13] Corrected lint errors and warnings --- internal/http/services/mentix/mentix.go | 13 +++++++++---- pkg/mentix/config/ids.go | 6 +++--- pkg/mentix/connectors/gocdb.go | 2 +- pkg/mentix/exporters/exporter.go | 2 +- pkg/mentix/exporters/promfilesd.go | 6 ++++-- pkg/mentix/exporters/webapi.go | 7 ++++--- pkg/mentix/mentix.go | 6 +----- pkg/mentix/meshdata/meshdata.go | 5 ++++- 8 files changed, 27 insertions(+), 20 deletions(-) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 75049dab8c..e93a8afd09 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -61,8 +61,9 @@ func (s *svc) Prefix() string { func (s *svc) Unprotected() []string { // Get all endpoints exposed by the RequestExporters - var endpoints []string - for _, exporter := range s.mntx.GetRequestExporters() { + exporters := s.mntx.GetRequestExporters() + endpoints := make([]string, len(exporters)) + for _, exporter := range exporters { endpoints = append(endpoints, exporter.Endpoint()) } return endpoints @@ -75,7 +76,11 @@ func (s *svc) Handler() http.Handler { func (s *svc) startBackgroundService() { // Just run Mentix in the background - go s.mntx.Run(s.stopSignal) + go func() { + if err := s.mntx.Run(s.stopSignal); err != nil { + s.log.Err(err).Msg("error while running mentix") + } + }() } func parseConfig(m map[string]interface{}) (*config.Configuration, error) { @@ -95,7 +100,7 @@ func applyDefaultConfig(*config.Configuration) { } if conf.Connector == "" { - conf.Connector = config.ConnectorID_GOCDB // Use GOCDB + conf.Connector = config.ConnectorIDGOCDB // Use GOCDB } if len(conf.Exporters) == 0 { diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index f9ba7b8f23..f8aed9c0dd 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -19,10 +19,10 @@ package config const ( - ConnectorID_GOCDB = "gocdb" + ConnectorIDGOCDB = "gocdb" ) const ( - ExporterID_WebAPI = "webapi" - ExporterID_PrometheusFileSD = "prom_filesd" + ExporterIDWebAPI = "webapi" + ExporterIDPrometheusFileSD = "prom_filesd" ) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 320bfbb119..d10690f15b 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -199,5 +199,5 @@ func (connector *GOCDBConnector) GetName() string { } func init() { - registerConnector(config.ConnectorID_GOCDB, &GOCDBConnector{}) + registerConnector(config.ConnectorIDGOCDB, &GOCDBConnector{}) } diff --git a/pkg/mentix/exporters/exporter.go b/pkg/mentix/exporters/exporter.go index e1de66229e..502b83d48c 100755 --- a/pkg/mentix/exporters/exporter.go +++ b/pkg/mentix/exporters/exporter.go @@ -118,7 +118,7 @@ func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { } func RegisteredExporterIDs() []string { - var keys []string + keys := make([]string, len(registeredExporters)) for k := range registeredExporters { keys = append(keys, k) } diff --git a/pkg/mentix/exporters/promfilesd.go b/pkg/mentix/exporters/promfilesd.go index c60142645c..c2906ac1e4 100755 --- a/pkg/mentix/exporters/promfilesd.go +++ b/pkg/mentix/exporters/promfilesd.go @@ -51,7 +51,9 @@ func (exporter *PrometheusFileSDExporter) Activate(conf *config.Configuration, l } // Create the output directory - os.MkdirAll(filepath.Dir(exporter.outputFilename), os.ModePerm) + if err := os.MkdirAll(filepath.Dir(exporter.outputFilename), os.ModePerm); err != nil { + return fmt.Errorf("unable to create directory tree") + } return nil } @@ -138,5 +140,5 @@ func (exporter *PrometheusFileSDExporter) GetName() string { } func init() { - registerExporter(config.ExporterID_PrometheusFileSD, &PrometheusFileSDExporter{}) + registerExporter(config.ExporterIDPrometheusFileSD, &PrometheusFileSDExporter{}) } diff --git a/pkg/mentix/exporters/webapi.go b/pkg/mentix/exporters/webapi.go index e1a2ff2f5d..939bb17739 100755 --- a/pkg/mentix/exporters/webapi.go +++ b/pkg/mentix/exporters/webapi.go @@ -50,9 +50,10 @@ func (exporter *WebAPIExporter) HandleRequest(resp http.ResponseWriter, req *htt data, err := webapi.HandleQuery(exporter.meshData, req.URL.Query()) if err == nil { - resp.Write(data) + if _, err := resp.Write(data); err != nil { + return fmt.Errorf("error writing the API request response: %v", err) + } } else { - resp.Write([]byte(fmt.Sprintf("Error while serving API request: %v", err))) return fmt.Errorf("error while serving API request: %v", err) } @@ -64,5 +65,5 @@ func (exporter *WebAPIExporter) GetName() string { } func init() { - registerExporter(config.ExporterID_WebAPI, &WebAPIExporter{}) + registerExporter(config.ExporterIDWebAPI, &WebAPIExporter{}) } diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 0eb472c748..60c1fb4bf6 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -81,7 +81,7 @@ func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) mntx.meshData = meshdata.New() // Log some infos - var exporterNames []string + exporterNames := make([]string, len(mntx.exporters)) for _, exporter := range mntx.exporters { exporterNames = append(exporterNames, exporter.GetName()) } @@ -112,10 +112,6 @@ func (mntx *Mentix) initExporters() error { if err != nil { return fmt.Errorf("unable to get registered exporters: %v", err) } - var names []string - for _, exporter := range exporters { - names = append(names, exporter.GetName()) - } mntx.exporters = exporters // Activate all exporters diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index c53d658ca4..a7b5b2a424 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -55,7 +55,10 @@ func (meshData *MeshData) Clone() *MeshData { // To avoid any "deep copy" packages, use JSON en- and decoding instead data, err := meshData.ToJSON() if err == nil { - clone.FromJSON(data) + if err := clone.FromJSON(data); err != nil { + // In case of an error, clear the data + clone.Clear() + } } return clone From 4d64f799e8a5240cbe737f7056415116604c94a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 19 May 2020 10:37:50 +0200 Subject: [PATCH 10/13] Added comments to most exported methods, functions and types --- internal/http/services/mentix/mentix.go | 2 +- pkg/mentix/config/config.go | 1 + pkg/mentix/connectors/connector.go | 5 +++++ pkg/mentix/connectors/gocdb/query.go | 1 + pkg/mentix/connectors/gocdb/types.go | 10 ++++++++++ pkg/mentix/exporters/exporter.go | 9 +++++++++ pkg/mentix/exporters/prometheus/types.go | 1 + pkg/mentix/exporters/reqexporter.go | 5 +++++ pkg/mentix/exporters/webapi/query.go | 1 + pkg/mentix/mentix.go | 5 +++++ pkg/mentix/meshdata/meshdata.go | 7 +++++++ pkg/mentix/meshdata/service.go | 3 +++ pkg/mentix/meshdata/site.go | 1 + pkg/mentix/network/utils.go | 3 +++ 14 files changed, 53 insertions(+), 1 deletion(-) diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index e93a8afd09..988f998921 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -120,7 +120,7 @@ func applyDefaultConfig(*config.Configuration) { } } -// New returns a new Mentix service +// New returns a new Mentix service. func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { // Prepare the configuration conf, err := parseConfig(m) diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index d24f9470c6..f2d627f3e4 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -18,6 +18,7 @@ package config +// Configuration holds the general Mentix configuration. type Configuration struct { Prefix string `mapstructure:"prefix"` diff --git a/pkg/mentix/connectors/connector.go b/pkg/mentix/connectors/connector.go index 3aa56dce4e..d5f3d6cd59 100755 --- a/pkg/mentix/connectors/connector.go +++ b/pkg/mentix/connectors/connector.go @@ -32,13 +32,18 @@ var ( registeredConnectors = map[string]Connector{} ) +// Connector is the interface that all connectors must implement. type Connector interface { + // Activate activates a connector. Activate(conf *config.Configuration, log *zerolog.Logger) error + // RetrieveMeshData fetches new mesh data. RetrieveMeshData() (*meshdata.MeshData, error) + // GetName returns the display name of the connector. GetName() string } +// BaseConnector implements basic connector functionality common to all connectors. type BaseConnector struct { conf *config.Configuration log *zerolog.Logger diff --git a/pkg/mentix/connectors/gocdb/query.go b/pkg/mentix/connectors/gocdb/query.go index f1bb720254..e633990bbf 100755 --- a/pkg/mentix/connectors/gocdb/query.go +++ b/pkg/mentix/connectors/gocdb/query.go @@ -24,6 +24,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/network" ) +// QueryGOCDB retrieves data from one of GOCDB's endpoints. func QueryGOCDB(address string, method string, isPrivate bool, scope string, params network.URLParams) ([]byte, error) { // The method must always be specified params["method"] = method diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go index 0ead7df2bb..cc09e6e107 100755 --- a/pkg/mentix/connectors/gocdb/types.go +++ b/pkg/mentix/connectors/gocdb/types.go @@ -18,24 +18,29 @@ package gocdb +// Extension represents Key-Value pairs in GOCDB. type Extension struct { Key string `xml:"KEY"` Value string `xml:"VALUE"` } +// Extensions is a list of Extension objects. type Extensions struct { Extensions []*Extension `xml:"EXTENSION"` } +// ServiceType represents a service type in GOCDB. type ServiceType struct { Name string `xml:"SERVICE_TYPE_NAME"` Description string `xml:"SERVICE_TYPE_DESC"` } +// ServiceTypes is a list of ServiceType objects. type ServiceTypes struct { Types []*ServiceType `xml:"SERVICE_TYPE"` } +// Site represents a site in GOCDB. type Site struct { ShortName string `xml:"SHORT_NAME"` OfficialName string `xml:"OFFICIAL_NAME"` @@ -46,10 +51,12 @@ type Site struct { Extensions Extensions `xml:"EXTENSIONS"` } +// Sites is a list of Site objects. type Sites struct { Sites []*Site `xml:"SITE"` } +// ServiceEndpoint represents an additional service endpoint of a service in GOCDB. type ServiceEndpoint struct { Name string `xml:"NAME"` URL string `xml:"URL"` @@ -58,10 +65,12 @@ type ServiceEndpoint struct { Extensions Extensions `xml:"EXTENSIONS"` } +// ServiceEndpoints is a list of ServiceEndpoint objects. type ServiceEndpoints struct { Endpoints []*ServiceEndpoint `xml:"ENDPOINT"` } +// Service represents a service in GOCDB. type Service struct { Host string `xml:"HOSTNAME"` Type string `xml:"SERVICE_TYPE"` @@ -71,6 +80,7 @@ type Service struct { Extensions Extensions `xml:"EXTENSIONS"` } +// Services is a list of Service objects. type Services struct { Services []*Service `xml:"SERVICE_ENDPOINT"` } diff --git a/pkg/mentix/exporters/exporter.go b/pkg/mentix/exporters/exporter.go index 502b83d48c..938d28a349 100755 --- a/pkg/mentix/exporters/exporter.go +++ b/pkg/mentix/exporters/exporter.go @@ -32,16 +32,23 @@ var ( registeredExporters = map[string]Exporter{} ) +// Exporter is the interface that all exporters must implement. type Exporter interface { + // Activate activates the exporter. Activate(conf *config.Configuration, log *zerolog.Logger) error + // Start starts the exporter; only exporters which perform periodical background tasks should do something here. Start() error + // Stop stops any running background activities of the exporter. Stop() + // UpdateMeshData is called whenever the mesh data has changed to reflect these changes. UpdateMeshData(*meshdata.MeshData) error + // GetName returns the display name of the exporter. GetName() string } +// BaseExporter implements basic exporter functionality common to all exporters. type BaseExporter struct { conf *config.Configuration log *zerolog.Logger @@ -98,6 +105,7 @@ func registerExporter(id string, exporter Exporter) { registeredExporters[id] = exporter } +// AvailableExporters returns a list of all exporters that are enabled in the configuration. func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { // Try to add all exporters configured in the environment var exporters []Exporter @@ -117,6 +125,7 @@ func AvailableExporters(conf *config.Configuration) ([]Exporter, error) { return exporters, nil } +// RegisteredExporterIDs returns a list of all registered exporter IDs. func RegisteredExporterIDs() []string { keys := make([]string, len(registeredExporters)) for k := range registeredExporters { diff --git a/pkg/mentix/exporters/prometheus/types.go b/pkg/mentix/exporters/prometheus/types.go index d55ab47d41..1b341b592e 100755 --- a/pkg/mentix/exporters/prometheus/types.go +++ b/pkg/mentix/exporters/prometheus/types.go @@ -18,6 +18,7 @@ package prometheus +// ScrapeConfig represents a scrape configuration in Prometheus. type ScrapeConfig struct { Targets []string `json:"targets"` Labels map[string]string `json:"labels"` diff --git a/pkg/mentix/exporters/reqexporter.go b/pkg/mentix/exporters/reqexporter.go index 69d0d54fb9..6a8d676e16 100644 --- a/pkg/mentix/exporters/reqexporter.go +++ b/pkg/mentix/exporters/reqexporter.go @@ -23,14 +23,19 @@ import ( "strings" ) +// RequestExporter is the interface implemented by exporters that offer an HTTP endpoint. type RequestExporter interface { Exporter + // Endpoint returns the (relative) endpoint of the exporter. Endpoint() string + // WantsRequest returns whether the exporter wants to handle the incoming request. WantsRequest(r *http.Request) bool + // HandleRequest handles the actual HTTP request. HandleRequest(resp http.ResponseWriter, req *http.Request) error } +// BaseRequestExporter implements basic exporter functionality common to all request exporters. type BaseRequestExporter struct { BaseExporter diff --git a/pkg/mentix/exporters/webapi/query.go b/pkg/mentix/exporters/webapi/query.go index 03799ce825..16d4bb6945 100755 --- a/pkg/mentix/exporters/webapi/query.go +++ b/pkg/mentix/exporters/webapi/query.go @@ -31,6 +31,7 @@ const ( queryMethodDefault = "" ) +// HandleQuery handles an HTTP request based on the provided 'method' parameter. func HandleQuery(meshData *meshdata.MeshData, params url.Values) ([]byte, error) { method := params.Get("method") switch strings.ToLower(method) { diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 60c1fb4bf6..2186118ff8 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -147,6 +147,8 @@ func (mntx *Mentix) destroy() { mntx.stopExporters() } +// Run starts the Mentix service that will periodically pull the configured data source and publish this data +// through the enabled exporters. func (mntx *Mentix) Run(stopSignal <-chan struct{}) error { defer mntx.destroy() @@ -212,6 +214,7 @@ func (mntx *Mentix) applyMeshData(meshData *meshdata.MeshData) error { return nil } +// GetRequestExporters returns all exporters that can handle HTTP requests. func (mntx *Mentix) GetRequestExporters() []exporters.RequestExporter { // Return all exporters that implement the RequestExporter interface var reqExporters []exporters.RequestExporter @@ -223,6 +226,8 @@ func (mntx *Mentix) GetRequestExporters() []exporters.RequestExporter { return reqExporters } +// RequestHandler handles any incoming HTTP requests by asking each RequestExporter whether it wants to +// handle the request (usually based on the relative URL path). func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { log := appctx.GetLogger(r.Context()) diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index a7b5b2a424..1d593c523e 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -23,16 +23,19 @@ import ( "fmt" ) +// MeshData holds the entire mesh data managed by Mentix. type MeshData struct { Sites []*Site ServiceTypes []*ServiceType } +// Clear removes all saved data, leaving an empty mesh. func (meshData *MeshData) Clear() { meshData.Sites = nil meshData.ServiceTypes = nil } +// ToJSON converts the data to JSON. func (meshData *MeshData) ToJSON() (string, error) { data, err := json.MarshalIndent(meshData, "", "\t") if err != nil { @@ -41,6 +44,7 @@ func (meshData *MeshData) ToJSON() (string, error) { return string(data), nil } +// FromJSON converts JSON data to mesh data. func (meshData *MeshData) FromJSON(data string) error { meshData.Clear() if err := json.Unmarshal([]byte(data), meshData); err != nil { @@ -49,6 +53,7 @@ func (meshData *MeshData) FromJSON(data string) error { return nil } +// Clone creates an exact copy of the mesh data. func (meshData *MeshData) Clone() *MeshData { clone := &MeshData{} @@ -64,6 +69,7 @@ func (meshData *MeshData) Clone() *MeshData { return clone } +// Compare checks whether the stored data equals the data of another MeshData object. func (meshData *MeshData) Compare(other *MeshData) bool { // To avoid cumbersome comparisons, just compare the JSON-encoded data json1, _ := meshData.ToJSON() @@ -71,6 +77,7 @@ func (meshData *MeshData) Compare(other *MeshData) bool { return json1 == json2 } +// New returns a new (empty) MeshData object. func New() *MeshData { meshData := &MeshData{} meshData.Clear() diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index 6bdfaa0633..4ff18ec63e 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -18,6 +18,7 @@ package meshdata +// Service represents a service managed by Mentix. type Service struct { ServiceEndpoint @@ -25,11 +26,13 @@ type Service struct { AdditionalEndpoints []*ServiceEndpoint } +// ServiceType represents a service type managed by Mentix. type ServiceType struct { Name string Description string } +// ServiceEndpoint represents a service endpoint managed by Mentix. type ServiceEndpoint struct { Type *ServiceType Name string diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index bdb61f0cab..6fba7eced0 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -18,6 +18,7 @@ package meshdata +// Site represents a single site managed by Mentix. type Site struct { Name string FullName string diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/network/utils.go index 1d968d405d..a9e14d06ef 100644 --- a/pkg/mentix/network/utils.go +++ b/pkg/mentix/network/utils.go @@ -26,8 +26,10 @@ import ( p "path" ) +// URLParams holds Key-Value URL parameters; it is a simpler form of url.Values. type URLParams map[string]string +// GenerateURL creates a URL object from a host, path and optional parameters. func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { fullURL, err := url.Parse(host) if err != nil { @@ -45,6 +47,7 @@ func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { return fullURL, nil } +// ReadEndpoint reads data from an HTTP endpoint. func ReadEndpoint(host string, path string, params URLParams) ([]byte, error) { endpointURL, err := GenerateURL(host, path, params) if err != nil { From 9b9ecf12f8b6b17f5341abbc61016deebe282da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 19 May 2020 10:47:40 +0200 Subject: [PATCH 11/13] Removed dependency on the 'path' package --- go.sum | 5 +++++ pkg/mentix/network/utils.go | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 45746ab446..a8f6d354a7 100644 --- a/go.sum +++ b/go.sum @@ -339,6 +339,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -348,6 +349,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -387,7 +389,9 @@ gopkg.in/Acconut/lockfile.v1 v1.1.0/go.mod h1:6UCz3wJ8tSFUsPR6uP/j8uegEtDuEEqFxl gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= @@ -398,6 +402,7 @@ gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/network/utils.go index a9e14d06ef..4dce8993f1 100644 --- a/pkg/mentix/network/utils.go +++ b/pkg/mentix/network/utils.go @@ -23,12 +23,23 @@ import ( "io/ioutil" "net/http" "net/url" - p "path" + "strings" ) // URLParams holds Key-Value URL parameters; it is a simpler form of url.Values. type URLParams map[string]string +func combineURLPath(base string, paths ...string) string { + fullPath := base + for _, path := range paths { + if !strings.HasSuffix(fullPath, "/") && !strings.HasPrefix(path, "/") { + fullPath += "/" + } + fullPath += path + } + return fullPath +} + // GenerateURL creates a URL object from a host, path and optional parameters. func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { fullURL, err := url.Parse(host) @@ -36,7 +47,7 @@ func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { return nil, fmt.Errorf("unable to generate URL: base=%v, path=%v, params=%v", host, path, params) } - fullURL.Path = p.Join(fullURL.Path, path) + fullURL.Path = combineURLPath(fullURL.Path, path) query := make(url.Values) for key, value := range params { From 1ab39ef4416d70f8f913f7dd6c71223e6d43e34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 20 May 2020 10:17:28 +0200 Subject: [PATCH 12/13] Revert "Removed dependency on the 'path' package" This reverts commit 9b9ecf12f8b6b17f5341abbc61016deebe282da1. --- go.sum | 5 ----- pkg/mentix/network/utils.go | 15 ++------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/go.sum b/go.sum index a8f6d354a7..45746ab446 100644 --- a/go.sum +++ b/go.sum @@ -339,7 +339,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -349,7 +348,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -389,9 +387,7 @@ gopkg.in/Acconut/lockfile.v1 v1.1.0/go.mod h1:6UCz3wJ8tSFUsPR6uP/j8uegEtDuEEqFxl gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= @@ -402,7 +398,6 @@ gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/mentix/network/utils.go b/pkg/mentix/network/utils.go index 4dce8993f1..a9e14d06ef 100644 --- a/pkg/mentix/network/utils.go +++ b/pkg/mentix/network/utils.go @@ -23,23 +23,12 @@ import ( "io/ioutil" "net/http" "net/url" - "strings" + p "path" ) // URLParams holds Key-Value URL parameters; it is a simpler form of url.Values. type URLParams map[string]string -func combineURLPath(base string, paths ...string) string { - fullPath := base - for _, path := range paths { - if !strings.HasSuffix(fullPath, "/") && !strings.HasPrefix(path, "/") { - fullPath += "/" - } - fullPath += path - } - return fullPath -} - // GenerateURL creates a URL object from a host, path and optional parameters. func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { fullURL, err := url.Parse(host) @@ -47,7 +36,7 @@ func GenerateURL(host string, path string, params URLParams) (*url.URL, error) { return nil, fmt.Errorf("unable to generate URL: base=%v, path=%v, params=%v", host, path, params) } - fullURL.Path = combineURLPath(fullURL.Path, path) + fullURL.Path = p.Join(fullURL.Path, path) query := make(url.Values) for key, value := range params { From 6218560366f25e94288c97c926d6421867317a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Wed, 20 May 2020 10:21:54 +0200 Subject: [PATCH 13/13] The Mentix type wasn't commented --- pkg/mentix/mentix.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 2186118ff8..90ccdd4606 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -33,6 +33,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/meshdata" ) +// Mentix represents the main Mentix service object. type Mentix struct { conf *config.Configuration log *zerolog.Logger @@ -241,6 +242,7 @@ func (mntx *Mentix) RequestHandler(w http.ResponseWriter, r *http.Request) { } } +// New creates a new Mentix service instance. func New(conf *config.Configuration, log *zerolog.Logger) (*Mentix, error) { mntx := new(Mentix) if err := mntx.initialize(conf, log); err != nil {