Mentix service inference (#2251)

changelog/unreleased/mentix-svc-inf.md

@@ -0,0 +1,5 @@
+Enhancement: Mentix service inference
+
+Previously, 4 different services per site had to be created in the GOCDB. This PR removes this redundancy by infering all endpoints from a single service entity, making site administration a lot easier.
+
+https://github.com/cs3org/reva/pull/2251

docs/content/en/docs/config/http/services/mentix/cs3api/_index.md

@@ -33,3 +33,12 @@ A list of all enabled connectors for the exporter.
 enabled_connectors = ["gocdb"]
 {{< /highlight >}}
 {{% /dir %}}
+
+{{% dir name="elevated_service_types" type="[]string" default="[GATEWAY,OCM,WEBDAV]" %}}
+When processing additional endpoints of a service, any service type listed here will be elevated to a standalone service.
+{{< highlight toml >}}
+[http.services.mentix.exporters.cs3api]
+elevated_service_types = ["METRICS", "WEBDAV"]
+{{< /highlight >}}
+{{% /dir %}}
+

internal/http/services/mentix/mentix.go

@@ -21,6 +21,7 @@ package mentix
 import (
 	"net/http"
 
+	"github.com/cs3org/reva/pkg/mentix/meshdata"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -157,6 +158,9 @@ func applyDefaultConfig(conf *config.Configuration) {
 		conf.Exporters.CS3API.Endpoint = "/cs3"
 	}
 	addDefaultConnector(&conf.Exporters.CS3API.EnabledConnectors)
+	if len(conf.Exporters.CS3API.ElevatedServiceTypes) == 0 {
+		conf.Exporters.CS3API.ElevatedServiceTypes = append(conf.Exporters.CS3API.ElevatedServiceTypes, meshdata.EndpointGateway, meshdata.EndpointOCM, meshdata.EndpointWebdav)
+	}
 
 	if conf.Exporters.SiteLocations.Endpoint == "" {
 		conf.Exporters.SiteLocations.Endpoint = "/loc"

pkg/mentix/config/config.go

@@ -57,9 +57,10 @@ type Configuration struct {
 		} `mapstructure:"webapi"`
 
 		CS3API struct {
-			Endpoint          string   `mapstructure:"endpoint"`
-			EnabledConnectors []string `mapstructure:"enabled_connectors"`
-			IsProtected       bool     `mapstructure:"is_protected"`
+			Endpoint             string   `mapstructure:"endpoint"`
+			EnabledConnectors    []string `mapstructure:"enabled_connectors"`
+			IsProtected          bool     `mapstructure:"is_protected"`
+			ElevatedServiceTypes []string `mapstructure:"elevated_service_types"`
 		} `mapstructure:"cs3api"`
 
 		SiteLocations struct {

pkg/mentix/connectors/gocdb.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cs3org/reva/pkg/mentix/utils"
 	"github.com/rs/zerolog"
 
 	"github.com/cs3org/reva/pkg/mentix/config"
@@ -200,6 +201,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site
 			endpoints = append(endpoints, &meshdata.ServiceEndpoint{
 				Type:        connector.findServiceType(meshData, endpoint.Type),
 				Name:        endpoint.Name,
+				RawURL:      endpoint.URL,
 				URL:         getServiceURLString(service, endpoint, host),
 				IsMonitored: strings.EqualFold(endpoint.IsMonitored, "Y"),
 				Properties:  connector.extensionsToMap(&endpoint.Extensions),
@@ -210,7 +212,8 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site
 		site.Services = append(site.Services, &meshdata.Service{
 			ServiceEndpoint: &meshdata.ServiceEndpoint{
 				Type:        connector.findServiceType(meshData, service.Type),
-				Name:        fmt.Sprintf("%v - %v", service.Host, service.Type),
+				Name:        service.Type,
+				RawURL:      service.URL,
 				URL:         getServiceURLString(service, nil, host),
 				IsMonitored: strings.EqualFold(service.IsMonitored, "Y"),
 				Properties:  connector.extensionsToMap(&service.Extensions),
@@ -239,10 +242,8 @@ func (connector *GOCDBConnector) queryDowntimes(meshData *meshdata.MeshData, sit
 		services := make([]string, 0, len(dt.AffectedServices.Services))
 		for _, service := range dt.AffectedServices.Services {
 			// Only add critical services to the list of affected services
-			for _, svcType := range connector.conf.Services.CriticalTypes {
-				if strings.EqualFold(svcType, service.Type) {
-					services = append(services, service.Type)
-				}
+			if utils.FindInStringArray(service.Type, connector.conf.Services.CriticalTypes, false) != -1 {
+				services = append(services, service.Type)
 			}
 		}
 

pkg/mentix/exchangers/exchanger.go

@@ -76,7 +76,7 @@ func (exchanger *BaseExchanger) Activate(conf *config.Configuration, log *zerolo
 	return nil
 }
 
-// Start starts the exchanger; only exchanger which perform periodical background tasks should do something here.
+// Start starts the exchanger; only exchangers which perform periodical background tasks should do something here.
 func (exchanger *BaseExchanger) Start() error {
 	return nil
 }

pkg/mentix/exchangers/exporters/cs3api/query.go

@@ -25,16 +25,17 @@ import (
 	"net/url"
 
 	ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1"
+	"github.com/cs3org/reva/pkg/mentix/utils"
 	"github.com/rs/zerolog"
 
 	"github.com/cs3org/reva/pkg/mentix/config"
 	"github.com/cs3org/reva/pkg/mentix/meshdata"
 )
 
 // HandleDefaultQuery processes a basic query.
-func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (int, []byte, error) {
+func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (int, []byte, error) {
 	// Convert the mesh data
-	ocmData, err := convertMeshDataToOCMData(meshData)
+	ocmData, err := convertMeshDataToOCMData(meshData, conf.Exporters.CS3API.ElevatedServiceTypes)
 	if err != nil {
 		return http.StatusBadRequest, []byte{}, fmt.Errorf("unable to convert the mesh data to OCM data structures: %v", err)
 	}
@@ -48,25 +49,37 @@ func HandleDefaultQuery(meshData *meshdata.MeshData, params url.Values, _ *confi
 	return http.StatusOK, data, nil
 }
 
-func convertMeshDataToOCMData(meshData *meshdata.MeshData) ([]*ocmprovider.ProviderInfo, error) {
+func convertMeshDataToOCMData(meshData *meshdata.MeshData, elevatedServiceTypes []string) ([]*ocmprovider.ProviderInfo, error) {
 	// Convert the mesh data into the corresponding OCM data structures
 	providers := make([]*ocmprovider.ProviderInfo, 0, len(meshData.Sites))
 	for _, site := range meshData.Sites {
 		// Gather all services from the site
 		services := make([]*ocmprovider.Service, 0, len(site.Services))
+
+		addService := func(host string, endpoint *meshdata.ServiceEndpoint, addEndpoints []*ocmprovider.ServiceEndpoint, apiVersion string) {
+			services = append(services, &ocmprovider.Service{
+				Host:                host,
+				Endpoint:            convertServiceEndpointToOCMData(endpoint),
+				AdditionalEndpoints: addEndpoints,
+				ApiVersion:          apiVersion,
+			})
+		}
+
 		for _, service := range site.Services {
+			apiVersion := meshdata.GetPropertyValue(service.Properties, meshdata.PropertyAPIVersion, "")
+
 			// Gather all additional endpoints of the service
 			addEndpoints := make([]*ocmprovider.ServiceEndpoint, 0, len(service.AdditionalEndpoints))
 			for _, endpoint := range service.AdditionalEndpoints {
-				addEndpoints = append(addEndpoints, convertServiceEndpointToOCMData(endpoint))
+				if utils.FindInStringArray(endpoint.Type.Name, elevatedServiceTypes, false) != -1 {
+					endpointURL, _ := url.Parse(endpoint.URL)
+					addService(endpointURL.Host, endpoint, nil, apiVersion)
+				} else {
+					addEndpoints = append(addEndpoints, convertServiceEndpointToOCMData(endpoint))
+				}
 			}
 
-			services = append(services, &ocmprovider.Service{
-				Host:                service.Host,
-				Endpoint:            convertServiceEndpointToOCMData(service.ServiceEndpoint),
-				AdditionalEndpoints: addEndpoints,
-				ApiVersion:          meshdata.GetPropertyValue(service.Properties, meshdata.PropertyAPIVersion, ""),
-			})
+			addService(service.Host, service.ServiceEndpoint, addEndpoints, apiVersion)
 		}
 
 		// Copy the site info into a ProviderInfo

pkg/mentix/exchangers/exporters/exporter.go

@@ -20,6 +20,7 @@ package exporters
 
 import (
 	"github.com/cs3org/reva/pkg/mentix/exchangers"
+	"github.com/cs3org/reva/pkg/mentix/meshdata"
 )
 
 // Exporter is the interface that all exporters must implement.
@@ -31,3 +32,10 @@ type Exporter interface {
 type BaseExporter struct {
 	exchangers.BaseExchanger
 }
+
+// Start starts the exporter.
+func (exporter *BaseExporter) Start() error {
+	// Initialize the exporter with empty data
+	_ = exporter.Update(meshdata.Map{})
+	return nil
+}

pkg/mentix/exchangers/exporters/promsd.go

@@ -25,19 +25,20 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
-	"strings"
 
+	"github.com/cs3org/reva/pkg/mentix/utils"
 	"github.com/rs/zerolog"
 
 	"github.com/cs3org/reva/pkg/mentix/config"
 	"github.com/cs3org/reva/pkg/mentix/exchangers/exporters/prometheus"
 	"github.com/cs3org/reva/pkg/mentix/meshdata"
 )
 
-type prometheusSDScrapeCreatorCallback = func(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig
+type prometheusSDScrapeCreatorCallback = func(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig
 type prometheusSDScrapeCreator struct {
 	outputFilename  string
 	creatorCallback prometheusSDScrapeCreatorCallback
+	serviceFilter   []string
 }
 
 // PrometheusSDExporter implements various Prometheus Service Discovery scrape config exporters.
@@ -47,64 +48,42 @@ type PrometheusSDExporter struct {
 	scrapeCreators map[string]prometheusSDScrapeCreator
 }
 
-func createMetricsSDScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig {
-	labels := getScrapeTargetLabels(site, host, endpoint)
-
-	// Support both HTTP and HTTPS endpoints by setting the scheme label accordingly
-	if len(endpoint.URL) > 0 {
-		if url, err := url.Parse(endpoint.URL); err == nil && (url.Scheme == "http" || url.Scheme == "https") {
-			labels["__scheme__"] = url.Scheme
-		}
-	}
-	// If a metrics path was specified as a property, use that one by setting the corresponding label
-	if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 {
-		labels["__metrics_path__"] = metricsPath
-	}
-
-	return &prometheus.ScrapeConfig{
-		Targets: []string{host},
-		Labels:  labels,
-	}
-}
-
-func createBlackboxSDScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig {
-	// The URL of the service is used as the actual target; it must be configured properly
-	target := endpoint.URL
-	if target == "" {
-		return nil
-	}
-
-	// Check if health checks are enabled for the endpoint; if they aren't, skip this endpoint
-	if enableHealthChecks := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyEnableHealthChecks, "false"); !strings.EqualFold(enableHealthChecks, "true") {
-		return nil
-	}
-
-	labels := getScrapeTargetLabels(site, host, endpoint)
-
-	// For health checks, the gRPC port must be set
-	if _, ok := labels["__meta_mentix_grpc_port"]; !ok {
-		return nil
-	}
+const (
+	labelSiteName    = "__meta_mentix_site"
+	labelSiteType    = "__meta_mentix_site_type"
+	labelSiteID      = "__meta_mentix_site_id"
+	labelSiteCountry = "__meta_mentix_site_country"
+	labelType        = "__meta_mentix_type"
+	labelScheme      = "__meta_mentix_scheme"
+	labelHost        = "__meta_mentix_host"
+	labelPort        = "__meta_mentix_port"
+	labelPath        = "__meta_mentix_path"
+	labelServiceHost = "__meta_mentix_service_host"
+	labelServiceURL  = "__meta_mentix_service_url"
+)
 
+func createGenericScrapeConfig(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig {
+	endpointURL, _ := url.Parse(endpoint.URL)
+	labels := getScrapeTargetLabels(site, service, endpoint)
 	return &prometheus.ScrapeConfig{
-		Targets: []string{target},
+		Targets: []string{endpointURL.Host},
 		Labels:  labels,
 	}
 }
-
-func getScrapeTargetLabels(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) map[string]string {
+func getScrapeTargetLabels(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) map[string]string {
+	endpointURL, _ := url.Parse(endpoint.URL)
 	labels := map[string]string{
-		"__meta_mentix_site":         site.Name,
-		"__meta_mentix_site_type":    meshdata.GetSiteTypeName(site.Type),
-		"__meta_mentix_site_id":      site.ID,
-		"__meta_mentix_host":         host,
-		"__meta_mentix_country":      site.CountryCode,
-		"__meta_mentix_service_type": endpoint.Type.Name,
-	}
-
-	// Get the gRPC port if the corresponding property has been set
-	if port := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyGRPCPort, ""); len(port) > 0 {
-		labels["__meta_mentix_grpc_port"] = port
+		labelSiteName:    site.Name,
+		labelSiteType:    meshdata.GetSiteTypeName(site.Type),
+		labelSiteID:      site.ID,
+		labelSiteCountry: site.CountryCode,
+		labelType:        endpoint.Type.Name,
+		labelScheme:      endpointURL.Scheme,
+		labelHost:        endpointURL.Hostname(),
+		labelPort:        endpointURL.Port(),
+		labelPath:        endpointURL.Path,
+		labelServiceHost: service.Host,
+		labelServiceURL:  service.URL,
 	}
 
 	return labels
@@ -113,11 +92,12 @@ func getScrapeTargetLabels(site *meshdata.Site, host string, endpoint *meshdata.
 func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Configuration) error {
 	exporter.scrapeCreators = make(map[string]prometheusSDScrapeCreator)
 
-	registerCreator := func(name string, outputFilename string, creator prometheusSDScrapeCreatorCallback) error {
+	registerCreator := func(name string, outputFilename string, creator prometheusSDScrapeCreatorCallback, serviceFilter []string) error {
 		if len(outputFilename) > 0 { // Only register the creator if an output filename was configured
 			exporter.scrapeCreators[name] = prometheusSDScrapeCreator{
 				outputFilename:  outputFilename,
 				creatorCallback: creator,
+				serviceFilter:   serviceFilter,
 			}
 
 			// Create the output directory for the target file so it exists when exporting
@@ -130,11 +110,11 @@ func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Config
 	}
 
 	// Register all scrape creators
-	if err := registerCreator("metrics", conf.Exporters.PrometheusSD.MetricsOutputFile, createMetricsSDScrapeConfig); err != nil {
+	if err := registerCreator("metrics", conf.Exporters.PrometheusSD.MetricsOutputFile, createGenericScrapeConfig, []string{meshdata.EndpointMetrics}); err != nil {
 		return fmt.Errorf("unable to register the 'metrics' scrape config creator: %v", err)
 	}
 
-	if err := registerCreator("blackbox", conf.Exporters.PrometheusSD.BlackboxOutputFile, createBlackboxSDScrapeConfig); err != nil {
+	if err := registerCreator("blackbox", conf.Exporters.PrometheusSD.BlackboxOutputFile, createGenericScrapeConfig, []string{meshdata.EndpointGateway}); err != nil {
 		return fmt.Errorf("unable to register the 'blackbox' scrape config creator: %v", err)
 	}
 
@@ -181,7 +161,7 @@ func (exporter *PrometheusSDExporter) exportMeshData() {
 	defer exporter.Locker().RUnlock()
 
 	for name, creator := range exporter.scrapeCreators {
-		scrapes := exporter.createScrapeConfigs(creator.creatorCallback)
+		scrapes := exporter.createScrapeConfigs(creator.creatorCallback, creator.serviceFilter)
 		if err := exporter.exportScrapeConfig(creator.outputFilename, scrapes); err != nil {
 			exporter.Log().Err(err).Str("kind", name).Str("file", creator.outputFilename).Msg("error exporting Prometheus SD scrape config")
 		} else {
@@ -190,11 +170,13 @@ func (exporter *PrometheusSDExporter) exportMeshData() {
 	}
 }
 
-func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback prometheusSDScrapeCreatorCallback) []*prometheus.ScrapeConfig {
+func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback prometheusSDScrapeCreatorCallback, serviceFilter []string) []*prometheus.ScrapeConfig {
 	var scrapes []*prometheus.ScrapeConfig
-	var addScrape = func(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) {
-		if scrape := creatorCallback(site, host, endpoint); scrape != nil {
-			scrapes = append(scrapes, scrape)
+	var addScrape = func(site *meshdata.Site, service *meshdata.Service, endpoint *meshdata.ServiceEndpoint) {
+		if len(serviceFilter) == 0 || utils.FindInStringArray(endpoint.Type.Name, serviceFilter, false) != -1 {
+			if scrape := creatorCallback(site, service, endpoint); scrape != nil {
+				scrapes = append(scrapes, scrape)
+			}
 		}
 	}
 
@@ -206,12 +188,12 @@ func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback promet
 			}
 
 			// Add the "main" service to the scrapes
-			addScrape(site, service.Host, service.ServiceEndpoint)
+			addScrape(site, service, service.ServiceEndpoint)
 
 			// Add all additional endpoints as well
 			for _, endpoint := range service.AdditionalEndpoints {
 				if endpoint.IsMonitored {
-					addScrape(site, service.Host, endpoint)
+					addScrape(site, service, endpoint)
 				}
 			}
 		}

pkg/mentix/exchangers/importers/sitereg/types.go

@@ -100,8 +100,7 @@ func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier,
 		revaURL = URL.String()
 	}
 
-	properties := make(map[string]string, 1)
-	meshdata.SetPropertyValue(&properties, meshdata.PropertyMetricsPath, siteData.Reva.MetricsPath)
+	properties := make(map[string]string)
 
 	revaService := &meshdata.Service{
 		ServiceEndpoint: &meshdata.ServiceEndpoint{