introduce metadata gateway

.drone.star

@@ -1710,7 +1710,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "PROXY_ENABLE_BASIC_AUTH": True,
             "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
             "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
-            "OCIS_LOG_LEVEL": "error",
+            "OCIS_LOG_LEVEL": "debug",
             "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings",
             "IDM_CREATE_DEMO_USERS": True,
             "IDM_ADMIN_PASSWORD": "admin",  # override the random admin password from `ocis init`

.vscode/launch.json

@@ -8,7 +8,7 @@
             "mode": "debug",
             "program": "${workspaceFolder}/ocis/cmd/ocis",
             "args": [
-                "server"
+                "storage-metadata"
             ],
             "env": {
                 // log settings for human developers
@@ -17,23 +17,6 @@
                 "OCIS_LOG_COLOR": "true",
                 // enable basic auth for dev setup so that we can use curl for testing
                 "PROXY_ENABLE_BASIC_AUTH": "true",
-                // set insecure options because we don't have valid certificates in dev environments
-                "OCIS_INSECURE": "true",
-                // set some hardcoded secrets
-                "OCIS_JWT_SECRET": "some-ocis-jwt-secret",
-                "STORAGE_TRANSFER_SECRET": "some-ocis-transfer-secret",
-                "OCIS_MACHINE_AUTH_API_KEY": "some-ocis-machine-auth-api-key",
-                // idm ldap
-                "IDM_SVC_PASSWORD": "some-ldap-idm-password",
-                "GRAPH_LDAP_BIND_PASSWORD": "some-ldap-idm-password",
-                // reva ldap
-                "IDM_REVASVC_PASSWORD": "some-ldap-reva-password",
-                "GROUPS_LDAP_BIND_PASSWORD": "some-ldap-reva-password",
-                "USERS_LDAP_BIND_PASSWORD": "some-ldap-reva-password",
-                "AUTH_BASIC_LDAP_BIND_PASSWORD": "some-ldap-reva-password",
-                // idp ldap
-                "IDM_IDPSVC_PASSWORD": "some-ldap-idp-password",
-                "IDP_LDAP_BIND_PASSWORD": "some-ldap-idp-password",
                 // admin user default password
                 "IDM_ADMIN_PASSWORD": "admin",
                 // demo users

changelog/unreleased/metadata-gateway.md

@@ -0,0 +1,5 @@
+Enhancement: wrap metadata storage with dedicated reva gateway
+
+We wrapped the metadata storage in a minimal reva instance with a dedicated gateway, including static storage registry, static auth registry, in memory userprovider, machine authprovider and demo permissions service. This allows us to preconfigure the service user for the ocis settings service, share and public share providers.
+
+https://github.com/owncloud/ocis/pull/3602

changelog/unreleased/update-reva.md

@@ -7,6 +7,7 @@ Updated reva to version 2.x.x. This update includes:
 https://github.com/owncloud/ocis/pull/3552
 https://github.com/owncloud/ocis/pull/3570
 https://github.com/owncloud/ocis/pull/3601
+https://github.com/owncloud/ocis/pull/3602
 https://github.com/owncloud/ocis/pull/3605
 https://github.com/owncloud/ocis/pull/3611
 https://github.com/owncloud/ocis/pull/3621

extensions/settings/pkg/config/config.go

@@ -40,6 +40,6 @@ type Metadata struct {
 	StorageAddress string `yaml:"storage_addr" env:"STORAGE_GRPC_ADDR"`
 
 	ServiceUserID     string `yaml:"service_user_id" env:"METADATA_SERVICE_USER_UUID"`
-	ServiceUserIDP    string `yaml:"service_user_idp" env:"OCIS_URL;METADATA_SERVICE_USER_IDP"`
+	ServiceUserIDP    string `yaml:"service_user_idp" env:"METADATA_SERVICE_USER_IDP"`
 	MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY"`
 }

extensions/settings/pkg/config/defaults/defaultconfig.go

@@ -50,10 +50,10 @@ func DefaultConfig() *config.Config {
 		},
 
 		Metadata: config.Metadata{
-			GatewayAddress: "127.0.0.1:9142",
+			GatewayAddress: "127.0.0.1:9215",
 			StorageAddress: "127.0.0.1:9215",
 			ServiceUserID:  "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-			ServiceUserIDP: "https://localhost:9200",
+			ServiceUserIDP: "internal",
 		},
 	}
 }

extensions/storage-metadata/pkg/command/command.go

@@ -140,10 +140,74 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
 		"grpc": map[string]interface{}{
 			"network": cfg.GRPC.Protocol,
 			"address": cfg.GRPC.Addr,
-			"interceptors": map[string]interface{}{
-				"log": map[string]interface{}{},
-			},
 			"services": map[string]interface{}{
+				"gateway": map[string]interface{}{
+					// registries are located on the gateway
+					"authregistrysvc":    cfg.GRPC.Addr,
+					"storageregistrysvc": cfg.GRPC.Addr,
+					// user metadata is located on the users services
+					"userprovidersvc":  cfg.GRPC.Addr,
+					"groupprovidersvc": cfg.GRPC.Addr,
+					"permissionssvc":   cfg.GRPC.Addr,
+					// other
+					"disable_home_creation_on_login": true, // metadata manually creates a space
+					// metadata always uses the simple upload, so no transfer secret or datagateway needed
+				},
+				"userprovider": map[string]interface{}{
+					"driver": "memory",
+					"drivers": map[string]interface{}{
+						"memory": map[string]interface{}{
+							"users": map[string]interface{}{
+								"serviceuser": map[string]interface{}{
+									"id": map[string]interface{}{
+										"opaqueId": "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", // FIXME generate service user id
+										"idp":      "internal",
+										"type":     1, // user.UserType_USER_TYPE_PRIMARY
+									},
+									"username":     "serviceuser",
+									"display_name": "System User",
+								},
+							},
+						},
+					},
+				},
+				"authregistry": map[string]interface{}{
+					"driver": "static",
+					"drivers": map[string]interface{}{
+						"static": map[string]interface{}{
+							"rules": map[string]interface{}{
+								"machine": cfg.GRPC.Addr,
+							},
+						},
+					},
+				},
+				"authprovider": map[string]interface{}{
+					"auth_manager": "machine",
+					"auth_managers": map[string]interface{}{
+						"machine": map[string]interface{}{
+							"api_key":      cfg.MachineAuthAPIKey,
+							"gateway_addr": cfg.GRPC.Addr,
+						},
+					},
+				},
+				"permissions": map[string]interface{}{
+					"driver": "demo",
+					"drivers": map[string]interface{}{
+						"demo": map[string]interface{}{},
+					},
+				},
+				"storageregistry": map[string]interface{}{
+					"driver": "static",
+					"drivers": map[string]interface{}{
+						"static": map[string]interface{}{
+							"rules": map[string]interface{}{
+								"/": map[string]interface{}{
+									"address": cfg.GRPC.Addr,
+								},
+							},
+						},
+					},
+				},
 				"storageprovider": map[string]interface{}{
 					"driver":          cfg.Driver,
 					"drivers":         config.MetadataDrivers(cfg),
@@ -155,7 +219,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
 		"http": map[string]interface{}{
 			"network": cfg.HTTP.Protocol,
 			"address": cfg.HTTP.Addr,
-			// TODO build services dynamically
+			// no datagateway needed as the metadata clients directly talk to the dataprovider with the simple protocol
 			"services": map[string]interface{}{
 				"dataprovider": map[string]interface{}{
 					"prefix":      "data",

extensions/storage-metadata/pkg/config/config.go

@@ -19,8 +19,9 @@ type Config struct {
 
 	Context context.Context `yaml:"context"`
 
-	TokenManager *TokenManager `yaml:"token_manager"`
-	Reva         *Reva         `yaml:"reva"`
+	TokenManager      *TokenManager `yaml:"token_manager"`
+	Reva              *Reva         `yaml:"reva"`
+	MachineAuthAPIKey string        `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;STORAGE_METADATA_MACHINE_AUTH_API_KEY"`
 
 	SkipUserGroupsInToken bool    `yaml:"skip_user_groups_in_token"`
 	Driver                string  `yaml:"driver" env:"STORAGE_METADATA_DRIVER" desc:"The driver which should be used by the service"`
@@ -60,8 +61,8 @@ type GRPCConfig struct {
 }
 
 type HTTPConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_METADATA_HTTP_ADDR" desc:"The address of the http service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_HTTP_PROTOCOL" desc:"The transport protocol of the http service."`
 }
 
 type Drivers struct {

extensions/storage-metadata/pkg/config/defaults/defaultconfig.go

@@ -59,7 +59,7 @@ func DefaultConfig() *config.Config {
 				SecProtocol:         "",
 				Keytab:              "",
 				SingleUsername:      "",
-				GatewaySVC:          "127.0.0.1:9142",
+				GatewaySVC:          "127.0.0.1:9215",
 			},
 			Local: config.LocalDriver{
 				Root: filepath.Join(defaults.BaseDataPath(), "storage", "local", "metadata"),
@@ -71,12 +71,12 @@ func DefaultConfig() *config.Config {
 				Root:                filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
 				UserLayout:          "{{.Id.OpaqueId}}",
 				Region:              "default",
-				PermissionsEndpoint: "127.0.0.1:9191",
+				PermissionsEndpoint: "127.0.0.1:9215",
 			},
 			OCIS: config.OCISDriver{
 				Root:                filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
 				UserLayout:          "{{.Id.OpaqueId}}",
-				PermissionsEndpoint: "127.0.0.1:9191",
+				PermissionsEndpoint: "127.0.0.1:9215",
 			},
 		},
 	}
@@ -121,6 +121,10 @@ func EnsureDefaults(cfg *config.Config) {
 	} else if cfg.TokenManager == nil {
 		cfg.TokenManager = &config.TokenManager{}
 	}
+
+	if cfg.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" {
+		cfg.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey
+	}
 }
 
 func Sanitize(cfg *config.Config) {

extensions/storage-metadata/pkg/config/parser/parse.go

@@ -38,5 +38,8 @@ func Validate(cfg *config.Config) error {
 		return shared.MissingJWTTokenError(cfg.Service.Name)
 	}
 
+	if cfg.MachineAuthAPIKey == "" {
+		return shared.MissingMachineAuthApiKeyError(cfg.Service.Name)
+	}
 	return nil
 }

extensions/storage-publiclink/pkg/config/config.go

@@ -26,33 +26,33 @@ type Config struct {
 	StorageProvider       StorageProvider `yaml:"storage_provider"`
 }
 type Tracing struct {
-	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."`
-	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_METADATA_TRACING_TYPE"`
-	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_METADATA_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
-	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_METADATA_TRACING_COLLECTOR"`
+	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_PUBLICLINK_TRACING_ENABLED" desc:"Activates tracing."`
+	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_PUBLICLINK_TRACING_TYPE"`
+	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_PUBLICLINK_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
+	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_PUBLICLINK_TRACING_COLLECTOR"`
 }
 
 type Logging struct {
-	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_METADATA_LOG_LEVEL" desc:"The log level."`
-	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_METADATA_LOG_PRETTY" desc:"Activates pretty log output."`
-	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_METADATA_LOG_COLOR" desc:"Activates colorized log output."`
-	File   string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_METADATA_LOG_FILE" desc:"The target log file."`
+	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_PUBLICLINK_LOG_LEVEL" desc:"The log level."`
+	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_PUBLICLINK_LOG_PRETTY" desc:"Activates pretty log output."`
+	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_PUBLICLINK_LOG_COLOR" desc:"Activates colorized log output."`
+	File   string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_PUBLICLINK_LOG_FILE" desc:"The target log file."`
 }
 
 type Service struct {
 	Name string `yaml:"-"`
 }
 
 type Debug struct {
-	Addr   string `yaml:"addr" env:"STORAGE_METADATA_DEBUG_ADDR"`
-	Token  string `yaml:"token" env:"STORAGE_METADATA_DEBUG_TOKEN"`
-	Pprof  bool   `yaml:"pprof" env:"STORAGE_METADATA_DEBUG_PPROF"`
-	Zpages bool   `yaml:"zpages" env:"STORAGE_METADATA_DEBUG_ZPAGES"`
+	Addr   string `yaml:"addr" env:"STORAGE_PUBLICLINK_DEBUG_ADDR"`
+	Token  string `yaml:"token" env:"STORAGE_PUBLICLINK_DEBUG_TOKEN"`
+	Pprof  bool   `yaml:"pprof" env:"STORAGE_PUBLICLINK_DEBUG_PPROF"`
+	Zpages bool   `yaml:"zpages" env:"STORAGE_PUBLICLINK_DEBUG_ZPAGES"`
 }
 
 type GRPCConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_PUBLICLINK_GRPC_ADDR" desc:"The address of the grpc service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_PUBLICLINK_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
 }
 
 type AuthProvider struct {

extensions/storage-shares/pkg/config/config.go

@@ -26,36 +26,36 @@ type Config struct {
 	SharesProviderEndpoint string          `yaml:"shares_provider_endpoint"`
 }
 type Tracing struct {
-	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."`
-	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_METADATA_TRACING_TYPE"`
-	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_METADATA_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
-	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_METADATA_TRACING_COLLECTOR"`
+	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_SHARES_TRACING_ENABLED" desc:"Activates tracing."`
+	Type      string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_SHARES_TRACING_TYPE"`
+	Endpoint  string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_SHARES_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."`
+	Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_SHARES_TRACING_COLLECTOR"`
 }
 
 type Logging struct {
-	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_METADATA_LOG_LEVEL" desc:"The log level."`
-	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_METADATA_LOG_PRETTY" desc:"Activates pretty log output."`
-	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_METADATA_LOG_COLOR" desc:"Activates colorized log output."`
-	File   string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_METADATA_LOG_FILE" desc:"The target log file."`
+	Level  string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_SHARES_LOG_LEVEL" desc:"The log level."`
+	Pretty bool   `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_SHARES_LOG_PRETTY" desc:"Activates pretty log output."`
+	Color  bool   `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_SHARES_LOG_COLOR" desc:"Activates colorized log output."`
+	File   string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_SHARES_LOG_FILE" desc:"The target log file."`
 }
 
 type Service struct {
 	Name string `yaml:"-"`
 }
 
 type Debug struct {
-	Addr   string `yaml:"addr" env:"STORAGE_METADATA_DEBUG_ADDR"`
-	Token  string `yaml:"token" env:"STORAGE_METADATA_DEBUG_TOKEN"`
-	Pprof  bool   `yaml:"pprof" env:"STORAGE_METADATA_DEBUG_PPROF"`
-	Zpages bool   `yaml:"zpages" env:"STORAGE_METADATA_DEBUG_ZPAGES"`
+	Addr   string `yaml:"addr" env:"STORAGE_SHARES_DEBUG_ADDR"`
+	Token  string `yaml:"token" env:"STORAGE_SHARES_DEBUG_TOKEN"`
+	Pprof  bool   `yaml:"pprof" env:"STORAGE_SHARES_DEBUG_PPROF"`
+	Zpages bool   `yaml:"zpages" env:"STORAGE_SHARES_DEBUG_ZPAGES"`
 }
 
 type GRPCConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_SHARES_GRPC_ADDR" desc:"The address of the grpc service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_SHARES_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
 }
 
 type HTTPConfig struct {
-	Addr     string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."`
-	Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."`
+	Addr     string `yaml:"addr" env:"STORAGE_SHARES_HTTP_ADDR" desc:"The address of the grpc service."`
+	Protocol string `yaml:"protocol" env:"STORAGE_SHARES_HTTP_PROTOCOL" desc:"The transport protocol of the grpc service."`
 }

extensions/storage/pkg/config/defaults/defaultconfig.go

@@ -331,9 +331,9 @@ func DefaultConfig() *config.Config {
 					GRPCAddr:    "127.0.0.1:9150",
 					Services:    []string{"usershareprovider", "publicshareprovider"},
 				},
-				CS3ProviderAddr:                  "127.0.0.1:9215",
+				CS3ProviderAddr:                  "127.0.0.1:9215", // metadata storage
 				CS3ServiceUser:                   "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-				CS3ServiceUserIdp:                "https://localhost:9200",
+				CS3ServiceUserIdp:                "internal",
 				UserDriver:                       "json",
 				UserJSONFile:                     path.Join(defaults.BaseDataPath(), "storage", "shares.json"),
 				UserSQLUsername:                  "",