introduce metadata gateway

.drone.star

@@ -1710,7 +1710,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "PROXY_ENABLE_BASIC_AUTH": True,
             "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
             "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
-            "OCIS_LOG_LEVEL": "error",
+            "OCIS_LOG_LEVEL": "debug",
             "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings",
             "OCIS_INSECURE": "true",
             "IDM_CREATE_DEMO_USERS": True,

changelog/unreleased/metadata-gateway.md

@@ -0,0 +1,5 @@
+Enhancement: wrap metadata storage with dedicated reva gateway
+
+We wrapped the metadata storage in a minimal reva instance with a dedicated gateway, including static storage registry, static auth registry, in memory userprovider, machine authprovider and demo permissions service. This allows us to preconfigure the service user for the ocis settings service, share and public share providers.
+
+https://github.com/owncloud/ocis/pull/3602

changelog/unreleased/update-reva.md

@@ -7,5 +7,6 @@ Updated reva to version 2.x.x. This update includes:
 https://github.com/owncloud/ocis/pull/3552
 https://github.com/owncloud/ocis/pull/3570
 https://github.com/owncloud/ocis/pull/3601
+https://github.com/owncloud/ocis/pull/3602
 https://github.com/owncloud/ocis/pull/3605
 https://github.com/owncloud/ocis/pull/3611

extensions/settings/pkg/config/defaults/defaultconfig.go

@@ -55,10 +55,10 @@ func DefaultConfig() *config.Config {
 		},
 
 		Metadata: config.Metadata{
-			GatewayAddress:    "127.0.0.1:9142",
+			GatewayAddress:    "127.0.0.1:9215",
 			StorageAddress:    "127.0.0.1:9215",
 			ServiceUserID:     "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-			ServiceUserIDP:    "https://localhost:9200",
+			ServiceUserIDP:    "internal",
 			MachineAuthAPIKey: "change-me-please",
 		},
 	}

extensions/storage-metadata/pkg/command/command.go

@@ -125,16 +125,80 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
 		},
 		"shared": map[string]interface{}{
 			"jwt_secret":                cfg.JWTSecret,
-			"gatewaysvc":                cfg.GatewayEndpoint,
+			"gatewaysvc":                cfg.GRPC.Addr,
 			"skip_user_groups_in_token": cfg.SkipUserGroupsInToken,
 		},
 		"grpc": map[string]interface{}{
 			"network": cfg.GRPC.Protocol,
 			"address": cfg.GRPC.Addr,
-			"interceptors": map[string]interface{}{
-				"log": map[string]interface{}{},
-			},
 			"services": map[string]interface{}{
+				"gateway": map[string]interface{}{
+					// registries are located on the gateway
+					"authregistrysvc":    cfg.GRPC.Addr,
+					"storageregistrysvc": cfg.GRPC.Addr,
+					// user metadata is located on the users services
+					"userprovidersvc":  cfg.GRPC.Addr,
+					"groupprovidersvc": cfg.GRPC.Addr,
+					"permissionssvc":   cfg.GRPC.Addr,
+					// other
+					"disable_home_creation_on_login": true, // metadata manually creates a space
+					// metadata always uses the simple upload, so no transfer secret or datagateway needed
+				},
+				"userprovider": map[string]interface{}{
+					"driver": "memory",
+					"drivers": map[string]interface{}{
+						"memory": map[string]interface{}{
+							"users": map[string]interface{}{
+								"serviceuser": map[string]interface{}{
+									"id": map[string]interface{}{
+										"opaqueId": "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
+										"idp":      "internal",
+										"type":     1, // user.UserType_USER_TYPE_PRIMARY
+									},
+									"username":     "serviceuser",
+									"display_name": "System User",
+								},
+							},
+						},
+					},
+				},
+				"authregistry": map[string]interface{}{
+					"driver": "static",
+					"drivers": map[string]interface{}{
+						"static": map[string]interface{}{
+							"rules": map[string]interface{}{
+								"machine": cfg.GRPC.Addr,
+							},
+						},
+					},
+				},
+				"authprovider": map[string]interface{}{
+					"auth_manager": "machine",
+					"auth_managers": map[string]interface{}{
+						"machine": map[string]interface{}{
+							"api_key":      cfg.MachineAuthAPIKey,
+							"gateway_addr": cfg.GRPC.Addr,
+						},
+					},
+				},
+				"permissions": map[string]interface{}{
+					"driver": "demo",
+					"drivers": map[string]interface{}{
+						"demo": map[string]interface{}{},
+					},
+				},
+				"storageregistry": map[string]interface{}{
+					"driver": "static",
+					"drivers": map[string]interface{}{
+						"static": map[string]interface{}{
+							"rules": map[string]interface{}{
+								"/": map[string]interface{}{
+									"address": cfg.GRPC.Addr,
+								},
+							},
+						},
+					},
+				},
 				"storageprovider": map[string]interface{}{
 					"driver":          cfg.Driver,
 					"drivers":         config.MetadataDrivers(cfg),
@@ -146,7 +210,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in
 		"http": map[string]interface{}{
 			"network": cfg.HTTP.Protocol,
 			"address": cfg.HTTP.Addr,
-			// TODO build services dynamically
+			// no datagateway needed as the metadata clients directly talk to the dataprovider with the simple protocol
 			"services": map[string]interface{}{
 				"dataprovider": map[string]interface{}{
 					"prefix":      "data",

extensions/storage-metadata/pkg/config/config.go

@@ -19,13 +19,13 @@ type Config struct {
 
 	Context               context.Context
 	JWTSecret             string
-	GatewayEndpoint       string
 	SkipUserGroupsInToken bool
 	Driver                string  `yaml:"driver" env:"STORAGE_METADATA_DRIVER" desc:"The driver which should be used by the service"`
 	Drivers               Drivers `yaml:"drivers"`
 	DataServerURL         string
 	TempFolder            string
-	DataProviderInsecure  bool `env:"OCIS_INSECURE;STORAGE_METADATA_DATAPROVIDER_INSECURE"`
+	DataProviderInsecure  bool   `env:"OCIS_INSECURE;STORAGE_METADATA_DATAPROVIDER_INSECURE"`
+	MachineAuthAPIKey     string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;STORAGE_METADATA_MACHINE_AUTH_API_KEY"`
 }
 type Tracing struct {
 	Enabled   bool   `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."`

extensions/storage-metadata/pkg/config/defaults/defaultconfig.go

@@ -35,11 +35,11 @@ func DefaultConfig() *config.Config {
 		Service: config.Service{
 			Name: "storage-metadata",
 		},
-		GatewayEndpoint: "127.0.0.1:9142",
-		JWTSecret:       "Pive-Fumkiu4",
-		TempFolder:      filepath.Join(defaults.BaseDataPath(), "tmp", "metadata"),
-		DataServerURL:   "http://localhost:9216/data",
-		Driver:          "ocis",
+		JWTSecret:         "Pive-Fumkiu4",
+		TempFolder:        filepath.Join(defaults.BaseDataPath(), "tmp", "metadata"),
+		DataServerURL:     "http://localhost:9216/data",
+		MachineAuthAPIKey: "change-me-please",
+		Driver:            "ocis",
 		Drivers: config.Drivers{
 			EOS: config.EOSDriver{
 				Root:                "/eos/dockertest/reva",
@@ -59,7 +59,7 @@ func DefaultConfig() *config.Config {
 				SecProtocol:         "",
 				Keytab:              "",
 				SingleUsername:      "",
-				GatewaySVC:          "127.0.0.1:9142",
+				GatewaySVC:          "127.0.0.1:9215",
 			},
 			Local: config.LocalDriver{
 				Root: filepath.Join(defaults.BaseDataPath(), "storage", "local", "metadata"),
@@ -71,12 +71,12 @@ func DefaultConfig() *config.Config {
 				Root:                filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
 				UserLayout:          "{{.Id.OpaqueId}}",
 				Region:              "default",
-				PermissionsEndpoint: "127.0.0.1:9191",
+				PermissionsEndpoint: "127.0.0.1:9215",
 			},
 			OCIS: config.OCISDriver{
 				Root:                filepath.Join(defaults.BaseDataPath(), "storage", "metadata"),
 				UserLayout:          "{{.Id.OpaqueId}}",
-				PermissionsEndpoint: "127.0.0.1:9191",
+				PermissionsEndpoint: "127.0.0.1:9215",
 			},
 		},
 	}

extensions/storage/pkg/config/defaults/defaultconfig.go

@@ -339,9 +339,9 @@ func DefaultConfig() *config.Config {
 					GRPCAddr:    "127.0.0.1:9150",
 					Services:    []string{"usershareprovider", "publicshareprovider"},
 				},
-				CS3ProviderAddr:                  "127.0.0.1:9215",
+				CS3ProviderAddr:                  "127.0.0.1:9215", // metadata storage
 				CS3ServiceUser:                   "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad",
-				CS3ServiceUserIdp:                "https://localhost:9200",
+				CS3ServiceUserIdp:                "internal",
 				UserDriver:                       "json",
 				UserJSONFile:                     path.Join(defaults.BaseDataPath(), "storage", "shares.json"),
 				UserSQLUsername:                  "",

go.sum

@@ -318,8 +318,6 @@ github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde h1:WrD9O8ZaWvsm0
 github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
 github.com/cs3org/reva v1.18.0 h1:MbPS5ZAa8RzKcTxAVeSDdISB3XXqLIxqB03BTN5ReBY=
 github.com/cs3org/reva v1.18.0/go.mod h1:e5VDUDu4vVWIeVkZcW//n6UZzhGGMa+Tz/whCiX3N6o=
-github.com/cs3org/reva/v2 v2.0.0-20220427133111-618964eed515 h1:8pPCLxNXVz/q7PMM6Zq1lff3P8SFAu8/CXwB2eA21xc=
-github.com/cs3org/reva/v2 v2.0.0-20220427133111-618964eed515/go.mod h1:2e/4HcIy54Mic3V7Ow0bz4n5dkZU0dHIZSWomFe5vng=
 github.com/cs3org/reva/v2 v2.0.0-20220427203355-0164880ac7d3 h1:6sKjGI0AUW5tBXWBduaBoc+9sNYZWQR894G0oFCbus0=
 github.com/cs3org/reva/v2 v2.0.0-20220427203355-0164880ac7d3/go.mod h1:2e/4HcIy54Mic3V7Ow0bz4n5dkZU0dHIZSWomFe5vng=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=