Integrate the s3ng storage driver

changelog/unreleased/enable-s3ng-storage-driver.md

@@ -0,0 +1,6 @@
+Enhancement: Enable the s3ng storage driver
+
+We made it possible to use the new s3ng storage driver by adding according
+commandline flags and environment variables.
+
+https://github.com/owncloud/ocis/pull/1886

storage/pkg/command/drivers.go

@@ -101,5 +101,15 @@ func drivers(cfg *config.Config) map[string]interface{} {
 			"bucket":     cfg.Reva.Storages.S3.Bucket,
 			"prefix":     cfg.Reva.Storages.S3.Root,
 		},
+		"s3ng": map[string]interface{}{
+			"root":          cfg.Reva.Storages.Common.Root,
+			"enable_home":   cfg.Reva.Storages.Common.EnableHome,
+			"user_layout":   cfg.Reva.Storages.Common.UserLayout,
+			"s3.region":     cfg.Reva.Storages.S3NG.Region,
+			"s3.access_key": cfg.Reva.Storages.S3NG.AccessKey,
+			"s3.secret_key": cfg.Reva.Storages.S3NG.SecretKey,
+			"s3.endpoint":   cfg.Reva.Storages.S3NG.Endpoint,
+			"s3.bucket":     cfg.Reva.Storages.S3NG.Bucket,
+		},
 	}
 }

storage/pkg/command/storagehome.go

@@ -50,6 +50,7 @@ func StorageHome(cfg *config.Config) *cli.Command {
 				cfg.Reva.Storages.Local.EnableHome = true
 				cfg.Reva.Storages.OwnCloud.EnableHome = true
 				cfg.Reva.Storages.S3.EnableHome = true
+				cfg.Reva.Storages.S3NG.EnableHome = true
 			}
 			rcfg := storageHomeConfigFromStruct(c, cfg)
 

storage/pkg/command/storageusers.go

@@ -50,6 +50,7 @@ func StorageUsers(cfg *config.Config) *cli.Command {
 				cfg.Reva.Storages.Local.EnableHome = true
 				cfg.Reva.Storages.OwnCloud.EnableHome = true
 				cfg.Reva.Storages.S3.EnableHome = true
+				cfg.Reva.Storages.S3NG.EnableHome = true
 			}
 			rcfg := storageUsersConfigFromStruct(c, cfg)
 

storage/pkg/config/config.go

@@ -164,6 +164,7 @@ type StorageConfig struct {
 	Local    DriverCommon
 	OwnCloud DriverOwnCloud
 	S3       DriverS3
+	S3NG     DriverS3NG
 	Common   DriverCommon
 	OCIS     DriverOCIS
 	// TODO checksums ... figure out what that is supposed to do
@@ -268,6 +269,17 @@ type DriverS3 struct {
 	Bucket    string
 }
 
+// DriverS3NG defines the available s3ng storage driver configuration.
+type DriverS3NG struct {
+	DriverCommon
+
+	Region    string
+	AccessKey string
+	SecretKey string
+	Endpoint  string
+	Bucket    string
+}
+
 // OIDC defines the available OpenID Connect configuration.
 type OIDC struct {
 	Issuer   string

storage/pkg/flagset/drivers3ng.go

@@ -0,0 +1,69 @@
+package flagset
+
+import (
+	"github.com/micro/cli/v2"
+	"github.com/owncloud/ocis/ocis-pkg/flags"
+	"github.com/owncloud/ocis/storage/pkg/config"
+)
+
+// DriverS3NGWithConfig applies cfg to the root flagset
+func DriverS3NGWithConfig(cfg *config.Config) []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "storage-s3ng-root",
+			Value:       flags.OverrideDefaultString(cfg.Reva.Storages.Common.Root, "/var/tmp/ocis/storage/users"),
+			Usage:       "the path to the local storage root",
+			EnvVars:     []string{"STORAGE_DRIVER_S3NG_ROOT"},
+			Destination: &cfg.Reva.Storages.Common.Root,
+		},
+		&cli.BoolFlag{
+			Name:        "storage-s3ng-enable-home",
+			Value:       flags.OverrideDefaultBool(cfg.Reva.Storages.Common.EnableHome, false),
+			Usage:       "enable the creation of home storages",
+			EnvVars:     []string{"STORAGE_DRIVER_S3NG_ENABLE_HOME"},
+			Destination: &cfg.Reva.Storages.Common.EnableHome,
+		},
+		&cli.StringFlag{
+			Name:        "storage-s3ng-layout",
+			Value:       flags.OverrideDefaultString(cfg.Reva.Storages.Common.UserLayout, "{{.Id.OpaqueId}}"),
+			Usage:       `"layout of the users home dir path on disk, in addition to {{.Username}}, {{.Mail}}, {{.Id.OpaqueId}}, {{.Id.Idp}} also supports prefixing dirs: "{{substr 0 1 .Username}}/{{.Username}}" will turn "Einstein" into "Ei/Einstein" `,
+			EnvVars:     []string{"STORAGE_DRIVER_S3NG_LAYOUT"},
+			Destination: &cfg.Reva.Storages.Common.UserLayout,
+		},
+		&cli.StringFlag{
+			Name:        "storage-s3ng-region",
+			Value:       "default",
+			Usage:       `"the s3 region" `,
+			EnvVars:     []string{"STORAGE_DRIVER_S3NG_REGION"},
+			Destination: &cfg.Reva.Storages.S3NG.Region,
+		},
+		&cli.StringFlag{
+			Name:        "storage-s3ng-accesskey",
+			Value:       "",
+			Usage:       `"the s3 access key" `,
+			EnvVars:     []string{"STORAGE_DRIVER_S3NG_ACCESS_KEY"},
+			Destination: &cfg.Reva.Storages.S3NG.AccessKey,
+		},
+		&cli.StringFlag{
+			Name:        "storage-s3ng-secretkey",
+			Value:       "",
+			Usage:       `"the secret s3 api key" `,
+			EnvVars:     []string{"STORAGE_DRIVER_S3NG_SECRET_KEY"},
+			Destination: &cfg.Reva.Storages.S3NG.SecretKey,
+		},
+		&cli.StringFlag{
+			Name:        "storage-s3ng-endpoint",
+			Value:       "",
+			Usage:       `"s3 compatible API endpoint" `,
+			EnvVars:     []string{"STORAGE_DRIVER_S3NG_ENDPOINT"},
+			Destination: &cfg.Reva.Storages.S3NG.Endpoint,
+		},
+		&cli.StringFlag{
+			Name:        "storage-s3ng-bucket",
+			Value:       "",
+			Usage:       `"bucket where the data will be stored in`,
+			EnvVars:     []string{"STORAGE_DRIVER_S3NG_BUCKET"},
+			Destination: &cfg.Reva.Storages.S3NG.Bucket,
+		},
+	}
+}

storage/pkg/flagset/storagehome.go

@@ -156,6 +156,7 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag {
 	flags = append(flags, DriverLocalWithConfig(cfg)...)
 	flags = append(flags, DriverOwnCloudWithConfig(cfg)...)
 	flags = append(flags, DriverOCISWithConfig(cfg)...)
+	flags = append(flags, DriverS3NGWithConfig(cfg)...)
 
 	return flags
 }

storage/pkg/flagset/storagemetadata.go

@@ -105,6 +105,7 @@ func StorageMetadata(cfg *config.Config) []cli.Flag {
 	f = append(f, DriverLocalWithConfig(cfg)...)
 	f = append(f, DriverOwnCloudWithConfig(cfg)...)
 	f = append(f, DriverOCISWithConfig(cfg)...)
+	f = append(f, DriverS3NGWithConfig(cfg)...)
 	return f
 
 }

storage/pkg/flagset/storageusers.go

@@ -146,6 +146,7 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag {
 	flags = append(flags, DriverLocalWithConfig(cfg)...)
 	flags = append(flags, DriverOwnCloudWithConfig(cfg)...)
 	flags = append(flags, DriverOCISWithConfig(cfg)...)
+	flags = append(flags, DriverS3NGWithConfig(cfg)...)
 
 	return flags
 }

storage/templates/CONFIGURATION.tmpl

@@ -111,3 +111,7 @@ Example: Set the home and users Storage Provider to `ocis`
 ### Ocis Driver
 
 {{ template "option" (list .Options "DriverOCISWithConfig") -}}
+
+### S3ng Driver
+
+{{ template "option" (list .Options "DriverS3NGWithConfig") -}}

tests/acceptance/docker/Makefile

@@ -54,13 +54,20 @@ help:
 	@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-ocis\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}"
 	@echo -e "\tmake localApiTests-apiBugDemonstration-ocis\t\t${BLUE}run apiBugDemonstration test suite${RESET}"
 	@echo
+	@echo -e "${GREEN}Run full oCIS test suites against oCIS with s3ng storage:${RESET}\n"
+	@echo -e "\tmake localApiTests-apiBasic-s3ng\t\t${BLUE}run apiBasic test suite${RESET}"
+	@echo -e "\tmake localApiTests-apiOcisSpecific-s3ng\t\t${BLUE}run apiOcisSPecific test suite${RESET}"
+	@echo
 	@echo -e "${GREEN}Run full oCIS test suites against oCIS with ownCloud storage:${RESET}\n"
 	@echo -e "\tmake localApiTests-apiAccountsHashDifficulty-owncloud\t\t${BLUE}run apiAccountsHashDifficulty test suite${RESET}"
 	@echo -e "\tmake localApiTests-apiBugDemonstration-owncloud\t${BLUE}run apiBugDemonstration test suite${RESET}"
 	@echo
 	@echo -e "${GREEN}Run full ownCloud test suites against oCIS with oCIS storage:${RESET}\n"
 	@echo -e "\tmake Core-API-Tests-ocis-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}"
 	@echo
+	@echo -e "${GREEN}Run full ownCloud test suites against oCIS with s3ng storage:${RESET}\n"
+	@echo -e "\tmake Core-API-Tests-s3ng-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}"
+	@echo
 	@echo -e "${GREEN}Run full ownCloud test suites against oCIS with ownCloud storage:${RESET}\n"
 	@echo -e "\tmake Core-API-Tests-owncloud-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}"
 	@echo
@@ -70,6 +77,12 @@ help:
 	@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
 	@echo -e "\texample: ${RED}tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature${RESET}"
 	@echo
+	@echo -e "${GREEN}Run an oCIS feature test against oCIS with s3ng storage:${RESET}\n"
+	@echo -e "\tmake test-ocis-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
+	@echo
+	@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
+	@echo -e "\texample: ${RED}tests/acceptance/features/apiOcisSpecific/apiAuthOcs-ocsDELETEAuth.feature${RESET}"
+	@echo
 	@echo -e "${GREEN}Run an oCIS feature test against oCIS with owncloud storage:${RESET}\n"
 	@echo -e "\tmake test-ocis-feature-owncloud-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
 	@echo
@@ -82,6 +95,12 @@ help:
 	@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
 	@echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}"
 	@echo
+	@echo -e "${GREEN}Run an ownCloud feature test against oCIS with s3ng storage:${RESET}\n"
+	@echo -e "\tmake test-oc10-feature-s3ng-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
+	@echo
+	@echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition."
+	@echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}"
+	@echo
 	@echo -e "${GREEN}Run an ownCloud feature test against oCIS with owncloud storage:${RESET}\n"
 	@echo -e "\tmake test-oc10-feature-owncloud-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}"
 	@echo
@@ -105,6 +124,14 @@ test-ocis-feature-ocis-storage: ## test a ocis feature with oCIS storage, useage
 	BEHAT_FEATURE=$(BEHAT_FEATURE) \
 	$(MAKE) --no-print-directory testSuite
 
+.PHONY: test-ocis-feature-s3ng-storage
+test-ocis-feature-s3ng-storage: ## test a ocis feature with s3ng storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiOcisSpecific/apiAuthOcs-ocsDELETEAuth.feature:7'
+	@TEST_SOURCE=ocis \
+	STORAGE=s3ng \
+	BEHAT_FEATURE=$(BEHAT_FEATURE) \
+	START_CEPH=1 \
+	$(MAKE) --no-print-directory testSuite
+
 .PHONY: test-ocis-feature-owncloud-storage
 test-ocis-feature-owncloud-storage: ## test a ocis feature with oc10 storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiBugDemonstration/apiAuthOcs-ocsDELETEAuth.feature:7'
 	@TEST_SOURCE=ocis \
@@ -119,6 +146,14 @@ test-oc10-feature-ocis-storage: ## test a oC10 feature with oCIS storage, useage
 	BEHAT_FEATURE=$(BEHAT_FEATURE) \
 	$(MAKE) --no-print-directory testSuite
 
+.PHONY: test-oc10-feature-s3ng-storage
+test-oc10-feature-s3ng-storage: ## test a oC10 feature with s3ng storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature'
+	@TEST_SOURCE=oc10 \
+	STORAGE=s3ng \
+	BEHAT_FEATURE=$(BEHAT_FEATURE) \
+	START_CEPH=1 \
+	$(MAKE) --no-print-directory testSuite
+
 .PHONY: test-oc10-feature-owncloud-storage
 test-oc10-feature-owncloud-storage: ## test a oC10 feature with oc10 storage, useage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature'
 	@TEST_SOURCE=oc10 \
@@ -154,6 +189,23 @@ localApiTests-apiAccountsHashDifficulty-ocis: ## run apiAccountsHashDifficulty t
 	BEHAT_SUITE=apiAccountsHashDifficulty \
 	$(MAKE) --no-print-directory testSuite
 
+.PHONY: localApiTests-apiOcisSpecific-s3ng
+localApiTests-apiOcisSpecific-s3ng: ## run apiOcisSPecific test suite with s3ng storage
+	@TEST_SOURCE=ocis \
+	STORAGE=s3ng \
+	BEHAT_SUITE=apiOcisSpecific \
+	START_CEPH=1 \
+	$(MAKE) --no-print-directory testSuite
+
+
+.PHONY: localApiTests-apiBasic-s3ng
+localApiTests-apiBasic-s3ng: ## run apiBasic test suite with s3ng storage
+	@TEST_SOURCE=ocis \
+	STORAGE=s3ng \
+	BEHAT_SUITE=apiBasic \
+	START_CEPH=1 \
+	$(MAKE) --no-print-directory testSuite
+
 targets = $(addprefix Core-API-Tests-owncloud-storage-,$(PARTS))
 .PHONY: $(targets)
 $(targets):
@@ -174,7 +226,12 @@ $(targets):
 
 .PHONY: testSuite
 testSuite: clean-docker-container
-	@COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) \
+	@if [ -n "${START_CEPH}" ]; then \
+		COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) \
+		COMPOSE_FILE=src/ceph.yml \
+		docker-compose run start_ceph; \
+	fi; \
+	COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) \
 	COMPOSE_FILE=$(COMPOSE_FILE) \
 	STORAGE=$(STORAGE) \
 	TEST_SOURCE=$(TEST_SOURCE) \
@@ -183,7 +240,7 @@ testSuite: clean-docker-container
 	BEHAT_FEATURE=$(BEHAT_FEATURE) \
 	DIVIDE_INTO_NUM_PARTS=$(DIVIDE_INTO_NUM_PARTS) \
 	RUN_PART=$(RUN_PART) \
-	docker-compose up -d --build --remove-orphans --force-recreate
+	docker-compose up -d --build --force-recreate
 
 .PHONY: show-test-logs
 show-test-logs: ## show logs of test

tests/acceptance/docker/src/ceph.yml

@@ -0,0 +1,17 @@
+services:
+  start_ceph:
+    image: dadarek/wait-for-dependencies
+    depends_on:
+      - ceph
+    command: ceph:5000
+  ceph:
+    image: ceph/daemon
+    command: demo
+    environment:
+      MON_IP: 127.0.0.1
+      CEPH_PUBLIC_NETWORK: 0.0.0.0/0
+      CEPH_DEMO_UID: test-user
+      CEPH_DEMO_ACCESS_KEY: test
+      CEPH_DEMO_SECRET_KEY: test
+      CEPH_DEMO_BUCKET: test
+      RGW_NAME: ceph

tests/acceptance/docker/src/ocis-base.yml

@@ -22,6 +22,12 @@ services:
       IDP_ISS: https://ocis-server:9200
       IDP_TLS: "true"
       ACCOUNTS_HASH_DIFFICULTY: 4
+      # s3ng specific settings
+      STORAGE_DRIVER_S3NG_ENDPOINT: http://ceph:8080
+      STORAGE_DRIVER_S3NG_REGION: default
+      STORAGE_DRIVER_S3NG_ACCESS_KEY: test
+      STORAGE_DRIVER_S3NG_SECRET_KEY: test
+      STORAGE_DRIVER_S3NG_BUCKET: test
     volumes:
       - ../../../config:/drone/src/ocis/tests/config
       - oCISownCloud10testsuite:/srv

tests/acceptance/docker/src/run-tests.sh

@@ -62,6 +62,12 @@ then
         export DELETE_USER_DATA_CMD='rm -rf /srv/app/tmp/ocis/storage/users/nodes/root/* /srv/app/tmp/ocis/storage/users/nodes/*-*-*-*'
         export OCIS_REVA_DATA_ROOT=''
         export OCIS_SKELETON_STRATEGY='upload'
+    elif [ "$STORAGE" = "s3ng" ]
+    then
+        export BEHAT_FILTER_TAGS='~@skipOnOcis-S3NG-Storage'
+        export DELETE_USER_DATA_CMD='rm -rf /srv/app/tmp/ocis/storage/users/nodes/root/* /srv/app/tmp/ocis/storage/users/nodes/*-*-*-*'
+        export OCIS_REVA_DATA_ROOT=''
+        export OCIS_SKELETON_STRATEGY='upload'
     else
         echo "non existing storage selected"
         exit 1