From a4fb51f3155627c0496050f680a4590c35a52526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 3 Nov 2021 12:44:37 +0100 Subject: [PATCH] fix idp schema, add virtual setup (#2219) --- .drone.star | 63 +++++++++- .../unreleased/add-virtual-view-tests.md | 3 + examples/meshdirectory/groups.demo.json | 32 ++--- examples/meshdirectory/users.demo.json | 6 +- examples/oc-phoenix/groups.demo.json | 32 ++--- examples/oc-phoenix/users.demo.json | 6 +- examples/ocmd/users.demo.json | 8 +- examples/plugin/users.demo.json | 3 +- examples/standalone/groups.demo.json | 32 ++--- examples/standalone/users.demo.json | 6 +- examples/storage-references/groups.demo.json | 32 ++--- examples/storage-references/users.demo.json | 8 +- examples/two-server-setup/groups.demo.json | 32 ++--- .../grpc/services/gateway/storageprovider.go | 18 ++- .../storageprovider/storageprovider.go | 9 +- .../utils/decomposedfs/decomposedfs.go | 14 ++- .../utils/decomposedfs/options/options.go | 4 +- pkg/storage/utils/decomposedfs/tree/tree.go | 18 +-- pkg/storage/utils/localfs/localfs.go | 3 +- tests/acceptance/config/behat.yml | 14 +++ .../apiVirtualViews/virtualViews.feature | 65 ++++++++++ .../integration/grpc/fixtures/users.demo.json | 6 +- tests/integration/grpc/userprovider_test.go | 2 +- .../drone/frontend-global.toml | 117 ++++++++++++++++++ tests/oc-integration-tests/drone/gateway.toml | 7 ++ .../drone/storage-local-1.toml | 46 +++++++ .../drone/storage-local-2.toml | 46 +++++++ .../drone/users.demo.json | 29 ++++- .../local/frontend-global.toml | 117 ++++++++++++++++++ tests/oc-integration-tests/local/gateway.toml | 7 ++ .../local/storage-local-1.toml | 46 +++++++ .../local/storage-local-2.toml | 46 +++++++ .../local/users.demo.json | 29 ++++- 33 files changed, 784 insertions(+), 122 deletions(-) create mode 100644 changelog/unreleased/add-virtual-view-tests.md create mode 100644 tests/acceptance/features/apiVirtualViews/virtualViews.feature create mode 100644 tests/oc-integration-tests/drone/frontend-global.toml create mode 100644 tests/oc-integration-tests/drone/storage-local-1.toml create mode 100644 tests/oc-integration-tests/drone/storage-local-2.toml create mode 100644 tests/oc-integration-tests/local/frontend-global.toml create mode 100644 tests/oc-integration-tests/local/storage-local-1.toml create mode 100644 tests/oc-integration-tests/local/storage-local-2.toml diff --git a/.drone.star b/.drone.star index 494c282bb0..101f46a0da 100644 --- a/.drone.star +++ b/.drone.star @@ -105,6 +105,7 @@ def main(ctx): litmusOcisOldWebdav(), litmusOcisNewWebdav(), litmusOcisSpacesDav(), + virtualViews(), ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12) @@ -484,6 +485,64 @@ def release(): "depends_on": ['changelog'], } +def virtualViews(): + return { + "kind": "pipeline", + "type": "docker", + "name": "virtual-views", + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + "tag", + ], + }, + }, + "steps": [ + makeStep("build-ci"), + { + "name": "revad-services", + "image": "registry.cern.ch/docker.io/library/golang:1.17", + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend-global.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-local-1.toml &", + "/drone/src/cmd/revad/revad -c storage-local-2.toml &", + "/drone/src/cmd/revad/revad -c users.toml", + ], + }, + cloneOc10TestReposStep(), + { + "name": "oC10APIAcceptanceTestsOcisStorage", + "image": "registry.cern.ch/docker.io/owncloudci/php:7.4", + "commands": [ + "cd /drone/src", + "make test-acceptance-api", + ], + "environment": { + "PATH_TO_CORE": "/drone/src/tmp/testrunner", + "TEST_SERVER_URL": "http://revad-services:20180", + "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", + "STORAGE_DRIVER": "OCIS", + "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", + "TEST_REVA": "true", + "REGULAR_USER_PASSWORD": "relativity", + "SEND_SCENARIO_LINE_REFERENCES": "true", + "BEHAT_SUITE": "apiVirtualViews", + }, + }, + ], + "depends_on": ['changelog'], + } + def litmusOcisOldWebdav(): return { "kind": "pipeline", @@ -701,7 +760,7 @@ def ocisIntegrationTests(parallelRuns, skipExceptParts = []): "environment": { "TEST_SERVER_URL": "http://revad-services:20080", "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-*", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", "STORAGE_DRIVER": "OCIS", "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", "TEST_WITH_LDAP": "true", @@ -776,7 +835,7 @@ def s3ngIntegrationTests(parallelRuns, skipExceptParts = []): "environment": { "TEST_SERVER_URL": "http://revad-services:20080", "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", - "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-*", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", "STORAGE_DRIVER": "S3NG", "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", "TEST_WITH_LDAP": "true", diff --git a/changelog/unreleased/add-virtual-view-tests.md b/changelog/unreleased/add-virtual-view-tests.md new file mode 100644 index 0000000000..17bdf6a721 --- /dev/null +++ b/changelog/unreleased/add-virtual-view-tests.md @@ -0,0 +1,3 @@ +Enhancement: add virtual view tests + +https://github.com/cs3org/reva/pull/2219 \ No newline at end of file diff --git a/examples/meshdirectory/groups.demo.json b/examples/meshdirectory/groups.demo.json index f78da191bc..42fd1af0c2 100644 --- a/examples/meshdirectory/groups.demo.json +++ b/examples/meshdirectory/groups.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "sailing-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "sailing-lovers", "mail": "sailing-lovers@example.org", @@ -12,7 +12,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -20,7 +20,7 @@ { "id": { "opaque_id": "violin-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "violin-haters", "mail": "violin-haters@example.org", @@ -30,7 +30,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -38,7 +38,7 @@ { "id": { "opaque_id": "radium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "radium-lovers", "mail": "radium-lovers@example.org", @@ -48,7 +48,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -56,7 +56,7 @@ { "id": { "opaque_id": "polonium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "polonium-lovers", "mail": "polonium-lovers@example.org", @@ -66,7 +66,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -74,7 +74,7 @@ { "id": { "opaque_id": "quantum-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "quantum-lovers", "mail": "quantum-lovers@example.org", @@ -84,7 +84,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -92,7 +92,7 @@ { "id": { "opaque_id": "philosophy-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "philosophy-haters", "mail": "philosophy-haters@example.org", @@ -102,7 +102,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -110,7 +110,7 @@ { "id": { "opaque_id": "physics-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "physics-lovers", "mail": "physics-lovers@example.org", @@ -120,19 +120,19 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] diff --git a/examples/meshdirectory/users.demo.json b/examples/meshdirectory/users.demo.json index 12c784f7eb..893d69b99f 100644 --- a/examples/meshdirectory/users.demo.json +++ b/examples/meshdirectory/users.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "einstein", @@ -14,7 +14,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "marie", @@ -26,7 +26,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "richard", diff --git a/examples/oc-phoenix/groups.demo.json b/examples/oc-phoenix/groups.demo.json index f78da191bc..42fd1af0c2 100644 --- a/examples/oc-phoenix/groups.demo.json +++ b/examples/oc-phoenix/groups.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "sailing-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "sailing-lovers", "mail": "sailing-lovers@example.org", @@ -12,7 +12,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -20,7 +20,7 @@ { "id": { "opaque_id": "violin-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "violin-haters", "mail": "violin-haters@example.org", @@ -30,7 +30,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -38,7 +38,7 @@ { "id": { "opaque_id": "radium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "radium-lovers", "mail": "radium-lovers@example.org", @@ -48,7 +48,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -56,7 +56,7 @@ { "id": { "opaque_id": "polonium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "polonium-lovers", "mail": "polonium-lovers@example.org", @@ -66,7 +66,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -74,7 +74,7 @@ { "id": { "opaque_id": "quantum-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "quantum-lovers", "mail": "quantum-lovers@example.org", @@ -84,7 +84,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -92,7 +92,7 @@ { "id": { "opaque_id": "philosophy-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "philosophy-haters", "mail": "philosophy-haters@example.org", @@ -102,7 +102,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -110,7 +110,7 @@ { "id": { "opaque_id": "physics-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "physics-lovers", "mail": "physics-lovers@example.org", @@ -120,19 +120,19 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] diff --git a/examples/oc-phoenix/users.demo.json b/examples/oc-phoenix/users.demo.json index 12c784f7eb..893d69b99f 100644 --- a/examples/oc-phoenix/users.demo.json +++ b/examples/oc-phoenix/users.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "einstein", @@ -14,7 +14,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "marie", @@ -26,7 +26,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "richard", diff --git a/examples/ocmd/users.demo.json b/examples/ocmd/users.demo.json index e1b52cb806..0f96aeece1 100644 --- a/examples/ocmd/users.demo.json +++ b/examples/ocmd/users.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "cernbox.cern.ch", + "idp": "https://cernbox.cern.ch", "type": 1 }, "username": "einstein", @@ -28,7 +28,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "cesnet.cz", + "idp": "https://cesnet.cz", "type": 1 }, "username": "marie", @@ -54,7 +54,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "example.org", + "idp": "https://example.org", "type": 1 }, "username": "richard", @@ -80,7 +80,7 @@ { "id": { "opaque_id": "932b4522-139b-4815-8ef4-42cdf82c3d51", - "idp": "example.com", + "idp": "https://example.com", "type": 1 }, "username": "test", diff --git a/examples/plugin/users.demo.json b/examples/plugin/users.demo.json index 21479c1ec7..99a04a2554 100644 --- a/examples/plugin/users.demo.json +++ b/examples/plugin/users.demo.json @@ -2,7 +2,8 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "cernbox.cern.ch" + "idp": "https://cernbox.cern.ch", + "type": 1 }, "username": "einstein", "secret": "relativity", diff --git a/examples/standalone/groups.demo.json b/examples/standalone/groups.demo.json index f78da191bc..42fd1af0c2 100644 --- a/examples/standalone/groups.demo.json +++ b/examples/standalone/groups.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "sailing-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "sailing-lovers", "mail": "sailing-lovers@example.org", @@ -12,7 +12,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -20,7 +20,7 @@ { "id": { "opaque_id": "violin-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "violin-haters", "mail": "violin-haters@example.org", @@ -30,7 +30,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -38,7 +38,7 @@ { "id": { "opaque_id": "radium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "radium-lovers", "mail": "radium-lovers@example.org", @@ -48,7 +48,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -56,7 +56,7 @@ { "id": { "opaque_id": "polonium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "polonium-lovers", "mail": "polonium-lovers@example.org", @@ -66,7 +66,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -74,7 +74,7 @@ { "id": { "opaque_id": "quantum-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "quantum-lovers", "mail": "quantum-lovers@example.org", @@ -84,7 +84,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -92,7 +92,7 @@ { "id": { "opaque_id": "philosophy-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "philosophy-haters", "mail": "philosophy-haters@example.org", @@ -102,7 +102,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -110,7 +110,7 @@ { "id": { "opaque_id": "physics-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "physics-lovers", "mail": "physics-lovers@example.org", @@ -120,19 +120,19 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] diff --git a/examples/standalone/users.demo.json b/examples/standalone/users.demo.json index 12c784f7eb..893d69b99f 100644 --- a/examples/standalone/users.demo.json +++ b/examples/standalone/users.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "einstein", @@ -14,7 +14,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "marie", @@ -26,7 +26,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "richard", diff --git a/examples/storage-references/groups.demo.json b/examples/storage-references/groups.demo.json index f78da191bc..42fd1af0c2 100644 --- a/examples/storage-references/groups.demo.json +++ b/examples/storage-references/groups.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "sailing-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "sailing-lovers", "mail": "sailing-lovers@example.org", @@ -12,7 +12,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -20,7 +20,7 @@ { "id": { "opaque_id": "violin-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "violin-haters", "mail": "violin-haters@example.org", @@ -30,7 +30,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -38,7 +38,7 @@ { "id": { "opaque_id": "radium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "radium-lovers", "mail": "radium-lovers@example.org", @@ -48,7 +48,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -56,7 +56,7 @@ { "id": { "opaque_id": "polonium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "polonium-lovers", "mail": "polonium-lovers@example.org", @@ -66,7 +66,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -74,7 +74,7 @@ { "id": { "opaque_id": "quantum-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "quantum-lovers", "mail": "quantum-lovers@example.org", @@ -84,7 +84,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -92,7 +92,7 @@ { "id": { "opaque_id": "philosophy-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "philosophy-haters", "mail": "philosophy-haters@example.org", @@ -102,7 +102,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -110,7 +110,7 @@ { "id": { "opaque_id": "physics-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "physics-lovers", "mail": "physics-lovers@example.org", @@ -120,19 +120,19 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] diff --git a/examples/storage-references/users.demo.json b/examples/storage-references/users.demo.json index 426ba07b75..38932b65a0 100644 --- a/examples/storage-references/users.demo.json +++ b/examples/storage-references/users.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "einstein", @@ -14,7 +14,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "marie", @@ -26,7 +26,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 1 }, "username": "richard", @@ -38,7 +38,7 @@ { "id": { "opaque_id": "0e4d9dc1-8349-49fe-8afc-6b844aec1cf6", - "idp": "localhost:20080", + "idp": "http://localhost:20080", "type": 7 }, "username": "lwaccount", diff --git a/examples/two-server-setup/groups.demo.json b/examples/two-server-setup/groups.demo.json index f78da191bc..42fd1af0c2 100644 --- a/examples/two-server-setup/groups.demo.json +++ b/examples/two-server-setup/groups.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "sailing-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "sailing-lovers", "mail": "sailing-lovers@example.org", @@ -12,7 +12,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -20,7 +20,7 @@ { "id": { "opaque_id": "violin-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "violin-haters", "mail": "violin-haters@example.org", @@ -30,7 +30,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -38,7 +38,7 @@ { "id": { "opaque_id": "radium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "radium-lovers", "mail": "radium-lovers@example.org", @@ -48,7 +48,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -56,7 +56,7 @@ { "id": { "opaque_id": "polonium-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "polonium-lovers", "mail": "polonium-lovers@example.org", @@ -66,7 +66,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -74,7 +74,7 @@ { "id": { "opaque_id": "quantum-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "quantum-lovers", "mail": "quantum-lovers@example.org", @@ -84,7 +84,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -92,7 +92,7 @@ { "id": { "opaque_id": "philosophy-haters", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "philosophy-haters", "mail": "philosophy-haters@example.org", @@ -102,7 +102,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] @@ -110,7 +110,7 @@ { "id": { "opaque_id": "physics-lovers", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "group_name": "physics-lovers", "mail": "physics-lovers@example.org", @@ -120,19 +120,19 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } }, { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" } } ] diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index dcfd7be3e7..5292ac309a 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -1334,7 +1334,14 @@ func (s *svc) statAcrossProviders(ctx context.Context, req *provider.StatRequest } if resp.Info != nil { info.Size += resp.Info.Size - info.Mtime = utils.LaterTS(info.Mtime, resp.Info.Mtime) + if utils.TSToUnixNano(resp.Info.Mtime) > utils.TSToUnixNano(info.Mtime) { + info.Mtime = resp.Info.Mtime + info.Etag = resp.Info.Etag + info.Checksum = resp.Info.Checksum + } + if info.Etag == "" && info.Etag != resp.Info.Etag { + info.Etag = resp.Info.Etag + } } } @@ -1705,7 +1712,14 @@ func (s *svc) listContainerAcrossProviders(ctx context.Context, req *provider.Li } // TODO(ishank011): aggregrate properties such as etag, checksum, etc. p.Size += info.Size - p.Mtime = utils.LaterTS(p.Mtime, info.Mtime) + if utils.TSToUnixNano(info.Mtime) > utils.TSToUnixNano(p.Mtime) { + p.Mtime = info.Mtime + p.Etag = info.Etag + p.Checksum = info.Checksum + } + if p.Etag == "" && p.Etag != info.Etag { + p.Etag = info.Etag + } p.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER p.MimeType = "httpd/unix-directory" } else { diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index a7e067147b..aac7f68453 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -852,7 +852,13 @@ func (s *service) listVirtualView(ctx context.Context, ref *provider.Reference) if p, ok := nestedInfos[parent]; ok { p.Size += info.Size - p.Mtime = utils.LaterTS(p.Mtime, info.Mtime) + if utils.TSToUnixNano(info.Mtime) > utils.TSToUnixNano(p.Mtime) { + p.Mtime = info.Mtime + p.Etag = info.Etag + } + if p.Etag == "" && p.Etag != info.Etag { + p.Etag = info.Etag + } } else { nestedInfos[parent] = &provider.ResourceInfo{ Path: parent, @@ -862,6 +868,7 @@ func (s *service) listVirtualView(ctx context.Context, ref *provider.Reference) }, Size: info.Size, Mtime: info.Mtime, + Etag: info.Etag, MimeType: "httpd/unix-directory", } } diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index ebbb7f9a30..447f9f8593 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -32,6 +32,7 @@ import ( "strings" "syscall" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" @@ -58,7 +59,7 @@ type PermissionsChecker interface { // Tree is used to manage a tree hierarchy type Tree interface { - Setup(owner string) error + Setup(owner *userpb.UserId, propagateToRoot bool) error GetMD(ctx context.Context, node *node.Node) (os.FileInfo, error) ListFolder(ctx context.Context, node *node.Node) ([]*node.Node, error) @@ -102,10 +103,19 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore) (storage.FS, error) return New(o, lu, p, tp) } +// when enable home is false we want propagation to root if tree size or mtime accounting is enabled +func enablePropagationForRoot(o *options.Options) bool { + return (!o.EnableHome && (o.TreeSizeAccounting || o.TreeTimeAccounting)) +} + // New returns an implementation of the storage.FS interface that talks to // a local filesystem. func New(o *options.Options, lu *Lookup, p PermissionsChecker, tp Tree) (storage.FS, error) { - err := tp.Setup(o.Owner) + err := tp.Setup(&userpb.UserId{ + OpaqueId: o.Owner, + Idp: o.OwnerIDP, + Type: userpb.UserType(userpb.UserType_value[o.OwnerType]), + }, enablePropagationForRoot(o)) if err != nil { logger.New().Error().Err(err). Msg("could not setup tree") diff --git a/pkg/storage/utils/decomposedfs/options/options.go b/pkg/storage/utils/decomposedfs/options/options.go index d48a178730..181d20f19e 100644 --- a/pkg/storage/utils/decomposedfs/options/options.go +++ b/pkg/storage/utils/decomposedfs/options/options.go @@ -50,7 +50,9 @@ type Options struct { TreeSizeAccounting bool `mapstructure:"treesize_accounting"` // set an owner for the root node - Owner string `mapstructure:"owner"` + Owner string `mapstructure:"owner"` + OwnerIDP string `mapstructure:"owner_idp"` + OwnerType string `mapstructure:"owner_type"` } // New returns a new Options instance for the given configuration diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index 30752f16cb..9984480b0e 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -95,7 +95,7 @@ func New(root string, tta bool, tsa bool, lu PathLookup, bs Blobstore) *Tree { } // Setup prepares the tree structure -func (t *Tree) Setup(owner string) error { +func (t *Tree) Setup(owner *userpb.UserId, propagateToRoot bool) error { // create data paths for internal layout dataPaths := []string{ filepath.Join(t.root, "nodes"), @@ -114,16 +114,20 @@ func (t *Tree) Setup(owner string) error { // the root node has an empty name // the root node has no parent n := node.New("root", "", "", 0, "", nil, t.lookup) - err := t.createNode( - n, - &userpb.UserId{ - OpaqueId: owner, - }, - ) + err := t.createNode(n, owner) if err != nil { return err } + // set propagation flag + v := []byte("0") + if propagateToRoot { + v = []byte("1") + } + if err = xattr.Set(n.InternalPath(), xattrs.PropagationAttr, v); err != nil { + return err + } + // create spaces folder and iterate over existing nodes to populate it spacesPath := filepath.Join(t.root, "spaces") fi, err := os.Stat(spacesPath) diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index a28f98e33d..9f807d9ad2 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -778,7 +778,8 @@ func (fs *localfs) CreateDir(ctx context.Context, ref *provider.Reference) error } return errors.Wrap(err, "localfs: error creating dir "+fn) } - return nil + + return fs.propagate(ctx, path.Dir(fn)) } func (fs *localfs) Delete(ctx context.Context, ref *provider.Reference) error { diff --git a/tests/acceptance/config/behat.yml b/tests/acceptance/config/behat.yml index c75fd5d367..535ab3e151 100644 --- a/tests/acceptance/config/behat.yml +++ b/tests/acceptance/config/behat.yml @@ -2,6 +2,20 @@ default: autoload: '': '%paths.base%/../features/bootstrap' + suites: + apiVirtualViews: + paths: + - '%paths.base%/../features/apiVirtualViews' + contexts: + - RevaContext: + - FeatureContext: &common_feature_context_params + baseUrl: http://localhost:20180 + adminUsername: admin + adminPassword: admin + regularUserPassword: relativity + ocPath: apps/testing/api/v1/occ + - WebDavPropertiesContext: + extensions: jarnaiz\JUnitFormatter\JUnitFormatterExtension: filename: report.xml diff --git a/tests/acceptance/features/apiVirtualViews/virtualViews.feature b/tests/acceptance/features/apiVirtualViews/virtualViews.feature new file mode 100644 index 0000000000..3baa7b0fb5 --- /dev/null +++ b/tests/acceptance/features/apiVirtualViews/virtualViews.feature @@ -0,0 +1,65 @@ +@api @virtual-views-required +Feature: virtual views + As admin + I want to be able to shard large folders over multiple storage providers + So that I can scale large numbers of users better. + + Background: + Given user "einstein" deletes everything from folder "virtual/" using the WebDAV API + And user "einstein" has created the following folders + | path | + | virtual/a | + | virtual/a/alice | + | virtual/b | + | virtual/c | + | virtual/k | + | virtual/l | + | virtual/z | + + Scenario: list large folder + Given using old DAV path + When user "einstein" lists the resources in "/virtual" with depth "0" using the WebDAV API + Then the HTTP status code should be "207" + And the last DAV response for user "einstein" should not contain these nodes + | name | + | virtual/a | + | virtual/b | + | virtual/c | + | virtual/k | + | virtual/l | + | virtual/z | + When user "einstein" lists the resources in "/virtual" with depth 1 using the WebDAV API + Then the HTTP status code should be "207" + And the last DAV response for user "einstein" should contain these nodes + | name | + | virtual/a | + | virtual/b | + | virtual/c | + | virtual/k | + | virtual/l | + | virtual/z | + And the last DAV response for user "einstein" should not contain these nodes + | name | + | virtual/a/alice | + When user "einstein" lists the resources in "/virtual" with depth "infinity" using the WebDAV API + Then the HTTP status code should be "207" + And the last DAV response for user "einstein" should contain these nodes + | name | + | virtual/a | + | virtual/a/alice | + | virtual/b | + | virtual/c | + | virtual/k | + | virtual/l | + | virtual/z | + + Scenario: etag changes when adding a folder + Given user "einstein" has stored etag of element "/" + And user "einstein" has stored etag of element "/virtual" + And user "einstein" has stored etag of element "/virtual/b" + When user "einstein" creates folder "/virtual/b/bar" using the WebDAV API + Then these etags should have changed: + | user | path | + | einstein | / | + | einstein | /virtual | + | einstein | /virtual/b | \ No newline at end of file diff --git a/tests/integration/grpc/fixtures/users.demo.json b/tests/integration/grpc/fixtures/users.demo.json index 62b7442aee..0ade748d03 100644 --- a/tests/integration/grpc/fixtures/users.demo.json +++ b/tests/integration/grpc/fixtures/users.demo.json @@ -2,7 +2,7 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "username": "einstein", "secret": "relativity", @@ -14,7 +14,7 @@ { "id": { "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "username": "marie", "secret": "radioactivity", @@ -25,7 +25,7 @@ { "id": { "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "localhost:20080" + "idp": "http://localhost:20080" }, "username": "richard", "secret": "superfluidity", diff --git a/tests/integration/grpc/userprovider_test.go b/tests/integration/grpc/userprovider_test.go index 3108b1ad56..c0252e457b 100644 --- a/tests/integration/grpc/userprovider_test.go +++ b/tests/integration/grpc/userprovider_test.go @@ -246,7 +246,7 @@ var _ = Describe("user providers", func() { dependencies = map[string]string{ "users": "userprovider-json.toml", } - existingIdp = "localhost:20080" + existingIdp = "http://localhost:20080" }) assertFindUsersResponses() diff --git a/tests/oc-integration-tests/drone/frontend-global.toml b/tests/oc-integration-tests/drone/frontend-global.toml new file mode 100644 index 0000000000..3b47aa9efb --- /dev/null +++ b/tests/oc-integration-tests/drone/frontend-global.toml @@ -0,0 +1,117 @@ +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +# This frontend-global.toml config file will start a reva service that: +# - serves as the entrypoint for owncloud APIs but with a globally accessible namespace. +# - serves http endpoints on port 20180 +# - / --------------- ocdav +# - /ocs ------------ ocs +# - /oauth2 --------- oidcprovider +# - /.well-known ---- wellknown service to announce openid-configuration +# - TODO(diocas): ocm +# - authenticates requests using oidc bearer auth and basic auth as fallback +# - serves the grpc services on port 20199 +[grpc] +address = "0.0.0.0:20199" + +[grpc.services.authprovider] +auth_manager = "oidc" + +# If you want to use your own openid provider change this config +[grpc.services.authprovider.auth_managers.oidc] +issuer = "http://localhost:20180" + +[http] +address = "0.0.0.0:20180" + +[http.middlewares.cors] +allow_credentials = true + +[http.services.ocdav] +# serve ocdav on the root path +prefix = "" +chunk_folder = "/drone/src/tmp/reva/chunks" +# for user lookups +# prefix the path of requests to /dav/files with this namespace +# While owncloud has only listed usernames at this endpoint CERN has +# been exposing more than just usernames. For owncloud deployments we +# can prefix the path to jail the requests to the correct CS3 namespace. +# In this deployment we mounted the owncloud storage provider at /users. It +# expects a username as the first path segment. +# currently, only the desktop client will use this endpoint, but only if +# the dav.chunking capability is available +# TODO implement a path wrapper that rewrites `` into the path +# layout for the users home? +# no, use GetHome? +# for eos we need to rewrite the path +# TODO strip the username from the path so the CS3 namespace can be mounted +# at the files/ endpoint? what about migration? separate reva instance +files_namespace = "/" + +# similar to the dav/files endpoint we can configure a prefix for the old webdav endpoint +# we use the old webdav endpoint to present the cs3 namespace +# note: this changes the tree that is rendered at remote.php/webdav from the users home to the cs3 namespace +# use webdav_namespace = "/home" to use the old namespace that only exposes the users files +# this endpoint should not affect the desktop client sync but will present different folders for the other clients: +# - the desktop clients use a hardcoded remote.php/dav/files/ if the dav.chunkung capability is present +# - the ios ios uses the core.webdav-root capability which points to remote.php/webdav in oc10 +# - the oc js sdk is hardcoded to the remote.php/webdav so it will see the new tree +# - TODO android? no sync ... but will see different tree +webdav_namespace = "/" + +[http.services.ocs] + +[http.services.ocs.capabilities.capabilities.core.status] +version = "10.0.11.5" +versionstring = "10.0.11" + +[http.services.ocs.capabilities.capabilities.files_sharing] +api_enabled = true +resharing = true +group_sharing = true +auto_accept_share = true +share_with_group_members_only = true +share_with_membership_groups_only = true +default_permissions = 22 +search_min_length = 3 + +[http.services.ocs.capabilities.capabilities.files_sharing.public] +enabled = true +send_mail = true +social_share = true +upload = true +multiple = true +supports_upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password] +enforced = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password.enforced_for] +read_only = true +read_write = true +upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.expire_date] +enabled = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user] +send_mail = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user_enumeration] +enabled = true +group_members_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.federation] +outgoing = true +incoming = true + +[http.services.ocs.capabilities.capabilities.notifications] +endpoints = [] + +[http.services.ocs.capabilities.capabilities.files.tus_support] +version = "1.0.0" +resumable = "1.0.0" +extension = "creation,creation-with-upload" +http_method_override = "" +max_chunk_size = 0 diff --git a/tests/oc-integration-tests/drone/gateway.toml b/tests/oc-integration-tests/drone/gateway.toml index 916ad90e01..6d2577c2ca 100644 --- a/tests/oc-integration-tests/drone/gateway.toml +++ b/tests/oc-integration-tests/drone/gateway.toml @@ -69,6 +69,13 @@ home_provider = "/home" # mount a storage provider without a path wrapper for direct access to users. "/users" = {"address" = "localhost:11000"} "123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:11000"} + +# the /virtual namespace is only accessible via the frontend-global service +"/virtual/[a-k]" = {"address" = "localhost:11100"} +"virtual-a-k" = {"address" = "localhost:11100"} +"/virtual/[l-z]" = {"address" = "localhost:11110"} +"virtual-l-z" = {"address" = "localhost:11110"} + # another mount point might be "/projects/" "/public" = {"address" = "localhost:13000"} diff --git a/tests/oc-integration-tests/drone/storage-local-1.toml b/tests/oc-integration-tests/drone/storage-local-1.toml new file mode 100644 index 0000000000..1ff744ab35 --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-local-1.toml @@ -0,0 +1,46 @@ +# This storage.toml config file will start a reva service that: +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +# - authenticates grpc storage provider requests using the internal jwt token +# - authenticates http upload and download requests requests using basic auth +# - serves the storage provider on grpc port 11000 +# - serves http dataprovider for this storage on port 11001 +# - /data - dataprovider: file up and download +[grpc] +address = "0.0.0.0:11100" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/virtual" +mount_id = "virtual-a-k" +expose_data_server = true +data_server_url = "http://localhost:11101/data" + +[grpc.services.storageprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data-local-1" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true + +[http] +address = "0.0.0.0:11101" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/drone/src/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data-local-1" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true diff --git a/tests/oc-integration-tests/drone/storage-local-2.toml b/tests/oc-integration-tests/drone/storage-local-2.toml new file mode 100644 index 0000000000..9b689acfbe --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-local-2.toml @@ -0,0 +1,46 @@ +# This storage.toml config file will start a reva service that: +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +# - authenticates grpc storage provider requests using the internal jwt token +# - authenticates http upload and download requests requests using basic auth +# - serves the storage provider on grpc port 11000 +# - serves http dataprovider for this storage on port 11001 +# - /data - dataprovider: file up and download +[grpc] +address = "0.0.0.0:11110" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/virtual" +mount_id = "virtual-l-z" +expose_data_server = true +data_server_url = "http://localhost:11111/data" + +[grpc.services.storageprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data-local-2" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true + +[http] +address = "0.0.0.0:11111" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/drone/src/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data-local-2" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true diff --git a/tests/oc-integration-tests/drone/users.demo.json b/tests/oc-integration-tests/drone/users.demo.json index 5a68de8de3..893d69b99f 100644 --- a/tests/oc-integration-tests/drone/users.demo.json +++ b/tests/oc-integration-tests/drone/users.demo.json @@ -2,12 +2,37 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "http://localhost:20080" + "idp": "http://localhost:20080", + "type": 1 }, "username": "einstein", "secret": "relativity", "mail": "einstein@example.org", "display_name": "Albert Einstein", - "groups": [] + "groups": ["sailing-lovers", "violin-haters", "physics-lovers"] + }, + { + "id": { + "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + "idp": "http://localhost:20080", + "type": 1 + }, + "username": "marie", + "secret": "radioactivity", + "mail": "marie@example.org", + "display_name": "Marie Curie", + "groups": ["radium-lovers", "polonium-lovers", "physics-lovers"] + }, + { + "id": { + "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", + "idp": "http://localhost:20080", + "type": 1 + }, + "username": "richard", + "secret": "superfluidity", + "mail": "richard@example.org", + "display_name": "Richard Feynman", + "groups": ["quantum-lovers", "philosophy-haters", "physics-lovers"] } ] diff --git a/tests/oc-integration-tests/local/frontend-global.toml b/tests/oc-integration-tests/local/frontend-global.toml new file mode 100644 index 0000000000..f6a68a090c --- /dev/null +++ b/tests/oc-integration-tests/local/frontend-global.toml @@ -0,0 +1,117 @@ +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +# This frontend-global.toml config file will start a reva service that: +# - serves as the entrypoint for owncloud APIs but with a globally accessible namespace. +# - serves http endpoints on port 20180 +# - / --------------- ocdav +# - /ocs ------------ ocs +# - /oauth2 --------- oidcprovider +# - /.well-known ---- wellknown service to announce openid-configuration +# - TODO(diocas): ocm +# - authenticates requests using oidc bearer auth and basic auth as fallback +# - serves the grpc services on port 20199 +[grpc] +address = "0.0.0.0:20199" + +[grpc.services.authprovider] +auth_manager = "oidc" + +# If you want to use your own openid provider change this config +[grpc.services.authprovider.auth_managers.oidc] +issuer = "http://localhost:20180" + +[http] +address = "0.0.0.0:20180" + +[http.middlewares.cors] +allow_credentials = true + +[http.services.ocdav] +# serve ocdav on the root path +prefix = "" +chunk_folder = "/var/tmp/reva/chunks" +# for user lookups +# prefix the path of requests to /dav/files with this namespace +# While owncloud has only listed usernames at this endpoint CERN has +# been exposing more than just usernames. For owncloud deployments we +# can prefix the path to jail the requests to the correct CS3 namespace. +# In this deployment we mounted the owncloud storage provider at /users. It +# expects a username as the first path segment. +# currently, only the desktop client will use this endpoint, but only if +# the dav.chunking capability is available +# TODO implement a path wrapper that rewrites `` into the path +# layout for the users home? +# no, use GetHome? +# for eos we need to rewrite the path +# TODO strip the username from the path so the CS3 namespace can be mounted +# at the files/ endpoint? what about migration? separate reva instance +files_namespace = "/" + +# similar to the dav/files endpoint we can configure a prefix for the old webdav endpoint +# we use the old webdav endpoint to present the cs3 namespace +# note: this changes the tree that is rendered at remote.php/webdav from the users home to the cs3 namespace +# use webdav_namespace = "/home" to use the old namespace that only exposes the users files +# this endpoint should not affect the desktop client sync but will present different folders for the other clients: +# - the desktop clients use a hardcoded remote.php/dav/files/ if the dav.chunkung capability is present +# - the ios ios uses the core.webdav-root capability which points to remote.php/webdav in oc10 +# - the oc js sdk is hardcoded to the remote.php/webdav so it will see the new tree +# - TODO android? no sync ... but will see different tree +webdav_namespace = "/" + +[http.services.ocs] + +[http.services.ocs.capabilities.capabilities.core.status] +version = "10.0.11.5" +versionstring = "10.0.11" + +[http.services.ocs.capabilities.capabilities.files_sharing] +api_enabled = true +resharing = true +group_sharing = true +auto_accept_share = true +share_with_group_members_only = true +share_with_membership_groups_only = true +default_permissions = 22 +search_min_length = 3 + +[http.services.ocs.capabilities.capabilities.files_sharing.public] +enabled = true +send_mail = true +social_share = true +upload = true +multiple = true +supports_upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password] +enforced = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password.enforced_for] +read_only = true +read_write = true +upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.expire_date] +enabled = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user] +send_mail = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user_enumeration] +enabled = true +group_members_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.federation] +outgoing = true +incoming = true + +[http.services.ocs.capabilities.capabilities.notifications] +endpoints = [] + +[http.services.ocs.capabilities.capabilities.files.tus_support] +version = "1.0.0" +resumable = "1.0.0" +extension = "creation,creation-with-upload" +http_method_override = "" +max_chunk_size = 0 diff --git a/tests/oc-integration-tests/local/gateway.toml b/tests/oc-integration-tests/local/gateway.toml index 71289e9308..717bc766ca 100644 --- a/tests/oc-integration-tests/local/gateway.toml +++ b/tests/oc-integration-tests/local/gateway.toml @@ -62,6 +62,13 @@ home_provider = "/home" # mount a storage provider without a path wrapper for direct access to users. "/users" = {"address" = "localhost:11000"} "123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:11000"} + +# the /virtual namespace is only accessible via the frontend-global service +"/virtual/[a-k]" = {"address" = "localhost:11100"} +"virtual-a-k" = {"address" = "localhost:11100"} +"/virtual/[l-z]" = {"address" = "localhost:11110"} +"virtual-l-z" = {"address" = "localhost:11110"} + # another mount point might be "/projects/" "/public" = {"address" = "localhost:13000"} diff --git a/tests/oc-integration-tests/local/storage-local-1.toml b/tests/oc-integration-tests/local/storage-local-1.toml new file mode 100644 index 0000000000..6e688e1a19 --- /dev/null +++ b/tests/oc-integration-tests/local/storage-local-1.toml @@ -0,0 +1,46 @@ +# This storage.toml config file will start a reva service that: +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +# - authenticates grpc storage provider requests using the internal jwt token +# - authenticates http upload and download requests requests using basic auth +# - serves the storage provider on grpc port 11000 +# - serves http dataprovider for this storage on port 11001 +# - /data - dataprovider: file up and download +[grpc] +address = "0.0.0.0:11100" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/virtual" +mount_id = "virtual-a-k" +expose_data_server = true +data_server_url = "http://localhost:11101/data" + +[grpc.services.storageprovider.drivers.ocis] +root = "/var/tmp/reva/data-local-1" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true + +[http] +address = "0.0.0.0:11101" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/var/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/var/tmp/reva/data-local-1" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true diff --git a/tests/oc-integration-tests/local/storage-local-2.toml b/tests/oc-integration-tests/local/storage-local-2.toml new file mode 100644 index 0000000000..fb1e035c8c --- /dev/null +++ b/tests/oc-integration-tests/local/storage-local-2.toml @@ -0,0 +1,46 @@ +# This storage.toml config file will start a reva service that: +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +# - authenticates grpc storage provider requests using the internal jwt token +# - authenticates http upload and download requests requests using basic auth +# - serves the storage provider on grpc port 11000 +# - serves http dataprovider for this storage on port 11001 +# - /data - dataprovider: file up and download +[grpc] +address = "0.0.0.0:11110" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/virtual" +mount_id = "virtual-l-z" +expose_data_server = true +data_server_url = "http://localhost:11111/data" + +[grpc.services.storageprovider.drivers.ocis] +root = "/var/tmp/reva/data-local-2" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true + +[http] +address = "0.0.0.0:11111" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/var/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/var/tmp/reva/data-local-2" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true diff --git a/tests/oc-integration-tests/local/users.demo.json b/tests/oc-integration-tests/local/users.demo.json index 5a68de8de3..893d69b99f 100644 --- a/tests/oc-integration-tests/local/users.demo.json +++ b/tests/oc-integration-tests/local/users.demo.json @@ -2,12 +2,37 @@ { "id": { "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "http://localhost:20080" + "idp": "http://localhost:20080", + "type": 1 }, "username": "einstein", "secret": "relativity", "mail": "einstein@example.org", "display_name": "Albert Einstein", - "groups": [] + "groups": ["sailing-lovers", "violin-haters", "physics-lovers"] + }, + { + "id": { + "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + "idp": "http://localhost:20080", + "type": 1 + }, + "username": "marie", + "secret": "radioactivity", + "mail": "marie@example.org", + "display_name": "Marie Curie", + "groups": ["radium-lovers", "polonium-lovers", "physics-lovers"] + }, + { + "id": { + "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", + "idp": "http://localhost:20080", + "type": 1 + }, + "username": "richard", + "secret": "superfluidity", + "mail": "richard@example.org", + "display_name": "Richard Feynman", + "groups": ["quantum-lovers", "philosophy-haters", "physics-lovers"] } ]