From 4a9e5171c673722d5b87a122d7cdb0d0a9cf6a33 Mon Sep 17 00:00:00 2001 From: Javier Ferrer Date: Fri, 12 May 2023 09:02:38 +0200 Subject: [PATCH 01/33] Add air config example --- .gitignore | 3 +++ examples/air/.air.toml | 47 ++++++++++++++++++++++++++++++++++++++++++ examples/air/README.md | 15 ++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 examples/air/.air.toml create mode 100644 examples/air/README.md diff --git a/.gitignore b/.gitignore index 5ed50f2891..119d753fd0 100644 --- a/.gitignore +++ b/.gitignore @@ -48,4 +48,7 @@ tests/ocis/tests/acceptance/work_tmp # drone .drone.yml +# air config file +.air.toml + toolchain/ diff --git a/examples/air/.air.toml b/examples/air/.air.toml new file mode 100644 index 0000000000..9cea546481 --- /dev/null +++ b/examples/air/.air.toml @@ -0,0 +1,47 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] +args_bin = [] +bin = "/usr/bin/supervisorctl restart revad:*" +cmd = "make revad && /usr/bin/supervisorctl stop revad:* && cp cmd/revad/revad /usr/bin/revad" +delay = 1000 +exclude_dir = [ + "assets", + "tmp", + "vendor", + "testdata", + "docs", + "changelog", + "examples", + "tests", + "vendor-bin", +] +exclude_file = ["cmd/revad/revad", "cmd/reva/reva"] +exclude_regex = ["_test.go"] +exclude_unchanged = false +follow_symlink = false +full_bin = "" +include_dir = [] +include_ext = ["go", "tpl", "tmpl", "html"] +kill_delay = "0s" +log = "build-errors.log" +send_interrupt = false +stop_on_error = false + +[color] +app = "" +build = "yellow" +main = "magenta" +runner = "green" +watcher = "cyan" + +[log] +time = false + +[misc] +clean_on_exit = false + +[screen] +clear_on_rebuild = false diff --git a/examples/air/README.md b/examples/air/README.md new file mode 100644 index 0000000000..4771e45e85 --- /dev/null +++ b/examples/air/README.md @@ -0,0 +1,15 @@ +# Air config file + +Put this config file in the root of the project and start the Reva daemon by +running [Air](https://github.com/cosmtrek/air) to enable live-reload for it. + +This config follows the setup used at CERN which leverages [Supervisord](http://supervisord.org/) +for controlling the services running. It is easy to make changes to use the more +common [systemd.service]. + +To do this, just replace the following two lines in the file: + +``` +bin = "/usr/bin/systemctl start revad" +cmd = "make revad && /usr/bin/systemctl stop revad && cp cmd/revad/revad /usr/bin/revad" +``` From 2b3033dd2c9b1caf81e625f1213a6fddd82a0870 Mon Sep 17 00:00:00 2001 From: Hugo Labrador Date: Fri, 21 Jul 2023 13:02:25 +0200 Subject: [PATCH 02/33] temporary workaround for EOS-5754 (#45) --- pkg/eosclient/eosbinary/eosbinary.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index 1a47aa8240..bbb0f1f036 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -1176,18 +1176,15 @@ func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, if mtimesec, err = strconv.ParseUint(stimeSplit[0], 10, 64); err == nil { mtimeSet = true } + if mtimenanos, err = strconv.ParseUint(stimeSplit[1], 10, 32); err != nil { mtimeSet = false } } if !mtimeSet { mtimeSplit := strings.Split(kv["mtime"], ".") - if mtimesec, err = strconv.ParseUint(mtimeSplit[0], 10, 64); err != nil { - return nil, err - } - if mtimenanos, err = strconv.ParseUint(mtimeSplit[1], 10, 32); err != nil { - return nil, err - } + mtimesec, _ = strconv.ParseUint(mtimeSplit[0], 10, 64) + mtimenanos, _ = strconv.ParseUint(mtimeSplit[1], 10, 32) } var ctimesec, ctimenanos uint64 From f3635514a28f60cddbc4ca88b496cf3302bb00be Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 27 Jul 2023 14:50:00 +0200 Subject: [PATCH 03/33] Initial implementation of an app provider for Overleaf --- .../packages/app/provider/overleaf/_index.md | 90 +++++++++++++++ pkg/app/provider/loader/loader.go | 3 +- pkg/app/provider/overleaf/overleaf.go | 105 ++++++++++++++++++ 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 docs/content/en/docs/config/packages/app/provider/overleaf/_index.md create mode 100644 pkg/app/provider/overleaf/overleaf.go diff --git a/docs/content/en/docs/config/packages/app/provider/overleaf/_index.md b/docs/content/en/docs/config/packages/app/provider/overleaf/_index.md new file mode 100644 index 0000000000..280e4bdef8 --- /dev/null +++ b/docs/content/en/docs/config/packages/app/provider/overleaf/_index.md @@ -0,0 +1,90 @@ +--- +title: "overleaf" +linkTitle: "overleaf" +weight: 10 +description: > + Configuration for the overleaf service +--- + +# _struct: config_ + +{{% dir name="mime_types" type="[]string" default=nil %}} +Inherited from the appprovider. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L69) +{{< highlight toml >}} +[app.provider.overleaf] +mime_types = nil +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="iop_secret" type="string" default="" %}} +The IOP secret used to connect to the wopiserver. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L70) +{{< highlight toml >}} +[app.provider.overleaf] +iop_secret = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="app_name" type="string" default="" %}} +The App user-friendly name. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L71) +{{< highlight toml >}} +[app.provider.overleaf] +app_name = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="app_icon_uri" type="string" default="" %}} +A URI to a static asset which represents the app icon. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L72) +{{< highlight toml >}} +[app.provider.overleaf] +app_icon_uri = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="folder_base_url" type="string" default="" %}} +The base URL to generate links to navigate back to the containing folder. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L73) +{{< highlight toml >}} +[app.provider.overleaf] +folder_base_url = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="app_url" type="string" default="" %}} +The App URL. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L74) +{{< highlight toml >}} +[app.provider.overleaf] +app_url = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="app_int_url" type="string" default="" %}} +The internal app URL in case of dockerized deployments. Defaults to AppURL [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L75) +{{< highlight toml >}} +[app.provider.overleaf] +app_int_url = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="app_api_key" type="string" default="" %}} +The API key used by the app, if applicable. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L76) +{{< highlight toml >}} +[app.provider.overleaf] +app_api_key = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="jwt_secret" type="string" default="" %}} +The JWT secret to be used to retrieve the token TTL. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L77) +{{< highlight toml >}} +[app.provider.overleaf] +jwt_secret = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="app_desktop_only" type="bool" default=false %}} +Specifies if the app can be opened only on desktop. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/overleaf/overleaf.go#L78) +{{< highlight toml >}} +[app.provider.overleaf] +app_desktop_only = false +{{< /highlight >}} +{{% /dir %}} + diff --git a/pkg/app/provider/loader/loader.go b/pkg/app/provider/loader/loader.go index 8809c109ad..c138f7b79b 100644 --- a/pkg/app/provider/loader/loader.go +++ b/pkg/app/provider/loader/loader.go @@ -19,8 +19,9 @@ package loader import ( - // Load core application providers. + // Importing app providers. _ "github.com/cs3org/reva/pkg/app/provider/demo" + _ "github.com/cs3org/reva/pkg/app/provider/overleaf" _ "github.com/cs3org/reva/pkg/app/provider/wopi" // Add your own here. ) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go new file mode 100644 index 0000000000..aaf0e2be84 --- /dev/null +++ b/pkg/app/provider/overleaf/overleaf.go @@ -0,0 +1,105 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package overleaf + +import ( + "context" + "fmt" + "net/http" + "strings" + + appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" + appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/app" + "github.com/cs3org/reva/pkg/app/provider/registry" + "github.com/cs3org/reva/pkg/appctx" + "github.com/mitchellh/mapstructure" +) + +type overleafProvider struct { + conf *config +} + +func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.ResourceInfo, viewMode appprovider.ViewMode, token string, opaqueMap map[string]*typespb.OpaqueEntry, language string) (*appprovider.OpenInAppURL, error) { + log := appctx.GetLogger(ctx) + + // we need to generate a more restricted token + restrictedToken := token + + // getting file/folder name so as not to expose authentication token in project name + name := resource.Path[strings.LastIndex(resource.Path, "/")+1:] + name = strings.Split(name, ".")[0] // removing extension resource has one + + url := fmt.Sprintf("%s/docs?snip_uri=%s/archiver?id=%s!%s&access_token=%s&snip_name=%s", p.conf.AppURL, p.conf.FolderBaseURL, resource.Id.StorageId, resource.Id.OpaqueId, restrictedToken, name) + + log.Debug().Str("url", url).Msg("Returning URL for creating a project") + return &appprovider.OpenInAppURL{ + AppUrl: url, + Method: http.MethodGet, + }, nil +} + +func (p *overleafProvider) GetAppProviderInfo(ctx context.Context) (*appregistry.ProviderInfo, error) { + return &appregistry.ProviderInfo{ + Name: "Overleaf", + MimeTypes: p.conf.MimeTypes, + DesktopOnly: p.conf.AppDesktopOnly, + Icon: p.conf.AppIconURI, + Description: "Export to", + }, nil +} + +func init() { + registry.Register("overleaf", New) +} + +type config struct { + MimeTypes []string `mapstructure:"mime_types" docs:"nil;Inherited from the appprovider."` + IOPSecret string `mapstructure:"iop_secret" docs:";The IOP secret used to connect to the wopiserver."` + AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` + AppIconURI string `mapstructure:"app_icon_uri" docs:";A URI to a static asset which represents the app icon."` + FolderBaseURL string `mapstructure:"folder_base_url" docs:";The base URL to generate links to navigate back to the containing folder."` + AppURL string `mapstructure:"app_url" docs:";The App URL."` + AppIntURL string `mapstructure:"app_int_url" docs:";The internal app URL in case of dockerized deployments. Defaults to AppURL"` + AppAPIKey string `mapstructure:"app_api_key" docs:";The API key used by the app, if applicable."` + JWTSecret string `mapstructure:"jwt_secret" docs:";The JWT secret to be used to retrieve the token TTL."` + AppDesktopOnly bool `mapstructure:"app_desktop_only" docs:"false;Specifies if the app can be opened only on desktop."` + InsecureConnections bool `mapstructure:"insecure_connections"` +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + return nil, err + } + return c, nil +} + +// New returns an implementation to of the app.Provider interface that +// connects to an application in the backend. +func New(m map[string]interface{}) (app.Provider, error) { + c, err := parseConfig(m) + if err != nil { + return nil, err + } + + return &overleafProvider{conf: c}, nil +} From f5b5946e5255acb2f10f1e169d6d4c4320f09f4e Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 27 Jul 2023 14:52:08 +0200 Subject: [PATCH 04/33] Added changelog --- changelog/unreleased/overleaf.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/overleaf.md diff --git a/changelog/unreleased/overleaf.md b/changelog/unreleased/overleaf.md new file mode 100644 index 0000000000..de015e12d4 --- /dev/null +++ b/changelog/unreleased/overleaf.md @@ -0,0 +1,5 @@ +Enhancement: implementation of an app provider for Overleaf + +WIP + +https://github.com/cs3org/reva/pull/4084 From 2eea3ea937d2f642dbb6c47e8dc3b76e6889bd88 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 27 Jul 2023 15:23:03 +0200 Subject: [PATCH 05/33] Fixed New function and updated config --- pkg/app/provider/overleaf/overleaf.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index aaf0e2be84..4950e3f905 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -61,7 +61,6 @@ func (p *overleafProvider) GetAppProviderInfo(ctx context.Context) (*appregistry return &appregistry.ProviderInfo{ Name: "Overleaf", MimeTypes: p.conf.MimeTypes, - DesktopOnly: p.conf.AppDesktopOnly, Icon: p.conf.AppIconURI, Description: "Export to", }, nil @@ -73,15 +72,11 @@ func init() { type config struct { MimeTypes []string `mapstructure:"mime_types" docs:"nil;Inherited from the appprovider."` - IOPSecret string `mapstructure:"iop_secret" docs:";The IOP secret used to connect to the wopiserver."` AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` AppIconURI string `mapstructure:"app_icon_uri" docs:";A URI to a static asset which represents the app icon."` FolderBaseURL string `mapstructure:"folder_base_url" docs:";The base URL to generate links to navigate back to the containing folder."` AppURL string `mapstructure:"app_url" docs:";The App URL."` AppIntURL string `mapstructure:"app_int_url" docs:";The internal app URL in case of dockerized deployments. Defaults to AppURL"` - AppAPIKey string `mapstructure:"app_api_key" docs:";The API key used by the app, if applicable."` - JWTSecret string `mapstructure:"jwt_secret" docs:";The JWT secret to be used to retrieve the token TTL."` - AppDesktopOnly bool `mapstructure:"app_desktop_only" docs:"false;Specifies if the app can be opened only on desktop."` InsecureConnections bool `mapstructure:"insecure_connections"` } @@ -95,7 +90,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an implementation to of the app.Provider interface that // connects to an application in the backend. -func New(m map[string]interface{}) (app.Provider, error) { +func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { c, err := parseConfig(m) if err != nil { return nil, err From 4929172dc6f45167b1f3850951e719a9da3a75d3 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 27 Jul 2023 15:42:10 +0200 Subject: [PATCH 06/33] Updated description of folder_base_url to better fit use case --- pkg/app/provider/overleaf/overleaf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index 4950e3f905..f5d89f14a4 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -74,7 +74,7 @@ type config struct { MimeTypes []string `mapstructure:"mime_types" docs:"nil;Inherited from the appprovider."` AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` AppIconURI string `mapstructure:"app_icon_uri" docs:";A URI to a static asset which represents the app icon."` - FolderBaseURL string `mapstructure:"folder_base_url" docs:";The base URL to generate links to navigate back to the containing folder."` + FolderBaseURL string `mapstructure:"folder_base_url" docs:"; Public internet facing URL used to serve the files to Overleaf."` AppURL string `mapstructure:"app_url" docs:";The App URL."` AppIntURL string `mapstructure:"app_int_url" docs:";The internal app URL in case of dockerized deployments. Defaults to AppURL"` InsecureConnections bool `mapstructure:"insecure_connections"` From 5fd9014117dc42547e53eec5f9bb0d2fc554b517 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Mon, 31 Jul 2023 17:47:36 +0200 Subject: [PATCH 07/33] Improved request and made use of Target and Action fields --- pkg/app/provider/overleaf/overleaf.go | 49 +++++++++++++++++++++------ 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index f5d89f14a4..de9e3b2f73 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -20,8 +20,8 @@ package overleaf import ( "context" - "fmt" "net/http" + "path/filepath" "strings" appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" @@ -31,6 +31,7 @@ import ( "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rhttp" "github.com/mitchellh/mapstructure" ) @@ -44,25 +45,53 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res // we need to generate a more restricted token restrictedToken := token + // Setting up archiver request + archHttpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.FolderBaseURL+"/archiver", nil) + if err != nil { + return nil, err + } + + archQuery := archHttpReq.URL.Query() + archQuery.Add("id", resource.Id.StorageId+"!"+resource.Id.OpaqueId) + archQuery.Add("access_token", restrictedToken) + archQuery.Add("arch_type", "zip") + + archHttpReq.URL.RawQuery = archQuery.Encode() + log.Debug().Str("Archiver url", archHttpReq.URL.String()).Msg("URL for downloading zipped resource from archiver") + + // Setting up Overleaf request + httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.AppURL, nil) + if err != nil { + return nil, err + } + + q := httpReq.URL.Query() + + // snip_uri is link to archiver request + q.Add("snip_uri", archHttpReq.URL.String()) + // getting file/folder name so as not to expose authentication token in project name - name := resource.Path[strings.LastIndex(resource.Path, "/")+1:] - name = strings.Split(name, ".")[0] // removing extension resource has one + name := strings.TrimSuffix(filepath.Base(resource.Path), filepath.Ext(resource.Path)) + q.Add("snip_name", name) - url := fmt.Sprintf("%s/docs?snip_uri=%s/archiver?id=%s!%s&access_token=%s&snip_name=%s", p.conf.AppURL, p.conf.FolderBaseURL, resource.Id.StorageId, resource.Id.OpaqueId, restrictedToken, name) + httpReq.URL.RawQuery = q.Encode() + url := httpReq.URL.String() + log.Debug().Str("Full Overleaf url", url).Msg("URL for exporting file to Overleaf") - log.Debug().Str("url", url).Msg("Returning URL for creating a project") + //url := fmt.Sprintf("%s/docs?snip_uri=%s/archiver?id=%s!%s&access_token=%s&arch_type=zip&snip_name=%s", p.conf.AppURL, p.conf.FolderBaseURL, resource.Id.StorageId, resource.Id.OpaqueId, restrictedToken, name) return &appprovider.OpenInAppURL{ AppUrl: url, Method: http.MethodGet, + Target: appprovider.Target_TARGET_BLANK, }, nil } func (p *overleafProvider) GetAppProviderInfo(ctx context.Context) (*appregistry.ProviderInfo, error) { return &appregistry.ProviderInfo{ - Name: "Overleaf", - MimeTypes: p.conf.MimeTypes, - Icon: p.conf.AppIconURI, - Description: "Export to", + Name: "Overleaf", + MimeTypes: p.conf.MimeTypes, + Icon: p.conf.AppIconURI, + Action: "Export to", }, nil } @@ -90,7 +119,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an implementation to of the app.Provider interface that // connects to an application in the backend. -func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { +func New(m map[string]interface{}) (app.Provider, error) { c, err := parseConfig(m) if err != nil { return nil, err From e432edda64b1a6160a61c179c8df11485f0da285 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Wed, 2 Aug 2023 12:10:08 +0200 Subject: [PATCH 08/33] Added checks for when a project is exported twice --- pkg/app/provider/overleaf/overleaf.go | 59 +++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index de9e3b2f73..b794dc883d 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -20,18 +20,24 @@ package overleaf import ( "context" + "fmt" "net/http" "path/filepath" "strings" + "time" appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/sharedconf" "github.com/mitchellh/mapstructure" ) @@ -42,7 +48,35 @@ type overleafProvider struct { func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.ResourceInfo, viewMode appprovider.ViewMode, token string, opaqueMap map[string]*typespb.OpaqueEntry, language string) (*appprovider.OpenInAppURL, error) { log := appctx.GetLogger(ctx) - // we need to generate a more restricted token + // client used to set and get arbitrary metadata to keep track whether project has already been exported + client, err := pool.GetGatewayServiceClient(pool.Endpoint(sharedconf.GetGatewaySVC(""))) + if err != nil { + return nil, err + } + + if _, ok := opaqueMap["override"]; !ok { + log.Debug().Msg("not ok") + // Check if resource has already been exported to Overleaf + + statRes, err := client.Stat(ctx, &provider.StatRequest{ + Ref: &provider.Reference{ + ResourceId: resource.Id, + }, + }) + if err != nil { + return nil, err + } + + creationTime, alreadySet := statRes.Info.GetArbitraryMetadata().Metadata["overleaf-exported"] + + if alreadySet { + return nil, errtypes.AlreadyExists("Project was already exported on " + creationTime + ".") + } + } else { + log.Debug().Msg("ok") + } + + // TODO: generate and use a more restricted token restrictedToken := token // Setting up archiver request @@ -76,9 +110,28 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res httpReq.URL.RawQuery = q.Encode() url := httpReq.URL.String() - log.Debug().Str("Full Overleaf url", url).Msg("URL for exporting file to Overleaf") - //url := fmt.Sprintf("%s/docs?snip_uri=%s/archiver?id=%s!%s&access_token=%s&arch_type=zip&snip_name=%s", p.conf.AppURL, p.conf.FolderBaseURL, resource.Id.StorageId, resource.Id.OpaqueId, restrictedToken, name) + req := &provider.SetArbitraryMetadataRequest{ + Ref: &provider.Reference{ + ResourceId: resource.Id, + }, + ArbitraryMetadata: &provider.ArbitraryMetadata{ + Metadata: map[string]string{ + "overleaf-exported": time.Now().Format("2006-01-02"), + }, + }, + } + + res, err := client.SetArbitraryMetadata(ctx, req) + + if err != nil { + return nil, err + } + + if res.Status.Code != rpc.Code_CODE_OK { + return nil, fmt.Errorf("error: code=%+v msg=%q support_trace=%q", res.Status.Code, res.Status.Message, res.Status.Trace) + } + return &appprovider.OpenInAppURL{ AppUrl: url, Method: http.MethodGet, From 72537544d0ef6503ce08747e179920abd16270d6 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Wed, 2 Aug 2023 14:56:06 +0200 Subject: [PATCH 09/33] Improved date format and naming of meta data --- pkg/app/provider/overleaf/overleaf.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index b794dc883d..1b9fc1412f 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "path/filepath" + "strconv" "strings" "time" @@ -55,7 +56,6 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res } if _, ok := opaqueMap["override"]; !ok { - log.Debug().Msg("not ok") // Check if resource has already been exported to Overleaf statRes, err := client.Stat(ctx, &provider.StatRequest{ @@ -67,13 +67,11 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res return nil, err } - creationTime, alreadySet := statRes.Info.GetArbitraryMetadata().Metadata["overleaf-exported"] + creationTime, alreadySet := statRes.Info.GetArbitraryMetadata().Metadata["reva.overleaf.time"] if alreadySet { - return nil, errtypes.AlreadyExists("Project was already exported on " + creationTime + ".") + return nil, errtypes.AlreadyExists("Project was already exported on:" + creationTime) } - } else { - log.Debug().Msg("ok") } // TODO: generate and use a more restricted token @@ -117,7 +115,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res }, ArbitraryMetadata: &provider.ArbitraryMetadata{ Metadata: map[string]string{ - "overleaf-exported": time.Now().Format("2006-01-02"), + "reva.overleaf.time": strconv.Itoa(int(time.Now().Unix())), }, }, } From 6c967c87dc7f73bbb83cbe032056f9c275527650 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Wed, 2 Aug 2023 15:04:17 +0200 Subject: [PATCH 10/33] wrapped errors to be more descriptive --- pkg/app/provider/overleaf/overleaf.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index 1b9fc1412f..dcacad8561 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -40,6 +40,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/sharedconf" "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" ) type overleafProvider struct { @@ -52,7 +53,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res // client used to set and get arbitrary metadata to keep track whether project has already been exported client, err := pool.GetGatewayServiceClient(pool.Endpoint(sharedconf.GetGatewaySVC(""))) if err != nil { - return nil, err + return nil, errors.Wrap(err, "overleaf: error fetching gateway service client.") } if _, ok := opaqueMap["override"]; !ok { @@ -64,7 +65,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res }, }) if err != nil { - return nil, err + return nil, errors.Wrap(err, "overleaf: error statting file.") } creationTime, alreadySet := statRes.Info.GetArbitraryMetadata().Metadata["reva.overleaf.time"] @@ -80,7 +81,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res // Setting up archiver request archHttpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.FolderBaseURL+"/archiver", nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "overleaf: error setting up http request.") } archQuery := archHttpReq.URL.Query() @@ -94,7 +95,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res // Setting up Overleaf request httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.AppURL, nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "overleaf: error setting up http request") } q := httpReq.URL.Query() @@ -123,7 +124,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res res, err := client.SetArbitraryMetadata(ctx, req) if err != nil { - return nil, err + return nil, errors.Wrap(err, "overleaf: error setting arbitrary metadata") } if res.Status.Code != rpc.Code_CODE_OK { From f62ae9084d09565d01cf31f0e5751bdd15bc71da Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 3 Aug 2023 11:21:39 +0200 Subject: [PATCH 11/33] Improved config --- pkg/app/provider/overleaf/overleaf.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index dcacad8561..13c0e0a147 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -79,7 +79,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res restrictedToken := token // Setting up archiver request - archHttpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.FolderBaseURL+"/archiver", nil) + archHttpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.ArchiverURL, nil) if err != nil { return nil, errors.Wrap(err, "overleaf: error setting up http request.") } @@ -155,7 +155,7 @@ type config struct { MimeTypes []string `mapstructure:"mime_types" docs:"nil;Inherited from the appprovider."` AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` AppIconURI string `mapstructure:"app_icon_uri" docs:";A URI to a static asset which represents the app icon."` - FolderBaseURL string `mapstructure:"folder_base_url" docs:"; Public internet facing URL used to serve the files to Overleaf."` + ArchiverURL string `mapstructure:"archiver_url" docs:"; Internet-facing URL of the archiver service, used to serve the files to Overleaf."` AppURL string `mapstructure:"app_url" docs:";The App URL."` AppIntURL string `mapstructure:"app_int_url" docs:";The internal app URL in case of dockerized deployments. Defaults to AppURL"` InsecureConnections bool `mapstructure:"insecure_connections"` From 62235963da9b9dc831ac45cc5d54f3652920798b Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 3 Aug 2023 12:19:05 +0200 Subject: [PATCH 12/33] Adding context to New --- pkg/app/provider/overleaf/overleaf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index 13c0e0a147..afc8d5e972 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -171,7 +171,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an implementation to of the app.Provider interface that // connects to an application in the backend. -func New(m map[string]interface{}) (app.Provider, error) { +func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { c, err := parseConfig(m) if err != nil { return nil, err From 2ee5f9f1f9c8759664bf360125d228ecce349b14 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 3 Aug 2023 15:01:15 +0200 Subject: [PATCH 13/33] Linting --- pkg/app/provider/overleaf/overleaf.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index afc8d5e972..492a03a230 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -53,7 +53,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res // client used to set and get arbitrary metadata to keep track whether project has already been exported client, err := pool.GetGatewayServiceClient(pool.Endpoint(sharedconf.GetGatewaySVC(""))) if err != nil { - return nil, errors.Wrap(err, "overleaf: error fetching gateway service client.") + return nil, errors.Wrap(err, "overleaf: error fetching gateway service client") } if _, ok := opaqueMap["override"]; !ok { @@ -65,13 +65,13 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res }, }) if err != nil { - return nil, errors.Wrap(err, "overleaf: error statting file.") + return nil, errors.Wrap(err, "overleaf: error statting file") } creationTime, alreadySet := statRes.Info.GetArbitraryMetadata().Metadata["reva.overleaf.time"] if alreadySet { - return nil, errtypes.AlreadyExists("Project was already exported on:" + creationTime) + return nil, errtypes.AlreadyExists("Project was already exported on: " + creationTime) } } @@ -79,18 +79,18 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res restrictedToken := token // Setting up archiver request - archHttpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.ArchiverURL, nil) + archHTTPReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.ArchiverURL, nil) if err != nil { - return nil, errors.Wrap(err, "overleaf: error setting up http request.") + return nil, errors.Wrap(err, "overleaf: error setting up http request") } - archQuery := archHttpReq.URL.Query() + archQuery := archHTTPReq.URL.Query() archQuery.Add("id", resource.Id.StorageId+"!"+resource.Id.OpaqueId) archQuery.Add("access_token", restrictedToken) archQuery.Add("arch_type", "zip") - archHttpReq.URL.RawQuery = archQuery.Encode() - log.Debug().Str("Archiver url", archHttpReq.URL.String()).Msg("URL for downloading zipped resource from archiver") + archHTTPReq.URL.RawQuery = archQuery.Encode() + log.Debug().Str("Archiver url", archHTTPReq.URL.String()).Msg("URL for downloading zipped resource from archiver") // Setting up Overleaf request httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.AppURL, nil) @@ -101,7 +101,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res q := httpReq.URL.Query() // snip_uri is link to archiver request - q.Add("snip_uri", archHttpReq.URL.String()) + q.Add("snip_uri", archHTTPReq.URL.String()) // getting file/folder name so as not to expose authentication token in project name name := strings.TrimSuffix(filepath.Base(resource.Path), filepath.Ext(resource.Path)) From 23b1f15747d12c251c5f108c0c4c1cef7fda3077 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Fri, 4 Aug 2023 10:41:10 +0200 Subject: [PATCH 14/33] Removing a check in http appprovider service so it allows folders --- internal/http/services/appprovider/appprovider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 24fb6d3dc4..c903f3731e 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -371,7 +371,7 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { return } - if statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_FILE { + if statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_FILE || statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_CONTAINER { writeError(w, r, appErrorInvalidParameter, "the given file id does not point to a file", nil) return } From ae197f8480a01eaa2f94eb55c127a12ef6541bf0 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Fri, 4 Aug 2023 10:47:58 +0200 Subject: [PATCH 15/33] Fixed check --- internal/http/services/appprovider/appprovider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index c903f3731e..30d934405c 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -371,7 +371,7 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { return } - if statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_FILE || statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_CONTAINER { + if statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_FILE && statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_CONTAINER { writeError(w, r, appErrorInvalidParameter, "the given file id does not point to a file", nil) return } From e6da495084941439c10549df3ead4b1971d9ea06 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Mon, 7 Aug 2023 14:16:48 +0200 Subject: [PATCH 16/33] Improving external attribute names and adding name attribute --- pkg/app/provider/overleaf/overleaf.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go index 492a03a230..792e604c2c 100644 --- a/pkg/app/provider/overleaf/overleaf.go +++ b/pkg/app/provider/overleaf/overleaf.go @@ -68,7 +68,7 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res return nil, errors.Wrap(err, "overleaf: error statting file") } - creationTime, alreadySet := statRes.Info.GetArbitraryMetadata().Metadata["reva.overleaf.time"] + creationTime, alreadySet := statRes.Info.GetArbitraryMetadata().Metadata["reva.overleaf.exporttime"] if alreadySet { return nil, errtypes.AlreadyExists("Project was already exported on: " + creationTime) @@ -116,7 +116,8 @@ func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.Res }, ArbitraryMetadata: &provider.ArbitraryMetadata{ Metadata: map[string]string{ - "reva.overleaf.time": strconv.Itoa(int(time.Now().Unix())), + "reva.overleaf.exporttime": strconv.Itoa(int(time.Now().Unix())), + "reva.overleaf.name": name, }, }, } From fc5816dbd7b2e138544d432d7791d18d8c862fd4 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 10 Aug 2023 15:49:38 +0200 Subject: [PATCH 17/33] Moved overleaf app provider to a service --- internal/http/services/loader/loader.go | 1 + internal/http/services/overleaf/overleaf.go | 283 ++++++++++++++++++++ pkg/app/provider/overleaf/overleaf.go | 182 ------------- 3 files changed, 284 insertions(+), 182 deletions(-) create mode 100644 internal/http/services/overleaf/overleaf.go delete mode 100644 pkg/app/provider/overleaf/overleaf.go diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index 3fc02ce9bb..c15d96b68a 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -29,6 +29,7 @@ import ( _ "github.com/cs3org/reva/internal/http/services/metrics" _ "github.com/cs3org/reva/internal/http/services/ocmd" _ "github.com/cs3org/reva/internal/http/services/ocmprovider" + _ "github.com/cs3org/reva/internal/http/services/overleaf" _ "github.com/cs3org/reva/internal/http/services/owncloud/ocdav" _ "github.com/cs3org/reva/internal/http/services/owncloud/ocs" _ "github.com/cs3org/reva/internal/http/services/plugins" diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go new file mode 100644 index 0000000000..b4983cce0c --- /dev/null +++ b/internal/http/services/overleaf/overleaf.go @@ -0,0 +1,283 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package overleaf + +import ( + "encoding/json" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + storagepb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/internal/http/services/reqres" + "github.com/cs3org/reva/pkg/appctx" + ctxpkg "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/rhttp/global" + "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/resourceid" + "github.com/go-chi/chi/v5" + "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" +) + +type svc struct { + conf *config + gtwClient gateway.GatewayAPIClient + log *zerolog.Logger + router *chi.Mux +} + +type config struct { + Prefix string `mapstructure:"prefix"` + GatewaySvc string `mapstructure:"gatewaysvc"` + AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` + ArchiverURL string `mapstructure:"archiver_url" docs:";Internet-facing URL of the archiver service, used to serve the files to Overleaf."` + AppURL string `mapstructure:"app_url" docs:";The App URL."` + Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` +} + +func init() { + global.Register("overleaf", New) +} + +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { + log.Debug().Msg("Overleaf service is created") + + conf := &config{} + if err := mapstructure.Decode(m, conf); err != nil { + return nil, err + } + + conf.init() + + gtw, err := pool.GetGatewayServiceClient(pool.Endpoint(conf.GatewaySvc)) + if err != nil { + return nil, err + } + + r := chi.NewRouter() + + s := &svc{ + conf: conf, + gtwClient: gtw, + log: log, + router: r, + } + + if err := s.routerInit(); err != nil { + return nil, err + } + + return s, nil +} + +func (s *svc) routerInit() error { + s.router.Get("/import", s.handleImport) + s.router.Post("/export", s.handleExport) + return nil +} + +func (c *config) init() { + if c.Prefix == "" { + c.Prefix = "overleaf" + } + + c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) +} + +// Close performs cleanup. +func (s *svc) Close() error { + return nil +} + +func (s *svc) Prefix() string { + return s.conf.Prefix +} + +func (s *svc) Unprotected() []string { + return nil +} + +func (s *svc) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.router.ServeHTTP(w, r) + }) +} + +func (s *svc) handleImport(w http.ResponseWriter, r *http.Request) { + reqres.WriteError(w, r, reqres.APIErrorUnimplemented, "Overleaf import not yet supported", nil) +} + +func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := appctx.GetLogger(ctx) + + if err := r.ParseForm(); err != nil { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "parameters could not be parsed", nil) + return + } + + resourceID := r.Form.Get("resource_id") + + var resourceRef storagepb.Reference + if resourceID == "" { + path := r.Form.Get("path") + if path == "" { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "missing resource ID or path", nil) + return + } + resourceRef.Path = path + } else { + resourceID := resourceid.OwnCloudResourceIDUnwrap(resourceID) + if resourceID == nil { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "invalid resource ID", nil) + return + } + resourceRef.ResourceId = resourceID + } + + statRes, err := s.gtwClient.Stat(ctx, &storagepb.StatRequest{Ref: &resourceRef}) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "Internal error accessing the resource, please try again later", err) + return + } + + if statRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + reqres.WriteError(w, r, reqres.APIErrorNotFound, "resource does not exist", nil) + return + } else if statRes.Status.Code != rpc.Code_CODE_OK { + reqres.WriteError(w, r, reqres.APIErrorServerError, "failed to stat the resource", nil) + return + } + + resource := statRes.Info + + // User needs to have download rights to export to Overleaf + if !resource.PermissionSet.InitiateFileDownload { + reqres.WriteError(w, r, reqres.APIErrorUnauthenticated, "permission denied when accessing the file", err) + return + } + + if resource.Type != storagepb.ResourceType_RESOURCE_TYPE_FILE && resource.Type != storagepb.ResourceType_RESOURCE_TYPE_CONTAINER { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "invalid resource type, resource should be a file or a folder", nil) + return + } + + token, ok := ctxpkg.ContextGetToken(ctx) + if !ok || token == "" { + reqres.WriteError(w, r, reqres.APIErrorUnauthenticated, "Access token is invalid or empty", err) + return + } + + if r.Form.Get("override") == "" { + creationTime, alreadySet := resource.GetArbitraryMetadata().Metadata["reva.overleaf.exporttime"] + if alreadySet { + w.WriteHeader(http.StatusConflict) + if err := json.NewEncoder(w).Encode(map[string]any{ + "code": "ALREADY_EXISTS", + "message": "Project was already exported", + "export_time": creationTime, + }); err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "error marshalling JSON response", err) + return + } + w.Header().Set("Content-Type", "application/json") + return + } + } + + // TODO: generate and use a more restricted token + restrictedToken := token + + // Setting up archiver request + archHTTPReq, err := rhttp.NewRequest(ctx, http.MethodGet, s.conf.ArchiverURL, nil) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "overleaf: error setting up http request", nil) + return + } + + archQuery := archHTTPReq.URL.Query() + archQuery.Add("id", resource.Id.StorageId+"!"+resource.Id.OpaqueId) + archQuery.Add("access_token", restrictedToken) + archQuery.Add("arch_type", "zip") + + archHTTPReq.URL.RawQuery = archQuery.Encode() + log.Debug().Str("Archiver url", archHTTPReq.URL.String()).Msg("URL for downloading zipped resource from archiver") + + // Setting up Overleaf request + appUrl := s.conf.AppURL + "/docs" + httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, appUrl, nil) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "overleaf: error setting up http request", nil) + return + } + + q := httpReq.URL.Query() + + // snip_uri is link to archiver request + q.Add("snip_uri", archHTTPReq.URL.String()) + + // getting file/folder name so as not to expose authentication token in project name + name := strings.TrimSuffix(filepath.Base(resource.Path), filepath.Ext(resource.Path)) + q.Add("snip_name", name) + + httpReq.URL.RawQuery = q.Encode() + url := httpReq.URL.String() + + req := &provider.SetArbitraryMetadataRequest{ + Ref: &provider.Reference{ + ResourceId: resource.Id, + }, + ArbitraryMetadata: &provider.ArbitraryMetadata{ + Metadata: map[string]string{ + "reva.overleaf.exporttime": strconv.Itoa(int(time.Now().Unix())), + "reva.overleaf.name": name, + }, + }, + } + + res, err := s.gtwClient.SetArbitraryMetadata(ctx, req) + + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "overleaf: error setting arbitrary metadata", nil) + return + } + + if res.Status.Code != rpc.Code_CODE_OK { + reqres.WriteError(w, r, reqres.APIErrorServerError, "overleaf: error statting", nil) + return + } + + if err := json.NewEncoder(w).Encode(map[string]any{ + "app_url": url, + }); err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "error marshalling JSON response", err) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) +} diff --git a/pkg/app/provider/overleaf/overleaf.go b/pkg/app/provider/overleaf/overleaf.go deleted file mode 100644 index 792e604c2c..0000000000 --- a/pkg/app/provider/overleaf/overleaf.go +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package overleaf - -import ( - "context" - "fmt" - "net/http" - "path/filepath" - "strconv" - "strings" - "time" - - appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" - appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/cs3org/reva/pkg/app" - "github.com/cs3org/reva/pkg/app/provider/registry" - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp" - "github.com/cs3org/reva/pkg/sharedconf" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" -) - -type overleafProvider struct { - conf *config -} - -func (p *overleafProvider) GetAppURL(ctx context.Context, resource *provider.ResourceInfo, viewMode appprovider.ViewMode, token string, opaqueMap map[string]*typespb.OpaqueEntry, language string) (*appprovider.OpenInAppURL, error) { - log := appctx.GetLogger(ctx) - - // client used to set and get arbitrary metadata to keep track whether project has already been exported - client, err := pool.GetGatewayServiceClient(pool.Endpoint(sharedconf.GetGatewaySVC(""))) - if err != nil { - return nil, errors.Wrap(err, "overleaf: error fetching gateway service client") - } - - if _, ok := opaqueMap["override"]; !ok { - // Check if resource has already been exported to Overleaf - - statRes, err := client.Stat(ctx, &provider.StatRequest{ - Ref: &provider.Reference{ - ResourceId: resource.Id, - }, - }) - if err != nil { - return nil, errors.Wrap(err, "overleaf: error statting file") - } - - creationTime, alreadySet := statRes.Info.GetArbitraryMetadata().Metadata["reva.overleaf.exporttime"] - - if alreadySet { - return nil, errtypes.AlreadyExists("Project was already exported on: " + creationTime) - } - } - - // TODO: generate and use a more restricted token - restrictedToken := token - - // Setting up archiver request - archHTTPReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.ArchiverURL, nil) - if err != nil { - return nil, errors.Wrap(err, "overleaf: error setting up http request") - } - - archQuery := archHTTPReq.URL.Query() - archQuery.Add("id", resource.Id.StorageId+"!"+resource.Id.OpaqueId) - archQuery.Add("access_token", restrictedToken) - archQuery.Add("arch_type", "zip") - - archHTTPReq.URL.RawQuery = archQuery.Encode() - log.Debug().Str("Archiver url", archHTTPReq.URL.String()).Msg("URL for downloading zipped resource from archiver") - - // Setting up Overleaf request - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, p.conf.AppURL, nil) - if err != nil { - return nil, errors.Wrap(err, "overleaf: error setting up http request") - } - - q := httpReq.URL.Query() - - // snip_uri is link to archiver request - q.Add("snip_uri", archHTTPReq.URL.String()) - - // getting file/folder name so as not to expose authentication token in project name - name := strings.TrimSuffix(filepath.Base(resource.Path), filepath.Ext(resource.Path)) - q.Add("snip_name", name) - - httpReq.URL.RawQuery = q.Encode() - url := httpReq.URL.String() - - req := &provider.SetArbitraryMetadataRequest{ - Ref: &provider.Reference{ - ResourceId: resource.Id, - }, - ArbitraryMetadata: &provider.ArbitraryMetadata{ - Metadata: map[string]string{ - "reva.overleaf.exporttime": strconv.Itoa(int(time.Now().Unix())), - "reva.overleaf.name": name, - }, - }, - } - - res, err := client.SetArbitraryMetadata(ctx, req) - - if err != nil { - return nil, errors.Wrap(err, "overleaf: error setting arbitrary metadata") - } - - if res.Status.Code != rpc.Code_CODE_OK { - return nil, fmt.Errorf("error: code=%+v msg=%q support_trace=%q", res.Status.Code, res.Status.Message, res.Status.Trace) - } - - return &appprovider.OpenInAppURL{ - AppUrl: url, - Method: http.MethodGet, - Target: appprovider.Target_TARGET_BLANK, - }, nil -} - -func (p *overleafProvider) GetAppProviderInfo(ctx context.Context) (*appregistry.ProviderInfo, error) { - return &appregistry.ProviderInfo{ - Name: "Overleaf", - MimeTypes: p.conf.MimeTypes, - Icon: p.conf.AppIconURI, - Action: "Export to", - }, nil -} - -func init() { - registry.Register("overleaf", New) -} - -type config struct { - MimeTypes []string `mapstructure:"mime_types" docs:"nil;Inherited from the appprovider."` - AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` - AppIconURI string `mapstructure:"app_icon_uri" docs:";A URI to a static asset which represents the app icon."` - ArchiverURL string `mapstructure:"archiver_url" docs:"; Internet-facing URL of the archiver service, used to serve the files to Overleaf."` - AppURL string `mapstructure:"app_url" docs:";The App URL."` - AppIntURL string `mapstructure:"app_int_url" docs:";The internal app URL in case of dockerized deployments. Defaults to AppURL"` - InsecureConnections bool `mapstructure:"insecure_connections"` -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - -// New returns an implementation to of the app.Provider interface that -// connects to an application in the backend. -func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - - return &overleafProvider{conf: c}, nil -} From 8d6c4f9c1e8717a0434d3665c8f819c0d433d69d Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 10 Aug 2023 15:59:04 +0200 Subject: [PATCH 18/33] Adapted Overleaf New fn and removed from loaders --- internal/http/services/overleaf/overleaf.go | 6 ++---- pkg/app/provider/loader/loader.go | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index b4983cce0c..a8518c42c1 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -19,6 +19,7 @@ package overleaf import ( + "context" "encoding/json" "net/http" "path/filepath" @@ -63,9 +64,7 @@ func init() { global.Register("overleaf", New) } -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { - log.Debug().Msg("Overleaf service is created") - +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err @@ -83,7 +82,6 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) s := &svc{ conf: conf, gtwClient: gtw, - log: log, router: r, } diff --git a/pkg/app/provider/loader/loader.go b/pkg/app/provider/loader/loader.go index c138f7b79b..35280916a8 100644 --- a/pkg/app/provider/loader/loader.go +++ b/pkg/app/provider/loader/loader.go @@ -21,7 +21,6 @@ package loader import ( // Importing app providers. _ "github.com/cs3org/reva/pkg/app/provider/demo" - _ "github.com/cs3org/reva/pkg/app/provider/overleaf" _ "github.com/cs3org/reva/pkg/app/provider/wopi" // Add your own here. ) From aa756def07fe7ab9009241976c357a89e3a066ed Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Mon, 14 Aug 2023 14:06:55 +0200 Subject: [PATCH 19/33] Base encoding project name to avoid problem with spaces --- internal/http/services/overleaf/overleaf.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index a8518c42c1..791d7e6eb7 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -20,6 +20,7 @@ package overleaf import ( "context" + "encoding/base64" "encoding/json" "net/http" "path/filepath" @@ -252,7 +253,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { ArbitraryMetadata: &provider.ArbitraryMetadata{ Metadata: map[string]string{ "reva.overleaf.exporttime": strconv.Itoa(int(time.Now().Unix())), - "reva.overleaf.name": name, + "reva.overleaf.name": base64.StdEncoding.EncodeToString([]byte(name)), }, }, } From 2d25a9908a9ccf7a7c851c978a12d314ef8da35a Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Tue, 15 Aug 2023 14:58:31 +0200 Subject: [PATCH 20/33] Refactoring out features common to import and export --- internal/http/services/overleaf/overleaf.go | 79 ++++++++++++--------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 791d7e6eb7..0a307cdd95 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -22,6 +22,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "net/http" "path/filepath" "strconv" @@ -134,41 +135,9 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) - if err := r.ParseForm(); err != nil { - reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "parameters could not be parsed", nil) - return - } - - resourceID := r.Form.Get("resource_id") - - var resourceRef storagepb.Reference - if resourceID == "" { - path := r.Form.Get("path") - if path == "" { - reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "missing resource ID or path", nil) - return - } - resourceRef.Path = path - } else { - resourceID := resourceid.OwnCloudResourceIDUnwrap(resourceID) - if resourceID == nil { - reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "invalid resource ID", nil) - return - } - resourceRef.ResourceId = resourceID - } - - statRes, err := s.gtwClient.Stat(ctx, &storagepb.StatRequest{Ref: &resourceRef}) + statRes, err := s.validateQuery(w, r, ctx) if err != nil { - reqres.WriteError(w, r, reqres.APIErrorServerError, "Internal error accessing the resource, please try again later", err) - return - } - - if statRes.Status.Code == rpc.Code_CODE_NOT_FOUND { - reqres.WriteError(w, r, reqres.APIErrorNotFound, "resource does not exist", nil) - return - } else if statRes.Status.Code != rpc.Code_CODE_OK { - reqres.WriteError(w, r, reqres.APIErrorServerError, "failed to stat the resource", nil) + // Validate query handles errors return } @@ -280,3 +249,45 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) } + +func (s *svc) validateQuery(w http.ResponseWriter, r *http.Request, ctx context.Context) (*storagepb.StatResponse, error) { + if err := r.ParseForm(); err != nil { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "parameters could not be parsed", nil) + return nil, err + } + + resourceID := r.Form.Get("resource_id") + + var resourceRef storagepb.Reference + if resourceID == "" { + path := r.Form.Get("path") + if path == "" { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "missing resource ID or path", nil) + return nil, errors.New("missing resource ID or path") + } + resourceRef.Path = path + } else { + resourceID := resourceid.OwnCloudResourceIDUnwrap(resourceID) + if resourceID == nil { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "invalid resource ID", nil) + return nil, errors.New("invalid resource ID") + } + resourceRef.ResourceId = resourceID + } + + statRes, err := s.gtwClient.Stat(ctx, &storagepb.StatRequest{Ref: &resourceRef}) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "Internal error accessing the resource, please try again later", err) + return nil, errors.New("Internal error accessing the resource, please try again later") + } + + if statRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + reqres.WriteError(w, r, reqres.APIErrorNotFound, "resource does not exist", nil) + return nil, errors.New("resource does not exist") + } else if statRes.Status.Code != rpc.Code_CODE_OK { + reqres.WriteError(w, r, reqres.APIErrorServerError, "failed to stat the resource", nil) + return nil, errors.New("failed to stat the resource") + } + + return statRes, nil +} From 42424616d727845847f3a507d08179181b22c06c Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 08:33:53 +0200 Subject: [PATCH 21/33] Removing unnecessary files for air config --- examples/air/.air.toml | 47 ------------------------------------------ examples/air/README.md | 15 -------------- 2 files changed, 62 deletions(-) delete mode 100644 examples/air/.air.toml delete mode 100644 examples/air/README.md diff --git a/examples/air/.air.toml b/examples/air/.air.toml deleted file mode 100644 index 9cea546481..0000000000 --- a/examples/air/.air.toml +++ /dev/null @@ -1,47 +0,0 @@ -root = "." -testdata_dir = "testdata" -tmp_dir = "tmp" - -[build] -args_bin = [] -bin = "/usr/bin/supervisorctl restart revad:*" -cmd = "make revad && /usr/bin/supervisorctl stop revad:* && cp cmd/revad/revad /usr/bin/revad" -delay = 1000 -exclude_dir = [ - "assets", - "tmp", - "vendor", - "testdata", - "docs", - "changelog", - "examples", - "tests", - "vendor-bin", -] -exclude_file = ["cmd/revad/revad", "cmd/reva/reva"] -exclude_regex = ["_test.go"] -exclude_unchanged = false -follow_symlink = false -full_bin = "" -include_dir = [] -include_ext = ["go", "tpl", "tmpl", "html"] -kill_delay = "0s" -log = "build-errors.log" -send_interrupt = false -stop_on_error = false - -[color] -app = "" -build = "yellow" -main = "magenta" -runner = "green" -watcher = "cyan" - -[log] -time = false - -[misc] -clean_on_exit = false - -[screen] -clear_on_rebuild = false diff --git a/examples/air/README.md b/examples/air/README.md deleted file mode 100644 index 4771e45e85..0000000000 --- a/examples/air/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Air config file - -Put this config file in the root of the project and start the Reva daemon by -running [Air](https://github.com/cosmtrek/air) to enable live-reload for it. - -This config follows the setup used at CERN which leverages [Supervisord](http://supervisord.org/) -for controlling the services running. It is easy to make changes to use the more -common [systemd.service]. - -To do this, just replace the following two lines in the file: - -``` -bin = "/usr/bin/systemctl start revad" -cmd = "make revad && /usr/bin/systemctl stop revad && cp cmd/revad/revad /usr/bin/revad" -``` From f960ddb92a2044b9b9c14aaa310b883aa343b606 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 09:02:15 +0200 Subject: [PATCH 22/33] Updating parsing of config --- internal/http/services/overleaf/overleaf.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 0a307cdd95..54a38a146c 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -40,9 +40,9 @@ import ( "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/go-chi/chi/v5" - "github.com/mitchellh/mapstructure" "github.com/rs/zerolog" ) @@ -67,8 +67,8 @@ func init() { } func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var conf config + if err := cfg.Decode(m, &conf); err != nil { return nil, err } @@ -82,7 +82,7 @@ func New(ctx context.Context, m map[string]interface{}) (global.Service, error) r := chi.NewRouter() s := &svc{ - conf: conf, + conf: &conf, gtwClient: gtw, router: r, } From c300c652a816d908659bf351f2900fa664a9e975 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 13:19:48 +0200 Subject: [PATCH 23/33] Refactoring and adding required property to fields --- internal/http/services/overleaf/overleaf.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 54a38a146c..188f0848da 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -55,10 +55,10 @@ type svc struct { type config struct { Prefix string `mapstructure:"prefix"` - GatewaySvc string `mapstructure:"gatewaysvc"` - AppName string `mapstructure:"app_name" docs:";The App user-friendly name."` - ArchiverURL string `mapstructure:"archiver_url" docs:";Internet-facing URL of the archiver service, used to serve the files to Overleaf."` - AppURL string `mapstructure:"app_url" docs:";The App URL."` + GatewaySvc string `mapstructure:"gatewaysvc" validate:"required"` + AppName string `mapstructure:"app_name" docs:";The App user-friendly name." validate:"required"` + ArchiverURL string `mapstructure:"archiver_url" docs:";Internet-facing URL of the archiver service, used to serve the files to Overleaf." validate:"required"` + AppURL string `mapstructure:"app_url" docs:";The App URL." validate:"required"` Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` } @@ -72,8 +72,6 @@ func New(ctx context.Context, m map[string]interface{}) (global.Service, error) return nil, err } - conf.init() - gtw, err := pool.GetGatewayServiceClient(pool.Endpoint(conf.GatewaySvc)) if err != nil { return nil, err @@ -100,7 +98,7 @@ func (s *svc) routerInit() error { return nil } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "overleaf" } From 926d3faf7df37679de30c628cf5a71ee5203a8e1 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 13:48:09 +0200 Subject: [PATCH 24/33] Refactoring Handler, resource id and deleting unnecessary file --- internal/http/services/overleaf/overleaf.go | 6 +- pkg/eosclient/eosbinary/eosbinary.go | 1301 ------------------- 2 files changed, 2 insertions(+), 1305 deletions(-) delete mode 100644 pkg/eosclient/eosbinary/eosbinary.go diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 188f0848da..21eddd8501 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -120,9 +120,7 @@ func (s *svc) Unprotected() []string { } func (s *svc) Handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.router.ServeHTTP(w, r) - }) + return s.router } func (s *svc) handleImport(w http.ResponseWriter, r *http.Request) { @@ -186,7 +184,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { } archQuery := archHTTPReq.URL.Query() - archQuery.Add("id", resource.Id.StorageId+"!"+resource.Id.OpaqueId) + archQuery.Add("id", resourceid.OwnCloudResourceIDWrap(resource.Id)) archQuery.Add("access_token", restrictedToken) archQuery.Add("arch_type", "zip") diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go deleted file mode 100644 index bbb0f1f036..0000000000 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ /dev/null @@ -1,1301 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package eosbinary - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "os/exec" - "path" - "path/filepath" - "strconv" - "strings" - "syscall" - "time" - - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/eosclient" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/storage/utils/acl" - "github.com/google/uuid" - "github.com/pkg/errors" - "go.opentelemetry.io/otel/trace" -) - -const ( - versionPrefix = ".sys.v#." - userACLEvalKey = "eval.useracl" - favoritesKey = "http://owncloud.org/ns/favorite" -) - -func serializeAttribute(a *eosclient.Attribute) string { - return fmt.Sprintf("%s.%s=%s", attrTypeToString(a.Type), a.Key, a.Val) -} - -func attrTypeToString(at eosclient.AttrType) string { - switch at { - case eosclient.SystemAttr: - return "sys" - case eosclient.UserAttr: - return "user" - default: - return "invalid" - } -} - -func isValidAttribute(a *eosclient.Attribute) bool { - // validate that an attribute is correct. - if (a.Type != eosclient.SystemAttr && a.Type != eosclient.UserAttr) || a.Key == "" { - return false - } - return true -} - -// Options to configure the Client. -type Options struct { - - // ForceSingleUserMode forces all connections to use only one user. - // This is the case when access to EOS is done from FUSE under apache or www-data. - ForceSingleUserMode bool - - // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. - UseKeytab bool - - // Whether to maintain the same inode across various versions of a file. - // Requires extra metadata operations if set to true - VersionInvariant bool - - // SingleUsername is the username to use when connecting to EOS. - // Defaults to apache - SingleUsername string - - // Location of the eos binary. - // Default is /usr/bin/eos. - EosBinary string - - // Location of the xrdcopy binary. - // Default is /opt/eos/xrootd/bin/xrdcopy. - XrdcopyBinary string - - // URL of the EOS MGM. - // Default is root://eos-example.org - URL string - - // Location on the local fs where to store reads. - // Defaults to os.TempDir() - CacheDirectory string - - // Keytab is the location of the EOS keytab file. - Keytab string - - // SecProtocol is the comma separated list of security protocols used by xrootd. - // For example: "sss, unix" - // DEPRECATED - // This variable is no longer used. Only sss and unix protocols are possible. - // If UseKeytab is set to true the protocol will be set to "sss", else to "unix" - SecProtocol string - - // TokenExpiry stores in seconds the time after which generated tokens will expire - // Default is 3600 - TokenExpiry int -} - -func (opt *Options) ApplyDefaults() { - if opt.ForceSingleUserMode && opt.SingleUsername != "" { - opt.SingleUsername = "apache" - } - - if opt.EosBinary == "" { - opt.EosBinary = "/usr/bin/eos" - } - - if opt.XrdcopyBinary == "" { - opt.XrdcopyBinary = "/opt/eos/xrootd/bin/xrdcopy" - } - - if opt.URL == "" { - opt.URL = "root://eos-example.org" - } - - if opt.CacheDirectory == "" { - opt.CacheDirectory = os.TempDir() - } -} - -// Client performs actions against a EOS management node (MGM). -// It requires the eos-client and xrootd-client packages installed to work. -type Client struct { - opt *Options -} - -// New creates a new client with the given options. -func New(opt *Options) (*Client, error) { - opt.ApplyDefaults() - c := new(Client) - c.opt = opt - return c, nil -} - -// executeXRDCopy executes xrdcpy commands and returns the stdout, stderr and return code. -func (c *Client) executeXRDCopy(ctx context.Context, cmdArgs []string) (string, string, error) { - log := appctx.GetLogger(ctx) - - outBuf := &bytes.Buffer{} - errBuf := &bytes.Buffer{} - - cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, cmdArgs...) - cmd.Stdout = outBuf - cmd.Stderr = errBuf - cmd.Env = []string{ - "EOS_MGM_URL=" + c.opt.URL, - } - - if c.opt.UseKeytab { - cmd.Env = append(cmd.Env, "XrdSecPROTOCOL=sss") - cmd.Env = append(cmd.Env, "XrdSecSSSKT="+c.opt.Keytab) - } else { // we are a trusted gateway - cmd.Env = append(cmd.Env, "XrdSecPROTOCOL=unix") - cmd.Env = append(cmd.Env, "KRB5CCNAME=FILE:/dev/null") // do not try to use krb - } - - err := cmd.Run() - - var exitStatus int - if exiterr, ok := err.(*exec.ExitError); ok { - // The program has exited with an exit code != 0 - // This works on both Unix and Windows. Although package - // syscall is generally platform dependent, WaitStatus is - // defined for both Unix and Windows and in both cases has - // an ExitStatus() method with the same signature. - if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - exitStatus = status.ExitStatus() - switch exitStatus { - case 0: - err = nil - case int(syscall.ENOENT): - err = errtypes.NotFound(errBuf.String()) - } - } - } - - // check for operation not permitted error - if strings.Contains(errBuf.String(), "Operation not permitted") { - err = errtypes.InvalidCredentials("eosclient: no sufficient permissions for the operation") - } - - args := fmt.Sprintf("%s", cmd.Args) - env := fmt.Sprintf("%s", cmd.Env) - log.Info().Str("args", args).Str("env", env).Int("exit", exitStatus).Msg("eos cmd") - - return outBuf.String(), errBuf.String(), err -} - -// exec executes only EOS commands the command and returns the stdout, stderr and return code. -func (c *Client) executeEOS(ctx context.Context, cmdArgs []string, auth eosclient.Authorization) (string, string, error) { - log := appctx.GetLogger(ctx) - - outBuf := &bytes.Buffer{} - errBuf := &bytes.Buffer{} - - cmd := exec.CommandContext(ctx, c.opt.EosBinary) - cmd.Stdout = outBuf - cmd.Stderr = errBuf - cmd.Env = []string{ - "EOS_MGM_URL=" + c.opt.URL, - } - - if auth.Token != "" { - cmd.Env = append(cmd.Env, "EOSAUTHZ="+auth.Token) - } else if auth.Role.UID != "" && auth.Role.GID != "" { - cmd.Args = append(cmd.Args, []string{"-r", auth.Role.UID, auth.Role.GID}...) - } - - if c.opt.UseKeytab { - cmd.Env = append(cmd.Env, "XrdSecPROTOCOL=sss") - cmd.Env = append(cmd.Env, "XrdSecSSSKT="+c.opt.Keytab) - } else { // we are a trusted gateway - cmd.Env = append(cmd.Env, "XrdSecPROTOCOL=unix") - cmd.Env = append(cmd.Env, "KRB5CCNAME=FILE:/dev/null") // do not try to use krb - } - - // add application label - cmd.Args = append(cmd.Args, "-a", "reva_eosclient::meta") - - cmd.Args = append(cmd.Args, cmdArgs...) - - span := trace.SpanFromContext(ctx) - cmd.Args = append(cmd.Args, "--comment", span.SpanContext().TraceID().String()) - - err := cmd.Run() - - var exitStatus int - if exiterr, ok := err.(*exec.ExitError); ok { - // The program has exited with an exit code != 0 - // This works on both Unix and Windows. Although package - // syscall is generally platform dependent, WaitStatus is - // defined for both Unix and Windows and in both cases has - // an ExitStatus() method with the same signature. - if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - exitStatus = status.ExitStatus() - switch exitStatus { - case 0: - err = nil - case int(syscall.ENOENT): - err = errtypes.NotFound(errBuf.String()) - case int(syscall.EPERM), int(syscall.E2BIG), int(syscall.EINVAL): - // eos reports back error code 1 (EPERM) when ? - // eos reports back error code 7 (E2BIG) when the user is not allowed to read the directory - // eos reports back error code 22 (EINVAL) when the user is not allowed to enter the instance - err = errtypes.PermissionDenied(errBuf.String()) - } - } - } - - args := fmt.Sprintf("%s", cmd.Args) - env := fmt.Sprintf("%s", cmd.Env) - log.Info().Str("args", args).Str("env", env).Int("exit", exitStatus).Str("err", errBuf.String()).Msg("eos cmd") - - if err != nil && exitStatus != int(syscall.ENOENT) { // don't wrap the errtypes.NotFoundError - err = errors.Wrap(err, "eosclient: error while executing command") - } - - return outBuf.String(), errBuf.String(), err -} - -// AddACL adds an new acl to EOS with the given aclType. -func (c *Client) AddACL(ctx context.Context, auth, rootAuth eosclient.Authorization, path string, pos uint, a *acl.Entry) error { - finfo, err := c.getRawFileInfoByPath(ctx, auth, path) - if err != nil { - return err - } - - sysACL := a.CitrineSerialize() - args := []string{"acl", "--sys"} - if finfo.IsDir { - args = append(args, "--recursive") - } - - // set position of ACLs to add. The default is to append to the end, so no arguments will be added in this case - // the first position starts at 1 = eosclient.StartPosition - if pos != eosclient.EndPosition { - args = append(args, "--position", fmt.Sprint(pos)) - } - - args = append(args, sysACL, path) - - _, _, err = c.executeEOS(ctx, args, rootAuth) - return err -} - -// RemoveACL removes the acl from EOS. -func (c *Client) RemoveACL(ctx context.Context, auth, rootAuth eosclient.Authorization, path string, a *acl.Entry) error { - finfo, err := c.getRawFileInfoByPath(ctx, auth, path) - if err != nil { - return err - } - - a.Permissions = "" - sysACL := a.CitrineSerialize() - args := []string{"acl", "--sys"} - if finfo.IsDir { - args = append(args, "--recursive") - } - args = append(args, sysACL, path) - - _, _, err = c.executeEOS(ctx, args, rootAuth) - return err -} - -// UpdateACL updates the EOS acl. -func (c *Client) UpdateACL(ctx context.Context, auth, rootAuth eosclient.Authorization, path string, position uint, a *acl.Entry) error { - return c.AddACL(ctx, auth, rootAuth, path, position, a) -} - -// GetACL for a file. -func (c *Client) GetACL(ctx context.Context, auth eosclient.Authorization, path, aclType, target string) (*acl.Entry, error) { - acls, err := c.ListACLs(ctx, auth, path) - if err != nil { - return nil, err - } - for _, a := range acls { - if a.Type == aclType && a.Qualifier == target { - return a, nil - } - } - return nil, errtypes.NotFound(fmt.Sprintf("%s:%s", aclType, target)) -} - -// ListACLs returns the list of ACLs present under the given path. -// EOS returns uids/gid for Citrine version and usernames for older versions. -// For Citire we need to convert back the uid back to username. -func (c *Client) ListACLs(ctx context.Context, auth eosclient.Authorization, path string) ([]*acl.Entry, error) { - parsedACLs, err := c.getACLForPath(ctx, auth, path) - if err != nil { - return nil, err - } - - // EOS Citrine ACLs are stored with uid. The UID will be resolved to the - // user opaque ID at the eosfs level. - return parsedACLs.Entries, nil -} - -func (c *Client) getACLForPath(ctx context.Context, auth eosclient.Authorization, path string) (*acl.ACLs, error) { - finfo, err := c.GetFileInfoByPath(ctx, auth, path) - if err != nil { - return nil, err - } - - return finfo.SysACL, nil -} - -// GetFileInfoByInode returns the FileInfo by the given inode. -func (c *Client) GetFileInfoByInode(ctx context.Context, auth eosclient.Authorization, inode uint64) (*eosclient.FileInfo, error) { - args := []string{"file", "info", fmt.Sprintf("inode:%d", inode), "-m"} - stdout, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - return nil, err - } - info, err := c.parseFileInfo(ctx, stdout, true) - if err != nil { - return nil, err - } - - if c.opt.VersionInvariant && isVersionFolder(info.File) { - info, err = c.getFileInfoFromVersion(ctx, auth, info.File) - if err != nil { - return nil, err - } - info.Inode = inode - } - - return c.mergeACLsAndAttrsForFiles(ctx, auth, info), nil -} - -// GetFileInfoByFXID returns the FileInfo by the given file id in hexadecimal. -func (c *Client) GetFileInfoByFXID(ctx context.Context, auth eosclient.Authorization, fxid string) (*eosclient.FileInfo, error) { - args := []string{"file", "info", fmt.Sprintf("fxid:%s", fxid), "-m"} - stdout, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - return nil, err - } - - info, err := c.parseFileInfo(ctx, stdout, true) - if err != nil { - return nil, err - } - - return c.mergeACLsAndAttrsForFiles(ctx, auth, info), nil -} - -// GetFileInfoByPath returns the FilInfo at the given path. -func (c *Client) GetFileInfoByPath(ctx context.Context, auth eosclient.Authorization, path string) (*eosclient.FileInfo, error) { - args := []string{"file", "info", path, "-m"} - stdout, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - return nil, err - } - info, err := c.parseFileInfo(ctx, stdout, true) - if err != nil { - return nil, err - } - - if c.opt.VersionInvariant && !isVersionFolder(path) && !info.IsDir { - ownerAuth := eosclient.Authorization{Role: eosclient.Role{ - UID: strconv.FormatUint(info.UID, 10), - GID: strconv.FormatUint(info.GID, 10), - }} - if inode, err := c.getVersionFolderInode(ctx, auth, ownerAuth, path); err == nil { - info.Inode = inode - } - } - - return c.mergeACLsAndAttrsForFiles(ctx, auth, info), nil -} - -func (c *Client) getRawFileInfoByPath(ctx context.Context, auth eosclient.Authorization, path string) (*eosclient.FileInfo, error) { - args := []string{"file", "info", path, "-m"} - stdout, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - return nil, err - } - return c.parseFileInfo(ctx, stdout, false) -} - -func (c *Client) mergeACLsAndAttrsForFiles(ctx context.Context, auth eosclient.Authorization, info *eosclient.FileInfo) *eosclient.FileInfo { - // We need to inherit the ACLs for the parent directory as these are not available for files - // And the attributes from the version folders - if !info.IsDir { - parentInfo, err := c.getRawFileInfoByPath(ctx, auth, path.Dir(info.File)) - // Even if this call fails, at least return the current file object - if err == nil { - info.SysACL.Entries = append(info.SysACL.Entries, parentInfo.SysACL.Entries...) - } - - // We need to merge attrs set for the version folders, so get those resolved for the current user - versionFolderInfo, err := c.GetFileInfoByPath(ctx, auth, getVersionFolder(info.File)) - if err == nil { - info.SysACL.Entries = append(info.SysACL.Entries, versionFolderInfo.SysACL.Entries...) - for k, v := range versionFolderInfo.Attrs { - info.Attrs[k] = v - } - } - } - - return info -} - -// SetAttr sets an extended attributes on a path. -func (c *Client) SetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, errorIfExists, recursive bool, path string) error { - if !isValidAttribute(attr) { - return errors.New("eos: attr is invalid: " + serializeAttribute(attr)) - } - - var info *eosclient.FileInfo - var err error - // We need to set the attrs on the version folder as they are not persisted across writes - // Except for the sys.eval.useracl attr as EOS uses that to determine if it needs to obey - // the user ACLs set on the file - if !(attr.Type == eosclient.SystemAttr && attr.Key == userACLEvalKey) { - info, err = c.getRawFileInfoByPath(ctx, auth, path) - if err != nil { - return err - } - if !info.IsDir { - path = getVersionFolder(path) - } - } - - // Favorites need to be stored per user so handle these separately - if attr.Type == eosclient.UserAttr && attr.Key == favoritesKey { - return c.handleFavAttr(ctx, auth, attr, recursive, path, info, true) - } - return c.setEOSAttr(ctx, auth, attr, errorIfExists, recursive, path) -} - -func (c *Client) setEOSAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, errorIfExists, recursive bool, path string) error { - args := []string{"attr"} - if recursive { - args = append(args, "-r") - } - args = append(args, "set") - if errorIfExists { - args = append(args, "-c") - } - args = append(args, serializeAttribute(attr), path) - - _, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - var exErr *exec.ExitError - if errors.As(err, &exErr) && exErr.ExitCode() == 17 { - return eosclient.AttrAlreadyExistsError - } - return err - } - return nil -} - -func (c *Client) handleFavAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path string, info *eosclient.FileInfo, set bool) error { - var err error - u := ctxpkg.ContextMustGetUser(ctx) - if info == nil { - info, err = c.getRawFileInfoByPath(ctx, auth, path) - if err != nil { - return err - } - } - favStr := info.Attrs[favoritesKey] - favs, err := acl.Parse(favStr, acl.ShortTextForm) - if err != nil { - return err - } - if set { - err = favs.SetEntry(acl.TypeUser, u.Id.OpaqueId, "1") - if err != nil { - return err - } - } else { - favs.DeleteEntry(acl.TypeUser, u.Id.OpaqueId) - } - attr.Val = favs.Serialize() - return c.setEOSAttr(ctx, auth, attr, false, recursive, path) -} - -// UnsetAttr unsets an extended attribute on a path. -func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path string) error { - if !isValidAttribute(attr) { - return errors.New("eos: attr is invalid: " + serializeAttribute(attr)) - } - - var info *eosclient.FileInfo - var err error - // We need to set the attrs on the version folder as they are not persisted across writes - // Except for the sys.eval.useracl attr as EOS uses that to determine if it needs to obey - // the user ACLs set on the file - if !(attr.Type == eosclient.SystemAttr && attr.Key == userACLEvalKey) { - info, err = c.getRawFileInfoByPath(ctx, auth, path) - if err != nil { - return err - } - if !info.IsDir { - path = getVersionFolder(path) - } - } - - // Favorites need to be stored per user so handle these separately - if attr.Type == eosclient.UserAttr && attr.Key == favoritesKey { - return c.handleFavAttr(ctx, auth, attr, recursive, path, info, false) - } - - var args []string - if recursive { - args = []string{"attr", "-r", "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path} - } else { - args = []string{"attr", "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path} - } - _, _, err = c.executeEOS(ctx, args, auth) - if err != nil { - var exErr *exec.ExitError - if errors.As(err, &exErr) && exErr.ExitCode() == 61 { - return eosclient.AttrNotExistsError - } - return err - } - return nil -} - -// GetAttr returns the attribute specified by key. -func (c *Client) GetAttr(ctx context.Context, auth eosclient.Authorization, key, path string) (*eosclient.Attribute, error) { - // As SetAttr set the attr on the version folder, we will read the attribute on it - // if the resource is not a folder - info, err := c.getRawFileInfoByPath(ctx, auth, path) - if err != nil { - return nil, err - } - if !info.IsDir { - path = getVersionFolder(path) - } - - args := []string{"attr", "get", key, path} - attrOut, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - return nil, err - } - attr, err := deserializeAttribute(attrOut) - if err != nil { - return nil, err - } - return attr, nil -} - -// GetAttrs returns all the attributes of a resource. -func (c *Client) GetAttrs(ctx context.Context, auth eosclient.Authorization, path string) ([]*eosclient.Attribute, error) { - info, err := c.getRawFileInfoByPath(ctx, auth, path) - if err != nil { - return nil, err - } - if !info.IsDir { - path = getVersionFolder(path) - } - - args := []string{"attr", "ls", path} - attrOut, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - return nil, err - } - - attrsStr := strings.Split(attrOut, "\n") - attrs := make([]*eosclient.Attribute, 0, len(attrsStr)) - for _, line := range attrsStr { - attr, err := deserializeAttribute(line) - if err != nil { - return nil, err - } - attrs = append(attrs, attr) - } - return attrs, nil -} - -func deserializeAttribute(attrStr string) (*eosclient.Attribute, error) { - // the string is in the form sys.forced.checksum="adler" - keyValue := strings.SplitN(strings.TrimSpace(attrStr), "=", 2) // keyValue = ["sys.forced.checksum", "\"adler\""] - if len(keyValue) != 2 { - return nil, errtypes.InternalError("wrong attr format to deserialize") - } - type2key := strings.SplitN(keyValue[0], ".", 2) // type2key = ["sys", "forced.checksum"] - if len(type2key) != 2 { - return nil, errtypes.InternalError("wrong attr format to deserialize") - } - t, err := eosclient.AttrStringToType(type2key[0]) - if err != nil { - return nil, err - } - // trim \" from value - value := strings.Trim(keyValue[1], "\"") - return &eosclient.Attribute{Type: t, Key: type2key[1], Val: value}, nil -} - -// GetQuota gets the quota of a user on the quota node defined by path. -func (c *Client) GetQuota(ctx context.Context, username string, rootAuth eosclient.Authorization, path string) (*eosclient.QuotaInfo, error) { - args := []string{"quota", "ls", "-u", username, "-m"} - stdout, _, err := c.executeEOS(ctx, args, rootAuth) - if err != nil { - return nil, err - } - return c.parseQuota(path, stdout) -} - -// SetQuota sets the quota of a user on the quota node defined by path. -func (c *Client) SetQuota(ctx context.Context, rootAuth eosclient.Authorization, info *eosclient.SetQuotaInfo) error { - maxBytes := fmt.Sprintf("%d", info.MaxBytes) - maxFiles := fmt.Sprintf("%d", info.MaxFiles) - args := []string{"quota", "set", "-u", info.Username, "-p", info.QuotaNode, "-v", maxBytes, "-i", maxFiles} - _, _, err := c.executeEOS(ctx, args, rootAuth) - if err != nil { - return err - } - return nil -} - -// Touch creates a 0-size,0-replica file in the EOS namespace. -func (c *Client) Touch(ctx context.Context, auth eosclient.Authorization, path string) error { - args := []string{"file", "touch", path} - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// Chown given path. -func (c *Client) Chown(ctx context.Context, auth, chownauth eosclient.Authorization, path string) error { - args := []string{"chown", chownauth.Role.UID + ":" + chownauth.Role.GID, path} - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// Chmod given path. -func (c *Client) Chmod(ctx context.Context, auth eosclient.Authorization, mode, path string) error { - args := []string{"chmod", mode, path} - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// CreateDir creates a directory at the given path. -func (c *Client) CreateDir(ctx context.Context, auth eosclient.Authorization, path string) error { - args := []string{"mkdir", "-p", path} - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// Remove removes the resource at the given path. -func (c *Client) Remove(ctx context.Context, auth eosclient.Authorization, path string, noRecycle bool) error { - args := []string{"rm", "-r"} - if noRecycle { - args = append(args, "--no-recycle-bin") // do not put the file in the recycle bin - } - args = append(args, path) - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// Rename renames the resource referenced by oldPath to newPath. -func (c *Client) Rename(ctx context.Context, auth eosclient.Authorization, oldPath, newPath string) error { - args := []string{"file", "rename", oldPath, newPath} - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// List the contents of the directory given by path. -func (c *Client) List(ctx context.Context, auth eosclient.Authorization, path string) ([]*eosclient.FileInfo, error) { - args := []string{"find", "--fileinfo", "--maxdepth", "1", path} - stdout, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) - } - return c.parseFind(ctx, auth, path, stdout) -} - -// Read reads a file from the mgm. -func (c *Client) Read(ctx context.Context, auth eosclient.Authorization, path string) (io.ReadCloser, error) { - rand := "eosread-" + uuid.New().String() - localTarget := fmt.Sprintf("%s/%s", c.opt.CacheDirectory, rand) - defer os.RemoveAll(localTarget) - - xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) - args := []string{"--nopbar", "--silent", "-f", xrdPath, localTarget} - - if auth.Token != "" { - args[3] += "?authz=" + auth.Token - } else if auth.Role.UID != "" && auth.Role.GID != "" { - args = append(args, fmt.Sprintf("-OSeos.ruid=%s&eos.rgid=%s&eos.app=reva_eosclient::read", auth.Role.UID, auth.Role.GID)) - } - - _, _, err := c.executeXRDCopy(ctx, args) - if err != nil { - return nil, err - } - return os.Open(localTarget) -} - -// Write writes a stream to the mgm. -func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path string, stream io.ReadCloser) error { - fd, err := os.CreateTemp(c.opt.CacheDirectory, "eoswrite-") - if err != nil { - return err - } - defer fd.Close() - defer os.RemoveAll(fd.Name()) - - // copy stream to local temp file - _, err = io.Copy(fd, stream) - if err != nil { - return err - } - - return c.WriteFile(ctx, auth, path, fd.Name()) -} - -// WriteFile writes an existing file to the mgm. -func (c *Client) WriteFile(ctx context.Context, auth eosclient.Authorization, path, source string) error { - xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) - args := []string{"--nopbar", "--silent", "-f", source, xrdPath} - - if auth.Token != "" { - args[4] += "?authz=" + auth.Token - } else if auth.Role.UID != "" && auth.Role.GID != "" { - args = append(args, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s&eos.app=reva_eosclient::write", auth.Role.UID, auth.Role.GID)) - } - - _, _, err := c.executeXRDCopy(ctx, args) - return err -} - -// ListDeletedEntries returns a list of the deleted entries. -func (c *Client) ListDeletedEntries(ctx context.Context, auth eosclient.Authorization) ([]*eosclient.DeletedEntry, error) { - // TODO(labkode): add protection if slave is configured and alive to count how many files are in the trashbin before - // triggering the recycle ls call that could break the instance because of unavailable memory. - args := []string{"recycle", "ls", "-m"} - stdout, _, err := c.executeEOS(ctx, args, auth) - if err != nil { - return nil, err - } - return parseRecycleList(stdout) -} - -// RestoreDeletedEntry restores a deleted entry. -func (c *Client) RestoreDeletedEntry(ctx context.Context, auth eosclient.Authorization, key string) error { - args := []string{"recycle", "restore", key} - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// PurgeDeletedEntries purges all entries from the recycle bin. -func (c *Client) PurgeDeletedEntries(ctx context.Context, auth eosclient.Authorization) error { - args := []string{"recycle", "purge"} - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// ListVersions list all the versions for a given file. -func (c *Client) ListVersions(ctx context.Context, auth eosclient.Authorization, p string) ([]*eosclient.FileInfo, error) { - versionFolder := getVersionFolder(p) - finfos, err := c.List(ctx, auth, versionFolder) - if err != nil { - // we send back an empty list - return []*eosclient.FileInfo{}, nil - } - return finfos, nil -} - -// RollbackToVersion rollbacks a file to a previous version. -func (c *Client) RollbackToVersion(ctx context.Context, auth eosclient.Authorization, path, version string) error { - args := []string{"file", "versions", path, version} - _, _, err := c.executeEOS(ctx, args, auth) - return err -} - -// ReadVersion reads the version for the given file. -func (c *Client) ReadVersion(ctx context.Context, auth eosclient.Authorization, p, version string) (io.ReadCloser, error) { - versionFile := path.Join(getVersionFolder(p), version) - return c.Read(ctx, auth, versionFile) -} - -// GenerateToken returns a token on behalf of the resource owner to be used by lightweight accounts. -func (c *Client) GenerateToken(ctx context.Context, auth eosclient.Authorization, p string, a *acl.Entry) (string, error) { - expiration := strconv.FormatInt(time.Now().Add(time.Duration(c.opt.TokenExpiry)*time.Second).Unix(), 10) - args := []string{"token", "--permission", a.Permissions, "--tree", "--path", p, "--expires", expiration} - stdout, _, err := c.executeEOS(ctx, args, auth) - return strings.TrimSpace(stdout), err -} - -func (c *Client) getVersionFolderInode(ctx context.Context, auth, ownerAuth eosclient.Authorization, p string) (uint64, error) { - versionFolder := getVersionFolder(p) - md, err := c.getRawFileInfoByPath(ctx, auth, versionFolder) - if err != nil { - if err = c.CreateDir(ctx, ownerAuth, versionFolder); err != nil { - return 0, err - } - md, err = c.getRawFileInfoByPath(ctx, auth, versionFolder) - if err != nil { - return 0, err - } - } - return md.Inode, nil -} - -func (c *Client) getFileInfoFromVersion(ctx context.Context, auth eosclient.Authorization, p string) (*eosclient.FileInfo, error) { - file := getFileFromVersionFolder(p) - md, err := c.GetFileInfoByPath(ctx, auth, file) - if err != nil { - return nil, err - } - return md, nil -} - -func isVersionFolder(p string) bool { - return strings.HasPrefix(path.Base(p), versionPrefix) -} - -func getVersionFolder(p string) string { - return path.Join(path.Dir(p), versionPrefix+path.Base(p)) -} - -func getFileFromVersionFolder(p string) string { - return path.Join(path.Dir(p), strings.TrimPrefix(path.Base(p), versionPrefix)) -} - -func parseRecycleList(raw string) ([]*eosclient.DeletedEntry, error) { - entries := []*eosclient.DeletedEntry{} - rawLines := strings.FieldsFunc(raw, func(c rune) bool { - return c == '\n' - }) - for _, rl := range rawLines { - if rl == "" { - continue - } - entry, err := parseRecycleEntry(rl) - if err != nil { - return nil, err - } - entries = append(entries, entry) - } - return entries, nil -} - -// parse entries like these: -// recycle=ls recycle-bin=/eos/backup/proc/recycle/ uid=gonzalhu gid=it size=0 deletion-time=1510823151 type=recursive-dir keylength.restore-path=45 restore-path=/eos/scratch/user/g/gonzalhu/.sys.v#.app.ico/ restore-key=0000000000a35100 -// recycle=ls recycle-bin=/eos/backup/proc/recycle/ uid=gonzalhu gid=it size=381038 deletion-time=1510823151 type=file keylength.restore-path=36 restore-path=/eos/scratch/user/g/gonzalhu/app.ico restore-key=000000002544fdb3. -func parseRecycleEntry(raw string) (*eosclient.DeletedEntry, error) { - partsBySpace := strings.FieldsFunc(raw, func(c rune) bool { - return c == ' ' - }) - restoreKeyPair, partsBySpace := partsBySpace[len(partsBySpace)-1], partsBySpace[:len(partsBySpace)-1] - restorePathPair := strings.Join(partsBySpace[8:], " ") - - partsBySpace = partsBySpace[:8] - partsBySpace = append(partsBySpace, restorePathPair) - partsBySpace = append(partsBySpace, restoreKeyPair) - - kv := getMap(partsBySpace) - size, err := strconv.ParseUint(kv["size"], 10, 64) - if err != nil { - return nil, err - } - isDir := false - if kv["type"] == "recursive-dir" { - isDir = true - } - deletionMTime, err := strconv.ParseUint(strings.Split(kv["deletion-time"], ".")[0], 10, 64) - if err != nil { - return nil, err - } - entry := &eosclient.DeletedEntry{ - RestorePath: kv["restore-path"], - RestoreKey: kv["restore-key"], - Size: size, - DeletionMTime: deletionMTime, - IsDir: isDir, - } - return entry, nil -} - -func getMap(partsBySpace []string) map[string]string { - kv := map[string]string{} - for _, pair := range partsBySpace { - parts := strings.Split(pair, "=") - if len(parts) > 1 { - kv[parts[0]] = parts[1] - } - } - return kv -} - -func (c *Client) parseFind(ctx context.Context, auth eosclient.Authorization, dirPath, raw string) ([]*eosclient.FileInfo, error) { - log := appctx.GetLogger(ctx) - - finfos := []*eosclient.FileInfo{} - versionFolders := map[string]*eosclient.FileInfo{} - rawLines := strings.FieldsFunc(raw, func(c rune) bool { - return c == '\n' - }) - - var ownerAuth *eosclient.Authorization - - var parent *eosclient.FileInfo - for _, rl := range rawLines { - if rl == "" { - continue - } - fi, err := c.parseFileInfo(ctx, rl, true) - if err != nil { - return nil, err - } - // dirs in eos end with a slash, like /eos/user/g/gonzalhu/ - // we skip the current directory as eos find will return the directory we - // ask to find - if fi.File == path.Clean(dirPath) { - parent = fi - continue - } - - // If it's a version folder, store it in a map, so that for the corresponding file, - // we can return its inode instead - if isVersionFolder(fi.File) { - versionFolders[fi.File] = fi - } - - if ownerAuth == nil { - ownerAuth = &eosclient.Authorization{ - Role: eosclient.Role{ - UID: strconv.FormatUint(fi.UID, 10), - GID: strconv.FormatUint(fi.GID, 10), - }, - } - } - - finfos = append(finfos, fi) - } - - for _, fi := range finfos { - // For files, inherit ACLs from the parent - // And set the inode to that of their version folder - if !fi.IsDir && !isVersionFolder(dirPath) { - if parent != nil { - fi.SysACL.Entries = append(fi.SysACL.Entries, parent.SysACL.Entries...) - } - versionFolderPath := getVersionFolder(fi.File) - if vf, ok := versionFolders[versionFolderPath]; ok { - fi.Inode = vf.Inode - fi.SysACL.Entries = append(fi.SysACL.Entries, vf.SysACL.Entries...) - for k, v := range vf.Attrs { - fi.Attrs[k] = v - } - } else if err := c.CreateDir(ctx, *ownerAuth, versionFolderPath); err == nil { // Create the version folder if it doesn't exist - if md, err := c.getRawFileInfoByPath(ctx, auth, versionFolderPath); err == nil { - fi.Inode = md.Inode - } else { - log.Error().Err(err).Interface("auth", ownerAuth).Str("path", versionFolderPath).Msg("got error creating version folder") - } - } - } - } - - return finfos, nil -} - -func (c Client) parseQuotaLine(line string) map[string]string { - partsBySpace := strings.FieldsFunc(line, func(c rune) bool { - return c == ' ' - }) - m := getMap(partsBySpace) - return m -} -func (c *Client) parseQuota(path, raw string) (*eosclient.QuotaInfo, error) { - rawLines := strings.FieldsFunc(raw, func(c rune) bool { - return c == '\n' - }) - for _, rl := range rawLines { - if rl == "" { - continue - } - - m := c.parseQuotaLine(rl) - // map[maxbytes:2000000000000 maxlogicalbytes:1000000000000 percentageusedbytes:0.49 quota:node uid:gonzalhu space:/eos/scratch/user/ usedbytes:9829986500 usedlogicalbytes:4914993250 statusfiles:ok usedfiles:334 maxfiles:1000000 statusbytes:ok] - - space := m["space"] - if strings.HasPrefix(path, filepath.Clean(space)) { - maxBytesString := m["maxlogicalbytes"] - usedBytesString := m["usedlogicalbytes"] - maxBytes, _ := strconv.ParseUint(maxBytesString, 10, 64) - usedBytes, _ := strconv.ParseUint(usedBytesString, 10, 64) - - maxInodesString := m["maxfiles"] - usedInodesString := m["usedfiles"] - maxInodes, _ := strconv.ParseUint(maxInodesString, 10, 64) - usedInodes, _ := strconv.ParseUint(usedInodesString, 10, 64) - - qi := &eosclient.QuotaInfo{ - AvailableBytes: maxBytes, - UsedBytes: usedBytes, - AvailableInodes: maxInodes, - UsedInodes: usedInodes, - } - return qi, nil - } - } - return &eosclient.QuotaInfo{}, nil -} - -// TODO(labkode): better API to access extended attributes. -func (c *Client) parseFileInfo(ctx context.Context, raw string, parseFavoriteKey bool) (*eosclient.FileInfo, error) { - line := raw[15:] - index := strings.Index(line, " file=/") - lengthString := line[0:index] - length, err := strconv.ParseUint(lengthString, 10, 64) - if err != nil { - return nil, err - } - - line = line[index+6:] // skip ' file=' - name := line[0:length] - - kv := make(map[string]string) - attrs := make(map[string]string) - // strip trailing slash - kv["file"] = strings.TrimSuffix(name, "/") - - line = line[length+1:] - partsBySpace := strings.FieldsFunc(line, func(c rune) bool { // we have [size=45 container=3 ...} - return c == ' ' - }) - var previousXAttr = "" - for _, p := range partsBySpace { - partsByEqual := strings.SplitN(p, "=", 2) // we have kv pairs like [size 14] - if len(partsByEqual) == 2 { - // handle xattrn and xattrv special cases - switch { - case partsByEqual[0] == "xattrn": - previousXAttr = partsByEqual[1] - if previousXAttr != "user.acl" { - previousXAttr = strings.Replace(previousXAttr, "user.", "", 1) - } - case partsByEqual[0] == "xattrv": - attrs[previousXAttr] = strings.ToValidUTF8(partsByEqual[1], "") - previousXAttr = "" - default: - kv[partsByEqual[0]] = partsByEqual[1] - } - } - } - fi, err := c.mapToFileInfo(ctx, kv, attrs, parseFavoriteKey) - if err != nil { - return nil, err - } - return fi, nil -} - -// mapToFileInfo converts the dictionary to an usable structure. -// The kv has format: -// map[sys.forced.space:default files:0 mode:42555 ino:5 sys.forced.blocksize:4k sys.forced.layout:replica uid:0 fid:5 sys.forced.blockchecksum:crc32c sys.recycle:/eos/backup/proc/recycle/ fxid:00000005 pid:1 etag:5:0.000 keylength.file:4 file:/eos treesize:1931593933849913 container:3 gid:0 mtime:1498571294.108614409 ctime:1460121992.294326762 pxid:00000001 sys.forced.checksum:adler sys.forced.nstripes:2]. -func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, parseFavoriteKey bool) (*eosclient.FileInfo, error) { - inode, err := strconv.ParseUint(kv["ino"], 10, 64) - if err != nil { - return nil, err - } - fid, err := strconv.ParseUint(kv["fid"], 10, 64) - if err != nil { - return nil, err - } - uid, err := strconv.ParseUint(kv["uid"], 10, 64) - if err != nil { - return nil, err - } - gid, err := strconv.ParseUint(kv["gid"], 10, 64) - if err != nil { - return nil, err - } - - var treeSize uint64 - // treeSize is only for containers, so we check - if val, ok := kv["treesize"]; ok { - treeSize, err = strconv.ParseUint(val, 10, 64) - if err != nil { - return nil, err - } - } - var fileCounter uint64 - // fileCounter is only for containers - if val, ok := kv["files"]; ok { - fileCounter, err = strconv.ParseUint(val, 10, 64) - if err != nil { - return nil, err - } - } - var dirCounter uint64 - // dirCounter is only for containers - if val, ok := kv["container"]; ok { - dirCounter, err = strconv.ParseUint(val, 10, 64) - if err != nil { - return nil, err - } - } - - // treeCount is the number of entries under the tree - treeCount := fileCounter + dirCounter - - var size uint64 - if val, ok := kv["size"]; ok { - size, err = strconv.ParseUint(val, 10, 64) - if err != nil { - return nil, err - } - } - - // look for the stime first as mtime is not updated for parent dirs; if that isn't set, we use mtime - var mtimesec, mtimenanos uint64 - var mtimeSet bool - if val, ok := kv["stime"]; ok && val != "" { - stimeSplit := strings.Split(val, ".") - if mtimesec, err = strconv.ParseUint(stimeSplit[0], 10, 64); err == nil { - mtimeSet = true - } - - if mtimenanos, err = strconv.ParseUint(stimeSplit[1], 10, 32); err != nil { - mtimeSet = false - } - } - if !mtimeSet { - mtimeSplit := strings.Split(kv["mtime"], ".") - mtimesec, _ = strconv.ParseUint(mtimeSplit[0], 10, 64) - mtimenanos, _ = strconv.ParseUint(mtimeSplit[1], 10, 32) - } - - var ctimesec, ctimenanos uint64 - if val, ok := kv["ctime"]; ok && val != "" { - split := strings.Split(val, ".") - ctimesec, err = strconv.ParseUint(split[0], 10, 64) - if err != nil { - return nil, err - } - ctimenanos, _ = strconv.ParseUint(split[1], 10, 32) - if err != nil { - return nil, err - } - } - - var atimesec, atimenanos uint64 - if val, ok := kv["atime"]; ok && val != "" { - split := strings.Split(val, ".") - atimesec, err = strconv.ParseUint(split[0], 10, 64) - if err != nil { - return nil, err - } - atimenanos, err = strconv.ParseUint(split[1], 10, 32) - if err != nil { - return nil, err - } - } - - isDir := false - var xs *eosclient.Checksum - if _, ok := kv["files"]; ok { - isDir = true - } else { - xs = &eosclient.Checksum{ - XSSum: kv["xs"], - XSType: kv["xstype"], - } - } - - sysACL, err := acl.Parse(attrs["sys.acl"], acl.ShortTextForm) - if err != nil { - return nil, err - } - - // Temporary until we migrate the user ACLs to sys ACLs on our MGMs - // Read user ACLs if sys.eval.useracl is set - if userACLEval, ok := attrs["sys."+userACLEvalKey]; ok && userACLEval == "1" { - if userACL, ok := attrs["user.acl"]; ok { - userAcls, err := acl.Parse(userACL, acl.ShortTextForm) - if err != nil { - return nil, err - } - for _, e := range userAcls.Entries { - err = sysACL.SetEntry(e.Type, e.Qualifier, e.Permissions) - if err != nil { - return nil, err - } - } - } - } - - // Read the favorite attr - if parseFavoriteKey { - parseAndSetFavoriteAttr(ctx, attrs) - } - - fi := &eosclient.FileInfo{ - File: kv["file"], - Inode: inode, - FID: fid, - UID: uid, - GID: gid, - ETag: kv["etag"], - Size: size, - TreeSize: treeSize, - MTimeSec: mtimesec, - MTimeNanos: uint32(mtimenanos), - CTimeSec: ctimesec, - CTimeNanos: uint32(ctimenanos), - ATimeSec: atimesec, - ATimeNanos: uint32(atimenanos), - IsDir: isDir, - Instance: c.opt.URL, - SysACL: sysACL, - TreeCount: treeCount, - Attrs: attrs, - XS: xs, - } - - return fi, nil -} - -func parseAndSetFavoriteAttr(ctx context.Context, attrs map[string]string) { - // Read and correctly set the favorite attr - if user, ok := ctxpkg.ContextGetUser(ctx); ok { - if favAttrStr, ok := attrs[favoritesKey]; ok { - favUsers, err := acl.Parse(favAttrStr, acl.ShortTextForm) - if err != nil { - return - } - for _, u := range favUsers.Entries { - // Check if the current user has favorited this resource - if u.Qualifier == user.Id.OpaqueId { - // Set attr val to 1 - attrs[favoritesKey] = "1" - return - } - } - } - } - - // Delete the favorite attr from the response - delete(attrs, favoritesKey) -} From 26e5160bf53077b4d9647311dc9c50837911dbe8 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 15:38:49 +0200 Subject: [PATCH 25/33] Revert "Refactoring Handler, resource id and deleting unnecessary file" This reverts commit 926d3faf7df37679de30c628cf5a71ee5203a8e1. --- internal/http/services/overleaf/overleaf.go | 6 +- pkg/eosclient/eosbinary/eosbinary.go | 1301 +++++++++++++++++++ 2 files changed, 1305 insertions(+), 2 deletions(-) create mode 100644 pkg/eosclient/eosbinary/eosbinary.go diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 21eddd8501..188f0848da 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -120,7 +120,9 @@ func (s *svc) Unprotected() []string { } func (s *svc) Handler() http.Handler { - return s.router + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.router.ServeHTTP(w, r) + }) } func (s *svc) handleImport(w http.ResponseWriter, r *http.Request) { @@ -184,7 +186,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { } archQuery := archHTTPReq.URL.Query() - archQuery.Add("id", resourceid.OwnCloudResourceIDWrap(resource.Id)) + archQuery.Add("id", resource.Id.StorageId+"!"+resource.Id.OpaqueId) archQuery.Add("access_token", restrictedToken) archQuery.Add("arch_type", "zip") diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go new file mode 100644 index 0000000000..bbb0f1f036 --- /dev/null +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -0,0 +1,1301 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package eosbinary + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/cs3org/reva/pkg/appctx" + ctxpkg "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/eosclient" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/storage/utils/acl" + "github.com/google/uuid" + "github.com/pkg/errors" + "go.opentelemetry.io/otel/trace" +) + +const ( + versionPrefix = ".sys.v#." + userACLEvalKey = "eval.useracl" + favoritesKey = "http://owncloud.org/ns/favorite" +) + +func serializeAttribute(a *eosclient.Attribute) string { + return fmt.Sprintf("%s.%s=%s", attrTypeToString(a.Type), a.Key, a.Val) +} + +func attrTypeToString(at eosclient.AttrType) string { + switch at { + case eosclient.SystemAttr: + return "sys" + case eosclient.UserAttr: + return "user" + default: + return "invalid" + } +} + +func isValidAttribute(a *eosclient.Attribute) bool { + // validate that an attribute is correct. + if (a.Type != eosclient.SystemAttr && a.Type != eosclient.UserAttr) || a.Key == "" { + return false + } + return true +} + +// Options to configure the Client. +type Options struct { + + // ForceSingleUserMode forces all connections to use only one user. + // This is the case when access to EOS is done from FUSE under apache or www-data. + ForceSingleUserMode bool + + // UseKeyTabAuth changes will authenticate requests by using an EOS keytab. + UseKeytab bool + + // Whether to maintain the same inode across various versions of a file. + // Requires extra metadata operations if set to true + VersionInvariant bool + + // SingleUsername is the username to use when connecting to EOS. + // Defaults to apache + SingleUsername string + + // Location of the eos binary. + // Default is /usr/bin/eos. + EosBinary string + + // Location of the xrdcopy binary. + // Default is /opt/eos/xrootd/bin/xrdcopy. + XrdcopyBinary string + + // URL of the EOS MGM. + // Default is root://eos-example.org + URL string + + // Location on the local fs where to store reads. + // Defaults to os.TempDir() + CacheDirectory string + + // Keytab is the location of the EOS keytab file. + Keytab string + + // SecProtocol is the comma separated list of security protocols used by xrootd. + // For example: "sss, unix" + // DEPRECATED + // This variable is no longer used. Only sss and unix protocols are possible. + // If UseKeytab is set to true the protocol will be set to "sss", else to "unix" + SecProtocol string + + // TokenExpiry stores in seconds the time after which generated tokens will expire + // Default is 3600 + TokenExpiry int +} + +func (opt *Options) ApplyDefaults() { + if opt.ForceSingleUserMode && opt.SingleUsername != "" { + opt.SingleUsername = "apache" + } + + if opt.EosBinary == "" { + opt.EosBinary = "/usr/bin/eos" + } + + if opt.XrdcopyBinary == "" { + opt.XrdcopyBinary = "/opt/eos/xrootd/bin/xrdcopy" + } + + if opt.URL == "" { + opt.URL = "root://eos-example.org" + } + + if opt.CacheDirectory == "" { + opt.CacheDirectory = os.TempDir() + } +} + +// Client performs actions against a EOS management node (MGM). +// It requires the eos-client and xrootd-client packages installed to work. +type Client struct { + opt *Options +} + +// New creates a new client with the given options. +func New(opt *Options) (*Client, error) { + opt.ApplyDefaults() + c := new(Client) + c.opt = opt + return c, nil +} + +// executeXRDCopy executes xrdcpy commands and returns the stdout, stderr and return code. +func (c *Client) executeXRDCopy(ctx context.Context, cmdArgs []string) (string, string, error) { + log := appctx.GetLogger(ctx) + + outBuf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, cmdArgs...) + cmd.Stdout = outBuf + cmd.Stderr = errBuf + cmd.Env = []string{ + "EOS_MGM_URL=" + c.opt.URL, + } + + if c.opt.UseKeytab { + cmd.Env = append(cmd.Env, "XrdSecPROTOCOL=sss") + cmd.Env = append(cmd.Env, "XrdSecSSSKT="+c.opt.Keytab) + } else { // we are a trusted gateway + cmd.Env = append(cmd.Env, "XrdSecPROTOCOL=unix") + cmd.Env = append(cmd.Env, "KRB5CCNAME=FILE:/dev/null") // do not try to use krb + } + + err := cmd.Run() + + var exitStatus int + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + // This works on both Unix and Windows. Although package + // syscall is generally platform dependent, WaitStatus is + // defined for both Unix and Windows and in both cases has + // an ExitStatus() method with the same signature. + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + exitStatus = status.ExitStatus() + switch exitStatus { + case 0: + err = nil + case int(syscall.ENOENT): + err = errtypes.NotFound(errBuf.String()) + } + } + } + + // check for operation not permitted error + if strings.Contains(errBuf.String(), "Operation not permitted") { + err = errtypes.InvalidCredentials("eosclient: no sufficient permissions for the operation") + } + + args := fmt.Sprintf("%s", cmd.Args) + env := fmt.Sprintf("%s", cmd.Env) + log.Info().Str("args", args).Str("env", env).Int("exit", exitStatus).Msg("eos cmd") + + return outBuf.String(), errBuf.String(), err +} + +// exec executes only EOS commands the command and returns the stdout, stderr and return code. +func (c *Client) executeEOS(ctx context.Context, cmdArgs []string, auth eosclient.Authorization) (string, string, error) { + log := appctx.GetLogger(ctx) + + outBuf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + cmd := exec.CommandContext(ctx, c.opt.EosBinary) + cmd.Stdout = outBuf + cmd.Stderr = errBuf + cmd.Env = []string{ + "EOS_MGM_URL=" + c.opt.URL, + } + + if auth.Token != "" { + cmd.Env = append(cmd.Env, "EOSAUTHZ="+auth.Token) + } else if auth.Role.UID != "" && auth.Role.GID != "" { + cmd.Args = append(cmd.Args, []string{"-r", auth.Role.UID, auth.Role.GID}...) + } + + if c.opt.UseKeytab { + cmd.Env = append(cmd.Env, "XrdSecPROTOCOL=sss") + cmd.Env = append(cmd.Env, "XrdSecSSSKT="+c.opt.Keytab) + } else { // we are a trusted gateway + cmd.Env = append(cmd.Env, "XrdSecPROTOCOL=unix") + cmd.Env = append(cmd.Env, "KRB5CCNAME=FILE:/dev/null") // do not try to use krb + } + + // add application label + cmd.Args = append(cmd.Args, "-a", "reva_eosclient::meta") + + cmd.Args = append(cmd.Args, cmdArgs...) + + span := trace.SpanFromContext(ctx) + cmd.Args = append(cmd.Args, "--comment", span.SpanContext().TraceID().String()) + + err := cmd.Run() + + var exitStatus int + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + // This works on both Unix and Windows. Although package + // syscall is generally platform dependent, WaitStatus is + // defined for both Unix and Windows and in both cases has + // an ExitStatus() method with the same signature. + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + exitStatus = status.ExitStatus() + switch exitStatus { + case 0: + err = nil + case int(syscall.ENOENT): + err = errtypes.NotFound(errBuf.String()) + case int(syscall.EPERM), int(syscall.E2BIG), int(syscall.EINVAL): + // eos reports back error code 1 (EPERM) when ? + // eos reports back error code 7 (E2BIG) when the user is not allowed to read the directory + // eos reports back error code 22 (EINVAL) when the user is not allowed to enter the instance + err = errtypes.PermissionDenied(errBuf.String()) + } + } + } + + args := fmt.Sprintf("%s", cmd.Args) + env := fmt.Sprintf("%s", cmd.Env) + log.Info().Str("args", args).Str("env", env).Int("exit", exitStatus).Str("err", errBuf.String()).Msg("eos cmd") + + if err != nil && exitStatus != int(syscall.ENOENT) { // don't wrap the errtypes.NotFoundError + err = errors.Wrap(err, "eosclient: error while executing command") + } + + return outBuf.String(), errBuf.String(), err +} + +// AddACL adds an new acl to EOS with the given aclType. +func (c *Client) AddACL(ctx context.Context, auth, rootAuth eosclient.Authorization, path string, pos uint, a *acl.Entry) error { + finfo, err := c.getRawFileInfoByPath(ctx, auth, path) + if err != nil { + return err + } + + sysACL := a.CitrineSerialize() + args := []string{"acl", "--sys"} + if finfo.IsDir { + args = append(args, "--recursive") + } + + // set position of ACLs to add. The default is to append to the end, so no arguments will be added in this case + // the first position starts at 1 = eosclient.StartPosition + if pos != eosclient.EndPosition { + args = append(args, "--position", fmt.Sprint(pos)) + } + + args = append(args, sysACL, path) + + _, _, err = c.executeEOS(ctx, args, rootAuth) + return err +} + +// RemoveACL removes the acl from EOS. +func (c *Client) RemoveACL(ctx context.Context, auth, rootAuth eosclient.Authorization, path string, a *acl.Entry) error { + finfo, err := c.getRawFileInfoByPath(ctx, auth, path) + if err != nil { + return err + } + + a.Permissions = "" + sysACL := a.CitrineSerialize() + args := []string{"acl", "--sys"} + if finfo.IsDir { + args = append(args, "--recursive") + } + args = append(args, sysACL, path) + + _, _, err = c.executeEOS(ctx, args, rootAuth) + return err +} + +// UpdateACL updates the EOS acl. +func (c *Client) UpdateACL(ctx context.Context, auth, rootAuth eosclient.Authorization, path string, position uint, a *acl.Entry) error { + return c.AddACL(ctx, auth, rootAuth, path, position, a) +} + +// GetACL for a file. +func (c *Client) GetACL(ctx context.Context, auth eosclient.Authorization, path, aclType, target string) (*acl.Entry, error) { + acls, err := c.ListACLs(ctx, auth, path) + if err != nil { + return nil, err + } + for _, a := range acls { + if a.Type == aclType && a.Qualifier == target { + return a, nil + } + } + return nil, errtypes.NotFound(fmt.Sprintf("%s:%s", aclType, target)) +} + +// ListACLs returns the list of ACLs present under the given path. +// EOS returns uids/gid for Citrine version and usernames for older versions. +// For Citire we need to convert back the uid back to username. +func (c *Client) ListACLs(ctx context.Context, auth eosclient.Authorization, path string) ([]*acl.Entry, error) { + parsedACLs, err := c.getACLForPath(ctx, auth, path) + if err != nil { + return nil, err + } + + // EOS Citrine ACLs are stored with uid. The UID will be resolved to the + // user opaque ID at the eosfs level. + return parsedACLs.Entries, nil +} + +func (c *Client) getACLForPath(ctx context.Context, auth eosclient.Authorization, path string) (*acl.ACLs, error) { + finfo, err := c.GetFileInfoByPath(ctx, auth, path) + if err != nil { + return nil, err + } + + return finfo.SysACL, nil +} + +// GetFileInfoByInode returns the FileInfo by the given inode. +func (c *Client) GetFileInfoByInode(ctx context.Context, auth eosclient.Authorization, inode uint64) (*eosclient.FileInfo, error) { + args := []string{"file", "info", fmt.Sprintf("inode:%d", inode), "-m"} + stdout, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, err + } + info, err := c.parseFileInfo(ctx, stdout, true) + if err != nil { + return nil, err + } + + if c.opt.VersionInvariant && isVersionFolder(info.File) { + info, err = c.getFileInfoFromVersion(ctx, auth, info.File) + if err != nil { + return nil, err + } + info.Inode = inode + } + + return c.mergeACLsAndAttrsForFiles(ctx, auth, info), nil +} + +// GetFileInfoByFXID returns the FileInfo by the given file id in hexadecimal. +func (c *Client) GetFileInfoByFXID(ctx context.Context, auth eosclient.Authorization, fxid string) (*eosclient.FileInfo, error) { + args := []string{"file", "info", fmt.Sprintf("fxid:%s", fxid), "-m"} + stdout, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, err + } + + info, err := c.parseFileInfo(ctx, stdout, true) + if err != nil { + return nil, err + } + + return c.mergeACLsAndAttrsForFiles(ctx, auth, info), nil +} + +// GetFileInfoByPath returns the FilInfo at the given path. +func (c *Client) GetFileInfoByPath(ctx context.Context, auth eosclient.Authorization, path string) (*eosclient.FileInfo, error) { + args := []string{"file", "info", path, "-m"} + stdout, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, err + } + info, err := c.parseFileInfo(ctx, stdout, true) + if err != nil { + return nil, err + } + + if c.opt.VersionInvariant && !isVersionFolder(path) && !info.IsDir { + ownerAuth := eosclient.Authorization{Role: eosclient.Role{ + UID: strconv.FormatUint(info.UID, 10), + GID: strconv.FormatUint(info.GID, 10), + }} + if inode, err := c.getVersionFolderInode(ctx, auth, ownerAuth, path); err == nil { + info.Inode = inode + } + } + + return c.mergeACLsAndAttrsForFiles(ctx, auth, info), nil +} + +func (c *Client) getRawFileInfoByPath(ctx context.Context, auth eosclient.Authorization, path string) (*eosclient.FileInfo, error) { + args := []string{"file", "info", path, "-m"} + stdout, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, err + } + return c.parseFileInfo(ctx, stdout, false) +} + +func (c *Client) mergeACLsAndAttrsForFiles(ctx context.Context, auth eosclient.Authorization, info *eosclient.FileInfo) *eosclient.FileInfo { + // We need to inherit the ACLs for the parent directory as these are not available for files + // And the attributes from the version folders + if !info.IsDir { + parentInfo, err := c.getRawFileInfoByPath(ctx, auth, path.Dir(info.File)) + // Even if this call fails, at least return the current file object + if err == nil { + info.SysACL.Entries = append(info.SysACL.Entries, parentInfo.SysACL.Entries...) + } + + // We need to merge attrs set for the version folders, so get those resolved for the current user + versionFolderInfo, err := c.GetFileInfoByPath(ctx, auth, getVersionFolder(info.File)) + if err == nil { + info.SysACL.Entries = append(info.SysACL.Entries, versionFolderInfo.SysACL.Entries...) + for k, v := range versionFolderInfo.Attrs { + info.Attrs[k] = v + } + } + } + + return info +} + +// SetAttr sets an extended attributes on a path. +func (c *Client) SetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, errorIfExists, recursive bool, path string) error { + if !isValidAttribute(attr) { + return errors.New("eos: attr is invalid: " + serializeAttribute(attr)) + } + + var info *eosclient.FileInfo + var err error + // We need to set the attrs on the version folder as they are not persisted across writes + // Except for the sys.eval.useracl attr as EOS uses that to determine if it needs to obey + // the user ACLs set on the file + if !(attr.Type == eosclient.SystemAttr && attr.Key == userACLEvalKey) { + info, err = c.getRawFileInfoByPath(ctx, auth, path) + if err != nil { + return err + } + if !info.IsDir { + path = getVersionFolder(path) + } + } + + // Favorites need to be stored per user so handle these separately + if attr.Type == eosclient.UserAttr && attr.Key == favoritesKey { + return c.handleFavAttr(ctx, auth, attr, recursive, path, info, true) + } + return c.setEOSAttr(ctx, auth, attr, errorIfExists, recursive, path) +} + +func (c *Client) setEOSAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, errorIfExists, recursive bool, path string) error { + args := []string{"attr"} + if recursive { + args = append(args, "-r") + } + args = append(args, "set") + if errorIfExists { + args = append(args, "-c") + } + args = append(args, serializeAttribute(attr), path) + + _, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + var exErr *exec.ExitError + if errors.As(err, &exErr) && exErr.ExitCode() == 17 { + return eosclient.AttrAlreadyExistsError + } + return err + } + return nil +} + +func (c *Client) handleFavAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path string, info *eosclient.FileInfo, set bool) error { + var err error + u := ctxpkg.ContextMustGetUser(ctx) + if info == nil { + info, err = c.getRawFileInfoByPath(ctx, auth, path) + if err != nil { + return err + } + } + favStr := info.Attrs[favoritesKey] + favs, err := acl.Parse(favStr, acl.ShortTextForm) + if err != nil { + return err + } + if set { + err = favs.SetEntry(acl.TypeUser, u.Id.OpaqueId, "1") + if err != nil { + return err + } + } else { + favs.DeleteEntry(acl.TypeUser, u.Id.OpaqueId) + } + attr.Val = favs.Serialize() + return c.setEOSAttr(ctx, auth, attr, false, recursive, path) +} + +// UnsetAttr unsets an extended attribute on a path. +func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path string) error { + if !isValidAttribute(attr) { + return errors.New("eos: attr is invalid: " + serializeAttribute(attr)) + } + + var info *eosclient.FileInfo + var err error + // We need to set the attrs on the version folder as they are not persisted across writes + // Except for the sys.eval.useracl attr as EOS uses that to determine if it needs to obey + // the user ACLs set on the file + if !(attr.Type == eosclient.SystemAttr && attr.Key == userACLEvalKey) { + info, err = c.getRawFileInfoByPath(ctx, auth, path) + if err != nil { + return err + } + if !info.IsDir { + path = getVersionFolder(path) + } + } + + // Favorites need to be stored per user so handle these separately + if attr.Type == eosclient.UserAttr && attr.Key == favoritesKey { + return c.handleFavAttr(ctx, auth, attr, recursive, path, info, false) + } + + var args []string + if recursive { + args = []string{"attr", "-r", "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path} + } else { + args = []string{"attr", "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path} + } + _, _, err = c.executeEOS(ctx, args, auth) + if err != nil { + var exErr *exec.ExitError + if errors.As(err, &exErr) && exErr.ExitCode() == 61 { + return eosclient.AttrNotExistsError + } + return err + } + return nil +} + +// GetAttr returns the attribute specified by key. +func (c *Client) GetAttr(ctx context.Context, auth eosclient.Authorization, key, path string) (*eosclient.Attribute, error) { + // As SetAttr set the attr on the version folder, we will read the attribute on it + // if the resource is not a folder + info, err := c.getRawFileInfoByPath(ctx, auth, path) + if err != nil { + return nil, err + } + if !info.IsDir { + path = getVersionFolder(path) + } + + args := []string{"attr", "get", key, path} + attrOut, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, err + } + attr, err := deserializeAttribute(attrOut) + if err != nil { + return nil, err + } + return attr, nil +} + +// GetAttrs returns all the attributes of a resource. +func (c *Client) GetAttrs(ctx context.Context, auth eosclient.Authorization, path string) ([]*eosclient.Attribute, error) { + info, err := c.getRawFileInfoByPath(ctx, auth, path) + if err != nil { + return nil, err + } + if !info.IsDir { + path = getVersionFolder(path) + } + + args := []string{"attr", "ls", path} + attrOut, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, err + } + + attrsStr := strings.Split(attrOut, "\n") + attrs := make([]*eosclient.Attribute, 0, len(attrsStr)) + for _, line := range attrsStr { + attr, err := deserializeAttribute(line) + if err != nil { + return nil, err + } + attrs = append(attrs, attr) + } + return attrs, nil +} + +func deserializeAttribute(attrStr string) (*eosclient.Attribute, error) { + // the string is in the form sys.forced.checksum="adler" + keyValue := strings.SplitN(strings.TrimSpace(attrStr), "=", 2) // keyValue = ["sys.forced.checksum", "\"adler\""] + if len(keyValue) != 2 { + return nil, errtypes.InternalError("wrong attr format to deserialize") + } + type2key := strings.SplitN(keyValue[0], ".", 2) // type2key = ["sys", "forced.checksum"] + if len(type2key) != 2 { + return nil, errtypes.InternalError("wrong attr format to deserialize") + } + t, err := eosclient.AttrStringToType(type2key[0]) + if err != nil { + return nil, err + } + // trim \" from value + value := strings.Trim(keyValue[1], "\"") + return &eosclient.Attribute{Type: t, Key: type2key[1], Val: value}, nil +} + +// GetQuota gets the quota of a user on the quota node defined by path. +func (c *Client) GetQuota(ctx context.Context, username string, rootAuth eosclient.Authorization, path string) (*eosclient.QuotaInfo, error) { + args := []string{"quota", "ls", "-u", username, "-m"} + stdout, _, err := c.executeEOS(ctx, args, rootAuth) + if err != nil { + return nil, err + } + return c.parseQuota(path, stdout) +} + +// SetQuota sets the quota of a user on the quota node defined by path. +func (c *Client) SetQuota(ctx context.Context, rootAuth eosclient.Authorization, info *eosclient.SetQuotaInfo) error { + maxBytes := fmt.Sprintf("%d", info.MaxBytes) + maxFiles := fmt.Sprintf("%d", info.MaxFiles) + args := []string{"quota", "set", "-u", info.Username, "-p", info.QuotaNode, "-v", maxBytes, "-i", maxFiles} + _, _, err := c.executeEOS(ctx, args, rootAuth) + if err != nil { + return err + } + return nil +} + +// Touch creates a 0-size,0-replica file in the EOS namespace. +func (c *Client) Touch(ctx context.Context, auth eosclient.Authorization, path string) error { + args := []string{"file", "touch", path} + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// Chown given path. +func (c *Client) Chown(ctx context.Context, auth, chownauth eosclient.Authorization, path string) error { + args := []string{"chown", chownauth.Role.UID + ":" + chownauth.Role.GID, path} + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// Chmod given path. +func (c *Client) Chmod(ctx context.Context, auth eosclient.Authorization, mode, path string) error { + args := []string{"chmod", mode, path} + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// CreateDir creates a directory at the given path. +func (c *Client) CreateDir(ctx context.Context, auth eosclient.Authorization, path string) error { + args := []string{"mkdir", "-p", path} + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// Remove removes the resource at the given path. +func (c *Client) Remove(ctx context.Context, auth eosclient.Authorization, path string, noRecycle bool) error { + args := []string{"rm", "-r"} + if noRecycle { + args = append(args, "--no-recycle-bin") // do not put the file in the recycle bin + } + args = append(args, path) + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// Rename renames the resource referenced by oldPath to newPath. +func (c *Client) Rename(ctx context.Context, auth eosclient.Authorization, oldPath, newPath string) error { + args := []string{"file", "rename", oldPath, newPath} + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// List the contents of the directory given by path. +func (c *Client) List(ctx context.Context, auth eosclient.Authorization, path string) ([]*eosclient.FileInfo, error) { + args := []string{"find", "--fileinfo", "--maxdepth", "1", path} + stdout, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, errors.Wrapf(err, "eosclient: error listing fn=%s", path) + } + return c.parseFind(ctx, auth, path, stdout) +} + +// Read reads a file from the mgm. +func (c *Client) Read(ctx context.Context, auth eosclient.Authorization, path string) (io.ReadCloser, error) { + rand := "eosread-" + uuid.New().String() + localTarget := fmt.Sprintf("%s/%s", c.opt.CacheDirectory, rand) + defer os.RemoveAll(localTarget) + + xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) + args := []string{"--nopbar", "--silent", "-f", xrdPath, localTarget} + + if auth.Token != "" { + args[3] += "?authz=" + auth.Token + } else if auth.Role.UID != "" && auth.Role.GID != "" { + args = append(args, fmt.Sprintf("-OSeos.ruid=%s&eos.rgid=%s&eos.app=reva_eosclient::read", auth.Role.UID, auth.Role.GID)) + } + + _, _, err := c.executeXRDCopy(ctx, args) + if err != nil { + return nil, err + } + return os.Open(localTarget) +} + +// Write writes a stream to the mgm. +func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path string, stream io.ReadCloser) error { + fd, err := os.CreateTemp(c.opt.CacheDirectory, "eoswrite-") + if err != nil { + return err + } + defer fd.Close() + defer os.RemoveAll(fd.Name()) + + // copy stream to local temp file + _, err = io.Copy(fd, stream) + if err != nil { + return err + } + + return c.WriteFile(ctx, auth, path, fd.Name()) +} + +// WriteFile writes an existing file to the mgm. +func (c *Client) WriteFile(ctx context.Context, auth eosclient.Authorization, path, source string) error { + xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) + args := []string{"--nopbar", "--silent", "-f", source, xrdPath} + + if auth.Token != "" { + args[4] += "?authz=" + auth.Token + } else if auth.Role.UID != "" && auth.Role.GID != "" { + args = append(args, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s&eos.app=reva_eosclient::write", auth.Role.UID, auth.Role.GID)) + } + + _, _, err := c.executeXRDCopy(ctx, args) + return err +} + +// ListDeletedEntries returns a list of the deleted entries. +func (c *Client) ListDeletedEntries(ctx context.Context, auth eosclient.Authorization) ([]*eosclient.DeletedEntry, error) { + // TODO(labkode): add protection if slave is configured and alive to count how many files are in the trashbin before + // triggering the recycle ls call that could break the instance because of unavailable memory. + args := []string{"recycle", "ls", "-m"} + stdout, _, err := c.executeEOS(ctx, args, auth) + if err != nil { + return nil, err + } + return parseRecycleList(stdout) +} + +// RestoreDeletedEntry restores a deleted entry. +func (c *Client) RestoreDeletedEntry(ctx context.Context, auth eosclient.Authorization, key string) error { + args := []string{"recycle", "restore", key} + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// PurgeDeletedEntries purges all entries from the recycle bin. +func (c *Client) PurgeDeletedEntries(ctx context.Context, auth eosclient.Authorization) error { + args := []string{"recycle", "purge"} + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// ListVersions list all the versions for a given file. +func (c *Client) ListVersions(ctx context.Context, auth eosclient.Authorization, p string) ([]*eosclient.FileInfo, error) { + versionFolder := getVersionFolder(p) + finfos, err := c.List(ctx, auth, versionFolder) + if err != nil { + // we send back an empty list + return []*eosclient.FileInfo{}, nil + } + return finfos, nil +} + +// RollbackToVersion rollbacks a file to a previous version. +func (c *Client) RollbackToVersion(ctx context.Context, auth eosclient.Authorization, path, version string) error { + args := []string{"file", "versions", path, version} + _, _, err := c.executeEOS(ctx, args, auth) + return err +} + +// ReadVersion reads the version for the given file. +func (c *Client) ReadVersion(ctx context.Context, auth eosclient.Authorization, p, version string) (io.ReadCloser, error) { + versionFile := path.Join(getVersionFolder(p), version) + return c.Read(ctx, auth, versionFile) +} + +// GenerateToken returns a token on behalf of the resource owner to be used by lightweight accounts. +func (c *Client) GenerateToken(ctx context.Context, auth eosclient.Authorization, p string, a *acl.Entry) (string, error) { + expiration := strconv.FormatInt(time.Now().Add(time.Duration(c.opt.TokenExpiry)*time.Second).Unix(), 10) + args := []string{"token", "--permission", a.Permissions, "--tree", "--path", p, "--expires", expiration} + stdout, _, err := c.executeEOS(ctx, args, auth) + return strings.TrimSpace(stdout), err +} + +func (c *Client) getVersionFolderInode(ctx context.Context, auth, ownerAuth eosclient.Authorization, p string) (uint64, error) { + versionFolder := getVersionFolder(p) + md, err := c.getRawFileInfoByPath(ctx, auth, versionFolder) + if err != nil { + if err = c.CreateDir(ctx, ownerAuth, versionFolder); err != nil { + return 0, err + } + md, err = c.getRawFileInfoByPath(ctx, auth, versionFolder) + if err != nil { + return 0, err + } + } + return md.Inode, nil +} + +func (c *Client) getFileInfoFromVersion(ctx context.Context, auth eosclient.Authorization, p string) (*eosclient.FileInfo, error) { + file := getFileFromVersionFolder(p) + md, err := c.GetFileInfoByPath(ctx, auth, file) + if err != nil { + return nil, err + } + return md, nil +} + +func isVersionFolder(p string) bool { + return strings.HasPrefix(path.Base(p), versionPrefix) +} + +func getVersionFolder(p string) string { + return path.Join(path.Dir(p), versionPrefix+path.Base(p)) +} + +func getFileFromVersionFolder(p string) string { + return path.Join(path.Dir(p), strings.TrimPrefix(path.Base(p), versionPrefix)) +} + +func parseRecycleList(raw string) ([]*eosclient.DeletedEntry, error) { + entries := []*eosclient.DeletedEntry{} + rawLines := strings.FieldsFunc(raw, func(c rune) bool { + return c == '\n' + }) + for _, rl := range rawLines { + if rl == "" { + continue + } + entry, err := parseRecycleEntry(rl) + if err != nil { + return nil, err + } + entries = append(entries, entry) + } + return entries, nil +} + +// parse entries like these: +// recycle=ls recycle-bin=/eos/backup/proc/recycle/ uid=gonzalhu gid=it size=0 deletion-time=1510823151 type=recursive-dir keylength.restore-path=45 restore-path=/eos/scratch/user/g/gonzalhu/.sys.v#.app.ico/ restore-key=0000000000a35100 +// recycle=ls recycle-bin=/eos/backup/proc/recycle/ uid=gonzalhu gid=it size=381038 deletion-time=1510823151 type=file keylength.restore-path=36 restore-path=/eos/scratch/user/g/gonzalhu/app.ico restore-key=000000002544fdb3. +func parseRecycleEntry(raw string) (*eosclient.DeletedEntry, error) { + partsBySpace := strings.FieldsFunc(raw, func(c rune) bool { + return c == ' ' + }) + restoreKeyPair, partsBySpace := partsBySpace[len(partsBySpace)-1], partsBySpace[:len(partsBySpace)-1] + restorePathPair := strings.Join(partsBySpace[8:], " ") + + partsBySpace = partsBySpace[:8] + partsBySpace = append(partsBySpace, restorePathPair) + partsBySpace = append(partsBySpace, restoreKeyPair) + + kv := getMap(partsBySpace) + size, err := strconv.ParseUint(kv["size"], 10, 64) + if err != nil { + return nil, err + } + isDir := false + if kv["type"] == "recursive-dir" { + isDir = true + } + deletionMTime, err := strconv.ParseUint(strings.Split(kv["deletion-time"], ".")[0], 10, 64) + if err != nil { + return nil, err + } + entry := &eosclient.DeletedEntry{ + RestorePath: kv["restore-path"], + RestoreKey: kv["restore-key"], + Size: size, + DeletionMTime: deletionMTime, + IsDir: isDir, + } + return entry, nil +} + +func getMap(partsBySpace []string) map[string]string { + kv := map[string]string{} + for _, pair := range partsBySpace { + parts := strings.Split(pair, "=") + if len(parts) > 1 { + kv[parts[0]] = parts[1] + } + } + return kv +} + +func (c *Client) parseFind(ctx context.Context, auth eosclient.Authorization, dirPath, raw string) ([]*eosclient.FileInfo, error) { + log := appctx.GetLogger(ctx) + + finfos := []*eosclient.FileInfo{} + versionFolders := map[string]*eosclient.FileInfo{} + rawLines := strings.FieldsFunc(raw, func(c rune) bool { + return c == '\n' + }) + + var ownerAuth *eosclient.Authorization + + var parent *eosclient.FileInfo + for _, rl := range rawLines { + if rl == "" { + continue + } + fi, err := c.parseFileInfo(ctx, rl, true) + if err != nil { + return nil, err + } + // dirs in eos end with a slash, like /eos/user/g/gonzalhu/ + // we skip the current directory as eos find will return the directory we + // ask to find + if fi.File == path.Clean(dirPath) { + parent = fi + continue + } + + // If it's a version folder, store it in a map, so that for the corresponding file, + // we can return its inode instead + if isVersionFolder(fi.File) { + versionFolders[fi.File] = fi + } + + if ownerAuth == nil { + ownerAuth = &eosclient.Authorization{ + Role: eosclient.Role{ + UID: strconv.FormatUint(fi.UID, 10), + GID: strconv.FormatUint(fi.GID, 10), + }, + } + } + + finfos = append(finfos, fi) + } + + for _, fi := range finfos { + // For files, inherit ACLs from the parent + // And set the inode to that of their version folder + if !fi.IsDir && !isVersionFolder(dirPath) { + if parent != nil { + fi.SysACL.Entries = append(fi.SysACL.Entries, parent.SysACL.Entries...) + } + versionFolderPath := getVersionFolder(fi.File) + if vf, ok := versionFolders[versionFolderPath]; ok { + fi.Inode = vf.Inode + fi.SysACL.Entries = append(fi.SysACL.Entries, vf.SysACL.Entries...) + for k, v := range vf.Attrs { + fi.Attrs[k] = v + } + } else if err := c.CreateDir(ctx, *ownerAuth, versionFolderPath); err == nil { // Create the version folder if it doesn't exist + if md, err := c.getRawFileInfoByPath(ctx, auth, versionFolderPath); err == nil { + fi.Inode = md.Inode + } else { + log.Error().Err(err).Interface("auth", ownerAuth).Str("path", versionFolderPath).Msg("got error creating version folder") + } + } + } + } + + return finfos, nil +} + +func (c Client) parseQuotaLine(line string) map[string]string { + partsBySpace := strings.FieldsFunc(line, func(c rune) bool { + return c == ' ' + }) + m := getMap(partsBySpace) + return m +} +func (c *Client) parseQuota(path, raw string) (*eosclient.QuotaInfo, error) { + rawLines := strings.FieldsFunc(raw, func(c rune) bool { + return c == '\n' + }) + for _, rl := range rawLines { + if rl == "" { + continue + } + + m := c.parseQuotaLine(rl) + // map[maxbytes:2000000000000 maxlogicalbytes:1000000000000 percentageusedbytes:0.49 quota:node uid:gonzalhu space:/eos/scratch/user/ usedbytes:9829986500 usedlogicalbytes:4914993250 statusfiles:ok usedfiles:334 maxfiles:1000000 statusbytes:ok] + + space := m["space"] + if strings.HasPrefix(path, filepath.Clean(space)) { + maxBytesString := m["maxlogicalbytes"] + usedBytesString := m["usedlogicalbytes"] + maxBytes, _ := strconv.ParseUint(maxBytesString, 10, 64) + usedBytes, _ := strconv.ParseUint(usedBytesString, 10, 64) + + maxInodesString := m["maxfiles"] + usedInodesString := m["usedfiles"] + maxInodes, _ := strconv.ParseUint(maxInodesString, 10, 64) + usedInodes, _ := strconv.ParseUint(usedInodesString, 10, 64) + + qi := &eosclient.QuotaInfo{ + AvailableBytes: maxBytes, + UsedBytes: usedBytes, + AvailableInodes: maxInodes, + UsedInodes: usedInodes, + } + return qi, nil + } + } + return &eosclient.QuotaInfo{}, nil +} + +// TODO(labkode): better API to access extended attributes. +func (c *Client) parseFileInfo(ctx context.Context, raw string, parseFavoriteKey bool) (*eosclient.FileInfo, error) { + line := raw[15:] + index := strings.Index(line, " file=/") + lengthString := line[0:index] + length, err := strconv.ParseUint(lengthString, 10, 64) + if err != nil { + return nil, err + } + + line = line[index+6:] // skip ' file=' + name := line[0:length] + + kv := make(map[string]string) + attrs := make(map[string]string) + // strip trailing slash + kv["file"] = strings.TrimSuffix(name, "/") + + line = line[length+1:] + partsBySpace := strings.FieldsFunc(line, func(c rune) bool { // we have [size=45 container=3 ...} + return c == ' ' + }) + var previousXAttr = "" + for _, p := range partsBySpace { + partsByEqual := strings.SplitN(p, "=", 2) // we have kv pairs like [size 14] + if len(partsByEqual) == 2 { + // handle xattrn and xattrv special cases + switch { + case partsByEqual[0] == "xattrn": + previousXAttr = partsByEqual[1] + if previousXAttr != "user.acl" { + previousXAttr = strings.Replace(previousXAttr, "user.", "", 1) + } + case partsByEqual[0] == "xattrv": + attrs[previousXAttr] = strings.ToValidUTF8(partsByEqual[1], "") + previousXAttr = "" + default: + kv[partsByEqual[0]] = partsByEqual[1] + } + } + } + fi, err := c.mapToFileInfo(ctx, kv, attrs, parseFavoriteKey) + if err != nil { + return nil, err + } + return fi, nil +} + +// mapToFileInfo converts the dictionary to an usable structure. +// The kv has format: +// map[sys.forced.space:default files:0 mode:42555 ino:5 sys.forced.blocksize:4k sys.forced.layout:replica uid:0 fid:5 sys.forced.blockchecksum:crc32c sys.recycle:/eos/backup/proc/recycle/ fxid:00000005 pid:1 etag:5:0.000 keylength.file:4 file:/eos treesize:1931593933849913 container:3 gid:0 mtime:1498571294.108614409 ctime:1460121992.294326762 pxid:00000001 sys.forced.checksum:adler sys.forced.nstripes:2]. +func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, parseFavoriteKey bool) (*eosclient.FileInfo, error) { + inode, err := strconv.ParseUint(kv["ino"], 10, 64) + if err != nil { + return nil, err + } + fid, err := strconv.ParseUint(kv["fid"], 10, 64) + if err != nil { + return nil, err + } + uid, err := strconv.ParseUint(kv["uid"], 10, 64) + if err != nil { + return nil, err + } + gid, err := strconv.ParseUint(kv["gid"], 10, 64) + if err != nil { + return nil, err + } + + var treeSize uint64 + // treeSize is only for containers, so we check + if val, ok := kv["treesize"]; ok { + treeSize, err = strconv.ParseUint(val, 10, 64) + if err != nil { + return nil, err + } + } + var fileCounter uint64 + // fileCounter is only for containers + if val, ok := kv["files"]; ok { + fileCounter, err = strconv.ParseUint(val, 10, 64) + if err != nil { + return nil, err + } + } + var dirCounter uint64 + // dirCounter is only for containers + if val, ok := kv["container"]; ok { + dirCounter, err = strconv.ParseUint(val, 10, 64) + if err != nil { + return nil, err + } + } + + // treeCount is the number of entries under the tree + treeCount := fileCounter + dirCounter + + var size uint64 + if val, ok := kv["size"]; ok { + size, err = strconv.ParseUint(val, 10, 64) + if err != nil { + return nil, err + } + } + + // look for the stime first as mtime is not updated for parent dirs; if that isn't set, we use mtime + var mtimesec, mtimenanos uint64 + var mtimeSet bool + if val, ok := kv["stime"]; ok && val != "" { + stimeSplit := strings.Split(val, ".") + if mtimesec, err = strconv.ParseUint(stimeSplit[0], 10, 64); err == nil { + mtimeSet = true + } + + if mtimenanos, err = strconv.ParseUint(stimeSplit[1], 10, 32); err != nil { + mtimeSet = false + } + } + if !mtimeSet { + mtimeSplit := strings.Split(kv["mtime"], ".") + mtimesec, _ = strconv.ParseUint(mtimeSplit[0], 10, 64) + mtimenanos, _ = strconv.ParseUint(mtimeSplit[1], 10, 32) + } + + var ctimesec, ctimenanos uint64 + if val, ok := kv["ctime"]; ok && val != "" { + split := strings.Split(val, ".") + ctimesec, err = strconv.ParseUint(split[0], 10, 64) + if err != nil { + return nil, err + } + ctimenanos, _ = strconv.ParseUint(split[1], 10, 32) + if err != nil { + return nil, err + } + } + + var atimesec, atimenanos uint64 + if val, ok := kv["atime"]; ok && val != "" { + split := strings.Split(val, ".") + atimesec, err = strconv.ParseUint(split[0], 10, 64) + if err != nil { + return nil, err + } + atimenanos, err = strconv.ParseUint(split[1], 10, 32) + if err != nil { + return nil, err + } + } + + isDir := false + var xs *eosclient.Checksum + if _, ok := kv["files"]; ok { + isDir = true + } else { + xs = &eosclient.Checksum{ + XSSum: kv["xs"], + XSType: kv["xstype"], + } + } + + sysACL, err := acl.Parse(attrs["sys.acl"], acl.ShortTextForm) + if err != nil { + return nil, err + } + + // Temporary until we migrate the user ACLs to sys ACLs on our MGMs + // Read user ACLs if sys.eval.useracl is set + if userACLEval, ok := attrs["sys."+userACLEvalKey]; ok && userACLEval == "1" { + if userACL, ok := attrs["user.acl"]; ok { + userAcls, err := acl.Parse(userACL, acl.ShortTextForm) + if err != nil { + return nil, err + } + for _, e := range userAcls.Entries { + err = sysACL.SetEntry(e.Type, e.Qualifier, e.Permissions) + if err != nil { + return nil, err + } + } + } + } + + // Read the favorite attr + if parseFavoriteKey { + parseAndSetFavoriteAttr(ctx, attrs) + } + + fi := &eosclient.FileInfo{ + File: kv["file"], + Inode: inode, + FID: fid, + UID: uid, + GID: gid, + ETag: kv["etag"], + Size: size, + TreeSize: treeSize, + MTimeSec: mtimesec, + MTimeNanos: uint32(mtimenanos), + CTimeSec: ctimesec, + CTimeNanos: uint32(ctimenanos), + ATimeSec: atimesec, + ATimeNanos: uint32(atimenanos), + IsDir: isDir, + Instance: c.opt.URL, + SysACL: sysACL, + TreeCount: treeCount, + Attrs: attrs, + XS: xs, + } + + return fi, nil +} + +func parseAndSetFavoriteAttr(ctx context.Context, attrs map[string]string) { + // Read and correctly set the favorite attr + if user, ok := ctxpkg.ContextGetUser(ctx); ok { + if favAttrStr, ok := attrs[favoritesKey]; ok { + favUsers, err := acl.Parse(favAttrStr, acl.ShortTextForm) + if err != nil { + return + } + for _, u := range favUsers.Entries { + // Check if the current user has favorited this resource + if u.Qualifier == user.Id.OpaqueId { + // Set attr val to 1 + attrs[favoritesKey] = "1" + return + } + } + } + } + + // Delete the favorite attr from the response + delete(attrs, favoritesKey) +} From ec1863630539fa86f5a97ca3a04c329d5006a1af Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 15:39:04 +0200 Subject: [PATCH 26/33] Revert "temporary workaround for EOS-5754 (#45)" This reverts commit 2b3033dd2c9b1caf81e625f1213a6fddd82a0870. --- pkg/eosclient/eosbinary/eosbinary.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index bbb0f1f036..1a47aa8240 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -1176,15 +1176,18 @@ func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, if mtimesec, err = strconv.ParseUint(stimeSplit[0], 10, 64); err == nil { mtimeSet = true } - if mtimenanos, err = strconv.ParseUint(stimeSplit[1], 10, 32); err != nil { mtimeSet = false } } if !mtimeSet { mtimeSplit := strings.Split(kv["mtime"], ".") - mtimesec, _ = strconv.ParseUint(mtimeSplit[0], 10, 64) - mtimenanos, _ = strconv.ParseUint(mtimeSplit[1], 10, 32) + if mtimesec, err = strconv.ParseUint(mtimeSplit[0], 10, 64); err != nil { + return nil, err + } + if mtimenanos, err = strconv.ParseUint(mtimeSplit[1], 10, 32); err != nil { + return nil, err + } } var ctimesec, ctimenanos uint64 From f7abf7a816af7ccd9f891b12ffbbc629abbe8c02 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 15:45:17 +0200 Subject: [PATCH 27/33] Refactoring id and handler --- internal/http/services/overleaf/overleaf.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 188f0848da..21eddd8501 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -120,9 +120,7 @@ func (s *svc) Unprotected() []string { } func (s *svc) Handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - s.router.ServeHTTP(w, r) - }) + return s.router } func (s *svc) handleImport(w http.ResponseWriter, r *http.Request) { @@ -186,7 +184,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { } archQuery := archHTTPReq.URL.Query() - archQuery.Add("id", resource.Id.StorageId+"!"+resource.Id.OpaqueId) + archQuery.Add("id", resourceid.OwnCloudResourceIDWrap(resource.Id)) archQuery.Add("access_token", restrictedToken) archQuery.Add("arch_type", "zip") From c6f595afb0f58ce55c6baae2e69dac581e705e99 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 16:11:46 +0200 Subject: [PATCH 28/33] Restricting token to scope of resource --- internal/http/services/overleaf/overleaf.go | 27 +++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 21eddd8501..7bc522eb4a 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -29,17 +29,20 @@ import ( "strings" "time" + authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" storagepb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/reqres" "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/auth/scope" ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/token/manager/jwt" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/go-chi/chi/v5" @@ -60,6 +63,7 @@ type config struct { ArchiverURL string `mapstructure:"archiver_url" docs:";Internet-facing URL of the archiver service, used to serve the files to Overleaf." validate:"required"` AppURL string `mapstructure:"app_url" docs:";The App URL." validate:"required"` Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` + JWTSecret string `mapstructure:"jwt_secret"` } func init() { @@ -173,8 +177,27 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { } } - // TODO: generate and use a more restricted token - restrictedToken := token + tokenManager, err := jwt.New(map[string]interface{}{ + "secret": sharedconf.GetJWTSecret(s.conf.JWTSecret), + }) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "error fetching secret", err) + return + } + + u := ctxpkg.ContextMustGetUser(ctx) + + scope, err := scope.AddResourceInfoScope(resource, authpb.Role_ROLE_VIEWER, nil) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "error getting restricted token", err) + return + } + + restrictedToken, err := tokenManager.MintToken(context.Background(), u, scope) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "error getting restricted token", err) + return + } // Setting up archiver request archHTTPReq, err := rhttp.NewRequest(ctx, http.MethodGet, s.conf.ArchiverURL, nil) From 752f5d0fa74920362b1f14442a3ebeb93209a2d8 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 16:27:56 +0200 Subject: [PATCH 29/33] Added changelog --- changelog/unreleased/overleaf.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/changelog/unreleased/overleaf.md b/changelog/unreleased/overleaf.md index de015e12d4..84f43ee6cc 100644 --- a/changelog/unreleased/overleaf.md +++ b/changelog/unreleased/overleaf.md @@ -1,5 +1,10 @@ Enhancement: implementation of an app provider for Overleaf -WIP +This PR adds an app provider for Overleaf as a standalone http service. + +The app provider currently consists of support for the export to Overleaf +feature, which when called returns a URL to Overleaf that prompts Overleaf +to download the appropriate resource making use of the Archiver service, +and upload the files to a user's Overleaf account. https://github.com/cs3org/reva/pull/4084 From 03ef8538812fb31a4cae393e2c4dba51a549aa88 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 17:03:32 +0200 Subject: [PATCH 30/33] Change validateQuery to getExportRequest which validates parameters --- internal/http/services/overleaf/overleaf.go | 49 ++++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 7bc522eb4a..123f7928d2 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -135,7 +135,26 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) - statRes, err := s.validateQuery(w, r, ctx) + exportRequest, err := getExportRequest(w, r) + + if err != nil { + return + } + + statRes, err := s.gtwClient.Stat(ctx, &storagepb.StatRequest{Ref: &exportRequest.ResourceRef}) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "Internal error accessing the resource, please try again later", err) + return + } + + if statRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + reqres.WriteError(w, r, reqres.APIErrorNotFound, "resource does not exist", nil) + return + } else if statRes.Status.Code != rpc.Code_CODE_OK { + reqres.WriteError(w, r, reqres.APIErrorServerError, "failed to stat the resource", nil) + return + } + if err != nil { // Validate query handles errors return @@ -160,7 +179,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { return } - if r.Form.Get("override") == "" { + if !exportRequest.Override { creationTime, alreadySet := resource.GetArbitraryMetadata().Metadata["reva.overleaf.exporttime"] if alreadySet { w.WriteHeader(http.StatusConflict) @@ -269,7 +288,7 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func (s *svc) validateQuery(w http.ResponseWriter, r *http.Request, ctx context.Context) (*storagepb.StatResponse, error) { +func getExportRequest(w http.ResponseWriter, r *http.Request) (*exportRequest, error) { if err := r.ParseForm(); err != nil { reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "parameters could not be parsed", nil) return nil, err @@ -294,19 +313,15 @@ func (s *svc) validateQuery(w http.ResponseWriter, r *http.Request, ctx context. resourceRef.ResourceId = resourceID } - statRes, err := s.gtwClient.Stat(ctx, &storagepb.StatRequest{Ref: &resourceRef}) - if err != nil { - reqres.WriteError(w, r, reqres.APIErrorServerError, "Internal error accessing the resource, please try again later", err) - return nil, errors.New("Internal error accessing the resource, please try again later") - } - - if statRes.Status.Code == rpc.Code_CODE_NOT_FOUND { - reqres.WriteError(w, r, reqres.APIErrorNotFound, "resource does not exist", nil) - return nil, errors.New("resource does not exist") - } else if statRes.Status.Code != rpc.Code_CODE_OK { - reqres.WriteError(w, r, reqres.APIErrorServerError, "failed to stat the resource", nil) - return nil, errors.New("failed to stat the resource") - } + // Override is true if field is set + override := r.Form.Get("override") != "" + return &exportRequest{ + ResourceRef: resourceRef, + Override: override, + }, nil +} - return statRes, nil +type exportRequest struct { + ResourceRef storagepb.Reference `json:"resourceId"` + Override bool `json:"override"` } From 996367ad045476c998385aeb20b13964eb0c2620 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Thu, 24 Aug 2023 17:06:23 +0200 Subject: [PATCH 31/33] Amending error message to include case where resource is a container --- internal/http/services/appprovider/appprovider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 30d934405c..14289f9006 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -372,7 +372,7 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { } if statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_FILE && statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_CONTAINER { - writeError(w, r, appErrorInvalidParameter, "the given file id does not point to a file", nil) + writeError(w, r, appErrorInvalidParameter, "the given file id does not point to a file or a container", nil) return } From c176fe4bd7af188068b835716395004b98179d58 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Fri, 25 Aug 2023 08:49:38 +0200 Subject: [PATCH 32/33] Removing unnecessary leftover check --- internal/http/services/overleaf/overleaf.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 123f7928d2..3fe3af4650 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -155,11 +155,6 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { return } - if err != nil { - // Validate query handles errors - return - } - resource := statRes.Info // User needs to have download rights to export to Overleaf From 72958909abae4bb8d8ab6e060def4c3cb6580ca0 Mon Sep 17 00:00:00 2001 From: Adriana Baldacchino Date: Fri, 25 Aug 2023 18:18:16 +0200 Subject: [PATCH 33/33] Linting --- internal/http/services/overleaf/overleaf.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/internal/http/services/overleaf/overleaf.go b/internal/http/services/overleaf/overleaf.go index 3fe3af4650..ab0bf42312 100644 --- a/internal/http/services/overleaf/overleaf.go +++ b/internal/http/services/overleaf/overleaf.go @@ -32,7 +32,6 @@ import ( authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" storagepb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/reqres" "github.com/cs3org/reva/pkg/appctx" @@ -46,13 +45,11 @@ import ( "github.com/cs3org/reva/pkg/utils/cfg" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/go-chi/chi/v5" - "github.com/rs/zerolog" ) type svc struct { conf *config gtwClient gateway.GatewayAPIClient - log *zerolog.Logger router *chi.Mux } @@ -61,7 +58,7 @@ type config struct { GatewaySvc string `mapstructure:"gatewaysvc" validate:"required"` AppName string `mapstructure:"app_name" docs:";The App user-friendly name." validate:"required"` ArchiverURL string `mapstructure:"archiver_url" docs:";Internet-facing URL of the archiver service, used to serve the files to Overleaf." validate:"required"` - AppURL string `mapstructure:"app_url" docs:";The App URL." validate:"required"` + appURL string `mapstructure:"app_url" docs:";The App URL." validate:"required"` Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` JWTSecret string `mapstructure:"jwt_secret"` } @@ -229,8 +226,8 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { log.Debug().Str("Archiver url", archHTTPReq.URL.String()).Msg("URL for downloading zipped resource from archiver") // Setting up Overleaf request - appUrl := s.conf.AppURL + "/docs" - httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, appUrl, nil) + appURL := s.conf.appURL + "/docs" + httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, appURL, nil) if err != nil { reqres.WriteError(w, r, reqres.APIErrorServerError, "overleaf: error setting up http request", nil) return @@ -248,11 +245,11 @@ func (s *svc) handleExport(w http.ResponseWriter, r *http.Request) { httpReq.URL.RawQuery = q.Encode() url := httpReq.URL.String() - req := &provider.SetArbitraryMetadataRequest{ - Ref: &provider.Reference{ + req := &storagepb.SetArbitraryMetadataRequest{ + Ref: &storagepb.Reference{ ResourceId: resource.Id, }, - ArbitraryMetadata: &provider.ArbitraryMetadata{ + ArbitraryMetadata: &storagepb.ArbitraryMetadata{ Metadata: map[string]string{ "reva.overleaf.exporttime": strconv.Itoa(int(time.Now().Unix())), "reva.overleaf.name": base64.StdEncoding.EncodeToString([]byte(name)),