diff --git a/Makefile b/Makefile index 433db8cb7..4eb590b06 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ GOROOT ?= $(shell go env GOROOT) build: go build ./pulsar + go build ./pulsaradmin go build -o bin/pulsar-perf ./perf lint: bin/golangci-lint diff --git a/go.mod b/go.mod index 69862b208..f88d6ad57 100644 --- a/go.mod +++ b/go.mod @@ -27,9 +27,11 @@ require ( golang.org/x/mod v0.5.1 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.30.0 ) +require github.com/golang/protobuf v1.5.2 + require ( github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect @@ -39,7 +41,6 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.1 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect diff --git a/go.sum b/go.sum index 3a3987443..00a44917d 100644 --- a/go.sum +++ b/go.sum @@ -551,7 +551,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/pulsaradmin/CONTRIBUTING.md b/pulsaradmin/CONTRIBUTING.md new file mode 100644 index 000000000..662947929 --- /dev/null +++ b/pulsaradmin/CONTRIBUTING.md @@ -0,0 +1,60 @@ +# Contributing guidelines + +## Project structure + +The overall project structure is illustrated below: + +```text +├── pkg/ +│ ├── admin/ +│ │ ├── auth/ +│ │ ├── config/ +│ ├── rest/ +│ └── utils/ +└── alias.go +``` + +- The `alias.go` file in the root defines `pulsaradmin` package scope, which contains shortcuts of some types and functions from the `pkg`. +- The `pkg/admin` package contains all operations for pulsar admin resources. *Note: We should add a new file here if we wanna support a new resource.* + - The `pkg/admin/config` package contains configuration options for constructing a pulsar admin client. + - The `pkg/admin/auth` package contains auth providers which work in transport layer. +- The `pkg/rest` package contains a wrapped HTTP client for requesting pulsar REST API. +- The `pkg/utils` package contains common data structures and functions. + +## Contributing steps +1. Submit an issue describing your proposed change. +2. Discuss and wait for proposal to be accepted. +3. Fork this repo, develop and test your code changes. +4. Submit a pull request. + +## Conventions + +Please read through below conventions before contributions. + +### PullRequest conventions + +- Use [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) to standardize PR title. + +### Code conventions + +- [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) +- [Effective Go](https://golang.org/doc/effective_go.html) +- Know and avoid [Go landmines](https://gist.github.com/lavalamp/4bd23295a9f32706a48f) +- Commenting + - [Go's commenting conventions](http://blog.golang.org/godoc-documenting-go-code) + - If reviewers ask questions about why the code is the way it is, that's a sign that comments might be helpful. +- Naming + - Please consider package name when selecting an interface name, and avoid redundancy. For example, `storage.Interface` is better than `storage.StorageInterface`. + - Do not use uppercase characters, underscores, or dashes in package names. + - Please consider parent directory name when choosing a package name. For example, `pkg/controllers/autoscaler/foo.go` should say `package autoscaler` not `package autoscalercontroller`. + - Unless there's a good reason, the `package foo` line should match the name of the directory in which the `.go` file exists. + - Importers can use a different name if they need to disambiguate. + - Locks should be called `lock` and should never be embedded (always `lock sync.Mutex`). When multiple locks are present, give each lock a distinct name following Go conventions: `stateLock`, `mapLock` etc. + +### Folder and file conventions + +- All filenames should be lowercase. +- Go source files and directories use underscores, not dashes. + - Package directories should generally avoid using separators as much as possible. When package names are multiple words, they usually should be in nested subdirectories. +- Document directories and filenames should use dashes rather than underscores. +- All source files should add a license at the beginning. diff --git a/pulsaradmin/README.md b/pulsaradmin/README.md new file mode 100644 index 000000000..bf5ca739c --- /dev/null +++ b/pulsaradmin/README.md @@ -0,0 +1,114 @@ +# Pulsar Admin Go Library + +Pulsar-Admin-Go is a [Go](https://go.dev) library for [Apache Pulsar](https://pulsar.apache.org/). It provides a unified Go API for managing pulsar resources such as tenants, namespaces and topics, etc. + +## Motivation + +Currently, many projects (e.g, [terraform-provider-pulsar](https://github.com/streamnative/terraform-provider-pulsar) and [pulsar-resources-operator](https://github.com/streamnative/pulsar-resources-operator)) +that need to manipulate the pulsar admin resources rely on the [pulsarctl](https://github.com/streamnative/pulsarctl), +which poses challenges for dependency management and versioning as we have to release a new pulsarctl to get updates. +So we decoupled the pulsar admin related api from pulsarctl and created the [pulsar-admin-go](https://github.com/apache/pulsar-client-go/pulsaradmin) library based on it, +which also provides a clearer perspective and maintainability from an architectural perspective. + +## Quickstart + +### Prerequisite + +- go1.18+ +- pulsar-admin-go in go.mod + + ```shell + go get github.com/apache/pulsar-client-go + ``` + +### Manage pulsar tenants + +- List all tenants + +```go +import ( + "github.com/apache/pulsar-client-go/pulsaradmin" +) + +func main() { + cfg := &pulsaradmin.Config{} + admin, err := pulsaradmin.NewClient(cfg) + if err != nil { + panic(err) + } + + tenants, _ := admin.Tenants().List() +} +``` + +### Manage pulsar namespaces + +- List all namespaces + +```go +import ( + "github.com/apache/pulsar-client-go/pulsaradmin" +) + +func main() { + cfg := &pulsaradmin.Config{} + admin, err := pulsaradmin.NewClient(cfg) + if err != nil { + panic(err) + } + + namespaces, _ := admin.Namespaces().GetNamespaces("public") +} +``` + +- Create a new namespace + +```go +import ( + "github.com/apache/pulsar-client-go/pulsaradmin" +) + +func main() { + cfg := &pulsaradmin.Config{} + admin, err := pulsaradmin.NewClient(cfg) + if err != nil { + panic(err) + } + + admin.Namespaces().CreateNamespace("public/dev") +} +``` + +### Manage pulsar topics + +- Create a topic + +```go + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +func main() { + cfg := &pulsaradmin.Config{} + admin, err := pulsaradmin.NewClient(cfg) + if err != nil { + panic(err) + } + + topic, _ := utils.GetTopicName("public/dev/topic") + + admin.Topics().Create(*topic, 3) +} +``` + +## Contributing + +Contributions are warmly welcomed and greatly appreciated! +The project follows the typical GitHub pull request model. See [CONTRIBUTING.md](CONTRIBUTING.md) for more details. +Before starting any work, please either comment on an existing issue, or file a new one. + +## License + +Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). diff --git a/pulsaradmin/alias.go b/pulsaradmin/alias.go new file mode 100644 index 000000000..53c8d93e0 --- /dev/null +++ b/pulsaradmin/alias.go @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pulsaradmin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" +) + +// Client contains all admin interfaces for operating pulsar resources +type Client = admin.Client + +// Config are the arguments for creating a new admin Client +type Config = config.Config + +var ( + // NewClient returns a new admin Client for operating pulsar resources + NewClient = admin.New +) diff --git a/pulsaradmin/pkg/admin/admin.go b/pulsaradmin/pkg/admin/admin.go new file mode 100644 index 000000000..a22b22c62 --- /dev/null +++ b/pulsaradmin/pkg/admin/admin.go @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "net/http" + "net/url" + "path" + "time" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/auth" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/rest" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +const ( + DefaultWebServiceURL = "http://localhost:8080" + DefaultHTTPTimeOutDuration = 5 * time.Minute + ReleaseVersion = "None" +) + +type TLSOptions struct { + TrustCertsFilePath string + AllowInsecureConnection bool +} + +// Client provides a client to the Pulsar Restful API +type Client interface { + Clusters() Clusters + Functions() Functions + Tenants() Tenants + Topics() Topics + Subscriptions() Subscriptions + Sources() Sources + Sinks() Sinks + Namespaces() Namespaces + Schemas() Schema + NsIsolationPolicy() NsIsolationPolicy + Brokers() Brokers + BrokerStats() BrokerStats + ResourceQuotas() ResourceQuotas + FunctionsWorker() FunctionsWorker + Packages() Packages +} + +type pulsarClient struct { + Client *rest.Client + APIVersion config.APIVersion +} + +// New returns a new client +func New(config *config.Config) (Client, error) { + authProvider, err := auth.GetAuthProvider(config) + if err != nil { + return nil, err + } + return NewPulsarClientWithAuthProvider(config, authProvider) +} + +// NewWithAuthProvider creates a client with auth provider. +// Deprecated: Use NewPulsarClientWithAuthProvider instead. +func NewWithAuthProvider(config *config.Config, authProvider auth.Provider) Client { + client, err := NewPulsarClientWithAuthProvider(config, authProvider) + if err != nil { + panic(err) + } + return client +} + +// NewPulsarClientWithAuthProvider create a client with auth provider. +func NewPulsarClientWithAuthProvider(config *config.Config, authProvider auth.Provider) (Client, error) { + if len(config.WebServiceURL) == 0 { + config.WebServiceURL = DefaultWebServiceURL + } + + return &pulsarClient{ + APIVersion: config.PulsarAPIVersion, + Client: &rest.Client{ + ServiceURL: config.WebServiceURL, + VersionInfo: ReleaseVersion, + HTTPClient: &http.Client{ + Timeout: DefaultHTTPTimeOutDuration, + Transport: authProvider, + }, + }, + }, nil +} + +func (c *pulsarClient) endpoint(componentPath string, parts ...string) string { + escapedParts := make([]string, len(parts)) + for i, part := range parts { + escapedParts[i] = url.PathEscape(part) + } + return path.Join( + utils.MakeHTTPPath(c.APIVersion.String(), componentPath), + path.Join(escapedParts...), + ) +} diff --git a/pulsaradmin/pkg/admin/admin_test.go b/pulsaradmin/pkg/admin/admin_test.go new file mode 100644 index 000000000..c4fa52956 --- /dev/null +++ b/pulsaradmin/pkg/admin/admin_test.go @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/auth" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" +) + +func TestPulsarClientEndpointEscapes(t *testing.T) { + client := pulsarClient{Client: nil, APIVersion: config.V2} + actual := client.endpoint("/myendpoint", "abc%? /def", "ghi") + expected := "/admin/v2/myendpoint/abc%25%3F%20%2Fdef/ghi" + assert.Equal(t, expected, actual) +} + +func TestNew(t *testing.T) { + config := &config.Config{} + admin, err := New(config) + require.NoError(t, err) + require.NotNil(t, admin) +} + +func TestNewWithAuthProvider(t *testing.T) { + config := &config.Config{} + + tokenAuth, err := auth.NewAuthenticationToken("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9."+ + "eyJzdWIiOiJhZG1pbiIsImlhdCI6MTUxNjIzOTAyMn0.sVt6cyu3HKd89LcQvZVMNbqT0DTl3FvG9oYbj8hBDqU", nil) + require.NoError(t, err) + require.NotNil(t, tokenAuth) + + admin, err := NewPulsarClientWithAuthProvider(config, tokenAuth) + require.NoError(t, err) + require.NotNil(t, admin) +} + +type customAuthProvider struct { + transport http.RoundTripper +} + +var _ auth.Provider = &customAuthProvider{} + +func (c *customAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { + panic("implement me") +} + +func (c *customAuthProvider) Transport() http.RoundTripper { + return c.transport +} + +func (c *customAuthProvider) WithTransport(transport http.RoundTripper) { + c.transport = transport +} + +func TestNewWithCustomAuthProviderWithTransport(t *testing.T) { + config := &config.Config{} + defaultTransport, err := auth.NewDefaultTransport(config) + require.NoError(t, err) + + customAuthProvider := &customAuthProvider{ + transport: defaultTransport, + } + + admin, err := NewPulsarClientWithAuthProvider(config, customAuthProvider) + require.NoError(t, err) + require.NotNil(t, admin) + + // Expected the customAuthProvider will not be overwritten. + require.Equal(t, customAuthProvider, admin.(*pulsarClient).Client.HTTPClient.Transport) +} + +func TestNewWithTlsAllowInsecure(t *testing.T) { + config := &config.Config{ + TLSAllowInsecureConnection: true, + } + admin, err := New(config) + require.NoError(t, err) + require.NotNil(t, admin) + + pulsarClientS := admin.(*pulsarClient) + require.NotNil(t, pulsarClientS.Client.HTTPClient.Transport) + + ap := pulsarClientS.Client.HTTPClient.Transport.(*auth.DefaultProvider) + tr := ap.Transport().(*http.Transport) + require.NotNil(t, tr) + require.NotNil(t, tr.TLSClientConfig) + require.True(t, tr.TLSClientConfig.InsecureSkipVerify) +} diff --git a/pulsaradmin/pkg/admin/auth/oauth2.go b/pulsaradmin/pkg/admin/auth/oauth2.go new file mode 100644 index 000000000..eab64cace --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/oauth2.go @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "encoding/json" + "net/http" + "path/filepath" + + "github.com/99designs/keyring" + "github.com/apache/pulsar-client-go/oauth2" + "github.com/apache/pulsar-client-go/oauth2/cache" + clock2 "github.com/apache/pulsar-client-go/oauth2/clock" + "github.com/apache/pulsar-client-go/oauth2/store" + "github.com/pkg/errors" + xoauth2 "golang.org/x/oauth2" +) + +const ( + OAuth2PluginName = "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2" + OAuth2PluginShortName = "oauth2" +) + +type OAuth2ClientCredentials struct { + IssuerURL string `json:"issuerUrl,omitempty"` + Audience string `json:"audience,omitempty"` + Scope string `json:"scope,omitempty"` + PrivateKey string `json:"privateKey,omitempty"` + ClientID string `json:"clientId,omitempty"` +} + +type OAuth2Provider struct { + clock clock2.RealClock + issuer oauth2.Issuer + store store.Store + source cache.CachingTokenSource + defaultTransport http.RoundTripper + tokenTransport *transport +} + +func NewAuthenticationOAuth2(issuer oauth2.Issuer, store store.Store) (*OAuth2Provider, error) { + p := &OAuth2Provider{ + clock: clock2.RealClock{}, + issuer: issuer, + store: store, + } + + err := p.loadGrant() + if err != nil { + return nil, err + } + + return p, nil +} + +// NewAuthenticationOAuth2WithDefaultFlow uses memory to save the grant +func NewAuthenticationOAuth2WithDefaultFlow(issuer oauth2.Issuer, keyFile string) (Provider, error) { + st := store.NewMemoryStore() + flow, err := oauth2.NewDefaultClientCredentialsFlow(oauth2.ClientCredentialsFlowOptions{ + KeyFile: keyFile, + }) + if err != nil { + return nil, err + } + + grant, err := flow.Authorize(issuer.Audience) + if err != nil { + return nil, err + } + + err = st.SaveGrant(issuer.Audience, *grant) + if err != nil { + return nil, err + } + + p := &OAuth2Provider{ + clock: clock2.RealClock{}, + issuer: issuer, + store: st, + } + + return p, p.loadGrant() +} + +func NewAuthenticationOAuth2FromAuthParams(encodedAuthParam string, + transport http.RoundTripper) (*OAuth2Provider, error) { + + var paramsJSON OAuth2ClientCredentials + err := json.Unmarshal([]byte(encodedAuthParam), ¶msJSON) + if err != nil { + return nil, err + } + return NewAuthenticationOAuth2WithParams(paramsJSON.IssuerURL, paramsJSON.ClientID, paramsJSON.Audience, + paramsJSON.Scope, transport) +} + +func NewAuthenticationOAuth2WithParams( + issuerEndpoint, + clientID, + audience string, + scope string, + transport http.RoundTripper) (*OAuth2Provider, error) { + + issuer := oauth2.Issuer{ + IssuerEndpoint: issuerEndpoint, + ClientID: clientID, + Audience: audience, + } + + keyringStore, err := MakeKeyringStore() + if err != nil { + return nil, err + } + + p := &OAuth2Provider{ + clock: clock2.RealClock{}, + issuer: issuer, + store: keyringStore, + defaultTransport: transport, + } + + err = p.loadGrant() + if err != nil { + return nil, err + } + + return p, nil +} + +func (o *OAuth2Provider) loadGrant() error { + grant, err := o.store.LoadGrant(o.issuer.Audience) + if err != nil { + if err == store.ErrNoAuthenticationData { + return errors.New("oauth2 login required") + } + return err + } + return o.initCache(grant) +} + +func (o *OAuth2Provider) initCache(grant *oauth2.AuthorizationGrant) error { + refresher, err := o.getRefresher(grant.Type) + if err != nil { + return err + } + + source, err := cache.NewDefaultTokenCache(o.store, o.issuer.Audience, refresher) + if err != nil { + return err + } + o.source = source + o.tokenTransport = &transport{ + source: o.source, + wrapped: &xoauth2.Transport{ + Source: o.source, + Base: o.defaultTransport, + }, + } + return nil +} + +func (o *OAuth2Provider) RoundTrip(req *http.Request) (*http.Response, error) { + return o.tokenTransport.RoundTrip(req) +} + +func (o *OAuth2Provider) WithTransport(tripper http.RoundTripper) { + o.defaultTransport = tripper +} + +func (o *OAuth2Provider) Transport() http.RoundTripper { + return o.tokenTransport +} + +func (o *OAuth2Provider) getRefresher(t oauth2.AuthorizationGrantType) (oauth2.AuthorizationGrantRefresher, error) { + switch t { + case oauth2.GrantTypeClientCredentials: + return oauth2.NewDefaultClientCredentialsGrantRefresher(o.clock) + case oauth2.GrantTypeDeviceCode: + return oauth2.NewDefaultDeviceAuthorizationGrantRefresher(o.clock) + default: + return nil, store.ErrUnsupportedAuthData + } +} + +type transport struct { + source cache.CachingTokenSource + wrapped *xoauth2.Transport +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + if len(req.Header.Get("Authorization")) != 0 { + return t.wrapped.Base.RoundTrip(req) + } + + res, err := t.wrapped.RoundTrip(req) + if err != nil { + return nil, err + } + + if res.StatusCode == 401 { + err := t.source.InvalidateToken() + if err != nil { + return nil, err + } + } + + return res, nil +} + +func (t *transport) WrappedRoundTripper() http.RoundTripper { return t.wrapped.Base } + +const ( + serviceName = "pulsar" + keyChainName = "pulsarctl" +) + +func MakeKeyringStore() (store.Store, error) { + kr, err := makeKeyring() + if err != nil { + return nil, err + } + return store.NewKeyringStore(kr) +} + +func makeKeyring() (keyring.Keyring, error) { + return keyring.Open(keyring.Config{ + AllowedBackends: keyring.AvailableBackends(), + ServiceName: serviceName, + KeychainName: keyChainName, + KeychainTrustApplication: true, + FileDir: filepath.Join("~/.config/pulsar", "credentials"), + FilePasswordFunc: keyringPrompt, + }) +} + +func keyringPrompt(prompt string) (string, error) { + return "", nil +} diff --git a/pulsaradmin/pkg/admin/auth/oauth2_test.go b/pulsaradmin/pkg/admin/auth/oauth2_test.go new file mode 100644 index 000000000..b25e57611 --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/oauth2_test.go @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/apache/pulsar-client-go/oauth2" + "github.com/apache/pulsar-client-go/oauth2/store" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +// mockOAuthServer will mock a oauth service for the tests +func mockOAuthServer() *httptest.Server { + // prepare a port for the mocked server + server := httptest.NewUnstartedServer(http.DefaultServeMux) + + // mock the used REST path for the tests + mockedHandler := http.NewServeMux() + mockedHandler.HandleFunc("/.well-known/openid-configuration", func(writer http.ResponseWriter, request *http.Request) { + s := fmt.Sprintf(`{ + "issuer":"%s", + "authorization_endpoint":"%s/authorize", + "token_endpoint":"%s/oauth/token", + "device_authorization_endpoint":"%s/oauth/device/code" +}`, server.URL, server.URL, server.URL, server.URL) + fmt.Fprintln(writer, s) + }) + mockedHandler.HandleFunc("/oauth/token", func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprintln(writer, "{\n \"access_token\": \"token-content\",\n \"token_type\": \"Bearer\"\n}") + }) + mockedHandler.HandleFunc("/authorize", func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprintln(writer, "true") + }) + + server.Config.Handler = mockedHandler + server.Start() + + return server +} + +// mockKeyFile will mock a temp key file for testing. +func mockKeyFile(server string) (string, error) { + pwd, err := os.Getwd() + if err != nil { + return "", err + } + kf, err := os.CreateTemp(pwd, "test_oauth2") + if err != nil { + return "", err + } + _, err = kf.WriteString(fmt.Sprintf(`{ + "type":"sn_service_account", + "client_id":"client-id", + "client_secret":"client-secret", + "client_email":"oauth@test.org", + "issuer_url":"%s" +}`, server)) + + if err != nil { + return "", err + } + + return kf.Name(), nil +} + +func TestOauth2(t *testing.T) { + server := mockOAuthServer() + defer server.Close() + kf, err := mockKeyFile(server.URL) + defer os.Remove(kf) + if err != nil { + t.Fatal(errors.Wrap(err, "create mocked key file failed")) + } + + issuer := oauth2.Issuer{ + IssuerEndpoint: server.URL, + ClientID: "client-id", + Audience: server.URL, + } + + memoryStore := store.NewMemoryStore() + err = saveGrant(memoryStore, kf, issuer.Audience) + if err != nil { + t.Fatal(err) + } + + auth, err := NewAuthenticationOAuth2(issuer, memoryStore) + if err != nil { + t.Fatal(err) + } + + token, err := auth.source.Token() + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "token-content", token.AccessToken) +} + +func saveGrant(store store.Store, keyFile, audience string) error { + flow, err := oauth2.NewDefaultClientCredentialsFlow(oauth2.ClientCredentialsFlowOptions{ + KeyFile: keyFile, + AdditionalScopes: nil, + }) + if err != nil { + return err + } + + grant, err := flow.Authorize(audience) + if err != nil { + return err + } + + return store.SaveGrant(audience, *grant) +} diff --git a/pulsaradmin/pkg/admin/auth/provider.go b/pulsaradmin/pkg/admin/auth/provider.go new file mode 100644 index 000000000..24f74d265 --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/provider.go @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "net/http" + + "github.com/apache/pulsar-client-go/oauth2" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" +) + +// Provider provide a general method to add auth message +type Provider interface { + RoundTrip(req *http.Request) (*http.Response, error) + Transport() http.RoundTripper + WithTransport(tripper http.RoundTripper) +} + +type DefaultProvider struct { + transport http.RoundTripper +} + +func NewDefaultProvider(t http.RoundTripper) Provider { + return &DefaultProvider{ + transport: t, + } +} + +func (dp *DefaultProvider) RoundTrip(req *http.Request) (*http.Response, error) { + return dp.transport.RoundTrip(req) +} + +func (dp *DefaultProvider) Transport() http.RoundTripper { + return dp.transport +} + +func (dp *DefaultProvider) WithTransport(t http.RoundTripper) { + dp.transport = t +} + +func GetAuthProvider(config *config.Config) (Provider, error) { + var provider Provider + defaultTransport, err := NewDefaultTransport(config) + if err != nil { + return nil, err + } + switch config.AuthPlugin { + case TLSPluginShortName: + fallthrough + case TLSPluginName: + provider, err = NewAuthenticationTLSFromAuthParams(config.AuthParams, defaultTransport) + case TokenPluginName: + fallthrough + case TokePluginShortName: + provider, err = NewAuthenticationTokenFromAuthParams(config.AuthParams, defaultTransport) + case OAuth2PluginName: + fallthrough + case OAuth2PluginShortName: + provider, err = NewAuthenticationOAuth2WithDefaultFlow(oauth2.Issuer{ + IssuerEndpoint: config.IssuerEndpoint, + ClientID: config.ClientID, + Audience: config.Audience, + }, config.KeyFile) + default: + switch { + case len(config.TLSCertFile) > 0 && len(config.TLSKeyFile) > 0: + provider, err = NewAuthenticationTLS(config.TLSCertFile, config.TLSKeyFile, defaultTransport) + case len(config.Token) > 0: + provider, err = NewAuthenticationToken(config.Token, defaultTransport) + case len(config.TokenFile) > 0: + provider, err = NewAuthenticationTokenFromFile(config.TokenFile, defaultTransport) + case len(config.IssuerEndpoint) > 0 || len(config.ClientID) > 0 || len(config.Audience) > 0 || len(config.Scope) > 0: + provider, err = NewAuthenticationOAuth2WithParams( + config.IssuerEndpoint, config.ClientID, config.Audience, config.Scope, defaultTransport) + default: + provider = NewDefaultProvider(defaultTransport) + } + } + return provider, err +} diff --git a/pulsaradmin/pkg/admin/auth/tls.go b/pulsaradmin/pkg/admin/auth/tls.go new file mode 100644 index 000000000..7c11fe27e --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/tls.go @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "crypto/tls" + "encoding/json" + "net/http" + "strings" +) + +const ( + TLSPluginName = "org.apache.pulsar.client.impl.auth.AuthenticationTls" + TLSPluginShortName = "tls" +) + +type TLS struct { + TLSCertFile string `json:"tlsCertFile"` + TLSKeyFile string `json:"tlsKeyFile"` +} + +type TLSAuthProvider struct { + certificatePath string + privateKeyPath string + T http.RoundTripper +} + +// NewAuthenticationTLS initialize the authentication provider +func NewAuthenticationTLS(certificatePath string, privateKeyPath string, + transport http.RoundTripper) (*TLSAuthProvider, error) { + provider := &TLSAuthProvider{ + certificatePath: certificatePath, + privateKeyPath: privateKeyPath, + T: transport, + } + if err := provider.configTLS(); err != nil { + return nil, err + } + return provider, nil +} + +func NewAuthenticationTLSFromAuthParams(encodedAuthParams string, + transport http.RoundTripper) (*TLSAuthProvider, error) { + var certificatePath string + var privateKeyPath string + + var tlsJSON TLS + err := json.Unmarshal([]byte(encodedAuthParams), &tlsJSON) + if err != nil { + parts := strings.Split(encodedAuthParams, ",") + for _, part := range parts { + kv := strings.Split(part, ":") + switch kv[0] { + case "tlsCertFile": + certificatePath = kv[1] + case "tlsKeyFile": + privateKeyPath = kv[1] + } + } + } else { + certificatePath = tlsJSON.TLSCertFile + privateKeyPath = tlsJSON.TLSKeyFile + } + + return NewAuthenticationTLS(certificatePath, privateKeyPath, transport) +} + +func (p *TLSAuthProvider) GetTLSCertificate() (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(p.certificatePath, p.privateKeyPath) + return &cert, err +} + +func (p *TLSAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { + return p.T.RoundTrip(req) +} + +func (p *TLSAuthProvider) Transport() http.RoundTripper { + return p.T +} + +func (p *TLSAuthProvider) configTLS() error { + cert, err := p.GetTLSCertificate() + if err != nil { + return err + } + transport := p.T.(*http.Transport) + transport.TLSClientConfig.Certificates = []tls.Certificate{*cert} + return nil +} + +func (p *TLSAuthProvider) WithTransport(tripper http.RoundTripper) { + p.T = tripper +} diff --git a/pulsaradmin/pkg/admin/auth/token.go b/pulsaradmin/pkg/admin/auth/token.go new file mode 100644 index 000000000..ecf57ecc8 --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/token.go @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + + "github.com/pkg/errors" +) + +const ( + tokenPrefix = "token:" + filePrefix = "file:" + TokenPluginName = "org.apache.pulsar.client.impl.auth.AuthenticationToken" + TokePluginShortName = "token" +) + +type Token struct { + Token string `json:"token"` +} + +type TokenAuthProvider struct { + T http.RoundTripper + token string +} + +// NewAuthenticationToken return a interface of Provider with a string token. +func NewAuthenticationToken(token string, transport http.RoundTripper) (*TokenAuthProvider, error) { + if len(token) == 0 { + return nil, errors.New("No token provided") + } + return &TokenAuthProvider{token: token, T: transport}, nil +} + +// NewAuthenticationTokenFromFile return a interface of a Provider with a string token file path. +func NewAuthenticationTokenFromFile(tokenFilePath string, transport http.RoundTripper) (*TokenAuthProvider, error) { + data, err := os.ReadFile(tokenFilePath) + if err != nil { + return nil, err + } + token := strings.Trim(string(data), " \n") + return NewAuthenticationToken(token, transport) +} + +func NewAuthenticationTokenFromAuthParams(encodedAuthParam string, + transport http.RoundTripper) (*TokenAuthProvider, error) { + var tokenAuthProvider *TokenAuthProvider + var err error + + var tokenJSON Token + err = json.Unmarshal([]byte(encodedAuthParam), &tokenJSON) + if err != nil { + switch { + case strings.HasPrefix(encodedAuthParam, tokenPrefix): + tokenAuthProvider, err = NewAuthenticationToken(strings.TrimPrefix(encodedAuthParam, tokenPrefix), transport) + case strings.HasPrefix(encodedAuthParam, filePrefix): + tokenAuthProvider, err = NewAuthenticationTokenFromFile(strings.TrimPrefix(encodedAuthParam, filePrefix), transport) + default: + tokenAuthProvider, err = NewAuthenticationToken(encodedAuthParam, transport) + } + } else { + tokenAuthProvider, err = NewAuthenticationToken(tokenJSON.Token, transport) + } + return tokenAuthProvider, err +} + +func (p *TokenAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", p.token)) + return p.T.RoundTrip(req) +} + +func (p *TokenAuthProvider) Transport() http.RoundTripper { + return p.T +} + +func (p *TokenAuthProvider) WithTransport(tripper http.RoundTripper) { + p.T = tripper +} diff --git a/pulsaradmin/pkg/admin/auth/transport.go b/pulsaradmin/pkg/admin/auth/transport.go new file mode 100644 index 000000000..d96ab57f5 --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/transport.go @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "os" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" +) + +type Transport struct { + T http.RoundTripper +} + +// GetDefaultTransport gets a default transport. +// Deprecated: Use NewDefaultTransport instead. +func GetDefaultTransport(config *config.Config) http.RoundTripper { + transport, err := NewDefaultTransport(config) + if err != nil { + panic(err) + } + + return transport +} + +func NewDefaultTransport(config *config.Config) (http.RoundTripper, error) { + transport := http.DefaultTransport.(*http.Transport).Clone() + tlsConfig := &tls.Config{ + InsecureSkipVerify: config.TLSAllowInsecureConnection, + } + if len(config.TLSTrustCertsFilePath) > 0 { + rootCA, err := os.ReadFile(config.TLSTrustCertsFilePath) + if err != nil { + return nil, err + } + tlsConfig.RootCAs = x509.NewCertPool() + tlsConfig.RootCAs.AppendCertsFromPEM(rootCA) + } + transport.MaxIdleConnsPerHost = 10 + transport.TLSClientConfig = tlsConfig + return transport, nil +} diff --git a/pulsaradmin/pkg/admin/broker_stats.go b/pulsaradmin/pkg/admin/broker_stats.go new file mode 100644 index 000000000..c9f9cb01e --- /dev/null +++ b/pulsaradmin/pkg/admin/broker_stats.go @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// BrokerStats is admin interface for broker stats management +type BrokerStats interface { + // GetMetrics returns Monitoring metrics + GetMetrics() ([]utils.Metrics, error) + + // GetMBeans requests JSON string server mbean dump + GetMBeans() ([]utils.Metrics, error) + + // GetTopics returns JSON string topics stats + GetTopics() (string, error) + + // GetLoadReport returns load report of broker + GetLoadReport() (*utils.LocalBrokerData, error) + + // GetAllocatorStats returns stats from broker + GetAllocatorStats(allocatorName string) (*utils.AllocatorStats, error) +} + +type brokerStats struct { + pulsar *pulsarClient + basePath string +} + +// BrokerStats is used to access the broker stats endpoints +func (c *pulsarClient) BrokerStats() BrokerStats { + return &brokerStats{ + pulsar: c, + basePath: "/broker-stats", + } +} + +func (bs *brokerStats) GetMetrics() ([]utils.Metrics, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/metrics") + var response []utils.Metrics + err := bs.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, err + } + + return response, nil +} + +func (bs *brokerStats) GetMBeans() ([]utils.Metrics, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/mbeans") + var response []utils.Metrics + err := bs.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, err + } + + return response, nil +} + +func (bs *brokerStats) GetTopics() (string, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/topics") + buf, err := bs.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + if err != nil { + return "", err + } + + return string(buf), nil +} + +func (bs *brokerStats) GetLoadReport() (*utils.LocalBrokerData, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/load-report") + response := utils.NewLocalBrokerData() + err := bs.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, nil + } + return &response, nil +} + +func (bs *brokerStats) GetAllocatorStats(allocatorName string) (*utils.AllocatorStats, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/allocator-stats", allocatorName) + var allocatorStats utils.AllocatorStats + err := bs.pulsar.Client.Get(endpoint, &allocatorStats) + if err != nil { + return nil, err + } + return &allocatorStats, nil +} diff --git a/pulsaradmin/pkg/admin/brokers.go b/pulsaradmin/pkg/admin/brokers.go new file mode 100644 index 000000000..79fcb092e --- /dev/null +++ b/pulsaradmin/pkg/admin/brokers.go @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "fmt" + "net/url" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Brokers is admin interface for brokers management +type Brokers interface { + // GetActiveBrokers returns the list of active brokers in the cluster. + GetActiveBrokers(cluster string) ([]string, error) + + // GetDynamicConfigurationNames returns list of updatable configuration name + GetDynamicConfigurationNames() ([]string, error) + + // GetOwnedNamespaces returns the map of owned namespaces and their status from a single broker in the cluster + GetOwnedNamespaces(cluster, brokerURL string) (map[string]utils.NamespaceOwnershipStatus, error) + + // UpdateDynamicConfiguration updates dynamic configuration value in to Zk that triggers watch on + // brokers and all brokers can update {@link ServiceConfiguration} value locally + UpdateDynamicConfiguration(configName, configValue string) error + + // DeleteDynamicConfiguration deletes dynamic configuration value in to Zk. It will not impact current value + // in broker but next time when broker restarts, it applies value from configuration file only. + DeleteDynamicConfiguration(configName string) error + + // GetRuntimeConfigurations returns values of runtime configuration + GetRuntimeConfigurations() (map[string]string, error) + + // GetInternalConfigurationData returns the internal configuration data + GetInternalConfigurationData() (*utils.InternalConfigurationData, error) + + // GetAllDynamicConfigurations returns values of all overridden dynamic-configs + GetAllDynamicConfigurations() (map[string]string, error) + + // HealthCheck run a health check on the broker + HealthCheck() error +} + +type broker struct { + pulsar *pulsarClient + basePath string +} + +// Brokers is used to access the brokers endpoints +func (c *pulsarClient) Brokers() Brokers { + return &broker{ + pulsar: c, + basePath: "/brokers", + } +} + +func (b *broker) GetActiveBrokers(cluster string) ([]string, error) { + endpoint := b.pulsar.endpoint(b.basePath, cluster) + var res []string + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) GetDynamicConfigurationNames() ([]string, error) { + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/") + var res []string + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) GetOwnedNamespaces(cluster, brokerURL string) (map[string]utils.NamespaceOwnershipStatus, error) { + endpoint := b.pulsar.endpoint(b.basePath, cluster, brokerURL, "ownedNamespaces") + var res map[string]utils.NamespaceOwnershipStatus + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) UpdateDynamicConfiguration(configName, configValue string) error { + value := url.QueryEscape(configValue) + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", configName, value) + return b.pulsar.Client.Post(endpoint, nil) +} + +func (b *broker) DeleteDynamicConfiguration(configName string) error { + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", configName) + return b.pulsar.Client.Delete(endpoint) +} + +func (b *broker) GetRuntimeConfigurations() (map[string]string, error) { + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", "runtime") + var res map[string]string + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) GetInternalConfigurationData() (*utils.InternalConfigurationData, error) { + endpoint := b.pulsar.endpoint(b.basePath, "/internal-configuration") + var res utils.InternalConfigurationData + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return &res, nil +} + +func (b *broker) GetAllDynamicConfigurations() (map[string]string, error) { + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", "values") + var res map[string]string + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) HealthCheck() error { + endpoint := b.pulsar.endpoint(b.basePath, "/health") + + buf, err := b.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + if err != nil { + return err + } + + if !strings.EqualFold(string(buf), "ok") { + return fmt.Errorf("health check returned unexpected result: %s", string(buf)) + } + return nil +} diff --git a/pulsaradmin/pkg/admin/cluster.go b/pulsaradmin/pkg/admin/cluster.go new file mode 100644 index 000000000..b290e3ef3 --- /dev/null +++ b/pulsaradmin/pkg/admin/cluster.go @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Clusters is admin interface for clusters management +type Clusters interface { + // List returns the list of clusters + List() ([]string, error) + + // Get the configuration data for the specified cluster + Get(string) (utils.ClusterData, error) + + // Create a new cluster + Create(utils.ClusterData) error + + // Delete an existing cluster + Delete(string) error + + // Update the configuration for a cluster + Update(utils.ClusterData) error + + // UpdatePeerClusters updates peer cluster names. + UpdatePeerClusters(string, []string) error + + // GetPeerClusters returns peer-cluster names + GetPeerClusters(string) ([]string, error) + + // CreateFailureDomain creates a domain into cluster + CreateFailureDomain(utils.FailureDomainData) error + + // GetFailureDomain returns the domain registered into a cluster + GetFailureDomain(clusterName, domainName string) (utils.FailureDomainData, error) + + // ListFailureDomains returns all registered domains in cluster + ListFailureDomains(string) (utils.FailureDomainMap, error) + + // DeleteFailureDomain deletes a domain in cluster + DeleteFailureDomain(utils.FailureDomainData) error + + // UpdateFailureDomain updates a domain into cluster + UpdateFailureDomain(utils.FailureDomainData) error +} + +type clusters struct { + pulsar *pulsarClient + basePath string +} + +// Clusters is used to access the cluster endpoints. +func (c *pulsarClient) Clusters() Clusters { + return &clusters{ + pulsar: c, + basePath: "/clusters", + } +} + +func (c *clusters) List() ([]string, error) { + var clusters []string + err := c.pulsar.Client.Get(c.pulsar.endpoint(c.basePath), &clusters) + return clusters, err +} + +func (c *clusters) Get(name string) (utils.ClusterData, error) { + cdata := utils.ClusterData{} + endpoint := c.pulsar.endpoint(c.basePath, name) + err := c.pulsar.Client.Get(endpoint, &cdata) + return cdata, err +} + +func (c *clusters) Create(cdata utils.ClusterData) error { + endpoint := c.pulsar.endpoint(c.basePath, cdata.Name) + return c.pulsar.Client.Put(endpoint, &cdata) +} + +func (c *clusters) Delete(name string) error { + endpoint := c.pulsar.endpoint(c.basePath, name) + return c.pulsar.Client.Delete(endpoint) +} + +func (c *clusters) Update(cdata utils.ClusterData) error { + endpoint := c.pulsar.endpoint(c.basePath, cdata.Name) + return c.pulsar.Client.Post(endpoint, &cdata) +} + +func (c *clusters) GetPeerClusters(name string) ([]string, error) { + var peerClusters []string + endpoint := c.pulsar.endpoint(c.basePath, name, "peers") + err := c.pulsar.Client.Get(endpoint, &peerClusters) + return peerClusters, err +} + +func (c *clusters) UpdatePeerClusters(cluster string, peerClusters []string) error { + endpoint := c.pulsar.endpoint(c.basePath, cluster, "peers") + return c.pulsar.Client.Post(endpoint, peerClusters) +} + +func (c *clusters) CreateFailureDomain(data utils.FailureDomainData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) + return c.pulsar.Client.Post(endpoint, &data) +} + +func (c *clusters) GetFailureDomain(clusterName string, domainName string) (utils.FailureDomainData, error) { + var res utils.FailureDomainData + endpoint := c.pulsar.endpoint(c.basePath, clusterName, "failureDomains", domainName) + err := c.pulsar.Client.Get(endpoint, &res) + return res, err +} + +func (c *clusters) ListFailureDomains(clusterName string) (utils.FailureDomainMap, error) { + var domainData utils.FailureDomainMap + endpoint := c.pulsar.endpoint(c.basePath, clusterName, "failureDomains") + err := c.pulsar.Client.Get(endpoint, &domainData) + return domainData, err +} + +func (c *clusters) DeleteFailureDomain(data utils.FailureDomainData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) + return c.pulsar.Client.Delete(endpoint) +} +func (c *clusters) UpdateFailureDomain(data utils.FailureDomainData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) + return c.pulsar.Client.Post(endpoint, &data) +} diff --git a/pulsaradmin/pkg/admin/config/api_version.go b/pulsaradmin/pkg/admin/config/api_version.go new file mode 100644 index 000000000..95f670458 --- /dev/null +++ b/pulsaradmin/pkg/admin/config/api_version.go @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +type APIVersion int + +const ( + undefined APIVersion = iota + V1 + V2 + V3 +) + +const DefaultAPIVersion = "v2" + +func (v APIVersion) String() string { + switch v { + case undefined: + return DefaultAPIVersion + case V1: + return "" + case V2: + return "v2" + case V3: + return "v3" + } + + return DefaultAPIVersion +} diff --git a/pulsaradmin/pkg/admin/config/api_version_test.go b/pulsaradmin/pkg/admin/config/api_version_test.go new file mode 100644 index 000000000..e1dc7bdb6 --- /dev/null +++ b/pulsaradmin/pkg/admin/config/api_version_test.go @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestApiVersion_String(t *testing.T) { + assert.Equal(t, "", V1.String()) + assert.Equal(t, "v2", V2.String()) + assert.Equal(t, "v3", V3.String()) + var undefinedAPIVersion APIVersion + assert.Equal(t, DefaultAPIVersion, undefinedAPIVersion.String()) +} diff --git a/pulsaradmin/pkg/admin/config/config.go b/pulsaradmin/pkg/admin/config/config.go new file mode 100644 index 000000000..9428f3a90 --- /dev/null +++ b/pulsaradmin/pkg/admin/config/config.go @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +type Config struct { + // the web service url that pulsarctl connects to. Default is http://localhost:8080 + WebServiceURL string + + // the bookkeeper service url that pulsarctl connects to. + BKWebServiceURL string + // Set the path to the trusted TLS certificate file + TLSTrustCertsFilePath string + // Configure whether the Pulsar client accept untrusted TLS certificate from broker (default: false) + TLSAllowInsecureConnection bool + + TLSEnableHostnameVerification bool + + AuthPlugin string + + AuthParams string + + // TLS Cert and Key Files for authentication + TLSCertFile string + TLSKeyFile string + + // Token and TokenFile is used to config the pulsarctl using token to authentication + Token string + TokenFile string + PulsarAPIVersion APIVersion + + // OAuth2 configuration + IssuerEndpoint string + ClientID string + Audience string + KeyFile string + Scope string +} diff --git a/pulsaradmin/pkg/admin/functions.go b/pulsaradmin/pkg/admin/functions.go new file mode 100644 index 000000000..cbaaf6be4 --- /dev/null +++ b/pulsaradmin/pkg/admin/functions.go @@ -0,0 +1,686 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Functions is admin interface for functions management +type Functions interface { + // CreateFunc create a new function. + CreateFunc(data *utils.FunctionConfig, fileName string) error + + // CreateFuncWithURL create a new function by providing url from which fun-pkg can be downloaded. + // supported url: http/file + // eg: + // File: file:/dir/fileName.jar + // Http: http://www.repo.com/fileName.jar + // + // @param functionConfig + // the function configuration object + // @param pkgURL + // url from which pkg can be downloaded + CreateFuncWithURL(data *utils.FunctionConfig, pkgURL string) error + + // StopFunction stop all function instances + StopFunction(tenant, namespace, name string) error + + // StopFunctionWithID stop function instance + StopFunctionWithID(tenant, namespace, name string, instanceID int) error + + // DeleteFunction delete an existing function + DeleteFunction(tenant, namespace, name string) error + + // Download Function Code + // @param destinationFile + // file where data should be downloaded to + // @param path + // Path where data is located + DownloadFunction(path, destinationFile string) error + + // Download Function Code + // @param destinationFile + // file where data should be downloaded to + // @param tenant + // Tenant name + // @param namespace + // Namespace name + // @param function + // Function name + DownloadFunctionByNs(destinationFile, tenant, namespace, function string) error + + // StartFunction start all function instances + StartFunction(tenant, namespace, name string) error + + // StartFunctionWithID start function instance + StartFunctionWithID(tenant, namespace, name string, instanceID int) error + + // RestartFunction restart all function instances + RestartFunction(tenant, namespace, name string) error + + // RestartFunctionWithID restart function instance + RestartFunctionWithID(tenant, namespace, name string, instanceID int) error + + // GetFunctions returns the list of functions + GetFunctions(tenant, namespace string) ([]string, error) + + // GetFunction returns the configuration for the specified function + GetFunction(tenant, namespace, name string) (utils.FunctionConfig, error) + + // GetFunctionStatus returns the current status of a function + GetFunctionStatus(tenant, namespace, name string) (utils.FunctionStatus, error) + + // GetFunctionStatusWithInstanceID returns the current status of a function instance + GetFunctionStatusWithInstanceID(tenant, namespace, name string, instanceID int) ( + utils.FunctionInstanceStatusData, error) + + // GetFunctionStats returns the current stats of a function + GetFunctionStats(tenant, namespace, name string) (utils.FunctionStats, error) + + // GetFunctionStatsWithInstanceID gets the current stats of a function instance + GetFunctionStatsWithInstanceID(tenant, namespace, name string, instanceID int) (utils.FunctionInstanceStatsData, error) + + // GetFunctionState fetch the current state associated with a Pulsar Function + // + // Response Example: + // { "value : 12, version : 2"} + GetFunctionState(tenant, namespace, name, key string) (utils.FunctionState, error) + + // PutFunctionState puts the given state associated with a Pulsar Function + PutFunctionState(tenant, namespace, name string, state utils.FunctionState) error + + // TriggerFunction triggers the function by writing to the input topic + TriggerFunction(tenant, namespace, name, topic, triggerValue, triggerFile string) (string, error) + + // UpdateFunction updates the configuration for a function. + UpdateFunction(functionConfig *utils.FunctionConfig, fileName string, updateOptions *utils.UpdateOptions) error + + // UpdateFunctionWithURL updates the configuration for a function. + // + // Update a function by providing url from which fun-pkg can be downloaded. supported url: http/file + // eg: + // File: file:/dir/fileName.jar + // Http: http://www.repo.com/fileName.jar + UpdateFunctionWithURL(functionConfig *utils.FunctionConfig, pkgURL string, updateOptions *utils.UpdateOptions) error + + // Upload function to Pulsar + Upload(sourceFile, path string) error +} + +type functions struct { + pulsar *pulsarClient + basePath string +} + +// Functions is used to access the functions endpoints +func (c *pulsarClient) Functions() Functions { + return &functions{ + pulsar: c, + basePath: "/functions", + } +} + +func (f *functions) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "application/json") + return w.CreatePart(h) +} + +func (f *functions) createTextFromFiled(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "text/plain") + return w.CreatePart(h) +} + +func (f *functions) CreateFunc(funcConf *utils.FunctionConfig, fileName string) error { + endpoint := f.pulsar.endpoint(f.basePath, funcConf.Tenant, funcConf.Namespace, funcConf.Name) + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + jsonData, err := json.Marshal(funcConf) + if err != nil { + return err + } + + stringWriter, err := f.createStringFromField(multiPartWriter, "functionConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = f.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (f *functions) CreateFuncWithURL(funcConf *utils.FunctionConfig, pkgURL string) error { + endpoint := f.pulsar.endpoint(f.basePath, funcConf.Tenant, funcConf.Namespace, funcConf.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := f.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(funcConf) + if err != nil { + return err + } + + stringWriter, err := f.createStringFromField(multiPartWriter, "functionConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = f.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (f *functions) StopFunction(tenant, namespace, name string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + return f.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (f *functions) StopFunctionWithID(tenant, namespace, name string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + + return f.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (f *functions) DeleteFunction(tenant, namespace, name string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + return f.pulsar.Client.Delete(endpoint) +} + +func (f *functions) DownloadFunction(path, destinationFile string) error { + endpoint := f.pulsar.endpoint(f.basePath, "download") + _, err := os.Open(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("file %s already exists, please delete "+ + "the file first or change the file name", destinationFile) + } + } + file, err := os.Create(destinationFile) + if err != nil { + return err + } + + tmpMap := make(map[string]string) + tmpMap["path"] = path + + _, err = f.pulsar.Client.GetWithOptions(endpoint, nil, tmpMap, false, file) + if err != nil { + return err + } + return nil +} + +func (f *functions) DownloadFunctionByNs(destinationFile, tenant, namespace, function string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, function, "download") + _, err := os.Open(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("file %s already exists, please delete "+ + "the file first or change the file name", destinationFile) + } + } + file, err := os.Create(destinationFile) + if err != nil { + return err + } + + _, err = f.pulsar.Client.GetWithOptions(endpoint, nil, nil, false, file) + if err != nil { + return err + } + + return nil +} + +func (f *functions) StartFunction(tenant, namespace, name string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + return f.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (f *functions) StartFunctionWithID(tenant, namespace, name string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + + return f.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (f *functions) RestartFunction(tenant, namespace, name string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + return f.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (f *functions) RestartFunctionWithID(tenant, namespace, name string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + + return f.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (f *functions) GetFunctions(tenant, namespace string) ([]string, error) { + var functions []string + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace) + err := f.pulsar.Client.Get(endpoint, &functions) + return functions, err +} + +func (f *functions) GetFunction(tenant, namespace, name string) (utils.FunctionConfig, error) { + var functionConfig utils.FunctionConfig + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + err := f.pulsar.Client.Get(endpoint, &functionConfig) + return functionConfig, err +} + +func (f *functions) UpdateFunction(functionConfig *utils.FunctionConfig, fileName string, + updateOptions *utils.UpdateOptions) error { + endpoint := f.pulsar.endpoint(f.basePath, functionConfig.Tenant, functionConfig.Namespace, functionConfig.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + jsonData, err := json.Marshal(functionConfig) + if err != nil { + return err + } + + stringWriter, err := f.createStringFromField(multiPartWriter, "functionConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := f.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = f.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (f *functions) UpdateFunctionWithURL(functionConfig *utils.FunctionConfig, pkgURL string, + updateOptions *utils.UpdateOptions) error { + endpoint := f.pulsar.endpoint(f.basePath, functionConfig.Tenant, functionConfig.Namespace, functionConfig.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := f.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(functionConfig) + if err != nil { + return err + } + + stringWriter, err := f.createStringFromField(multiPartWriter, "functionConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := f.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = f.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (f *functions) GetFunctionStatus(tenant, namespace, name string) (utils.FunctionStatus, error) { + var functionStatus utils.FunctionStatus + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + err := f.pulsar.Client.Get(endpoint+"/status", &functionStatus) + return functionStatus, err +} + +func (f *functions) GetFunctionStatusWithInstanceID(tenant, namespace, name string, + instanceID int) (utils.FunctionInstanceStatusData, error) { + var functionInstanceStatusData utils.FunctionInstanceStatusData + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + err := f.pulsar.Client.Get(endpoint+"/status", &functionInstanceStatusData) + return functionInstanceStatusData, err +} + +func (f *functions) GetFunctionStats(tenant, namespace, name string) (utils.FunctionStats, error) { + var functionStats utils.FunctionStats + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + err := f.pulsar.Client.Get(endpoint+"/stats", &functionStats) + return functionStats, err +} + +func (f *functions) GetFunctionStatsWithInstanceID(tenant, namespace, name string, + instanceID int) (utils.FunctionInstanceStatsData, error) { + var functionInstanceStatsData utils.FunctionInstanceStatsData + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + err := f.pulsar.Client.Get(endpoint+"/stats", &functionInstanceStatsData) + return functionInstanceStatsData, err +} + +func (f *functions) GetFunctionState(tenant, namespace, name, key string) (utils.FunctionState, error) { + var functionState utils.FunctionState + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, "state", key) + err := f.pulsar.Client.Get(endpoint, &functionState) + return functionState, err +} + +func (f *functions) PutFunctionState(tenant, namespace, name string, state utils.FunctionState) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, "state", state.Key) + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + stateData, err := json.Marshal(state) + + if err != nil { + return err + } + + stateWriter, err := f.createStringFromField(multiPartWriter, "state") + if err != nil { + return err + } + + _, err = stateWriter.Write(stateData) + + if err != nil { + return err + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + + err = f.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + + if err != nil { + return err + } + + return nil +} + +func (f *functions) TriggerFunction(tenant, namespace, name, topic, triggerValue, triggerFile string) (string, error) { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, "trigger") + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + if triggerFile != "" { + file, err := os.Open(triggerFile) + if err != nil { + return "", err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("dataStream", filepath.Base(file.Name())) + + if err != nil { + return "", err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return "", err + } + } + + if triggerValue != "" { + valueWriter, err := f.createTextFromFiled(multiPartWriter, "data") + if err != nil { + return "", err + } + + _, err = valueWriter.Write([]byte(triggerValue)) + if err != nil { + return "", err + } + } + + if topic != "" { + topicWriter, err := f.createTextFromFiled(multiPartWriter, "topic") + if err != nil { + return "", err + } + + _, err = topicWriter.Write([]byte(topic)) + if err != nil { + return "", err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err := multiPartWriter.Close(); err != nil { + return "", err + } + + contentType := multiPartWriter.FormDataContentType() + var str string + err := f.pulsar.Client.PostWithMultiPart(endpoint, &str, bodyBuf, contentType) + if err != nil { + return "", err + } + + return str, nil +} + +func (f *functions) Upload(sourceFile, path string) error { + if strings.TrimSpace(sourceFile) == "" && strings.TrimSpace(path) == "" { + return fmt.Errorf("source file or path is empty") + } + file, err := os.Open(sourceFile) + if err != nil { + return err + } + defer file.Close() + endpoint := f.pulsar.endpoint(f.basePath, "upload") + var b bytes.Buffer + w := multipart.NewWriter(&b) + writer, err := w.CreateFormFile("data", file.Name()) + if err != nil { + return err + } + _, err = io.Copy(writer, file) + if err != nil { + return err + } + if err := w.WriteField("path", path); err != nil { + return err + } + err = w.Close() + if err != nil { + return err + } + return f.pulsar.Client.PostWithMultiPart(endpoint, nil, &b, w.FormDataContentType()) +} diff --git a/pulsaradmin/pkg/admin/functions_worker.go b/pulsaradmin/pkg/admin/functions_worker.go new file mode 100644 index 000000000..3cad65de8 --- /dev/null +++ b/pulsaradmin/pkg/admin/functions_worker.go @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +type FunctionsWorker interface { + // Get all functions stats on a worker + GetFunctionsStats() ([]*utils.WorkerFunctionInstanceStats, error) + + // Get worker metrics + GetMetrics() ([]*utils.Metrics, error) + + // Get List of all workers belonging to this cluster + GetCluster() ([]*utils.WorkerInfo, error) + + // Get the worker who is the leader of the clusterv + GetClusterLeader() (*utils.WorkerInfo, error) + + // Get the function assignment among the cluster + GetAssignments() (map[string][]string, error) +} + +type worker struct { + pulsar *pulsarClient + workerPath string + workerStatsPath string +} + +func (c *pulsarClient) FunctionsWorker() FunctionsWorker { + return &worker{ + pulsar: c, + workerPath: "/worker", + workerStatsPath: "/worker-stats", + } +} + +func (w *worker) GetFunctionsStats() ([]*utils.WorkerFunctionInstanceStats, error) { + endpoint := w.pulsar.endpoint(w.workerStatsPath, "functionsmetrics") + var workerStats []*utils.WorkerFunctionInstanceStats + err := w.pulsar.Client.Get(endpoint, &workerStats) + if err != nil { + return nil, err + } + return workerStats, nil +} + +func (w *worker) GetMetrics() ([]*utils.Metrics, error) { + endpoint := w.pulsar.endpoint(w.workerStatsPath, "metrics") + var metrics []*utils.Metrics + err := w.pulsar.Client.Get(endpoint, &metrics) + if err != nil { + return nil, err + } + return metrics, nil +} + +func (w *worker) GetCluster() ([]*utils.WorkerInfo, error) { + endpoint := w.pulsar.endpoint(w.workerPath, "cluster") + var workersInfo []*utils.WorkerInfo + err := w.pulsar.Client.Get(endpoint, &workersInfo) + if err != nil { + return nil, err + } + return workersInfo, nil +} + +func (w *worker) GetClusterLeader() (*utils.WorkerInfo, error) { + endpoint := w.pulsar.endpoint(w.workerPath, "cluster", "leader") + var workerInfo utils.WorkerInfo + err := w.pulsar.Client.Get(endpoint, &workerInfo) + if err != nil { + return nil, err + } + return &workerInfo, nil +} + +func (w *worker) GetAssignments() (map[string][]string, error) { + endpoint := w.pulsar.endpoint(w.workerPath, "assignments") + var assignments map[string][]string + err := w.pulsar.Client.Get(endpoint, &assignments) + if err != nil { + return nil, err + } + return assignments, nil +} diff --git a/pulsaradmin/pkg/admin/namespace.go b/pulsaradmin/pkg/admin/namespace.go new file mode 100644 index 000000000..732441e8c --- /dev/null +++ b/pulsaradmin/pkg/admin/namespace.go @@ -0,0 +1,875 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "net/url" + "strconv" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Namespaces is admin interface for namespaces management +type Namespaces interface { + // GetNamespaces returns the list of all the namespaces for a certain tenant + GetNamespaces(tenant string) ([]string, error) + + // GetTopics returns the list of all the topics under a certain namespace + GetTopics(namespace string) ([]string, error) + + // GetPolicies returns the dump all the policies specified for a namespace + GetPolicies(namespace string) (*utils.Policies, error) + + // CreateNamespace creates a new empty namespace with no policies attached + CreateNamespace(namespace string) error + + // CreateNsWithNumBundles creates a new empty namespace with no policies attached + CreateNsWithNumBundles(namespace string, numBundles int) error + + // CreateNsWithPolices creates a new namespace with the specified policies + CreateNsWithPolices(namespace string, polices utils.Policies) error + + // CreateNsWithBundlesData creates a new empty namespace with no policies attached + CreateNsWithBundlesData(namespace string, bundleData *utils.BundlesData) error + + // DeleteNamespace deletes an existing namespace + DeleteNamespace(namespace string) error + + // DeleteNamespaceBundle deletes an existing bundle in a namespace + DeleteNamespaceBundle(namespace string, bundleRange string) error + + // SetNamespaceMessageTTL sets the messages Time to Live for all the topics within a namespace + SetNamespaceMessageTTL(namespace string, ttlInSeconds int) error + + // GetNamespaceMessageTTL returns the message TTL for a namespace + GetNamespaceMessageTTL(namespace string) (int, error) + + // GetRetention returns the retention configuration for a namespace + GetRetention(namespace string) (*utils.RetentionPolicies, error) + + // SetRetention sets the retention configuration for all the topics on a namespace + SetRetention(namespace string, policy utils.RetentionPolicies) error + + // GetBacklogQuotaMap returns backlog quota map on a namespace + GetBacklogQuotaMap(namespace string) (map[utils.BacklogQuotaType]utils.BacklogQuota, error) + + // SetBacklogQuota sets a backlog quota for all the topics on a namespace + SetBacklogQuota(namespace string, backlogQuota utils.BacklogQuota, backlogQuotaType utils.BacklogQuotaType) error + + // RemoveBacklogQuota removes a backlog quota policy from a namespace + RemoveBacklogQuota(namespace string) error + + // SetTopicAutoCreation sets topic auto-creation config for a namespace, overriding broker settings + SetTopicAutoCreation(namespace utils.NameSpaceName, config utils.TopicAutoCreationConfig) error + + // RemoveTopicAutoCreation removes topic auto-creation config for a namespace, defaulting to broker settings + RemoveTopicAutoCreation(namespace utils.NameSpaceName) error + + // SetSchemaValidationEnforced sets schema validation enforced for namespace + SetSchemaValidationEnforced(namespace utils.NameSpaceName, schemaValidationEnforced bool) error + + // GetSchemaValidationEnforced returns schema validation enforced for namespace + GetSchemaValidationEnforced(namespace utils.NameSpaceName) (bool, error) + + // SetSchemaAutoUpdateCompatibilityStrategy sets the strategy used to check the a new schema provided + // by a producer is compatible with the current schema before it is installed + SetSchemaAutoUpdateCompatibilityStrategy(namespace utils.NameSpaceName, + strategy utils.SchemaCompatibilityStrategy) error + + // GetSchemaAutoUpdateCompatibilityStrategy returns the strategy used to check the a new schema provided + // by a producer is compatible with the current schema before it is installed + GetSchemaAutoUpdateCompatibilityStrategy(namespace utils.NameSpaceName) (utils.SchemaCompatibilityStrategy, error) + + // ClearOffloadDeleteLag clears the offload deletion lag for a namespace. + ClearOffloadDeleteLag(namespace utils.NameSpaceName) error + + // SetOffloadDeleteLag sets the offload deletion lag for a namespace + SetOffloadDeleteLag(namespace utils.NameSpaceName, timeMs int64) error + + // GetOffloadDeleteLag returns the offload deletion lag for a namespace, in milliseconds + GetOffloadDeleteLag(namespace utils.NameSpaceName) (int64, error) + + // SetOffloadThreshold sets the offloadThreshold for a namespace + SetOffloadThreshold(namespace utils.NameSpaceName, threshold int64) error + + // GetOffloadThreshold returns the offloadThreshold for a namespace + GetOffloadThreshold(namespace utils.NameSpaceName) (int64, error) + + // SetCompactionThreshold sets the compactionThreshold for a namespace + SetCompactionThreshold(namespace utils.NameSpaceName, threshold int64) error + + // GetCompactionThreshold returns the compactionThreshold for a namespace + GetCompactionThreshold(namespace utils.NameSpaceName) (int64, error) + + // SetMaxConsumersPerSubscription sets maxConsumersPerSubscription for a namespace. + SetMaxConsumersPerSubscription(namespace utils.NameSpaceName, max int) error + + // GetMaxConsumersPerSubscription returns the maxConsumersPerSubscription for a namespace. + GetMaxConsumersPerSubscription(namespace utils.NameSpaceName) (int, error) + + // SetMaxConsumersPerTopic sets maxConsumersPerTopic for a namespace. + SetMaxConsumersPerTopic(namespace utils.NameSpaceName, max int) error + + // GetMaxConsumersPerTopic returns the maxProducersPerTopic for a namespace. + GetMaxConsumersPerTopic(namespace utils.NameSpaceName) (int, error) + + // SetMaxProducersPerTopic sets maxProducersPerTopic for a namespace. + SetMaxProducersPerTopic(namespace utils.NameSpaceName, max int) error + + // GetMaxProducersPerTopic returns the maxProducersPerTopic for a namespace. + GetMaxProducersPerTopic(namespace utils.NameSpaceName) (int, error) + + // GetNamespaceReplicationClusters returns the replication clusters for a namespace + GetNamespaceReplicationClusters(namespace string) ([]string, error) + + // SetNamespaceReplicationClusters returns the replication clusters for a namespace + SetNamespaceReplicationClusters(namespace string, clusterIds []string) error + + // SetNamespaceAntiAffinityGroup sets anti-affinity group name for a namespace + SetNamespaceAntiAffinityGroup(namespace string, namespaceAntiAffinityGroup string) error + + // GetAntiAffinityNamespaces returns all namespaces that grouped with given anti-affinity group + GetAntiAffinityNamespaces(tenant, cluster, namespaceAntiAffinityGroup string) ([]string, error) + + // GetNamespaceAntiAffinityGroup returns anti-affinity group name for a namespace + GetNamespaceAntiAffinityGroup(namespace string) (string, error) + + // DeleteNamespaceAntiAffinityGroup deletes anti-affinity group name for a namespace + DeleteNamespaceAntiAffinityGroup(namespace string) error + + // SetDeduplicationStatus sets the deduplication status for all topics within a namespace + // When deduplication is enabled, the broker will prevent to store the same Message multiple times + SetDeduplicationStatus(namespace string, enableDeduplication bool) error + + // SetPersistence sets the persistence configuration for all the topics on a namespace + SetPersistence(namespace string, persistence utils.PersistencePolicies) error + + // GetPersistence returns the persistence configuration for a namespace + GetPersistence(namespace string) (*utils.PersistencePolicies, error) + + // SetBookieAffinityGroup sets bookie affinity group for a namespace to isolate namespace write to bookies that are + // part of given affinity group + SetBookieAffinityGroup(namespace string, bookieAffinityGroup utils.BookieAffinityGroupData) error + + // DeleteBookieAffinityGroup deletes bookie affinity group configured for a namespace + DeleteBookieAffinityGroup(namespace string) error + + // GetBookieAffinityGroup returns bookie affinity group configured for a namespace + GetBookieAffinityGroup(namespace string) (*utils.BookieAffinityGroupData, error) + + // Unload a namespace from the current serving broker + Unload(namespace string) error + + // UnloadNamespaceBundle unloads namespace bundle + UnloadNamespaceBundle(namespace, bundle string) error + + // SplitNamespaceBundle splits namespace bundle + SplitNamespaceBundle(namespace, bundle string, unloadSplitBundles bool) error + + // GetNamespacePermissions returns permissions on a namespace + GetNamespacePermissions(namespace utils.NameSpaceName) (map[string][]utils.AuthAction, error) + + // GrantNamespacePermission grants permission on a namespace. + GrantNamespacePermission(namespace utils.NameSpaceName, role string, action []utils.AuthAction) error + + // RevokeNamespacePermission revokes permissions on a namespace. + RevokeNamespacePermission(namespace utils.NameSpaceName, role string) error + + // GrantSubPermission grants permission to role to access subscription's admin-api + GrantSubPermission(namespace utils.NameSpaceName, sName string, roles []string) error + + // RevokeSubPermission revoke permissions on a subscription's admin-api access + RevokeSubPermission(namespace utils.NameSpaceName, sName, role string) error + + // SetSubscriptionAuthMode sets the given subscription auth mode on all topics on a namespace + SetSubscriptionAuthMode(namespace utils.NameSpaceName, mode utils.SubscriptionAuthMode) error + + // SetEncryptionRequiredStatus sets the encryption required status for all topics within a namespace + SetEncryptionRequiredStatus(namespace utils.NameSpaceName, encrypt bool) error + + // UnsubscribeNamespace unsubscribe the given subscription on all topics on a namespace + UnsubscribeNamespace(namespace utils.NameSpaceName, sName string) error + + // UnsubscribeNamespaceBundle unsubscribe the given subscription on all topics on a namespace bundle + UnsubscribeNamespaceBundle(namespace utils.NameSpaceName, bundle, sName string) error + + // ClearNamespaceBundleBacklogForSubscription clears backlog for a given subscription on all + // topics on a namespace bundle + ClearNamespaceBundleBacklogForSubscription(namespace utils.NameSpaceName, bundle, sName string) error + + // ClearNamespaceBundleBacklog clears backlog for all topics on a namespace bundle + ClearNamespaceBundleBacklog(namespace utils.NameSpaceName, bundle string) error + + // ClearNamespaceBacklogForSubscription clears backlog for a given subscription on all topics on a namespace + ClearNamespaceBacklogForSubscription(namespace utils.NameSpaceName, sName string) error + + // ClearNamespaceBacklog clears backlog for all topics on a namespace + ClearNamespaceBacklog(namespace utils.NameSpaceName) error + + // SetReplicatorDispatchRate sets replicator-Message-dispatch-rate (Replicators under this namespace + // can dispatch this many messages per second) + SetReplicatorDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error + + // Get replicator-Message-dispatch-rate (Replicators under this namespace + // can dispatch this many messages per second) + GetReplicatorDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) + + // SetSubscriptionDispatchRate sets subscription-Message-dispatch-rate (subscriptions under this namespace + // can dispatch this many messages per second) + SetSubscriptionDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error + + // GetSubscriptionDispatchRate returns subscription-Message-dispatch-rate (subscriptions under this namespace + // can dispatch this many messages per second) + GetSubscriptionDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) + + // SetSubscribeRate sets namespace-subscribe-rate (topics under this namespace will limit by subscribeRate) + SetSubscribeRate(namespace utils.NameSpaceName, rate utils.SubscribeRate) error + + // GetSubscribeRate returns namespace-subscribe-rate (topics under this namespace allow subscribe + // times per consumer in a period) + GetSubscribeRate(namespace utils.NameSpaceName) (utils.SubscribeRate, error) + + // SetDispatchRate sets Message-dispatch-rate (topics under this namespace can dispatch + // this many messages per second) + SetDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error + + // GetDispatchRate returns Message-dispatch-rate (topics under this namespace can dispatch + // this many messages per second) + GetDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) + + // SetPublishRate sets the maximum rate or number of messages that producers can publish to topics in this namespace + SetPublishRate(namespace utils.NameSpaceName, pubRate utils.PublishRate) error + + // GetPublishRate gets the maximum rate or number of messages that producer can publish to topics in the namespace + GetPublishRate(namespace utils.NameSpaceName) (utils.PublishRate, error) + + // SetIsAllowAutoUpdateSchema sets whether to allow auto update schema on a namespace + SetIsAllowAutoUpdateSchema(namespace utils.NameSpaceName, isAllowAutoUpdateSchema bool) error + + // GetIsAllowAutoUpdateSchema gets whether to allow auto update schema on a namespace + GetIsAllowAutoUpdateSchema(namespace utils.NameSpaceName) (bool, error) + + // GetInactiveTopicPolicies gets the inactive topic policies on a namespace + GetInactiveTopicPolicies(namespace utils.NameSpaceName) (utils.InactiveTopicPolicies, error) + + // RemoveInactiveTopicPolicies removes inactive topic policies from a namespace + RemoveInactiveTopicPolicies(namespace utils.NameSpaceName) error + + // SetInactiveTopicPolicies sets the inactive topic policies on a namespace + SetInactiveTopicPolicies(namespace utils.NameSpaceName, data utils.InactiveTopicPolicies) error +} + +type namespaces struct { + pulsar *pulsarClient + basePath string +} + +// Namespaces is used to access the namespaces endpoints +func (c *pulsarClient) Namespaces() Namespaces { + return &namespaces{ + pulsar: c, + basePath: "/namespaces", + } +} + +func (n *namespaces) GetNamespaces(tenant string) ([]string, error) { + var namespaces []string + endpoint := n.pulsar.endpoint(n.basePath, tenant) + err := n.pulsar.Client.Get(endpoint, &namespaces) + return namespaces, err +} + +func (n *namespaces) GetTopics(namespace string) ([]string, error) { + var topics []string + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String(), "topics") + err = n.pulsar.Client.Get(endpoint, &topics) + return topics, err +} + +func (n *namespaces) GetPolicies(namespace string) (*utils.Policies, error) { + var police utils.Policies + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + err = n.pulsar.Client.Get(endpoint, &police) + return &police, err +} + +func (n *namespaces) CreateNsWithNumBundles(namespace string, numBundles int) error { + return n.CreateNsWithBundlesData(namespace, utils.NewBundlesDataWithNumBundles(numBundles)) +} + +func (n *namespaces) CreateNsWithPolices(namespace string, policies utils.Policies) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + return n.pulsar.Client.Put(endpoint, &policies) +} + +func (n *namespaces) CreateNsWithBundlesData(namespace string, bundleData *utils.BundlesData) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + polices := new(utils.Policies) + polices.Bundles = bundleData + + return n.pulsar.Client.Put(endpoint, &polices) +} + +func (n *namespaces) CreateNamespace(namespace string) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + return n.pulsar.Client.Put(endpoint, nil) +} + +func (n *namespaces) DeleteNamespace(namespace string) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) DeleteNamespaceBundle(namespace string, bundleRange string) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String(), bundleRange) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) GetNamespaceMessageTTL(namespace string) (int, error) { + var ttl int + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return 0, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "messageTTL") + err = n.pulsar.Client.Get(endpoint, &ttl) + return ttl, err +} + +func (n *namespaces) SetNamespaceMessageTTL(namespace string, ttlInSeconds int) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "messageTTL") + return n.pulsar.Client.Post(endpoint, &ttlInSeconds) +} + +func (n *namespaces) SetRetention(namespace string, policy utils.RetentionPolicies) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "retention") + return n.pulsar.Client.Post(endpoint, &policy) +} + +func (n *namespaces) GetRetention(namespace string) (*utils.RetentionPolicies, error) { + var policy utils.RetentionPolicies + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "retention") + err = n.pulsar.Client.Get(endpoint, &policy) + return &policy, err +} + +func (n *namespaces) GetBacklogQuotaMap(namespace string) (map[utils.BacklogQuotaType]utils.BacklogQuota, error) { + var backlogQuotaMap map[utils.BacklogQuotaType]utils.BacklogQuota + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "backlogQuotaMap") + err = n.pulsar.Client.Get(endpoint, &backlogQuotaMap) + return backlogQuotaMap, err +} + +func (n *namespaces) SetBacklogQuota(namespace string, backlogQuota utils.BacklogQuota, + backlogQuotaType utils.BacklogQuotaType) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "backlogQuota") + params := make(map[string]string) + params["backlogQuotaType"] = string(backlogQuotaType) + return n.pulsar.Client.PostWithQueryParams(endpoint, &backlogQuota, params) +} + +func (n *namespaces) RemoveBacklogQuota(namespace string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "backlogQuota") + params := map[string]string{ + "backlogQuotaType": string(utils.DestinationStorage), + } + return n.pulsar.Client.DeleteWithQueryParams(endpoint, params) +} + +func (n *namespaces) SetTopicAutoCreation(namespace utils.NameSpaceName, config utils.TopicAutoCreationConfig) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "autoTopicCreation") + return n.pulsar.Client.Post(endpoint, &config) +} + +func (n *namespaces) RemoveTopicAutoCreation(namespace utils.NameSpaceName) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "autoTopicCreation") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetSchemaValidationEnforced(namespace utils.NameSpaceName, schemaValidationEnforced bool) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "schemaValidationEnforced") + return n.pulsar.Client.Post(endpoint, schemaValidationEnforced) +} + +func (n *namespaces) GetSchemaValidationEnforced(namespace utils.NameSpaceName) (bool, error) { + var result bool + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "schemaValidationEnforced") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetSchemaAutoUpdateCompatibilityStrategy(namespace utils.NameSpaceName, + strategy utils.SchemaCompatibilityStrategy) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "schemaAutoUpdateCompatibilityStrategy") + return n.pulsar.Client.Put(endpoint, strategy.String()) +} + +func (n *namespaces) GetSchemaAutoUpdateCompatibilityStrategy(namespace utils.NameSpaceName) ( + utils.SchemaCompatibilityStrategy, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "schemaAutoUpdateCompatibilityStrategy") + b, err := n.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + if err != nil { + return "", err + } + s, err := utils.ParseSchemaAutoUpdateCompatibilityStrategy(strings.ReplaceAll(string(b), "\"", "")) + if err != nil { + return "", err + } + return s, nil +} + +func (n *namespaces) ClearOffloadDeleteLag(namespace utils.NameSpaceName) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadDeletionLagMs") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetOffloadDeleteLag(namespace utils.NameSpaceName, timeMs int64) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadDeletionLagMs") + return n.pulsar.Client.Put(endpoint, timeMs) +} + +func (n *namespaces) GetOffloadDeleteLag(namespace utils.NameSpaceName) (int64, error) { + var result int64 + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadDeletionLagMs") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetMaxConsumersPerSubscription(namespace utils.NameSpaceName, max int) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxConsumersPerSubscription") + return n.pulsar.Client.Post(endpoint, max) +} + +func (n *namespaces) GetMaxConsumersPerSubscription(namespace utils.NameSpaceName) (int, error) { + var result int + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxConsumersPerSubscription") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetOffloadThreshold(namespace utils.NameSpaceName, threshold int64) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadThreshold") + return n.pulsar.Client.Put(endpoint, threshold) +} + +func (n *namespaces) GetOffloadThreshold(namespace utils.NameSpaceName) (int64, error) { + var result int64 + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadThreshold") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetMaxConsumersPerTopic(namespace utils.NameSpaceName, max int) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxConsumersPerTopic") + return n.pulsar.Client.Post(endpoint, max) +} + +func (n *namespaces) GetMaxConsumersPerTopic(namespace utils.NameSpaceName) (int, error) { + var result int + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxConsumersPerTopic") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetCompactionThreshold(namespace utils.NameSpaceName, threshold int64) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "compactionThreshold") + return n.pulsar.Client.Put(endpoint, threshold) +} + +func (n *namespaces) GetCompactionThreshold(namespace utils.NameSpaceName) (int64, error) { + var result int64 + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "compactionThreshold") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetMaxProducersPerTopic(namespace utils.NameSpaceName, max int) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxProducersPerTopic") + return n.pulsar.Client.Post(endpoint, max) +} + +func (n *namespaces) GetMaxProducersPerTopic(namespace utils.NameSpaceName) (int, error) { + var result int + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxProducersPerTopic") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) GetNamespaceReplicationClusters(namespace string) ([]string, error) { + var data []string + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "replication") + err = n.pulsar.Client.Get(endpoint, &data) + return data, err +} + +func (n *namespaces) SetNamespaceReplicationClusters(namespace string, clusterIds []string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "replication") + return n.pulsar.Client.Post(endpoint, &clusterIds) +} + +func (n *namespaces) SetNamespaceAntiAffinityGroup(namespace string, namespaceAntiAffinityGroup string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "antiAffinity") + return n.pulsar.Client.Post(endpoint, namespaceAntiAffinityGroup) +} + +func (n *namespaces) GetAntiAffinityNamespaces(tenant, cluster, namespaceAntiAffinityGroup string) ([]string, error) { + var data []string + endpoint := n.pulsar.endpoint(n.basePath, cluster, "antiAffinity", namespaceAntiAffinityGroup) + params := map[string]string{ + "property": tenant, + } + _, err := n.pulsar.Client.GetWithQueryParams(endpoint, &data, params, false) + return data, err +} + +func (n *namespaces) GetNamespaceAntiAffinityGroup(namespace string) (string, error) { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return "", err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "antiAffinity") + data, err := n.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + return string(data), err +} + +func (n *namespaces) DeleteNamespaceAntiAffinityGroup(namespace string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "antiAffinity") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetDeduplicationStatus(namespace string, enableDeduplication bool) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "deduplication") + return n.pulsar.Client.Post(endpoint, enableDeduplication) +} + +func (n *namespaces) SetPersistence(namespace string, persistence utils.PersistencePolicies) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence") + return n.pulsar.Client.Post(endpoint, &persistence) +} + +func (n *namespaces) SetBookieAffinityGroup(namespace string, bookieAffinityGroup utils.BookieAffinityGroupData) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence", "bookieAffinity") + return n.pulsar.Client.Post(endpoint, &bookieAffinityGroup) +} + +func (n *namespaces) DeleteBookieAffinityGroup(namespace string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence", "bookieAffinity") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) GetBookieAffinityGroup(namespace string) (*utils.BookieAffinityGroupData, error) { + var data utils.BookieAffinityGroupData + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence", "bookieAffinity") + err = n.pulsar.Client.Get(endpoint, &data) + return &data, err +} + +func (n *namespaces) GetPersistence(namespace string) (*utils.PersistencePolicies, error) { + var persistence utils.PersistencePolicies + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence") + err = n.pulsar.Client.Get(endpoint, &persistence) + return &persistence, err +} + +func (n *namespaces) Unload(namespace string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "unload") + return n.pulsar.Client.Put(endpoint, nil) +} + +func (n *namespaces) UnloadNamespaceBundle(namespace, bundle string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), bundle, "unload") + return n.pulsar.Client.Put(endpoint, nil) +} + +func (n *namespaces) SplitNamespaceBundle(namespace, bundle string, unloadSplitBundles bool) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), bundle, "split") + params := map[string]string{ + "unload": strconv.FormatBool(unloadSplitBundles), + } + return n.pulsar.Client.PutWithQueryParams(endpoint, nil, nil, params) +} + +func (n *namespaces) GetNamespacePermissions(namespace utils.NameSpaceName) (map[string][]utils.AuthAction, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions") + var permissions map[string][]utils.AuthAction + err := n.pulsar.Client.Get(endpoint, &permissions) + return permissions, err +} + +func (n *namespaces) GrantNamespacePermission(namespace utils.NameSpaceName, role string, + action []utils.AuthAction) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions", role) + s := make([]string, 0) + for _, v := range action { + s = append(s, v.String()) + } + return n.pulsar.Client.Post(endpoint, s) +} + +func (n *namespaces) RevokeNamespacePermission(namespace utils.NameSpaceName, role string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions", role) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) GrantSubPermission(namespace utils.NameSpaceName, sName string, roles []string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions", + "subscription", sName) + return n.pulsar.Client.Post(endpoint, roles) +} + +func (n *namespaces) RevokeSubPermission(namespace utils.NameSpaceName, sName, role string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions", + "subscription", sName, role) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetSubscriptionAuthMode(namespace utils.NameSpaceName, mode utils.SubscriptionAuthMode) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscriptionAuthMode") + return n.pulsar.Client.Post(endpoint, mode.String()) +} + +func (n *namespaces) SetEncryptionRequiredStatus(namespace utils.NameSpaceName, encrypt bool) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "encryptionRequired") + return n.pulsar.Client.Post(endpoint, strconv.FormatBool(encrypt)) +} + +func (n *namespaces) UnsubscribeNamespace(namespace utils.NameSpaceName, sName string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "unsubscribe", url.QueryEscape(sName)) + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) UnsubscribeNamespaceBundle(namespace utils.NameSpaceName, bundle, sName string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), bundle, "unsubscribe", url.QueryEscape(sName)) + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) ClearNamespaceBundleBacklogForSubscription(namespace utils.NameSpaceName, + bundle, sName string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), bundle, "clearBacklog", url.QueryEscape(sName)) + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) ClearNamespaceBundleBacklog(namespace utils.NameSpaceName, bundle string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), bundle, "clearBacklog") + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) ClearNamespaceBacklogForSubscription(namespace utils.NameSpaceName, sName string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "clearBacklog", url.QueryEscape(sName)) + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) ClearNamespaceBacklog(namespace utils.NameSpaceName) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "clearBacklog") + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) SetReplicatorDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "replicatorDispatchRate") + return n.pulsar.Client.Post(endpoint, rate) +} + +func (n *namespaces) GetReplicatorDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "replicatorDispatchRate") + var rate utils.DispatchRate + err := n.pulsar.Client.Get(endpoint, &rate) + return rate, err +} + +func (n *namespaces) SetSubscriptionDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscriptionDispatchRate") + return n.pulsar.Client.Post(endpoint, rate) +} + +func (n *namespaces) GetSubscriptionDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscriptionDispatchRate") + var rate utils.DispatchRate + err := n.pulsar.Client.Get(endpoint, &rate) + return rate, err +} + +func (n *namespaces) SetSubscribeRate(namespace utils.NameSpaceName, rate utils.SubscribeRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscribeRate") + return n.pulsar.Client.Post(endpoint, rate) +} + +func (n *namespaces) GetSubscribeRate(namespace utils.NameSpaceName) (utils.SubscribeRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscribeRate") + var rate utils.SubscribeRate + err := n.pulsar.Client.Get(endpoint, &rate) + return rate, err +} + +func (n *namespaces) SetDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "dispatchRate") + return n.pulsar.Client.Post(endpoint, rate) +} + +func (n *namespaces) GetDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "dispatchRate") + var rate utils.DispatchRate + err := n.pulsar.Client.Get(endpoint, &rate) + return rate, err +} + +func (n *namespaces) SetPublishRate(namespace utils.NameSpaceName, pubRate utils.PublishRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "publishRate") + return n.pulsar.Client.Post(endpoint, pubRate) +} + +func (n *namespaces) GetPublishRate(namespace utils.NameSpaceName) (utils.PublishRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "publishRate") + var pubRate utils.PublishRate + err := n.pulsar.Client.Get(endpoint, &pubRate) + return pubRate, err +} + +func (n *namespaces) SetIsAllowAutoUpdateSchema(namespace utils.NameSpaceName, isAllowAutoUpdateSchema bool) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "isAllowAutoUpdateSchema") + return n.pulsar.Client.Post(endpoint, &isAllowAutoUpdateSchema) +} + +func (n *namespaces) GetIsAllowAutoUpdateSchema(namespace utils.NameSpaceName) (bool, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "isAllowAutoUpdateSchema") + var result bool + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) GetInactiveTopicPolicies(namespace utils.NameSpaceName) (utils.InactiveTopicPolicies, error) { + var out utils.InactiveTopicPolicies + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "inactiveTopicPolicies") + err := n.pulsar.Client.Get(endpoint, &out) + return out, err +} + +func (n *namespaces) RemoveInactiveTopicPolicies(namespace utils.NameSpaceName) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "inactiveTopicPolicies") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetInactiveTopicPolicies(namespace utils.NameSpaceName, data utils.InactiveTopicPolicies) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "inactiveTopicPolicies") + return n.pulsar.Client.Post(endpoint, data) +} diff --git a/pulsaradmin/pkg/admin/ns_isolation_policy.go b/pulsaradmin/pkg/admin/ns_isolation_policy.go new file mode 100644 index 000000000..d8897f9f6 --- /dev/null +++ b/pulsaradmin/pkg/admin/ns_isolation_policy.go @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +type NsIsolationPolicy interface { + // Create a namespace isolation policy for a cluster + CreateNamespaceIsolationPolicy(cluster, policyName string, namespaceIsolationData utils.NamespaceIsolationData) error + + // Delete a namespace isolation policy for a cluster + DeleteNamespaceIsolationPolicy(cluster, policyName string) error + + // Get a single namespace isolation policy for a cluster + GetNamespaceIsolationPolicy(cluster, policyName string) (*utils.NamespaceIsolationData, error) + + // Get the namespace isolation policies of a cluster + GetNamespaceIsolationPolicies(cluster string) (map[string]utils.NamespaceIsolationData, error) + + // Returns list of active brokers with namespace-isolation policies attached to it. + GetBrokersWithNamespaceIsolationPolicy(cluster string) ([]utils.BrokerNamespaceIsolationData, error) + + // Returns active broker with namespace-isolation policies attached to it. + GetBrokerWithNamespaceIsolationPolicy(cluster, broker string) (*utils.BrokerNamespaceIsolationData, error) +} + +type nsIsolationPolicy struct { + pulsar *pulsarClient + basePath string +} + +func (c *pulsarClient) NsIsolationPolicy() NsIsolationPolicy { + return &nsIsolationPolicy{ + pulsar: c, + basePath: "/clusters", + } +} + +func (n *nsIsolationPolicy) CreateNamespaceIsolationPolicy(cluster, policyName string, + namespaceIsolationData utils.NamespaceIsolationData) error { + return n.setNamespaceIsolationPolicy(cluster, policyName, namespaceIsolationData) +} + +func (n *nsIsolationPolicy) setNamespaceIsolationPolicy(cluster, policyName string, + namespaceIsolationData utils.NamespaceIsolationData) error { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) + return n.pulsar.Client.Post(endpoint, &namespaceIsolationData) +} + +func (n *nsIsolationPolicy) DeleteNamespaceIsolationPolicy(cluster, policyName string) error { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *nsIsolationPolicy) GetNamespaceIsolationPolicy(cluster, policyName string) ( + *utils.NamespaceIsolationData, error) { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) + var nsIsolationData utils.NamespaceIsolationData + err := n.pulsar.Client.Get(endpoint, &nsIsolationData) + if err != nil { + return nil, err + } + return &nsIsolationData, nil +} + +func (n *nsIsolationPolicy) GetNamespaceIsolationPolicies(cluster string) ( + map[string]utils.NamespaceIsolationData, error) { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies") + var tmpMap map[string]utils.NamespaceIsolationData + err := n.pulsar.Client.Get(endpoint, &tmpMap) + if err != nil { + return nil, err + } + return tmpMap, nil +} + +func (n *nsIsolationPolicy) GetBrokersWithNamespaceIsolationPolicy(cluster string) ( + []utils.BrokerNamespaceIsolationData, error) { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", "brokers") + var res []utils.BrokerNamespaceIsolationData + err := n.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (n *nsIsolationPolicy) GetBrokerWithNamespaceIsolationPolicy(cluster, + broker string) (*utils.BrokerNamespaceIsolationData, error) { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", "brokers", broker) + var brokerNamespaceIsolationData utils.BrokerNamespaceIsolationData + err := n.pulsar.Client.Get(endpoint, &brokerNamespaceIsolationData) + if err != nil { + return nil, err + } + return &brokerNamespaceIsolationData, nil +} diff --git a/pulsaradmin/pkg/admin/packages.go b/pulsaradmin/pkg/admin/packages.go new file mode 100644 index 000000000..c7a0fd5ff --- /dev/null +++ b/pulsaradmin/pkg/admin/packages.go @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "os" + "path" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Packages is admin interface for functions management +type Packages interface { + // Download Function/Connector Package + // @param destinationFile + // file where data should be downloaded to + // @param packageURL + // the package URL + Download(packageURL, destinationFile string) error + + // Upload Function/Connector Package + // @param filePath + // file where data should be uploaded to + // @param packageURL + // type://tenant/namespace/packageName@version + // @param description + // descriptions of a package + // @param contact + // contact information of a package + // @param properties + // external infromations of a package + Upload(packageURL, filePath, description, contact string, properties map[string]string) error + + // List all the packages with the given type in a namespace + List(typeName, namespace string) ([]string, error) + + // ListVersions list all the versions of a package + ListVersions(packageURL string) ([]string, error) + + // Delete the specified package + Delete(packageURL string) error + + // GetMetadata get a package metadata information + GetMetadata(packageURL string) (utils.PackageMetadata, error) + + // UpdateMetadata update a package metadata information + UpdateMetadata(packageURL, description, contact string, properties map[string]string) error +} + +type packages struct { + pulsar *pulsarClient + basePath string +} + +func (p *packages) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "application/json") + return w.CreatePart(h) +} + +// Packages is used to access the functions endpoints +func (c *pulsarClient) Packages() Packages { + return &packages{ + pulsar: c, + basePath: "/packages", + } +} + +func (p packages) Download(packageURL, destinationFile string) error { + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) + + parent := path.Dir(destinationFile) + if parent != "." { + err = os.MkdirAll(parent, 0755) + if err != nil { + return fmt.Errorf("failed to create parent directory %s: %w", parent, err) + } + } + + _, err = os.Open(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("file %s already exists, please delete "+ + "the file first or change the file name", destinationFile) + } + } + file, err := os.Create(destinationFile) + if err != nil { + return err + } + + _, err = p.pulsar.Client.GetWithOptions(endpoint, nil, nil, false, file) + if err != nil { + return err + } + return nil +} + +func (p packages) Upload(packageURL, filePath, description, contact string, properties map[string]string) error { + if strings.TrimSpace(filePath) == "" { + return errors.New("file path is empty") + } + if strings.TrimSpace(packageURL) == "" { + return errors.New("package URL is empty") + } + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) + metadata := utils.PackageMetadata{ + Description: description, + Contact: contact, + Properties: properties, + } + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + metadataJSON, err := json.Marshal(metadata) + if err != nil { + return err + } + + stringWriter, err := p.createStringFromField(multiPartWriter, "metadata") + if err != nil { + return err + } + + _, err = stringWriter.Write(metadataJSON) + if err != nil { + return err + } + + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("file", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = p.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (p packages) List(typeName, namespace string) ([]string, error) { + var packageList []string + endpoint := p.pulsar.endpoint(p.basePath, typeName, namespace) + err := p.pulsar.Client.Get(endpoint, &packageList) + return packageList, err +} + +func (p packages) ListVersions(packageURL string) ([]string, error) { + var versionList []string + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return versionList, err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName()) + err = p.pulsar.Client.Get(endpoint, &versionList) + return versionList, err +} + +func (p packages) Delete(packageURL string) error { + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) + + return p.pulsar.Client.Delete(endpoint) +} + +func (p packages) GetMetadata(packageURL string) (utils.PackageMetadata, error) { + var metadata utils.PackageMetadata + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return metadata, err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion(), "metadata") + err = p.pulsar.Client.Get(endpoint, &metadata) + return metadata, err +} + +func (p packages) UpdateMetadata(packageURL, description, contact string, properties map[string]string) error { + metadata := utils.PackageMetadata{ + Description: description, + Contact: contact, + Properties: properties, + } + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion(), "metadata") + + return p.pulsar.Client.Put(endpoint, &metadata) +} diff --git a/pulsaradmin/pkg/admin/resource_quotas.go b/pulsaradmin/pkg/admin/resource_quotas.go new file mode 100644 index 000000000..fc5209b54 --- /dev/null +++ b/pulsaradmin/pkg/admin/resource_quotas.go @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +type ResourceQuotas interface { + // Get default resource quota for new resource bundles. + GetDefaultResourceQuota() (*utils.ResourceQuota, error) + + // Set default resource quota for new namespace bundles. + SetDefaultResourceQuota(quota utils.ResourceQuota) error + + // Get resource quota of a namespace bundle. + GetNamespaceBundleResourceQuota(namespace, bundle string) (*utils.ResourceQuota, error) + + // Set resource quota for a namespace bundle. + SetNamespaceBundleResourceQuota(namespace, bundle string, quota utils.ResourceQuota) error + + // Reset resource quota for a namespace bundle to default value. + ResetNamespaceBundleResourceQuota(namespace, bundle string) error +} + +type resource struct { + pulsar *pulsarClient + basePath string +} + +func (c *pulsarClient) ResourceQuotas() ResourceQuotas { + return &resource{ + pulsar: c, + basePath: "/resource-quotas", + } +} + +func (r *resource) GetDefaultResourceQuota() (*utils.ResourceQuota, error) { + endpoint := r.pulsar.endpoint(r.basePath) + var quota utils.ResourceQuota + err := r.pulsar.Client.Get(endpoint, "a) + if err != nil { + return nil, err + } + return "a, nil +} + +func (r *resource) SetDefaultResourceQuota(quota utils.ResourceQuota) error { + endpoint := r.pulsar.endpoint(r.basePath) + return r.pulsar.Client.Post(endpoint, "a) +} + +func (r *resource) GetNamespaceBundleResourceQuota(namespace, bundle string) (*utils.ResourceQuota, error) { + endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) + var quota utils.ResourceQuota + err := r.pulsar.Client.Get(endpoint, "a) + if err != nil { + return nil, err + } + return "a, nil +} + +func (r *resource) SetNamespaceBundleResourceQuota(namespace, bundle string, quota utils.ResourceQuota) error { + endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) + return r.pulsar.Client.Post(endpoint, "a) +} + +func (r *resource) ResetNamespaceBundleResourceQuota(namespace, bundle string) error { + endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) + return r.pulsar.Client.Delete(endpoint) +} diff --git a/pulsaradmin/pkg/admin/schema.go b/pulsaradmin/pkg/admin/schema.go new file mode 100644 index 000000000..146552684 --- /dev/null +++ b/pulsaradmin/pkg/admin/schema.go @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "fmt" + "strconv" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Schema is admin interface for schema management +type Schema interface { + // GetSchemaInfo retrieves the latest schema of a topic + GetSchemaInfo(topic string) (*utils.SchemaInfo, error) + + // GetSchemaInfoWithVersion retrieves the latest schema with version of a topic + GetSchemaInfoWithVersion(topic string) (*utils.SchemaInfoWithVersion, error) + + // GetSchemaInfoByVersion retrieves the schema of a topic at a given version + GetSchemaInfoByVersion(topic string, version int64) (*utils.SchemaInfo, error) + + // DeleteSchema deletes the schema associated with a given topic + DeleteSchema(topic string) error + + // CreateSchemaByPayload creates a schema for a given topic + CreateSchemaByPayload(topic string, schemaPayload utils.PostSchemaPayload) error +} + +type schemas struct { + pulsar *pulsarClient + basePath string +} + +// Schemas is used to access the schemas endpoints +func (c *pulsarClient) Schemas() Schema { + return &schemas{ + pulsar: c, + basePath: "/schemas", + } +} + +func (s *schemas) GetSchemaInfo(topic string) (*utils.SchemaInfo, error) { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return nil, err + } + var response utils.GetSchemaResponse + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), + topicName.GetLocalName(), "schema") + + err = s.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, err + } + + info := utils.ConvertGetSchemaResponseToSchemaInfo(topicName, response) + return info, nil +} + +func (s *schemas) GetSchemaInfoWithVersion(topic string) (*utils.SchemaInfoWithVersion, error) { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return nil, err + } + var response utils.GetSchemaResponse + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), + topicName.GetLocalName(), "schema") + + err = s.pulsar.Client.Get(endpoint, &response) + if err != nil { + fmt.Println("err:", err.Error()) + return nil, err + } + + info := utils.ConvertGetSchemaResponseToSchemaInfoWithVersion(topicName, response) + return info, nil +} + +func (s *schemas) GetSchemaInfoByVersion(topic string, version int64) (*utils.SchemaInfo, error) { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return nil, err + } + + var response utils.GetSchemaResponse + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), topicName.GetLocalName(), + "schema", strconv.FormatInt(version, 10)) + + err = s.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, err + } + + info := utils.ConvertGetSchemaResponseToSchemaInfo(topicName, response) + return info, nil +} + +func (s *schemas) DeleteSchema(topic string) error { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return err + } + + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), + topicName.GetLocalName(), "schema") + + fmt.Println(endpoint) + + return s.pulsar.Client.Delete(endpoint) +} + +func (s *schemas) CreateSchemaByPayload(topic string, schemaPayload utils.PostSchemaPayload) error { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return err + } + + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), + topicName.GetLocalName(), "schema") + + return s.pulsar.Client.Post(endpoint, &schemaPayload) +} diff --git a/pulsaradmin/pkg/admin/sinks.go b/pulsaradmin/pkg/admin/sinks.go new file mode 100644 index 000000000..acbf83113 --- /dev/null +++ b/pulsaradmin/pkg/admin/sinks.go @@ -0,0 +1,437 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Sinks is admin interface for sinks management +type Sinks interface { + // ListSinks returns the list of all the Pulsar Sinks. + ListSinks(tenant, namespace string) ([]string, error) + + // GetSink returns the configuration for the specified sink + GetSink(tenant, namespace, Sink string) (utils.SinkConfig, error) + + // CreateSink creates a new sink + CreateSink(config *utils.SinkConfig, fileName string) error + + // CreateSinkWithURL creates a new sink by providing url from which fun-pkg can be downloaded. supported url: http/file + CreateSinkWithURL(config *utils.SinkConfig, pkgURL string) error + + // UpdateSink updates the configuration for a sink. + UpdateSink(config *utils.SinkConfig, fileName string, options *utils.UpdateOptions) error + + // UpdateSinkWithURL updates a sink by providing url from which fun-pkg can be downloaded. supported url: http/file + UpdateSinkWithURL(config *utils.SinkConfig, pkgURL string, options *utils.UpdateOptions) error + + // DeleteSink deletes an existing sink + DeleteSink(tenant, namespace, Sink string) error + + // GetSinkStatus returns the current status of a sink. + GetSinkStatus(tenant, namespace, Sink string) (utils.SinkStatus, error) + + // GetSinkStatusWithID returns the current status of a sink instance. + GetSinkStatusWithID(tenant, namespace, Sink string, id int) (utils.SinkInstanceStatusData, error) + + // RestartSink restarts all sink instances + RestartSink(tenant, namespace, Sink string) error + + // RestartSinkWithID restarts sink instance + RestartSinkWithID(tenant, namespace, Sink string, id int) error + + // StopSink stops all sink instances + StopSink(tenant, namespace, Sink string) error + + // StopSinkWithID stops sink instance + StopSinkWithID(tenant, namespace, Sink string, id int) error + + // StartSink starts all sink instances + StartSink(tenant, namespace, Sink string) error + + // StartSinkWithID starts sink instance + StartSinkWithID(tenant, namespace, Sink string, id int) error + + // GetBuiltInSinks fetches a list of supported Pulsar IO sinks currently running in cluster mode + GetBuiltInSinks() ([]*utils.ConnectorDefinition, error) + + // ReloadBuiltInSinks reload the available built-in connectors, include Source and Sink + ReloadBuiltInSinks() error +} + +type sinks struct { + pulsar *pulsarClient + basePath string +} + +// Sinks is used to access the sinks endpoints +func (c *pulsarClient) Sinks() Sinks { + return &sinks{ + pulsar: c, + basePath: "/sinks", + } +} + +func (s *sinks) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "application/json") + return w.CreatePart(h) +} + +func (s *sinks) createTextFromFiled(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "text/plain") + return w.CreatePart(h) +} + +func (s *sinks) ListSinks(tenant, namespace string) ([]string, error) { + var sinks []string + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace) + err := s.pulsar.Client.Get(endpoint, &sinks) + return sinks, err +} + +func (s *sinks) GetSink(tenant, namespace, sink string) (utils.SinkConfig, error) { + var sinkConfig utils.SinkConfig + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + err := s.pulsar.Client.Get(endpoint, &sinkConfig) + return sinkConfig, err +} + +func (s *sinks) CreateSink(config *utils.SinkConfig, fileName string) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sinkConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sinks) CreateSinkWithURL(config *utils.SinkConfig, pkgURL string) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := s.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sinkConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sinks) UpdateSink(config *utils.SinkConfig, fileName string, updateOptions *utils.UpdateOptions) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sinkConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := s.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sinks) UpdateSinkWithURL(config *utils.SinkConfig, pkgURL string, updateOptions *utils.UpdateOptions) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := s.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sinkConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := s.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sinks) DeleteSink(tenant, namespace, sink string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + return s.pulsar.Client.Delete(endpoint) +} + +func (s *sinks) GetSinkStatus(tenant, namespace, sink string) (utils.SinkStatus, error) { + var sinkStatus utils.SinkStatus + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + err := s.pulsar.Client.Get(endpoint+"/status", &sinkStatus) + return sinkStatus, err +} + +func (s *sinks) GetSinkStatusWithID(tenant, namespace, sink string, id int) (utils.SinkInstanceStatusData, error) { + var sinkInstanceStatusData utils.SinkInstanceStatusData + instanceID := fmt.Sprintf("%d", id) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink, instanceID) + err := s.pulsar.Client.Get(endpoint+"/status", &sinkInstanceStatusData) + return sinkInstanceStatusData, err +} + +func (s *sinks) RestartSink(tenant, namespace, sink string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + return s.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (s *sinks) RestartSinkWithID(tenant, namespace, sink string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink, id) + + return s.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (s *sinks) StopSink(tenant, namespace, sink string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + return s.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (s *sinks) StopSinkWithID(tenant, namespace, sink string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink, id) + + return s.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (s *sinks) StartSink(tenant, namespace, sink string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + return s.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (s *sinks) StartSinkWithID(tenant, namespace, sink string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink, id) + + return s.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (s *sinks) GetBuiltInSinks() ([]*utils.ConnectorDefinition, error) { + var connectorDefinition []*utils.ConnectorDefinition + endpoint := s.pulsar.endpoint(s.basePath, "builtinsinks") + err := s.pulsar.Client.Get(endpoint, &connectorDefinition) + return connectorDefinition, err +} + +func (s *sinks) ReloadBuiltInSinks() error { + endpoint := s.pulsar.endpoint(s.basePath, "reloadBuiltInSinks") + return s.pulsar.Client.Post(endpoint, nil) +} diff --git a/pulsaradmin/pkg/admin/sources.go b/pulsaradmin/pkg/admin/sources.go new file mode 100644 index 000000000..e10d1da06 --- /dev/null +++ b/pulsaradmin/pkg/admin/sources.go @@ -0,0 +1,440 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Sources is admin interface for sources management +type Sources interface { + // ListSources returns the list of all the Pulsar Sources. + ListSources(tenant, namespace string) ([]string, error) + + // GetSource return the configuration for the specified source + GetSource(tenant, namespace, source string) (utils.SourceConfig, error) + + // CreateSource creates a new source + CreateSource(config *utils.SourceConfig, fileName string) error + + // CreateSourceWithURL creates a new source by providing url from which fun-pkg can be downloaded. + // supported url: http/file + CreateSourceWithURL(config *utils.SourceConfig, pkgURL string) error + + // UpdateSource updates the configuration for a source. + UpdateSource(config *utils.SourceConfig, fileName string, options *utils.UpdateOptions) error + + // UpdateSourceWithURL updates a source by providing url from which fun-pkg can be downloaded. supported url: http/file + UpdateSourceWithURL(config *utils.SourceConfig, pkgURL string, options *utils.UpdateOptions) error + + // DeleteSource deletes an existing source + DeleteSource(tenant, namespace, source string) error + + // GetSourceStatus returns the current status of a source. + GetSourceStatus(tenant, namespace, source string) (utils.SourceStatus, error) + + // GetSourceStatusWithID returns the current status of a source instance. + GetSourceStatusWithID(tenant, namespace, source string, id int) (utils.SourceInstanceStatusData, error) + + // RestartSource restarts all source instances + RestartSource(tenant, namespace, source string) error + + // RestartSourceWithID restarts source instance + RestartSourceWithID(tenant, namespace, source string, id int) error + + // StopSource stops all source instances + StopSource(tenant, namespace, source string) error + + // StopSourceWithID stops source instance + StopSourceWithID(tenant, namespace, source string, id int) error + + // StartSource starts all source instances + StartSource(tenant, namespace, source string) error + + // StartSourceWithID starts source instance + StartSourceWithID(tenant, namespace, source string, id int) error + + // GetBuiltInSources fetches a list of supported Pulsar IO sources currently running in cluster mode + GetBuiltInSources() ([]*utils.ConnectorDefinition, error) + + // ReloadBuiltInSources reloads the available built-in connectors, include Source and Sink + ReloadBuiltInSources() error +} + +type sources struct { + pulsar *pulsarClient + basePath string +} + +// Sources is used to access the sources endpoints +func (c *pulsarClient) Sources() Sources { + return &sources{ + pulsar: c, + basePath: "/sources", + } +} + +func (s *sources) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "application/json") + return w.CreatePart(h) +} + +func (s *sources) createTextFromFiled(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "text/plain") + return w.CreatePart(h) +} + +func (s *sources) ListSources(tenant, namespace string) ([]string, error) { + var sources []string + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace) + err := s.pulsar.Client.Get(endpoint, &sources) + return sources, err +} + +func (s *sources) GetSource(tenant, namespace, source string) (utils.SourceConfig, error) { + var sourceConfig utils.SourceConfig + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + err := s.pulsar.Client.Get(endpoint, &sourceConfig) + return sourceConfig, err +} + +func (s *sources) CreateSource(config *utils.SourceConfig, fileName string) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sourceConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sources) CreateSourceWithURL(config *utils.SourceConfig, pkgURL string) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := s.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sourceConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sources) UpdateSource(config *utils.SourceConfig, fileName string, updateOptions *utils.UpdateOptions) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sourceConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := s.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sources) UpdateSourceWithURL(config *utils.SourceConfig, pkgURL string, + updateOptions *utils.UpdateOptions) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := s.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sourceConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := s.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sources) DeleteSource(tenant, namespace, source string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + return s.pulsar.Client.Delete(endpoint) +} + +func (s *sources) GetSourceStatus(tenant, namespace, source string) (utils.SourceStatus, error) { + var sourceStatus utils.SourceStatus + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + err := s.pulsar.Client.Get(endpoint+"/status", &sourceStatus) + return sourceStatus, err +} + +func (s *sources) GetSourceStatusWithID(tenant, namespace, source string, id int) ( + utils.SourceInstanceStatusData, error) { + var sourceInstanceStatusData utils.SourceInstanceStatusData + instanceID := fmt.Sprintf("%d", id) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source, instanceID) + err := s.pulsar.Client.Get(endpoint+"/status", &sourceInstanceStatusData) + return sourceInstanceStatusData, err +} + +func (s *sources) RestartSource(tenant, namespace, source string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + return s.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (s *sources) RestartSourceWithID(tenant, namespace, source string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source, id) + + return s.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (s *sources) StopSource(tenant, namespace, source string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + return s.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (s *sources) StopSourceWithID(tenant, namespace, source string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source, id) + + return s.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (s *sources) StartSource(tenant, namespace, source string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + return s.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (s *sources) StartSourceWithID(tenant, namespace, source string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source, id) + + return s.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (s *sources) GetBuiltInSources() ([]*utils.ConnectorDefinition, error) { + var connectorDefinition []*utils.ConnectorDefinition + endpoint := s.pulsar.endpoint(s.basePath, "builtinsources") + err := s.pulsar.Client.Get(endpoint, &connectorDefinition) + return connectorDefinition, err +} + +func (s *sources) ReloadBuiltInSources() error { + endpoint := s.pulsar.endpoint(s.basePath, "reloadBuiltInSources") + return s.pulsar.Client.Post(endpoint, nil) +} diff --git a/pulsaradmin/pkg/admin/subscription.go b/pulsaradmin/pkg/admin/subscription.go new file mode 100644 index 000000000..456de46cd --- /dev/null +++ b/pulsaradmin/pkg/admin/subscription.go @@ -0,0 +1,316 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/binary" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/golang/protobuf/proto" //nolint:staticcheck + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Subscriptions is admin interface for subscriptions management +type Subscriptions interface { + // Create a new subscription on a topic + Create(utils.TopicName, string, utils.MessageID) error + + // Delete a subscription. + // Delete a persistent subscription from a topic. There should not be any active consumers on the subscription + Delete(utils.TopicName, string) error + + // ForceDelete deletes a subscription forcefully + ForceDelete(utils.TopicName, string) error + + // List returns the list of subscriptions + List(utils.TopicName) ([]string, error) + + // ResetCursorToMessageID resets cursor position on a topic subscription + // @param + // messageID reset subscription to messageId (or previous nearest messageId if given messageId is not valid) + ResetCursorToMessageID(utils.TopicName, string, utils.MessageID) error + + // ResetCursorToTimestamp resets cursor position on a topic subscription + // @param + // time reset subscription to position closest to time in ms since epoch + ResetCursorToTimestamp(utils.TopicName, string, int64) error + + // ClearBacklog skips all messages on a topic subscription + ClearBacklog(utils.TopicName, string) error + + // SkipMessages skips messages on a topic subscription + SkipMessages(utils.TopicName, string, int64) error + + // ExpireMessages expires all messages older than given N (expireTimeInSeconds) seconds for a given subscription + ExpireMessages(utils.TopicName, string, int64) error + + // ExpireAllMessages expires all messages older than given N (expireTimeInSeconds) seconds for all + // subscriptions of the persistent-topic + ExpireAllMessages(utils.TopicName, int64) error + + // PeekMessages peeks messages from a topic subscription + PeekMessages(utils.TopicName, string, int) ([]*utils.Message, error) + + // GetMessageByID gets message by its ledgerID and entryID + GetMessageByID(topic utils.TopicName, ledgerID, entryID int64) (*utils.Message, error) +} + +type subscriptions struct { + pulsar *pulsarClient + basePath string + SubPath string +} + +// Subscriptions is used to access the subscriptions endpoints +func (c *pulsarClient) Subscriptions() Subscriptions { + return &subscriptions{ + pulsar: c, + basePath: "", + SubPath: "subscription", + } +} + +func (s *subscriptions) Create(topic utils.TopicName, sName string, messageID utils.MessageID) error { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName)) + return s.pulsar.Client.Put(endpoint, messageID) +} + +func (s *subscriptions) delete(topic utils.TopicName, subName string, force bool) error { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(subName)) + queryParams := make(map[string]string) + queryParams["force"] = strconv.FormatBool(force) + return s.pulsar.Client.DeleteWithQueryParams(endpoint, queryParams) +} + +func (s *subscriptions) Delete(topic utils.TopicName, sName string) error { + return s.delete(topic, sName, false) +} + +func (s *subscriptions) ForceDelete(topic utils.TopicName, sName string) error { + return s.delete(topic, sName, true) +} + +func (s *subscriptions) List(topic utils.TopicName) ([]string, error) { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), "subscriptions") + var list []string + return list, s.pulsar.Client.Get(endpoint, &list) +} + +func (s *subscriptions) ResetCursorToMessageID(topic utils.TopicName, sName string, id utils.MessageID) error { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), "resetcursor") + return s.pulsar.Client.Post(endpoint, id) +} + +func (s *subscriptions) ResetCursorToTimestamp(topic utils.TopicName, sName string, timestamp int64) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), + "resetcursor", strconv.FormatInt(timestamp, 10)) + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) ClearBacklog(topic utils.TopicName, sName string) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), "skip_all") + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) SkipMessages(topic utils.TopicName, sName string, n int64) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), + "skip", strconv.FormatInt(n, 10)) + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) ExpireMessages(topic utils.TopicName, sName string, expire int64) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), + "expireMessages", strconv.FormatInt(expire, 10)) + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) ExpireAllMessages(topic utils.TopicName, expire int64) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), "all_subscription", + "expireMessages", strconv.FormatInt(expire, 10)) + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) PeekMessages(topic utils.TopicName, sName string, n int) ([]*utils.Message, error) { + var msgs []*utils.Message + + count := 1 + for n > 0 { + m, err := s.peekNthMessage(topic, sName, count) + if err != nil { + return nil, err + } + msgs = append(msgs, m...) + n -= len(m) + count++ + } + + return msgs, nil +} + +func (s *subscriptions) peekNthMessage(topic utils.TopicName, sName string, pos int) ([]*utils.Message, error) { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), "subscription", url.PathEscape(sName), + "position", strconv.Itoa(pos)) + + resp, err := s.pulsar.Client.MakeRequest(http.MethodGet, endpoint) + if err != nil { + return nil, err + } + defer safeRespClose(resp) + + return handleResp(topic, resp) +} + +func (s *subscriptions) GetMessageByID(topic utils.TopicName, ledgerID, entryID int64) (*utils.Message, error) { + ledgerIDStr := strconv.FormatInt(ledgerID, 10) + entryIDStr := strconv.FormatInt(entryID, 10) + + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), "ledger", ledgerIDStr, "entry", entryIDStr) + resp, err := s.pulsar.Client.MakeRequest(http.MethodGet, endpoint) + if err != nil { + return nil, err + } + defer safeRespClose(resp) + + messages, err := handleResp(topic, resp) + if err != nil { + return nil, err + } + + if len(messages) == 0 { + return nil, nil + } + return messages[0], nil +} + +// safeRespClose is used to close a response body +func safeRespClose(resp *http.Response) { + if resp != nil { + // ignore error since it is closing a response body + _ = resp.Body.Close() + } +} + +const ( + PublishTimeHeader = "X-Pulsar-Publish-Time" + BatchHeader = "X-Pulsar-Num-Batch-Message" + PropertyPrefix = "X-Pulsar-Property-" +) + +func handleResp(topic utils.TopicName, resp *http.Response) ([]*utils.Message, error) { + msgID := resp.Header.Get("X-Pulsar-Message-ID") + ID, err := utils.ParseMessageID(msgID) + if err != nil { + return nil, err + } + + // read data + payload, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + properties := make(map[string]string) + for k := range resp.Header { + switch { + case k == PublishTimeHeader: + h := resp.Header.Get(k) + if h != "" { + properties["publish-time"] = h + } + case k == BatchHeader: + h := resp.Header.Get(k) + if h != "" { + properties[BatchHeader] = h + } + return getIndividualMsgsFromBatch(topic, ID, payload, properties) + case strings.Contains(k, PropertyPrefix): + key := strings.TrimPrefix(k, PropertyPrefix) + properties[key] = resp.Header.Get(k) + } + } + + return []*utils.Message{utils.NewMessage(topic.String(), *ID, payload, properties)}, nil +} + +func getIndividualMsgsFromBatch(topic utils.TopicName, msgID *utils.MessageID, data []byte, + properties map[string]string) ([]*utils.Message, error) { + + batchSize, err := strconv.Atoi(properties[BatchHeader]) + if err != nil { + return nil, nil + } + + msgs := make([]*utils.Message, 0, batchSize) + + // read all messages in batch + buf32 := make([]byte, 4) + rdBuf := bytes.NewReader(data) + for i := 0; i < batchSize; i++ { + msgID.BatchIndex = i + // singleMetaSize + if _, err := io.ReadFull(rdBuf, buf32); err != nil { + return nil, err + } + singleMetaSize := binary.BigEndian.Uint32(buf32) + + // singleMeta + singleMetaBuf := make([]byte, singleMetaSize) + if _, err := io.ReadFull(rdBuf, singleMetaBuf); err != nil { + return nil, err + } + + singleMeta := new(utils.SingleMessageMetadata) + if err := proto.Unmarshal(singleMetaBuf, singleMeta); err != nil { + return nil, err + } + + if len(singleMeta.Properties) > 0 { + for _, v := range singleMeta.Properties { + k := *v.Key + property := *v.Value + properties[k] = property + } + } + + // payload + singlePayload := make([]byte, singleMeta.GetPayloadSize()) + if _, err := io.ReadFull(rdBuf, singlePayload); err != nil { + return nil, err + } + + msgs = append(msgs, &utils.Message{ + Topic: topic.String(), + MessageID: *msgID, + Payload: singlePayload, + Properties: properties, + }) + } + + return msgs, nil +} diff --git a/pulsaradmin/pkg/admin/tenant.go b/pulsaradmin/pkg/admin/tenant.go new file mode 100644 index 000000000..62e176bcb --- /dev/null +++ b/pulsaradmin/pkg/admin/tenant.go @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Tenants is admin interface for tenants management +type Tenants interface { + // Create a new tenant + Create(utils.TenantData) error + + // Delete an existing tenant + Delete(string) error + + // Update the admins for a tenant + Update(utils.TenantData) error + + // List returns the list of tenants + List() ([]string, error) + + // Get returns the config of the tenant. + Get(string) (utils.TenantData, error) +} + +type tenants struct { + pulsar *pulsarClient + basePath string +} + +// Tenants is used to access the tenants endpoints +func (c *pulsarClient) Tenants() Tenants { + return &tenants{ + pulsar: c, + basePath: "/tenants", + } +} + +func (c *tenants) Create(data utils.TenantData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.Name) + return c.pulsar.Client.Put(endpoint, &data) +} + +func (c *tenants) Delete(name string) error { + endpoint := c.pulsar.endpoint(c.basePath, name) + return c.pulsar.Client.Delete(endpoint) +} + +func (c *tenants) Update(data utils.TenantData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.Name) + return c.pulsar.Client.Post(endpoint, &data) +} + +func (c *tenants) List() ([]string, error) { + var tenantList []string + endpoint := c.pulsar.endpoint(c.basePath, "") + err := c.pulsar.Client.Get(endpoint, &tenantList) + return tenantList, err +} + +func (c *tenants) Get(name string) (utils.TenantData, error) { + var data utils.TenantData + endpoint := c.pulsar.endpoint(c.basePath, name) + err := c.pulsar.Client.Get(endpoint, &data) + return data, err +} diff --git a/pulsaradmin/pkg/admin/topic.go b/pulsaradmin/pkg/admin/topic.go new file mode 100644 index 000000000..c888827bf --- /dev/null +++ b/pulsaradmin/pkg/admin/topic.go @@ -0,0 +1,725 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "fmt" + "strconv" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Topics is admin interface for topics management +type Topics interface { + // Create a topic + Create(utils.TopicName, int) error + + // Delete a topic + Delete(utils.TopicName, bool, bool) error + + // Update number of partitions of a non-global partitioned topic + // It requires partitioned-topic to be already exist and number of new partitions must be greater than existing + // number of partitions. Decrementing number of partitions requires deletion of topic which is not supported. + Update(utils.TopicName, int) error + + // GetMetadata returns metadata of a partitioned topic + GetMetadata(utils.TopicName) (utils.PartitionedTopicMetadata, error) + + // List returns the list of topics under a namespace + List(utils.NameSpaceName) ([]string, []string, error) + + // GetInternalInfo returns the internal metadata info for the topic + GetInternalInfo(utils.TopicName) (utils.ManagedLedgerInfo, error) + + // GetPermissions returns permissions on a topic + // Retrieve the effective permissions for a topic. These permissions are defined by the permissions set at the + // namespace level combined (union) with any eventual specific permission set on the topic. + GetPermissions(utils.TopicName) (map[string][]utils.AuthAction, error) + + // GrantPermission grants a new permission to a client role on a single topic + GrantPermission(utils.TopicName, string, []utils.AuthAction) error + + // RevokePermission revokes permissions to a client role on a single topic. If the permission + // was not set at the topic level, but rather at the namespace level, this operation will + // return an error (HTTP status code 412). + RevokePermission(utils.TopicName, string) error + + // Lookup a topic returns the broker URL that serves the topic + Lookup(utils.TopicName) (utils.LookupData, error) + + // GetBundleRange returns a bundle range of a topic + GetBundleRange(utils.TopicName) (string, error) + + // GetLastMessageID returns the last commit message Id of a topic + GetLastMessageID(utils.TopicName) (utils.MessageID, error) + + // GetMessageID returns the message Id by timestamp(ms) of a topic + GetMessageID(utils.TopicName, int64) (utils.MessageID, error) + + // GetStats returns the stats for the topic + // All the rates are computed over a 1 minute window and are relative the last completed 1 minute period + GetStats(utils.TopicName) (utils.TopicStats, error) + + // GetInternalStats returns the internal stats for the topic. + GetInternalStats(utils.TopicName) (utils.PersistentTopicInternalStats, error) + + // GetPartitionedStats returns the stats for the partitioned topic + // All the rates are computed over a 1 minute window and are relative the last completed 1 minute period + GetPartitionedStats(utils.TopicName, bool) (utils.PartitionedTopicStats, error) + + // Terminate the topic and prevent any more messages being published on it + Terminate(utils.TopicName) (utils.MessageID, error) + + // Offload triggers offloading messages in topic to longterm storage + Offload(utils.TopicName, utils.MessageID) error + + // OffloadStatus checks the status of an ongoing offloading operation for a topic + OffloadStatus(utils.TopicName) (utils.OffloadProcessStatus, error) + + // Unload a topic + Unload(utils.TopicName) error + + // Compact triggers compaction to run for a topic. A single topic can only have one instance of compaction + // running at any time. Any attempt to trigger another will be met with a ConflictException. + Compact(utils.TopicName) error + + // CompactStatus checks the status of an ongoing compaction for a topic + CompactStatus(utils.TopicName) (utils.LongRunningProcessStatus, error) + + // GetMessageTTL Get the message TTL for a topic + GetMessageTTL(utils.TopicName) (int, error) + + // SetMessageTTL Set the message TTL for a topic + SetMessageTTL(utils.TopicName, int) error + + // RemoveMessageTTL Remove the message TTL for a topic + RemoveMessageTTL(utils.TopicName) error + + // GetMaxProducers Get max number of producers for a topic + GetMaxProducers(utils.TopicName) (int, error) + + // SetMaxProducers Set max number of producers for a topic + SetMaxProducers(utils.TopicName, int) error + + // RemoveMaxProducers Remove max number of producers for a topic + RemoveMaxProducers(utils.TopicName) error + + // GetMaxConsumers Get max number of consumers for a topic + GetMaxConsumers(utils.TopicName) (int, error) + + // SetMaxConsumers Set max number of consumers for a topic + SetMaxConsumers(utils.TopicName, int) error + + // RemoveMaxConsumers Remove max number of consumers for a topic + RemoveMaxConsumers(utils.TopicName) error + + // GetMaxUnackMessagesPerConsumer Get max unacked messages policy on consumer for a topic + GetMaxUnackMessagesPerConsumer(utils.TopicName) (int, error) + + // SetMaxUnackMessagesPerConsumer Set max unacked messages policy on consumer for a topic + SetMaxUnackMessagesPerConsumer(utils.TopicName, int) error + + // RemoveMaxUnackMessagesPerConsumer Remove max unacked messages policy on consumer for a topic + RemoveMaxUnackMessagesPerConsumer(utils.TopicName) error + + // GetMaxUnackMessagesPerSubscription Get max unacked messages policy on subscription for a topic + GetMaxUnackMessagesPerSubscription(utils.TopicName) (int, error) + + // SetMaxUnackMessagesPerSubscription Set max unacked messages policy on subscription for a topic + SetMaxUnackMessagesPerSubscription(utils.TopicName, int) error + + // RemoveMaxUnackMessagesPerSubscription Remove max unacked messages policy on subscription for a topic + RemoveMaxUnackMessagesPerSubscription(utils.TopicName) error + + // GetPersistence Get the persistence policies for a topic + GetPersistence(utils.TopicName) (*utils.PersistenceData, error) + + // SetPersistence Set the persistence policies for a topic + SetPersistence(utils.TopicName, utils.PersistenceData) error + + // RemovePersistence Remove the persistence policies for a topic + RemovePersistence(utils.TopicName) error + + // GetDelayedDelivery Get the delayed delivery policy for a topic + GetDelayedDelivery(utils.TopicName) (*utils.DelayedDeliveryData, error) + + // SetDelayedDelivery Set the delayed delivery policy on a topic + SetDelayedDelivery(utils.TopicName, utils.DelayedDeliveryData) error + + // RemoveDelayedDelivery Remove the delayed delivery policy on a topic + RemoveDelayedDelivery(utils.TopicName) error + + // GetDispatchRate Get message dispatch rate for a topic + GetDispatchRate(utils.TopicName) (*utils.DispatchRateData, error) + + // SetDispatchRate Set message dispatch rate for a topic + SetDispatchRate(utils.TopicName, utils.DispatchRateData) error + + // RemoveDispatchRate Remove message dispatch rate for a topic + RemoveDispatchRate(utils.TopicName) error + + // GetPublishRate Get message publish rate for a topic + GetPublishRate(utils.TopicName) (*utils.PublishRateData, error) + + // SetPublishRate Set message publish rate for a topic + SetPublishRate(utils.TopicName, utils.PublishRateData) error + + // RemovePublishRate Remove message publish rate for a topic + RemovePublishRate(utils.TopicName) error + + // GetDeduplicationStatus Get the deduplication policy for a topic + GetDeduplicationStatus(utils.TopicName) (bool, error) + + // SetDeduplicationStatus Set the deduplication policy for a topic + SetDeduplicationStatus(utils.TopicName, bool) error + + // RemoveDeduplicationStatus Remove the deduplication policy for a topic + RemoveDeduplicationStatus(utils.TopicName) error + + // GetRetention returns the retention configuration for a topic + GetRetention(utils.TopicName, bool) (*utils.RetentionPolicies, error) + + // RemoveRetention removes the retention configuration on a topic + RemoveRetention(utils.TopicName) error + + // SetRetention sets the retention policy for a topic + SetRetention(utils.TopicName, utils.RetentionPolicies) error + + // Get the compaction threshold for a topic + GetCompactionThreshold(topic utils.TopicName, applied bool) (int64, error) + + // Set the compaction threshold for a topic + SetCompactionThreshold(topic utils.TopicName, threshold int64) error + + // Remove compaction threshold for a topic + RemoveCompactionThreshold(utils.TopicName) error + + // GetBacklogQuotaMap returns backlog quota map for a topic + GetBacklogQuotaMap(topic utils.TopicName, applied bool) (map[utils.BacklogQuotaType]utils.BacklogQuota, error) + + // SetBacklogQuota sets a backlog quota for a topic + SetBacklogQuota(utils.TopicName, utils.BacklogQuota, utils.BacklogQuotaType) error + + // RemoveBacklogQuota removes a backlog quota policy from a topic + RemoveBacklogQuota(utils.TopicName, utils.BacklogQuotaType) error + + // GetInactiveTopicPolicies gets the inactive topic policies on a topic + GetInactiveTopicPolicies(topic utils.TopicName, applied bool) (utils.InactiveTopicPolicies, error) + + // RemoveInactiveTopicPolicies removes inactive topic policies from a topic + RemoveInactiveTopicPolicies(utils.TopicName) error + + // SetInactiveTopicPolicies sets the inactive topic policies on a topic + SetInactiveTopicPolicies(topic utils.TopicName, data utils.InactiveTopicPolicies) error + + // GetReplicationClusters get the replication clusters of a topic + GetReplicationClusters(topic utils.TopicName) ([]string, error) + + // SetReplicationClusters sets the replication clusters on a topic + SetReplicationClusters(topic utils.TopicName, data []string) error +} + +type topics struct { + pulsar *pulsarClient + basePath string + persistentPath string + nonPersistentPath string + lookupPath string +} + +// Check whether the topics struct implements the Topics interface. +var _ Topics = &topics{} + +// Topics is used to access the topics endpoints +func (c *pulsarClient) Topics() Topics { + return &topics{ + pulsar: c, + basePath: "", + persistentPath: "/persistent", + nonPersistentPath: "/non-persistent", + lookupPath: "/lookup/v2/topic", + } +} + +func (t *topics) Create(topic utils.TopicName, partitions int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitions") + data := &partitions + if partitions == 0 { + endpoint = t.pulsar.endpoint(t.basePath, topic.GetRestPath()) + data = nil + } + + return t.pulsar.Client.Put(endpoint, data) +} + +func (t *topics) Delete(topic utils.TopicName, force bool, nonPartitioned bool) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitions") + if nonPartitioned { + endpoint = t.pulsar.endpoint(t.basePath, topic.GetRestPath()) + } + params := map[string]string{ + "force": strconv.FormatBool(force), + } + return t.pulsar.Client.DeleteWithQueryParams(endpoint, params) +} + +func (t *topics) Update(topic utils.TopicName, partitions int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitions") + return t.pulsar.Client.Post(endpoint, partitions) +} + +func (t *topics) GetMetadata(topic utils.TopicName) (utils.PartitionedTopicMetadata, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitions") + var partitionedMeta utils.PartitionedTopicMetadata + err := t.pulsar.Client.Get(endpoint, &partitionedMeta) + return partitionedMeta, err +} + +func (t *topics) List(namespace utils.NameSpaceName) ([]string, []string, error) { + var partitionedTopics, nonPartitionedTopics []string + partitionedTopicsChan := make(chan []string) + nonPartitionedTopicsChan := make(chan []string) + errChan := make(chan error) + + pp := t.pulsar.endpoint(t.persistentPath, namespace.String(), "partitioned") + np := t.pulsar.endpoint(t.nonPersistentPath, namespace.String(), "partitioned") + p := t.pulsar.endpoint(t.persistentPath, namespace.String()) + n := t.pulsar.endpoint(t.nonPersistentPath, namespace.String()) + + go t.getTopics(pp, partitionedTopicsChan, errChan) + go t.getTopics(np, partitionedTopicsChan, errChan) + go t.getTopics(p, nonPartitionedTopicsChan, errChan) + go t.getTopics(n, nonPartitionedTopicsChan, errChan) + + requestCount := 4 + for { + select { + case err := <-errChan: + if err != nil { + return nil, nil, err + } + continue + case pTopic := <-partitionedTopicsChan: + requestCount-- + partitionedTopics = append(partitionedTopics, pTopic...) + case npTopic := <-nonPartitionedTopicsChan: + requestCount-- + nonPartitionedTopics = append(nonPartitionedTopics, npTopic...) + } + if requestCount == 0 { + break + } + } + return partitionedTopics, nonPartitionedTopics, nil +} + +func (t *topics) getTopics(endpoint string, out chan<- []string, err chan<- error) { + var topics []string + err <- t.pulsar.Client.Get(endpoint, &topics) + out <- topics +} + +func (t *topics) GetInternalInfo(topic utils.TopicName) (utils.ManagedLedgerInfo, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "internal-info") + var info utils.ManagedLedgerInfo + err := t.pulsar.Client.Get(endpoint, &info) + return info, err +} + +func (t *topics) GetPermissions(topic utils.TopicName) (map[string][]utils.AuthAction, error) { + var permissions map[string][]utils.AuthAction + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "permissions") + err := t.pulsar.Client.Get(endpoint, &permissions) + return permissions, err +} + +func (t *topics) GrantPermission(topic utils.TopicName, role string, action []utils.AuthAction) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "permissions", role) + s := []string{} + for _, v := range action { + s = append(s, v.String()) + } + return t.pulsar.Client.Post(endpoint, s) +} + +func (t *topics) RevokePermission(topic utils.TopicName, role string) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "permissions", role) + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) Lookup(topic utils.TopicName) (utils.LookupData, error) { + var lookup utils.LookupData + endpoint := fmt.Sprintf("%s/%s", t.lookupPath, topic.GetRestPath()) + err := t.pulsar.Client.Get(endpoint, &lookup) + return lookup, err +} + +func (t *topics) GetBundleRange(topic utils.TopicName) (string, error) { + endpoint := fmt.Sprintf("%s/%s/%s", t.lookupPath, topic.GetRestPath(), "bundle") + data, err := t.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + return string(data), err +} + +func (t *topics) GetLastMessageID(topic utils.TopicName) (utils.MessageID, error) { + var messageID utils.MessageID + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "lastMessageId") + err := t.pulsar.Client.Get(endpoint, &messageID) + return messageID, err +} + +func (t *topics) GetMessageID(topic utils.TopicName, timestamp int64) (utils.MessageID, error) { + var messageID utils.MessageID + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "messageid", strconv.FormatInt(timestamp, 10)) + err := t.pulsar.Client.Get(endpoint, &messageID) + return messageID, err +} + +func (t *topics) GetStats(topic utils.TopicName) (utils.TopicStats, error) { + var stats utils.TopicStats + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "stats") + err := t.pulsar.Client.Get(endpoint, &stats) + return stats, err +} + +func (t *topics) GetInternalStats(topic utils.TopicName) (utils.PersistentTopicInternalStats, error) { + var stats utils.PersistentTopicInternalStats + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "internalStats") + err := t.pulsar.Client.Get(endpoint, &stats) + return stats, err +} + +func (t *topics) GetPartitionedStats(topic utils.TopicName, perPartition bool) (utils.PartitionedTopicStats, error) { + var stats utils.PartitionedTopicStats + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitioned-stats") + params := map[string]string{ + "perPartition": strconv.FormatBool(perPartition), + } + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &stats, params, true) + return stats, err +} + +func (t *topics) Terminate(topic utils.TopicName) (utils.MessageID, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "terminate") + var messageID utils.MessageID + err := t.pulsar.Client.PostWithObj(endpoint, nil, &messageID) + return messageID, err +} + +func (t *topics) Offload(topic utils.TopicName, messageID utils.MessageID) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "offload") + return t.pulsar.Client.Put(endpoint, messageID) +} + +func (t *topics) OffloadStatus(topic utils.TopicName) (utils.OffloadProcessStatus, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "offload") + var status utils.OffloadProcessStatus + err := t.pulsar.Client.Get(endpoint, &status) + return status, err +} + +func (t *topics) Unload(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "unload") + return t.pulsar.Client.Put(endpoint, nil) +} + +func (t *topics) Compact(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compaction") + return t.pulsar.Client.Put(endpoint, nil) +} + +func (t *topics) CompactStatus(topic utils.TopicName) (utils.LongRunningProcessStatus, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compaction") + var status utils.LongRunningProcessStatus + err := t.pulsar.Client.Get(endpoint, &status) + return status, err +} + +func (t *topics) GetMessageTTL(topic utils.TopicName) (int, error) { + var ttl int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "messageTTL") + err := t.pulsar.Client.Get(endpoint, &ttl) + return ttl, err +} + +func (t *topics) SetMessageTTL(topic utils.TopicName, messageTTL int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "messageTTL") + var params = make(map[string]string) + params["messageTTL"] = strconv.Itoa(messageTTL) + err := t.pulsar.Client.PostWithQueryParams(endpoint, nil, params) + return err +} + +func (t *topics) RemoveMessageTTL(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "messageTTL") + var params = make(map[string]string) + params["messageTTL"] = strconv.Itoa(0) + err := t.pulsar.Client.DeleteWithQueryParams(endpoint, params) + return err +} + +func (t *topics) GetMaxProducers(topic utils.TopicName) (int, error) { + var maxProducers int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxProducers") + err := t.pulsar.Client.Get(endpoint, &maxProducers) + return maxProducers, err +} + +func (t *topics) SetMaxProducers(topic utils.TopicName, maxProducers int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxProducers") + err := t.pulsar.Client.Post(endpoint, &maxProducers) + return err +} + +func (t *topics) RemoveMaxProducers(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxProducers") + err := t.pulsar.Client.Delete(endpoint) + return err +} + +func (t *topics) GetMaxConsumers(topic utils.TopicName) (int, error) { + var maxConsumers int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxConsumers") + err := t.pulsar.Client.Get(endpoint, &maxConsumers) + return maxConsumers, err +} + +func (t *topics) SetMaxConsumers(topic utils.TopicName, maxConsumers int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxConsumers") + err := t.pulsar.Client.Post(endpoint, &maxConsumers) + return err +} + +func (t *topics) RemoveMaxConsumers(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxConsumers") + err := t.pulsar.Client.Delete(endpoint) + return err +} + +func (t *topics) GetMaxUnackMessagesPerConsumer(topic utils.TopicName) (int, error) { + var maxNum int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnConsumer") + err := t.pulsar.Client.Get(endpoint, &maxNum) + return maxNum, err +} + +func (t *topics) SetMaxUnackMessagesPerConsumer(topic utils.TopicName, maxUnackedNum int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnConsumer") + return t.pulsar.Client.Post(endpoint, &maxUnackedNum) +} + +func (t *topics) RemoveMaxUnackMessagesPerConsumer(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnConsumer") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetMaxUnackMessagesPerSubscription(topic utils.TopicName) (int, error) { + var maxNum int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnSubscription") + err := t.pulsar.Client.Get(endpoint, &maxNum) + return maxNum, err +} + +func (t *topics) SetMaxUnackMessagesPerSubscription(topic utils.TopicName, maxUnackedNum int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnSubscription") + return t.pulsar.Client.Post(endpoint, &maxUnackedNum) +} + +func (t *topics) RemoveMaxUnackMessagesPerSubscription(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnSubscription") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetPersistence(topic utils.TopicName) (*utils.PersistenceData, error) { + var persistenceData utils.PersistenceData + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "persistence") + err := t.pulsar.Client.Get(endpoint, &persistenceData) + return &persistenceData, err +} + +func (t *topics) SetPersistence(topic utils.TopicName, persistenceData utils.PersistenceData) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "persistence") + return t.pulsar.Client.Post(endpoint, &persistenceData) +} + +func (t *topics) RemovePersistence(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "persistence") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetDelayedDelivery(topic utils.TopicName) (*utils.DelayedDeliveryData, error) { + var delayedDeliveryData utils.DelayedDeliveryData + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "delayedDelivery") + err := t.pulsar.Client.Get(endpoint, &delayedDeliveryData) + return &delayedDeliveryData, err +} + +func (t *topics) SetDelayedDelivery(topic utils.TopicName, delayedDeliveryData utils.DelayedDeliveryData) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "delayedDelivery") + return t.pulsar.Client.Post(endpoint, &delayedDeliveryData) +} + +func (t *topics) RemoveDelayedDelivery(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "delayedDelivery") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetDispatchRate(topic utils.TopicName) (*utils.DispatchRateData, error) { + var dispatchRateData utils.DispatchRateData + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "dispatchRate") + err := t.pulsar.Client.Get(endpoint, &dispatchRateData) + return &dispatchRateData, err +} + +func (t *topics) SetDispatchRate(topic utils.TopicName, dispatchRateData utils.DispatchRateData) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "dispatchRate") + return t.pulsar.Client.Post(endpoint, &dispatchRateData) +} + +func (t *topics) RemoveDispatchRate(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "dispatchRate") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetPublishRate(topic utils.TopicName) (*utils.PublishRateData, error) { + var publishRateData utils.PublishRateData + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "publishRate") + err := t.pulsar.Client.Get(endpoint, &publishRateData) + return &publishRateData, err +} + +func (t *topics) SetPublishRate(topic utils.TopicName, publishRateData utils.PublishRateData) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "publishRate") + return t.pulsar.Client.Post(endpoint, &publishRateData) +} + +func (t *topics) RemovePublishRate(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "publishRate") + return t.pulsar.Client.Delete(endpoint) +} +func (t *topics) GetDeduplicationStatus(topic utils.TopicName) (bool, error) { + var enabled bool + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "deduplicationEnabled") + err := t.pulsar.Client.Get(endpoint, &enabled) + return enabled, err +} + +func (t *topics) SetDeduplicationStatus(topic utils.TopicName, enabled bool) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "deduplicationEnabled") + return t.pulsar.Client.Post(endpoint, enabled) +} +func (t *topics) RemoveDeduplicationStatus(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "deduplicationEnabled") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetRetention(topic utils.TopicName, applied bool) (*utils.RetentionPolicies, error) { + var policy utils.RetentionPolicies + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "retention") + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &policy, map[string]string{ + "applied": strconv.FormatBool(applied), + }, true) + return &policy, err +} + +func (t *topics) RemoveRetention(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "retention") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) SetRetention(topic utils.TopicName, data utils.RetentionPolicies) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "retention") + return t.pulsar.Client.Post(endpoint, data) +} + +func (t *topics) GetCompactionThreshold(topic utils.TopicName, applied bool) (int64, error) { + var threshold int64 + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compactionThreshold") + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &threshold, map[string]string{ + "applied": strconv.FormatBool(applied), + }, true) + return threshold, err +} + +func (t *topics) SetCompactionThreshold(topic utils.TopicName, threshold int64) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compactionThreshold") + err := t.pulsar.Client.Post(endpoint, threshold) + return err +} + +func (t *topics) RemoveCompactionThreshold(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compactionThreshold") + err := t.pulsar.Client.Delete(endpoint) + return err +} + +func (t *topics) GetBacklogQuotaMap(topic utils.TopicName, applied bool) (map[utils.BacklogQuotaType]utils.BacklogQuota, + error) { + var backlogQuotaMap map[utils.BacklogQuotaType]utils.BacklogQuota + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "backlogQuotaMap") + + queryParams := map[string]string{"applied": strconv.FormatBool(applied)} + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &backlogQuotaMap, queryParams, true) + + return backlogQuotaMap, err +} + +func (t *topics) SetBacklogQuota(topic utils.TopicName, backlogQuota utils.BacklogQuota, + backlogQuotaType utils.BacklogQuotaType) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "backlogQuota") + params := make(map[string]string) + params["backlogQuotaType"] = string(backlogQuotaType) + return t.pulsar.Client.PostWithQueryParams(endpoint, &backlogQuota, params) +} + +func (t *topics) RemoveBacklogQuota(topic utils.TopicName, backlogQuotaType utils.BacklogQuotaType) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "backlogQuota") + return t.pulsar.Client.DeleteWithQueryParams(endpoint, map[string]string{ + "backlogQuotaType": string(backlogQuotaType), + }) +} + +func (t *topics) GetInactiveTopicPolicies(topic utils.TopicName, applied bool) (utils.InactiveTopicPolicies, error) { + var out utils.InactiveTopicPolicies + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "inactiveTopicPolicies") + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &out, map[string]string{ + "applied": strconv.FormatBool(applied), + }, true) + return out, err +} + +func (t *topics) RemoveInactiveTopicPolicies(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "inactiveTopicPolicies") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) SetInactiveTopicPolicies(topic utils.TopicName, data utils.InactiveTopicPolicies) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "inactiveTopicPolicies") + return t.pulsar.Client.Post(endpoint, data) +} + +func (t *topics) SetReplicationClusters(topic utils.TopicName, data []string) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "replication") + return t.pulsar.Client.Post(endpoint, data) +} + +func (t *topics) GetReplicationClusters(topic utils.TopicName) ([]string, error) { + var data []string + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "replication") + err := t.pulsar.Client.Get(endpoint, &data) + return data, err +} diff --git a/pulsaradmin/pkg/rest/client.go b/pulsaradmin/pkg/rest/client.go new file mode 100644 index 000000000..a1a79737c --- /dev/null +++ b/pulsaradmin/pkg/rest/client.go @@ -0,0 +1,414 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rest + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/url" + "path" +) + +// Client is a base client that is used to make http request to the ServiceURL +type Client struct { + ServiceURL string + HTTPClient *http.Client + VersionInfo string +} + +func (c *Client) newRequest(method, path string) (*request, error) { + base, err := url.Parse(c.ServiceURL) + if err != nil { + return nil, err + } + u, err := url.Parse(path) + if err != nil { + return nil, err + } + + req := &request{ + method: method, + url: &url.URL{ + Scheme: base.Scheme, + User: base.User, + Host: base.Host, + Path: endpoint(base.Path, u.Path), + }, + params: make(url.Values), + } + return req, nil +} + +func (c *Client) doRequest(r *request) (*http.Response, error) { + req, err := r.toHTTP() + if err != nil { + return nil, err + } + + if r.contentType != "" { + req.Header.Set("Content-Type", r.contentType) + } else if req.Body != nil { + req.Header.Set("Content-Type", "application/json") + } + + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", c.useragent()) + hc := c.HTTPClient + if hc == nil { + hc = http.DefaultClient + } + + return hc.Do(req) +} + +// MakeRequest can make a simple request and handle the response by yourself +func (c *Client) MakeRequest(method, endpoint string) (*http.Response, error) { + req, err := c.newRequest(method, endpoint) + if err != nil { + return nil, err + } + + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *Client) Get(endpoint string, obj interface{}) error { + _, err := c.GetWithQueryParams(endpoint, obj, nil, true) + return err +} + +func (c *Client) GetWithQueryParams(endpoint string, obj interface{}, params map[string]string, + decode bool) ([]byte, error) { + return c.GetWithOptions(endpoint, obj, params, decode, nil) +} + +func (c *Client) GetWithOptions(endpoint string, obj interface{}, params map[string]string, + decode bool, file io.Writer) ([]byte, error) { + + req, err := c.newRequest(http.MethodGet, endpoint) + if err != nil { + return nil, err + } + + if params != nil { + query := req.url.Query() + for k, v := range params { + query.Add(k, v) + } + req.params = query + } + + //nolint:bodyclose + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return nil, err + } + defer safeRespClose(resp) + + if obj != nil { + if err := decodeJSONBody(resp, &obj); err != nil { + if err == io.EOF { + return nil, nil + } + return nil, err + } + } else if !decode { + if file != nil { + _, err := io.Copy(file, resp.Body) + if err != nil { + return nil, err + } + } else { + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, err + } + } + + return nil, err +} + +func (c *Client) useragent() string { + return c.VersionInfo +} + +func (c *Client) Put(endpoint string, in interface{}) error { + return c.PutWithQueryParams(endpoint, in, nil, nil) +} + +func (c *Client) PutWithQueryParams(endpoint string, in, obj interface{}, params map[string]string) error { + req, err := c.newRequest(http.MethodPut, endpoint) + if err != nil { + return err + } + req.obj = in + + if params != nil { + query := req.url.Query() + for k, v := range params { + query.Add(k, v) + } + req.params = query + } + + //nolint:bodyclose + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + if obj != nil { + if err := decodeJSONBody(resp, &obj); err != nil { + return err + } + } + + return nil +} + +func (c *Client) PutWithMultiPart(endpoint string, body io.Reader, contentType string) error { + req, err := c.newRequest(http.MethodPut, endpoint) + if err != nil { + return err + } + req.body = body + req.contentType = contentType + + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + return nil +} + +func (c *Client) Delete(endpoint string) error { + return c.DeleteWithQueryParams(endpoint, nil) +} + +func (c *Client) DeleteWithQueryParams(endpoint string, params map[string]string) error { + req, err := c.newRequest(http.MethodDelete, endpoint) + if err != nil { + return err + } + + if params != nil { + query := req.url.Query() + for k, v := range params { + query.Add(k, v) + } + req.params = query + } + + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + return nil +} + +func (c *Client) Post(endpoint string, in interface{}) error { + return c.PostWithObj(endpoint, in, nil) +} + +func (c *Client) PostWithObj(endpoint string, in, obj interface{}) error { + req, err := c.newRequest(http.MethodPost, endpoint) + if err != nil { + return err + } + req.obj = in + + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + if obj != nil { + if err := decodeJSONBody(resp, &obj); err != nil { + return err + } + } + + return nil +} + +func (c *Client) PostWithMultiPart(endpoint string, in interface{}, body io.Reader, contentType string) error { + req, err := c.newRequest(http.MethodPost, endpoint) + if err != nil { + return err + } + req.obj = in + req.body = body + req.contentType = contentType + + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + return nil +} + +func (c *Client) PostWithQueryParams(endpoint string, in interface{}, params map[string]string) error { + req, err := c.newRequest(http.MethodPost, endpoint) + if err != nil { + return err + } + if in != nil { + req.obj = in + } + if params != nil { + query := req.url.Query() + for k, v := range params { + query.Add(k, v) + } + req.params = query + } + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + return nil +} + +type request struct { + method string + contentType string + url *url.URL + params url.Values + + obj interface{} + body io.Reader +} + +func (r *request) toHTTP() (*http.Request, error) { + r.url.RawQuery = r.params.Encode() + + // add a request body if there is one + if r.body == nil && r.obj != nil { + body, err := encodeJSONBody(r.obj) + if err != nil { + return nil, err + } + r.body = body + } + + req, err := http.NewRequest(r.method, r.url.RequestURI(), r.body) + if err != nil { + return nil, err + } + + req.URL.Host = r.url.Host + req.URL.Scheme = r.url.Scheme + req.Host = r.url.Host + return req, nil +} + +// respIsOk is used to validate a successful http status code +func respIsOk(resp *http.Response) bool { + return resp.StatusCode >= http.StatusOK && resp.StatusCode <= http.StatusNoContent +} + +// checkSuccessful checks for a valid response and parses an error +func checkSuccessful(resp *http.Response, err error) (*http.Response, error) { + if err != nil { + safeRespClose(resp) + return nil, err + } + + if !respIsOk(resp) { + defer safeRespClose(resp) + return nil, responseError(resp) + } + + return resp, nil +} + +func endpoint(parts ...string) string { + return path.Join(parts...) +} + +// encodeJSONBody is used to JSON encode a body +func encodeJSONBody(obj interface{}) (io.Reader, error) { + b, err := json.Marshal(obj) + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil +} + +// decodeJSONBody is used to JSON decode a body +func decodeJSONBody(resp *http.Response, out interface{}) error { + if resp.ContentLength == 0 { + return nil + } + dec := json.NewDecoder(resp.Body) + return dec.Decode(out) +} + +// safeRespClose is used to close a response body +func safeRespClose(resp *http.Response) { + if resp != nil { + // ignore error since it is closing a response body + _ = resp.Body.Close() + } +} + +// responseError is used to parse a response into a client error +func responseError(resp *http.Response) error { + e := Error{ + Code: resp.StatusCode, + Reason: resp.Status, + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + e.Reason = err.Error() + return e + } + + err = json.Unmarshal(body, &e) + if err != nil { + if len(body) != 0 { + e.Reason = string(body) + } + return e + } + + return e +} diff --git a/pulsaradmin/pkg/rest/client_test.go b/pulsaradmin/pkg/rest/client_test.go new file mode 100644 index 000000000..50a7c5a84 --- /dev/null +++ b/pulsaradmin/pkg/rest/client_test.go @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rest + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEncodeJSONBody(t *testing.T) { + testcases := []struct { + obj interface{} + expected int + }{ + {obj: "1", expected: 3}, + {obj: "12", expected: 4}, + {obj: 1, expected: 1}, + {obj: 12, expected: 2}, + } + + for _, testcase := range testcases { + r, err := encodeJSONBody(testcase.obj) + require.NoError(t, err) + + b, err := io.ReadAll(r) + require.NoError(t, err) + + require.Equal(t, testcase.expected, len(b)) + } +} diff --git a/pulsaradmin/pkg/rest/errors.go b/pulsaradmin/pkg/rest/errors.go new file mode 100644 index 000000000..dc611ec38 --- /dev/null +++ b/pulsaradmin/pkg/rest/errors.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rest + +import "fmt" + +// Error is a admin error type +type Error struct { + Reason string `json:"reason"` + Code int +} + +func (e Error) Error() string { + return fmt.Sprintf("code: %d reason: %s", e.Code, e.Reason) +} + +func IsAdminError(err error) bool { + _, ok := err.(Error) + return ok +} diff --git a/pulsaradmin/pkg/utils/allocator_stats.go b/pulsaradmin/pkg/utils/allocator_stats.go new file mode 100644 index 000000000..4c2a7fb52 --- /dev/null +++ b/pulsaradmin/pkg/utils/allocator_stats.go @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type AllocatorStats struct { + NumDirectArenas int `json:"numDirectArenas"` + NumHeapArenas int `json:"numHeapArenas"` + NumThreadLocalCaches int `json:"numThreadLocalCaches"` + NormalCacheSize int `json:"normalCacheSize"` + SmallCacheSize int `json:"smallCacheSize"` + TinyCacheSize int `json:"tinyCacheSize"` + DirectArenas []PoolArenaStats `json:"directArenas"` + HeapArenas []PoolArenaStats `json:"heapArenas"` +} + +type PoolArenaStats struct { + NumTinySubpages int `json:"numTinySubpages"` + NumSmallSubpages int `json:"numSmallSubpages"` + NumChunkLists int `json:"numChunkLists"` + TinySubpages []PoolSubpageStats `json:"tinySubpages"` + SmallSubpages []PoolSubpageStats `json:"smallSubpages"` + ChunkLists []PoolChunkListStats `json:"chunkLists"` + NumAllocations int64 `json:"numAllocations"` + NumTinyAllocations int64 `json:"numTinyAllocations"` + NumSmallAllocations int64 `json:"numSmallAllocations"` + NumNormalAllocations int64 `json:"numNormalAllocations"` + NumHugeAllocations int64 `json:"numHugeAllocations"` + NumDeallocations int64 `json:"numDeallocations"` + NumTinyDeallocations int64 `json:"numTinyDeallocations"` + NumSmallDeallocations int64 `json:"numSmallDeallocations"` + NumNormalDeallocations int64 `json:"numNormalDeallocations"` + NumHugeDeallocations int64 `json:"numHugeDeallocations"` + NumActiveAllocations int64 `json:"numActiveAllocations"` + NumActiveTinyAllocations int64 `json:"numActiveTinyAllocations"` + NumActiveSmallAllocations int64 `json:"numActiveSmallAllocations"` + NumActiveNormalAllocations int64 `json:"numActiveNormalAllocations"` + NumActiveHugeAllocations int64 `json:"numActiveHugeAllocations"` +} + +type PoolSubpageStats struct { + MaxNumElements int `json:"maxNumElements"` + NumAvailable int `json:"numAvailable"` + ElementSize int `json:"elementSize"` + PageSize int `json:"pageSize"` +} + +type PoolChunkListStats struct { + MinUsage int `json:"minUsage"` + MaxUsage int `json:"maxUsage"` + Chunks []PoolChunkStats `json:"chunks"` +} + +type PoolChunkStats struct { + Usage int `json:"usage"` + ChunkSize int `json:"chunkSize"` + FreeBytes int `json:"freeBytes"` +} diff --git a/pulsaradmin/pkg/utils/auth_action.go b/pulsaradmin/pkg/utils/auth_action.go new file mode 100644 index 000000000..7f2bf2573 --- /dev/null +++ b/pulsaradmin/pkg/utils/auth_action.go @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type AuthAction string + +const ( + produce AuthAction = "produce" + consume AuthAction = "consume" + functionsAuth AuthAction = "functions" + packages AuthAction = "packages" + sinks AuthAction = "sinks" + sources AuthAction = "sources" +) + +func ParseAuthAction(action string) (AuthAction, error) { + switch action { + case "produce": + return produce, nil + case "consume": + return consume, nil + case "functions": + return functionsAuth, nil + case "packages": + return packages, nil + case "sinks": + return sinks, nil + case "sources": + return sources, nil + default: + return "", errors.Errorf("The auth action only can be specified as 'produce', "+ + "'consume', 'sources', 'sinks', 'packages', or 'functions'. Invalid auth action '%s'", action) + } +} + +func (a AuthAction) String() string { + return string(a) +} diff --git a/pulsaradmin/pkg/utils/auth_polices.go b/pulsaradmin/pkg/utils/auth_polices.go new file mode 100644 index 000000000..065b3c434 --- /dev/null +++ b/pulsaradmin/pkg/utils/auth_polices.go @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type AuthPolicies struct { + NamespaceAuth map[string][]AuthAction `json:"namespace_auth"` + DestinationAuth map[string]map[string][]AuthAction `json:"destination_auth"` + SubscriptionAuthRoles map[string][]string `json:"subscription_auth_roles"` +} + +func NewAuthPolicies() *AuthPolicies { + return &AuthPolicies{ + NamespaceAuth: make(map[string][]AuthAction), + DestinationAuth: make(map[string]map[string][]AuthAction), + SubscriptionAuthRoles: make(map[string][]string), + } +} diff --git a/pulsaradmin/pkg/utils/auth_polices_test.go b/pulsaradmin/pkg/utils/auth_polices_test.go new file mode 100644 index 000000000..014f594bd --- /dev/null +++ b/pulsaradmin/pkg/utils/auth_polices_test.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAuthPolicies(t *testing.T) { + testData := "{\n" + + " \"namespace_auth\": {\n" + + " \"persistent://public/default/ns_auth\": [\n" + + " \"produce\",\n" + + " \"consume\",\n" + + " \"function\"\n" + + " ]\n" + + " },\n" + + " \"destination_auth\": {\n" + + " \"persistent://public/default/dest_auth\": {\n" + + " \"admin-role\": [\n" + + " \"produce\",\n" + + " \"consume\",\n" + + " \"function\"\n" + + " ]\n" + + " },\n" + + " \"persistent://public/default/dest_auth_1\": {\n" + + " \"grant-partitioned-role\": [\n" + + " \"produce\",\n" + + " \"consume\"\n" + + " ]\n" + + " },\n" + + " \"persistent://public/default/test-revoke-partitioned-topic\": {},\n" + + " \"persistent://public/default/test-revoke-non-partitioned-topic\": {}\n },\n" + + " \"subscription_auth_roles\": {}\n" + + "}" + + policies := &AuthPolicies{} + err := json.Unmarshal([]byte(testData), policies) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 3, len(policies.NamespaceAuth["persistent://public/default/ns_auth"])) + assert.Equal(t, 4, len(policies.DestinationAuth)) + assert.Equal(t, 3, len(policies.DestinationAuth["persistent://public/default/dest_auth"]["admin-role"])) + assert.Equal(t, 0, len(policies.SubscriptionAuthRoles)) +} diff --git a/pulsaradmin/pkg/utils/backlog_quota.go b/pulsaradmin/pkg/utils/backlog_quota.go new file mode 100644 index 000000000..1930cad8c --- /dev/null +++ b/pulsaradmin/pkg/utils/backlog_quota.go @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type BacklogQuota struct { + LimitTime int64 `json:"limitTime"` + LimitSize int64 `json:"limitSize"` + Policy RetentionPolicy `json:"policy"` +} + +func NewBacklogQuota(limitSize int64, limitTime int64, policy RetentionPolicy) BacklogQuota { + return BacklogQuota{ + LimitSize: limitSize, + LimitTime: limitTime, + Policy: policy, + } +} + +type RetentionPolicy string + +const ( + ProducerRequestHold RetentionPolicy = "producer_request_hold" + ProducerException RetentionPolicy = "producer_exception" + ConsumerBacklogEviction RetentionPolicy = "consumer_backlog_eviction" +) + +func ParseRetentionPolicy(str string) (RetentionPolicy, error) { + switch str { + case ProducerRequestHold.String(): + return ProducerRequestHold, nil + case ProducerException.String(): + return ProducerException, nil + case ConsumerBacklogEviction.String(): + return ConsumerBacklogEviction, nil + default: + return "", errors.Errorf("Invalid retention policy %s", str) + } +} + +func (s RetentionPolicy) String() string { + return string(s) +} + +type BacklogQuotaType string + +const ( + DestinationStorage BacklogQuotaType = "destination_storage" + MessageAge BacklogQuotaType = "message_age" +) + +func ParseBacklogQuotaType(str string) (BacklogQuotaType, error) { + switch str { + case "": + fallthrough + case DestinationStorage.String(): + return DestinationStorage, nil + case MessageAge.String(): + return MessageAge, nil + default: + return "", errors.Errorf("Invalid backlog quota type: %s", str) + } +} + +func (b BacklogQuotaType) String() string { + return string(b) +} diff --git a/pulsaradmin/pkg/utils/batch_source_config.go b/pulsaradmin/pkg/utils/batch_source_config.go new file mode 100644 index 000000000..3db2db697 --- /dev/null +++ b/pulsaradmin/pkg/utils/batch_source_config.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +const ( + BatchsourceConfigKey string = "__BATCHSOURCECONFIGS__" + BatchsourceClassnameKey string = "__BATCHSOURCECLASSNAME__" +) + +type BatchSourceConfig struct { + DiscoveryTriggererClassName string `json:"discoveryTriggererClassName" yaml:"discoveryTriggererClassName"` + + DiscoveryTriggererConfig map[string]interface{} `json:"discoveryTriggererConfig" yaml:"discoveryTriggererConfig"` +} diff --git a/pulsaradmin/pkg/utils/broker_ns_isolation_data.go b/pulsaradmin/pkg/utils/broker_ns_isolation_data.go new file mode 100644 index 000000000..1381647b6 --- /dev/null +++ b/pulsaradmin/pkg/utils/broker_ns_isolation_data.go @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type BrokerNamespaceIsolationData struct { + BrokerName string `json:"brokerName"` + PolicyName string `json:"policyName"` + IsPrimary bool `json:"isPrimary"` + NamespaceRegex []string `json:"namespaceRegex"` +} diff --git a/pulsaradmin/pkg/utils/bundles_data.go b/pulsaradmin/pkg/utils/bundles_data.go new file mode 100644 index 000000000..2bba7ae49 --- /dev/null +++ b/pulsaradmin/pkg/utils/bundles_data.go @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type BundlesData struct { + Boundaries []string `json:"boundaries"` + NumBundles int `json:"numBundles"` +} + +func NewBundlesData(boundaries []string) BundlesData { + return BundlesData{ + Boundaries: boundaries, + NumBundles: len(boundaries) - 1, + } +} + +func NewBundlesDataWithNumBundles(numBundles int) *BundlesData { + return &BundlesData{ + Boundaries: nil, + NumBundles: numBundles, + } +} + +func NewDefaultBoundle() *BundlesData { + bundleData := NewBundlesDataWithNumBundles(1) + bundleData.Boundaries = append(bundleData.Boundaries, FirstBoundary, LastBoundary) + return bundleData +} diff --git a/pulsaradmin/pkg/utils/connector_definition.go b/pulsaradmin/pkg/utils/connector_definition.go new file mode 100644 index 000000000..72b12893c --- /dev/null +++ b/pulsaradmin/pkg/utils/connector_definition.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +// Basic information about a Pulsar connector +type ConnectorDefinition struct { + // The name of the connector type + Name string `json:"name"` + + // Description to be used for user help + Description string `json:"description"` + + // The class name for the connector source implementation + //

If not defined, it will be assumed this connector cannot act as a data source + SourceClass string `json:"sourceClass"` + + // The class name for the connector sink implementation + //

If not defined, it will be assumed this connector cannot act as a data sink + SinkClass string `json:"sinkClass"` +} diff --git a/pulsaradmin/pkg/utils/consumer_config.go b/pulsaradmin/pkg/utils/consumer_config.go new file mode 100644 index 000000000..f609ae48d --- /dev/null +++ b/pulsaradmin/pkg/utils/consumer_config.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type ConsumerConfig struct { + SchemaType string `json:"schemaType,omitempty" yaml:"schemaType"` + SerdeClassName string `json:"serdeClassName,omitempty" yaml:"serdeClassName"` + RegexPattern bool `json:"regexPattern,omitempty" yaml:"regexPattern"` + ReceiverQueueSize int `json:"receiverQueueSize,omitempty" yaml:"receiverQueueSize"` + SchemaProperties map[string]string `json:"schemaProperties,omitempty" yaml:"schemaProperties"` + ConsumerProperties map[string]string `json:"consumerProperties,omitempty" yaml:"consumerProperties"` + CryptoConfig *CryptoConfig `json:"cryptoConfig,omitempty" yaml:"cryptoConfig"` + PoolMessages bool `json:"poolMessages,omitempty" yaml:"poolMessages"` +} diff --git a/pulsaradmin/pkg/utils/crypto_config.go b/pulsaradmin/pkg/utils/crypto_config.go new file mode 100644 index 000000000..e411bb800 --- /dev/null +++ b/pulsaradmin/pkg/utils/crypto_config.go @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type CryptoConfig struct { + CryptoKeyReaderClassName string `json:"cryptoKeyReaderClassName" yaml:"cryptoKeyReaderClassName"` + CryptoKeyReaderConfig map[string]interface{} `json:"cryptoKeyReaderConfig" yaml:"cryptoKeyReaderConfig"` + + EncryptionKeys []string `json:"encryptionKeys" yaml:"encryptionKeys"` + ProducerCryptoFailureAction string `json:"producerCryptoFailureAction" yaml:"producerCryptoFailureAction"` + ConsumerCryptoFailureAction string `json:"consumerCryptoFailureAction" yaml:"consumerCryptoFailureAction"` +} diff --git a/pulsaradmin/pkg/utils/data.go b/pulsaradmin/pkg/utils/data.go new file mode 100644 index 000000000..55888aab2 --- /dev/null +++ b/pulsaradmin/pkg/utils/data.go @@ -0,0 +1,467 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +// ClusterData information on a cluster +type ClusterData struct { + Name string `json:"-"` + ServiceURL string `json:"serviceUrl"` + ServiceURLTls string `json:"serviceUrlTls"` + BrokerServiceURL string `json:"brokerServiceUrl"` + BrokerServiceURLTls string `json:"brokerServiceUrlTls"` + PeerClusterNames []string `json:"peerClusterNames"` + AuthenticationPlugin string `json:"authenticationPlugin"` + AuthenticationParameters string `json:"authenticationParameters"` + BrokerClientTrustCertsFilePath string `json:"brokerClientTrustCertsFilePath"` + BrokerClientTLSEnabled bool `json:"brokerClientTlsEnabled"` +} + +// FunctionData information for a Pulsar Function +type FunctionData struct { + UpdateAuthData bool `json:"updateAuthData"` + RetainOrdering bool `json:"retainOrdering"` + Watch bool `json:"watch"` + AutoAck bool `json:"autoAck"` + Parallelism int `json:"parallelism"` + WindowLengthCount int `json:"windowLengthCount"` + SlidingIntervalCount int `json:"slidingIntervalCount"` + MaxMessageRetries int `json:"maxMessageRetries"` + + TimeoutMs int64 `json:"timeoutMs"` + SlidingIntervalDurationMs int64 `json:"slidingIntervalDurationMs"` + WindowLengthDurationMs int64 `json:"windowLengthDurationMs"` + RAM int64 `json:"ram"` + Disk int64 `json:"disk"` + CPU float64 `json:"cpu"` + SubsName string `json:"subsName"` + DeadLetterTopic string `json:"deadLetterTopic"` + Key string `json:"key"` + State string `json:"state"` + TriggerValue string `json:"triggerValue"` + TriggerFile string `json:"triggerFile"` + Topic string `json:"topic"` + + UserCodeFile string `json:"-"` + FQFN string `json:"fqfn"` + Tenant string `json:"tenant"` + Namespace string `json:"namespace"` + FuncName string `json:"functionName"` + InstanceID string `json:"instance_id"` + ClassName string `json:"className"` + FunctionType string `json:"functionType"` + CleanupSubscription bool `json:"cleanupSubscription"` + Jar string `json:"jarFile"` + Py string `json:"pyFile"` + Go string `json:"goFile"` + Inputs string `json:"inputs"` + TopicsPattern string `json:"topicsPattern"` + Output string `json:"output"` + ProducerConfig string `json:"producerConfig"` + LogTopic string `json:"logTopic"` + SchemaType string `json:"schemaType"` + CustomSerDeInputs string `json:"customSerdeInputString"` + CustomSchemaInput string `json:"customSchemaInputString"` + CustomSchemaOutput string `json:"customSchemaOutputString"` + InputSpecs string `json:"inputSpecs"` + InputTypeClassName string `json:"inputTypeClassName"` + OutputSerDeClassName string `json:"outputSerdeClassName"` + OutputTypeClassName string `json:"outputTypeClassName"` + FunctionConfigFile string `json:"fnConfigFile"` + ProcessingGuarantees string `json:"processingGuarantees"` + UserConfig string `json:"userConfigString"` + RetainKeyOrdering bool `json:"retainKeyOrdering"` + BatchBuilder string `json:"batchBuilder"` + ForwardSourceMessageProperty bool `json:"forwardSourceMessageProperty"` + SubsPosition string `json:"subsPosition"` + SkipToLatest bool `json:"skipToLatest"` + CustomRuntimeOptions string `json:"customRuntimeOptions"` + Secrets string `json:"secretsString"` + DestinationFile string `json:"destinationFile"` + Path string `json:"path"` + FuncConf *FunctionConfig `json:"-"` +} + +// Failure Domain information +type FailureDomainData struct { + ClusterName string `json:"-"` + DomainName string `json:"-"` + BrokerList []string `json:"brokers"` +} + +type FailureDomainMap map[string]FailureDomainData + +// Tenant args +type TenantData struct { + Name string `json:"-"` + AdminRoles []string `json:"adminRoles"` + AllowedClusters []string `json:"allowedClusters"` +} + +type SourceData struct { + Tenant string `json:"tenant,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + SourceType string `json:"sourceType,omitempty"` + ProcessingGuarantees string `json:"processingGuarantees,omitempty"` + DestinationTopicName string `json:"destinationTopicName,omitempty"` + ProducerConfig string `json:"producerConfig,omitempty"` + BatchBuilder string `json:"batchBuilder,omitempty"` + DeserializationClassName string `json:"deserializationClassName,omitempty"` + SchemaType string `json:"schemaType,omitempty"` + Parallelism int `json:"parallelism,omitempty"` + Archive string `json:"archive,omitempty"` + ClassName string `json:"className,omitempty"` + SourceConfigFile string `json:"sourceConfigFile,omitempty"` + CPU float64 `json:"cpu,omitempty"` + RAM int64 `json:"ram,omitempty"` + Disk int64 `json:"disk,omitempty"` + SourceConfigString string `json:"sourceConfigString,omitempty"` + BatchSourceConfigString string `json:"batchSourceConfigString,omitempty"` + CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty"` + Secrets string `json:"secretsString,omitempty"` + + SourceConf *SourceConfig `json:"-,omitempty"` + InstanceID string `json:"instanceId,omitempty"` + + UpdateAuthData bool `json:"updateAuthData,omitempty"` +} + +type SinkData struct { + UpdateAuthData bool `json:"updateAuthData,omitempty"` + RetainOrdering bool `json:"retainOrdering,omitempty"` + AutoAck bool `json:"autoAck,omitempty"` + Parallelism int `json:"parallelism,omitempty"` + RAM int64 `json:"ram,omitempty"` + Disk int64 `json:"disk,omitempty"` + TimeoutMs int64 `json:"timeoutMs,omitempty"` + CPU float64 `json:"cpu,omitempty"` + Tenant string `json:"tenant,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + SinkType string `json:"sinkType,omitempty"` + CleanupSubscription bool `json:"cleanupSubscription"` + Inputs string `json:"inputs,omitempty"` + TopicsPattern string `json:"topicsPattern,omitempty"` + SubsName string `json:"subsName,omitempty"` + SubsPosition string `json:"subsPosition,omitempty"` + CustomSerdeInputString string `json:"customSerdeInputString,omitempty"` + CustomSchemaInputString string `json:"customSchemaInputString,omitempty"` + InputSpecs string `json:"inputSpecs,omitempty"` + MaxMessageRetries int `json:"maxMessageRetries,omitempty"` + DeadLetterTopic string `json:"deadLetterTopic,omitempty"` + ProcessingGuarantees string `json:"processingGuarantees,omitempty"` + RetainKeyOrdering bool `json:"retainKeyOrdering,omitempty"` + Archive string `json:"archive,omitempty"` + ClassName string `json:"className,omitempty"` + SinkConfigFile string `json:"sinkConfigFile,omitempty"` + SinkConfigString string `json:"sinkConfigString,omitempty"` + NegativeAckRedeliveryDelayMs int64 `json:"negativeAckRedeliveryDelayMs,omitempty"` + CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty"` + Secrets string `json:"secretsString,omitempty"` + InstanceID string `json:"instanceId,omitempty"` + TransformFunction string `json:"transformFunction,omitempty"` + TransformFunctionClassName string `json:"transformFunctionClassName,omitempty"` + TransformFunctionConfig string `json:"transformFunctionConfig,omitempty"` + SinkConf *SinkConfig `json:"-,omitempty"` +} + +// Topic data +type PartitionedTopicMetadata struct { + Partitions int `json:"partitions"` +} + +type ManagedLedgerInfoLedgerInfo struct { + LedgerID int64 `json:"ledgerId"` + Entries int64 `json:"entries"` + Size int64 `json:"size"` + Timestamp int64 `json:"timestamp"` + Offloaded bool `json:"isOffloaded"` + OffloadedContextUUID string `json:"offloadedContextUuid"` +} + +type ManagedLedgerInfo struct { + Version int `json:"version"` + CreationDate string `json:"creationDate"` + ModificationData string `json:"modificationData"` + Ledgers []ManagedLedgerInfoLedgerInfo `json:"ledgers"` + TerminatedPosition PositionInfo `json:"terminatedPosition"` + Cursors map[string]CursorInfo `json:"cursors"` +} + +type NamespacesData struct { + Enable bool `json:"enable"` + Unload bool `json:"unload"` + NumBundles int `json:"numBundles"` + BookkeeperEnsemble int `json:"bookkeeperEnsemble"` + BookkeeperWriteQuorum int `json:"bookkeeperWriteQuorum"` + MessageTTL int `json:"messageTTL"` + BookkeeperAckQuorum int `json:"bookkeeperAckQuorum"` + ManagedLedgerMaxMarkDeleteRate float64 `json:"managedLedgerMaxMarkDeleteRate"` + ClusterIds string `json:"clusterIds"` + RetentionTimeStr string `json:"retentionTimeStr"` + LimitStr string `json:"limitStr"` + LimitTime int64 `json:"limitTime"` + PolicyStr string `json:"policyStr"` + BacklogQuotaType string `json:"backlogQuotaType"` + AntiAffinityGroup string `json:"antiAffinityGroup"` + Tenant string `json:"tenant"` + Cluster string `json:"cluster"` + Bundle string `json:"bundle"` + Clusters []string `json:"clusters"` +} + +type TopicStats struct { + BacklogSize int64 `json:"backlogSize"` + MsgCounterIn int64 `json:"msgInCounter"` + MsgCounterOut int64 `json:"msgOutCounter"` + MsgRateIn float64 `json:"msgRateIn"` + MsgRateOut float64 `json:"msgRateOut"` + MsgThroughputIn float64 `json:"msgThroughputIn"` + MsgThroughputOut float64 `json:"msgThroughputOut"` + AverageMsgSize float64 `json:"averageMsgSize"` + StorageSize int64 `json:"storageSize"` + Publishers []PublisherStats `json:"publishers"` + Subscriptions map[string]SubscriptionStats `json:"subscriptions"` + Replication map[string]ReplicatorStats `json:"replication"` + DeDuplicationStatus string `json:"deduplicationStatus"` +} + +type PublisherStats struct { + ProducerID int64 `json:"producerId"` + MsgRateIn float64 `json:"msgRateIn"` + MsgThroughputIn float64 `json:"msgThroughputIn"` + AverageMsgSize float64 `json:"averageMsgSize"` + Metadata map[string]string `json:"metadata"` +} + +type SubscriptionStats struct { + BlockedSubscriptionOnUnackedMsgs bool `json:"blockedSubscriptionOnUnackedMsgs"` + IsReplicated bool `json:"isReplicated"` + LastConsumedFlowTimestamp int64 `json:"lastConsumedFlowTimestamp"` + LastConsumedTimestamp int64 `json:"lastConsumedTimestamp"` + LastAckedTimestamp int64 `json:"lastAckedTimestamp"` + MsgRateOut float64 `json:"msgRateOut"` + MsgThroughputOut float64 `json:"msgThroughputOut"` + MsgRateRedeliver float64 `json:"msgRateRedeliver"` + MsgRateExpired float64 `json:"msgRateExpired"` + MsgBacklog int64 `json:"msgBacklog"` + MsgBacklogNoDelayed int64 `json:"msgBacklogNoDelayed"` + MsgDelayed int64 `json:"msgDelayed"` + UnAckedMessages int64 `json:"unackedMessages"` + SubType string `json:"type"` + ActiveConsumerName string `json:"activeConsumerName"` + Consumers []ConsumerStats `json:"consumers"` +} + +type ConsumerStats struct { + BlockedConsumerOnUnAckedMsgs bool `json:"blockedConsumerOnUnackedMsgs"` + AvailablePermits int `json:"availablePermits"` + UnAckedMessages int `json:"unackedMessages"` + MsgRateOut float64 `json:"msgRateOut"` + MsgThroughputOut float64 `json:"msgThroughputOut"` + MsgRateRedeliver float64 `json:"msgRateRedeliver"` + ConsumerName string `json:"consumerName"` + Metadata map[string]string `json:"metadata"` +} + +type ReplicatorStats struct { + Connected bool `json:"connected"` + MsgRateIn float64 `json:"msgRateIn"` + MsgRateOut float64 `json:"msgRateOut"` + MsgThroughputIn float64 `json:"msgThroughputIn"` + MsgThroughputOut float64 `json:"msgThroughputOut"` + MsgRateExpired float64 `json:"msgRateExpired"` + ReplicationBacklog int64 `json:"replicationBacklog"` + ReplicationDelayInSeconds int64 `json:"replicationDelayInSeconds"` + InboundConnection string `json:"inboundConnection"` + InboundConnectedSince string `json:"inboundConnectedSince"` + OutboundConnection string `json:"outboundConnection"` + OutboundConnectedSince string `json:"outboundConnectedSince"` +} + +type PersistentTopicInternalStats struct { + WaitingCursorsCount int `json:"waitingCursorsCount"` + PendingAddEntriesCount int `json:"pendingAddEntriesCount"` + EntriesAddedCounter int64 `json:"entriesAddedCounter"` + NumberOfEntries int64 `json:"numberOfEntries"` + TotalSize int64 `json:"totalSize"` + CurrentLedgerEntries int64 `json:"currentLedgerEntries"` + CurrentLedgerSize int64 `json:"currentLedgerSize"` + LastLedgerCreatedTimestamp string `json:"lastLedgerCreatedTimestamp"` + LastLedgerCreationFailureTimestamp string `json:"lastLedgerCreationFailureTimestamp"` + LastConfirmedEntry string `json:"lastConfirmedEntry"` + State string `json:"state"` + Ledgers []LedgerInfo `json:"ledgers"` + Cursors map[string]CursorStats `json:"cursors"` + SchemaLedgers []SchemaLedger `json:"schemaLedgers"` + CompactedLedger CompactedLedger `json:"compactedLedger"` +} + +type LedgerInfo struct { + LedgerID int64 `json:"ledgerId"` + Entries int64 `json:"entries"` + Size int64 `json:"size"` + Timestamp int64 `json:"timestamp"` + Offloaded bool `json:"offloaded"` + MetaData string `json:"metadata"` + UnderReplicated bool `json:"underReplicated"` +} + +type CursorInfo struct { + Version int `json:"version"` + CreationDate string `json:"creationDate"` + ModificationDate string `json:"modificationDate"` + CursorsLedgerID int64 `json:"cursorsLedgerId"` + MarkDelete PositionInfo `json:"markDelete"` + IndividualDeletedMessages []MessageRangeInfo `json:"individualDeletedMessages"` + Properties map[string]int64 +} + +type PositionInfo struct { + LedgerID int64 `json:"ledgerId"` + EntryID int64 `json:"entryId"` +} + +type MessageRangeInfo struct { + From PositionInfo `json:"from"` + To PositionInfo `json:"to"` + Offloaded bool `json:"offloaded"` +} + +type CursorStats struct { + MarkDeletePosition string `json:"markDeletePosition"` + ReadPosition string `json:"readPosition"` + WaitingReadOp bool `json:"waitingReadOp"` + PendingReadOps int `json:"pendingReadOps"` + MessagesConsumedCounter int64 `json:"messagesConsumedCounter"` + CursorLedger int64 `json:"cursorLedger"` + CursorLedgerLastEntry int64 `json:"cursorLedgerLastEntry"` + IndividuallyDeletedMessages string `json:"individuallyDeletedMessages"` + LastLedgerWitchTimestamp string `json:"lastLedgerWitchTimestamp"` + State string `json:"state"` + NumberOfEntriesSinceFirstNotAckedMessage int64 `json:"numberOfEntriesSinceFirstNotAckedMessage"` + TotalNonContiguousDeletedMessagesRange int `json:"totalNonContiguousDeletedMessagesRange"` + Properties map[string]int64 `json:"properties"` +} + +type PartitionedTopicStats struct { + MsgRateIn float64 `json:"msgRateIn"` + MsgRateOut float64 `json:"msgRateOut"` + MsgThroughputIn float64 `json:"msgThroughputIn"` + MsgThroughputOut float64 `json:"msgThroughputOut"` + AverageMsgSize float64 `json:"averageMsgSize"` + StorageSize int64 `json:"storageSize"` + Publishers []PublisherStats `json:"publishers"` + Subscriptions map[string]SubscriptionStats `json:"subscriptions"` + Replication map[string]ReplicatorStats `json:"replication"` + DeDuplicationStatus string `json:"deduplicationStatus"` + Metadata PartitionedTopicMetadata `json:"metadata"` + Partitions map[string]TopicStats `json:"partitions"` +} + +type SchemaData struct { + Version int64 `json:"version"` + Filename string `json:"filename"` + Jar string `json:"jar"` + Type string `json:"type"` + Classname string `json:"classname"` + AlwaysAllowNull bool `json:"alwaysAllowNull"` + DryRun bool `json:"dryRun"` +} + +type LookupData struct { + BrokerURL string `json:"brokerUrl"` + BrokerURLTLS string `json:"brokerUrlTls"` + HTTPURL string `json:"httpUrl"` + HTTPURLTLS string `json:"httpUrlTls"` +} + +type NsIsolationPoliciesData struct { + Namespaces []string `json:"namespaces"` + Primary []string `json:"primary"` + Secondary []string `json:"secondary"` + AutoFailoverPolicyTypeName string `json:"autoFailoverPolicyTypeName"` + AutoFailoverPolicyParams string `json:"autoFailoverPolicyParams"` +} + +type BrokerData struct { + URL string `json:"brokerUrl"` + ConfigName string `json:"configName"` + ConfigValue string `json:"configValue"` +} + +type BrokerStatsData struct { + Indent bool `json:"indent"` +} + +type ResourceQuotaData struct { + Names string `json:"names"` + Bundle string `json:"bundle"` + MsgRateIn int64 `json:"msgRateIn"` + MsgRateOut int64 `json:"msgRateOut"` + BandwidthIn int64 `json:"bandwidthIn"` + BandwidthOut int64 `json:"bandwidthOut"` + Memory int64 `json:"memory"` + Dynamic bool `json:"dynamic"` +} + +type PersistenceData struct { + BookkeeperEnsemble int64 `json:"bookkeeperEnsemble"` + BookkeeperWriteQuorum int64 `json:"bookkeeperWriteQuorum"` + BookkeeperAckQuorum int64 `json:"bookkeeperAckQuorum"` + ManagedLedgerMaxMarkDeleteRate float64 `json:"managedLedgerMaxMarkDeleteRate"` +} + +type DelayedDeliveryCmdData struct { + Enable bool `json:"enable"` + Disable bool `json:"disable"` + DelayedDeliveryTimeStr string `json:"delayedDeliveryTimeStr"` +} + +type DelayedDeliveryData struct { + TickTime float64 `json:"tickTime"` + Active bool `json:"active"` +} + +type DispatchRateData struct { + DispatchThrottlingRateInMsg int64 `json:"dispatchThrottlingRateInMsg"` + DispatchThrottlingRateInByte int64 `json:"dispatchThrottlingRateInByte"` + RatePeriodInSecond int64 `json:"ratePeriodInSecond"` + RelativeToPublishRate bool `json:"relativeToPublishRate"` +} + +type PublishRateData struct { + PublishThrottlingRateInMsg int64 `json:"publishThrottlingRateInMsg"` + PublishThrottlingRateInByte int64 `json:"publishThrottlingRateInByte"` +} + +type SchemaLedger struct { + LedgerID int64 `json:"ledgerId"` + Entries int64 `json:"entries"` + Size int64 `json:"size"` + Timestamp int64 `json:"timestamp"` + IsOffloaded bool `json:"isOffloaded"` +} + +type CompactedLedger struct { + LedgerID int64 `json:"ledgerId"` + Entries int64 `json:"entries"` + Size int64 `json:"size"` + Offloaded bool `json:"offloaded"` + UnderReplicated bool `json:"underReplicated"` +} diff --git a/pulsaradmin/pkg/utils/dispatch_rate.go b/pulsaradmin/pkg/utils/dispatch_rate.go new file mode 100644 index 000000000..9c6ccce88 --- /dev/null +++ b/pulsaradmin/pkg/utils/dispatch_rate.go @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type DispatchRate struct { + DispatchThrottlingRateInMsg int `json:"dispatchThrottlingRateInMsg"` + DispatchThrottlingRateInByte int64 `json:"dispatchThrottlingRateInByte"` + RatePeriodInSecond int `json:"ratePeriodInSecond"` +} + +func NewDispatchRate() *DispatchRate { + return &DispatchRate{ + DispatchThrottlingRateInMsg: -1, + DispatchThrottlingRateInByte: -1, + RatePeriodInSecond: 1, + } +} + +type SubscribeRate struct { + SubscribeThrottlingRatePerConsumer int `json:"subscribeThrottlingRatePerConsumer"` + RatePeriodInSecond int `json:"ratePeriodInSecond"` +} + +func NewSubscribeRate() *SubscribeRate { + return &SubscribeRate{ + SubscribeThrottlingRatePerConsumer: -1, + RatePeriodInSecond: 30, + } +} diff --git a/pulsaradmin/pkg/utils/function_confg.go b/pulsaradmin/pkg/utils/function_confg.go new file mode 100644 index 000000000..02a00db3b --- /dev/null +++ b/pulsaradmin/pkg/utils/function_confg.go @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +const ( + JavaRuntime = "JAVA" + PythonRuntime = "PYTHON" + GoRuntime = "GO" +) + +type FunctionConfig struct { + TimeoutMs *int64 `json:"timeoutMs,omitempty" yaml:"timeoutMs"` + TopicsPattern *string `json:"topicsPattern,omitempty" yaml:"topicsPattern"` + // Whether the subscriptions the functions created/used should be deleted when the functions is deleted + CleanupSubscription bool `json:"cleanupSubscription" yaml:"cleanupSubscription"` + RetainOrdering bool `json:"retainOrdering" yaml:"retainOrdering"` + RetainKeyOrdering bool `json:"retainKeyOrdering" yaml:"retainKeyOrdering"` + BatchBuilder string `json:"batchBuilder,omitempty" yaml:"batchBuilder"` + ForwardSourceMessageProperty bool `json:"forwardSourceMessageProperty" yaml:"forwardSourceMessageProperty"` + AutoAck bool `json:"autoAck" yaml:"autoAck"` + Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"` + MaxMessageRetries *int `json:"maxMessageRetries,omitempty" yaml:"maxMessageRetries"` + + Output string `json:"output,omitempty" yaml:"output"` + + ProducerConfig *ProducerConfig `json:"producerConfig,omitempty" yaml:"producerConfig"` + CustomSchemaOutputs map[string]string `json:"customSchemaOutputs,omitempty" yaml:"customSchemaOutputs"` + + OutputSerdeClassName string `json:"outputSerdeClassName,omitempty" yaml:"outputSerdeClassName"` + LogTopic string `json:"logTopic,omitempty" yaml:"logTopic"` + ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"` + + // Represents either a builtin schema type (eg: 'avro', 'json', etc) or the class name for a Schema implementation + OutputSchemaType string `json:"outputSchemaType,omitempty" yaml:"outputSchemaType"` + OutputTypeClassName string `json:"outputTypeClassName,omitempty" yaml:"outputTypeClassName"` + + Runtime string `json:"runtime,omitempty" yaml:"runtime"` + DeadLetterTopic string `json:"deadLetterTopic,omitempty" yaml:"deadLetterTopic"` + SubName string `json:"subName,omitempty" yaml:"subName"` + FQFN string `json:"fqfn,omitempty" yaml:"fqfn"` + Jar *string `json:"jar,omitempty" yaml:"jar"` + Py *string `json:"py,omitempty" yaml:"py"` + Go *string `json:"go,omitempty" yaml:"go"` + FunctionType *string `json:"functionType,omitempty" yaml:"functionType"` + // Any flags that you want to pass to the runtime. + // note that in thread mode, these flags will have no impact + RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"` + + Tenant string `json:"tenant,omitempty" yaml:"tenant"` + Namespace string `json:"namespace,omitempty" yaml:"namespace"` + Name string `json:"name,omitempty" yaml:"name"` + ClassName string `json:"className,omitempty" yaml:"className"` + + Resources *Resources `json:"resources,omitempty" yaml:"resources"` + WindowConfig *WindowConfig `json:"windowConfig,omitempty" yaml:"windowConfig"` + Inputs []string `json:"inputs,omitempty" yaml:"inputs"` + UserConfig map[string]interface{} `json:"userConfig,omitempty" yaml:"userConfig"` + CustomSerdeInputs map[string]string `json:"customSerdeInputs,omitempty" yaml:"customSerdeInputs"` + CustomSchemaInputs map[string]string `json:"customSchemaInputs,omitempty" yaml:"customSchemaInputs"` + + // A generalized way of specifying inputs + InputSpecs map[string]ConsumerConfig `json:"inputSpecs,omitempty" yaml:"inputSpecs"` + InputTypeClassName string `json:"inputTypeClassName,omitempty" yaml:"inputTypeClassName"` + + CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"` + + // This is a map of secretName(aka how the secret is going to be + // accessed in the function via context) to an object that + // encapsulates how the secret is fetched by the underlying + // secrets provider. The type of an value here can be found by the + // SecretProviderConfigurator.getSecretObjectType() method. + Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"` + + MaxPendingAsyncRequests int `json:"maxPendingAsyncRequests,omitempty" yaml:"maxPendingAsyncRequests"` + //nolint + ExposePulsarAdminClientEnabled bool `json:"exposePulsarAdminClientEnabled" yaml:"exposePulsarAdminClientEnabled"` + SkipToLatest bool `json:"skipToLatest" yaml:"skipToLatest"` + SubscriptionPosition string `json:"subscriptionPosition,omitempty" yaml:"subscriptionPosition"` +} diff --git a/pulsaradmin/pkg/utils/function_state.go b/pulsaradmin/pkg/utils/function_state.go new file mode 100644 index 000000000..63fa15057 --- /dev/null +++ b/pulsaradmin/pkg/utils/function_state.go @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type FunctionState struct { + Key string `json:"key"` + StringValue string `json:"stringValue"` + ByteValue []byte `json:"byteValue"` + NumValue int64 `json:"numberValue"` + Version int64 `json:"version"` +} diff --git a/pulsaradmin/pkg/utils/function_status.go b/pulsaradmin/pkg/utils/function_status.go new file mode 100644 index 000000000..360e61b07 --- /dev/null +++ b/pulsaradmin/pkg/utils/function_status.go @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type FunctionStatus struct { + NumInstances int `json:"numInstances"` + NumRunning int `json:"numRunning"` + Instances []FunctionInstanceStatus `json:"instances"` +} + +type FunctionInstanceStatus struct { + InstanceID int `json:"instanceId"` + Status FunctionInstanceStatusData `json:"status"` +} + +type FunctionInstanceStatusData struct { + Running bool `json:"running"` + Err string `json:"error"` + NumRestarts int64 `json:"numRestarts"` + NumReceived int64 `json:"numReceived"` + NumSuccessfullyProcessed int64 `json:"numSuccessfullyProcessed"` + NumUserExceptions int64 `json:"numUserExceptions"` + LatestUserExceptions []ExceptionInformation `json:"latestUserExceptions"` + NumSystemExceptions int64 `json:"numSystemExceptions"` + LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"` + AverageLatency float64 `json:"averageLatency"` + LastInvocationTime int64 `json:"lastInvocationTime"` + WorkerID string `json:"workerId"` +} + +type ExceptionInformation struct { + ExceptionString string `json:"exceptionString"` + TimestampMs int64 `json:"timestampMs"` +} diff --git a/pulsaradmin/pkg/utils/functions_stats.go b/pulsaradmin/pkg/utils/functions_stats.go new file mode 100644 index 000000000..82761afe5 --- /dev/null +++ b/pulsaradmin/pkg/utils/functions_stats.go @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type FunctionStats struct { + // Overall total number of records function received from source + ReceivedTotal int64 `json:"receivedTotal"` + + // Overall total number of records successfully processed by user function + ProcessedSuccessfullyTotal int64 `json:"processedSuccessfullyTotal"` + + // Overall total number of system exceptions thrown + SystemExceptionsTotal int64 `json:"systemExceptionsTotal"` + + // Overall total number of user exceptions thrown + UserExceptionsTotal int64 `json:"userExceptionsTotal"` + + // Average process latency for function + AvgProcessLatency float64 `json:"avgProcessLatency"` + + // Timestamp of when the function was last invoked by any instance + LastInvocation int64 `json:"lastInvocation"` + + OneMin FunctionInstanceStatsDataBase `json:"oneMin"` + + Instances []FunctionInstanceStats `json:"instances"` + + FunctionInstanceStats +} + +type FunctionInstanceStats struct { + FunctionInstanceStatsDataBase + + InstanceID int64 `json:"instanceId"` + + Metrics FunctionInstanceStatsData `json:"metrics"` +} + +type FunctionInstanceStatsDataBase struct { + // Total number of records function received from source for instance + ReceivedTotal int64 `json:"receivedTotal"` + + // Total number of records successfully processed by user function for instance + ProcessedSuccessfullyTotal int64 `json:"processedSuccessfullyTotal"` + + // Total number of system exceptions thrown for instance + SystemExceptionsTotal int64 `json:"systemExceptionsTotal"` + + // Total number of user exceptions thrown for instance + UserExceptionsTotal int64 `json:"userExceptionsTotal"` + + // Average process latency for function for instance + AvgProcessLatency float64 `json:"avgProcessLatency"` +} + +type FunctionInstanceStatsData struct { + OneMin FunctionInstanceStatsDataBase `json:"oneMin"` + + // Timestamp of when the function was last invoked for instance + LastInvocation int64 `json:"lastInvocation"` + + // Map of user defined metrics + UserMetrics map[string]float64 `json:"userMetrics"` + + FunctionInstanceStatsDataBase +} + +func (fs *FunctionStats) AddInstance(functionInstanceStats FunctionInstanceStats) { + fs.Instances = append(fs.Instances, functionInstanceStats) +} + +func (fs *FunctionStats) CalculateOverall() *FunctionStats { + var ( + nonNullInstances int + nonNullInstancesOneMin int + ) + + for _, functionInstanceStats := range fs.Instances { + functionInstanceStatsData := functionInstanceStats.Metrics + fs.ReceivedTotal += functionInstanceStatsData.ReceivedTotal + fs.ProcessedSuccessfullyTotal += functionInstanceStatsData.ProcessedSuccessfullyTotal + fs.SystemExceptionsTotal += functionInstanceStatsData.SystemExceptionsTotal + fs.UserExceptionsTotal += functionInstanceStatsData.UserExceptionsTotal + + if functionInstanceStatsData.AvgProcessLatency != 0 { + if fs.AvgProcessLatency == 0 { + fs.AvgProcessLatency = 0.0 + } + + fs.AvgProcessLatency += functionInstanceStatsData.AvgProcessLatency + nonNullInstances++ + } + + fs.OneMin.ReceivedTotal += functionInstanceStatsData.OneMin.ReceivedTotal + fs.OneMin.ProcessedSuccessfullyTotal += functionInstanceStatsData.OneMin.ProcessedSuccessfullyTotal + fs.OneMin.SystemExceptionsTotal += functionInstanceStatsData.OneMin.SystemExceptionsTotal + fs.OneMin.UserExceptionsTotal += functionInstanceStatsData.OneMin.UserExceptionsTotal + + if functionInstanceStatsData.OneMin.AvgProcessLatency != 0 { + if fs.OneMin.AvgProcessLatency == 0 { + fs.OneMin.AvgProcessLatency = 0.0 + } + + fs.OneMin.AvgProcessLatency += functionInstanceStatsData.OneMin.AvgProcessLatency + nonNullInstancesOneMin++ + } + + if functionInstanceStatsData.LastInvocation != 0 { + if fs.LastInvocation == 0 || functionInstanceStatsData.LastInvocation > fs.LastInvocation { + fs.LastInvocation = functionInstanceStatsData.LastInvocation + } + } + } + + // calculate average from sum + if nonNullInstances > 0 { + fs.AvgProcessLatency /= float64(nonNullInstances) + } else { + fs.AvgProcessLatency = 0 + } + + // calculate 1min average from sum + if nonNullInstancesOneMin > 0 { + fs.OneMin.AvgProcessLatency /= float64(nonNullInstancesOneMin) + } else { + fs.AvgProcessLatency = 0 + } + + return fs +} diff --git a/pulsaradmin/pkg/utils/home_dir.go b/pulsaradmin/pkg/utils/home_dir.go new file mode 100644 index 000000000..330831c40 --- /dev/null +++ b/pulsaradmin/pkg/utils/home_dir.go @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "os" + "path/filepath" + "runtime" +) + +// HomeDir returns the home directory for the current user. +// On Windows: +// 1. the first of %HOME%, %HOMEDRIVE%%HOMEPATH%, %USERPROFILE% containing a `.pulsar\config` file is returned. +// 2. if none of those locations contain a `.pulsar\config` file, the first of +// %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% +// that exists and is writeable is returned. +// 3. if none of those locations are writeable, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% +// that exists is returned. +// 4. if none of those locations exists, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% +// that is set is returned. +func HomeDir() string { + if runtime.GOOS == "windows" { + home := os.Getenv("HOME") + homeDriveHomePath := "" + if homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH"); len(homeDrive) > 0 && len(homePath) > 0 { + homeDriveHomePath = homeDrive + homePath + } + userProfile := os.Getenv("USERPROFILE") + + // Return first of %HOME%, %HOMEDRIVE%/%HOMEPATH%, %USERPROFILE% that contains a `.pulsar\config` file. + // %HOMEDRIVE%/%HOMEPATH% is preferred over %USERPROFILE% for backwards-compatibility. + for _, p := range []string{home, homeDriveHomePath, userProfile} { + if len(p) == 0 { + continue + } + if _, err := os.Stat(filepath.Join(p, ".pulsar", "config")); err != nil { + continue + } + return p + } + + firstSetPath := "" + firstExistingPath := "" + + // Prefer %USERPROFILE% over %HOMEDRIVE%/%HOMEPATH% for compatibility with other auth-writing tools + for _, p := range []string{home, userProfile, homeDriveHomePath} { + if len(p) == 0 { + continue + } + if len(firstSetPath) == 0 { + // remember the first path that is set + firstSetPath = p + } + info, err := os.Stat(p) + if err != nil { + continue + } + if len(firstExistingPath) == 0 { + // remember the first path that exists + firstExistingPath = p + } + if info.IsDir() && info.Mode().Perm()&(1<<(uint(7))) != 0 { + // return first path that is writeable + return p + } + } + + // If none are writeable, return first location that exists + if len(firstExistingPath) > 0 { + return firstExistingPath + } + + // If none exist, return first location that is set + if len(firstSetPath) > 0 { + return firstSetPath + } + + // We've got nothing + return "" + } + return os.Getenv("HOME") +} diff --git a/pulsaradmin/pkg/utils/inactive_topic_policies.go b/pulsaradmin/pkg/utils/inactive_topic_policies.go new file mode 100644 index 000000000..05f81b664 --- /dev/null +++ b/pulsaradmin/pkg/utils/inactive_topic_policies.go @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type InactiveTopicDeleteMode string + +const ( + // The topic can be deleted when no subscriptions and no active producers. + DeleteWhenNoSubscriptions InactiveTopicDeleteMode = "delete_when_no_subscriptions" + // The topic can be deleted when all subscriptions catchup and no active producers/consumers. + DeleteWhenSubscriptionsCaughtUp InactiveTopicDeleteMode = "delete_when_subscriptions_caught_up" +) + +func (i InactiveTopicDeleteMode) String() string { + return string(i) +} + +func ParseInactiveTopicDeleteMode(str string) (InactiveTopicDeleteMode, error) { + switch str { + case DeleteWhenNoSubscriptions.String(): + return DeleteWhenNoSubscriptions, nil + case DeleteWhenSubscriptionsCaughtUp.String(): + return DeleteWhenSubscriptionsCaughtUp, nil + default: + return "", errors.Errorf("cannot parse %s to InactiveTopicDeleteMode type", str) + } +} + +type InactiveTopicPolicies struct { + InactiveTopicDeleteMode *InactiveTopicDeleteMode `json:"inactiveTopicDeleteMode"` + MaxInactiveDurationSeconds int `json:"maxInactiveDurationSeconds"` + DeleteWhileInactive bool `json:"deleteWhileInactive"` +} + +func NewInactiveTopicPolicies(inactiveTopicDeleteMode *InactiveTopicDeleteMode, maxInactiveDurationSeconds int, + deleteWhileInactive bool) InactiveTopicPolicies { + return InactiveTopicPolicies{ + InactiveTopicDeleteMode: inactiveTopicDeleteMode, + MaxInactiveDurationSeconds: maxInactiveDurationSeconds, + DeleteWhileInactive: deleteWhileInactive, + } +} diff --git a/pulsaradmin/pkg/utils/internal_configuration_data.go b/pulsaradmin/pkg/utils/internal_configuration_data.go new file mode 100644 index 000000000..75cad0e24 --- /dev/null +++ b/pulsaradmin/pkg/utils/internal_configuration_data.go @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type InternalConfigurationData struct { + ZookeeperServers string `json:"zookeeperServers"` + ConfigurationStoreServers string `json:"configurationStoreServers"` + LedgersRootPath string `json:"ledgersRootPath"` + StateStorageServiceURL string `json:"stateStorageServiceUrl"` +} diff --git a/pulsaradmin/pkg/utils/load_manage_report.go b/pulsaradmin/pkg/utils/load_manage_report.go new file mode 100644 index 000000000..5196da9b5 --- /dev/null +++ b/pulsaradmin/pkg/utils/load_manage_report.go @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "math" +) + +type LocalBrokerData struct { + // URLs to satisfy contract of ServiceLookupData (used by NamespaceService). + WebServiceURL string `json:"webServiceUrl"` + WebServiceURLTLS string `json:"webServiceUrlTls"` + PulsarServiceURL string `json:"pulsarServiceUrl"` + PulsarServiceURLTLS string `json:"pulsarServiceUrlTls"` + PersistentTopicsEnabled bool `json:"persistentTopicsEnabled"` + NonPersistentTopicsEnabled bool `json:"nonPersistentTopicsEnabled"` + + // Most recently available system resource usage. + CPU ResourceUsage `json:"cpu"` + Memory ResourceUsage `json:"memory"` + DirectMemory ResourceUsage `json:"directMemory"` + BandwidthIn ResourceUsage `json:"bandwidthIn"` + BandwidthOut ResourceUsage `json:"bandwidthOut"` + + // Message data from the most recent namespace bundle stats. + MsgThroughputIn float64 `json:"msgThroughputIn"` + MsgThroughputOut float64 `json:"msgThroughputOut"` + MsgRateIn float64 `json:"msgRateIn"` + MsgRateOut float64 `json:"msgRateOut"` + + // Timestamp of last update. + LastUpdate int64 `json:"lastUpdate"` + + // The stats given in the most recent invocation of update. + LastStats map[string]*NamespaceBundleStats `json:"lastStats"` + NumTopics int `json:"numTopics"` + NumBundles int `json:"numBundles"` + NumConsumers int `json:"numConsumers"` + NumProducers int `json:"numProducers"` + + // All bundles belonging to this broker. + Bundles []string `json:"bundles"` + + // The bundles gained since the last invocation of update. + LastBundleGains []string `json:"lastBundleGains"` + + // The bundles lost since the last invocation of update. + LastBundleLosses []string `json:"lastBundleLosses"` + + // The version string that this broker is running, obtained from the Maven build artifact in the POM + BrokerVersionString string `json:"brokerVersionString"` + + // This place-holder requires to identify correct LoadManagerReport type while deserializing + LoadReportType string `json:"loadReportType"` + + // the external protocol data advertised by protocol handlers. + Protocols map[string]string `json:"protocols"` +} + +func NewLocalBrokerData() LocalBrokerData { + lastStats := make(map[string]*NamespaceBundleStats) + lastStats[""] = NewNamespaceBundleStats() + return LocalBrokerData{ + LastStats: lastStats, + } +} + +type NamespaceBundleStats struct { + MsgRateIn float64 `json:"msgRateIn"` + MsgThroughputIn float64 `json:"msgThroughputIn"` + MsgRateOut float64 `json:"msgRateOut"` + MsgThroughputOut float64 `json:"msgThroughputOut"` + ConsumerCount int `json:"consumerCount"` + ProducerCount int `json:"producerCount"` + TopicsNum int64 `json:"topics"` + CacheSize int64 `json:"cacheSize"` + + // Consider the throughput equal if difference is less than 100 KB/s + ThroughputDifferenceThreshold float64 `json:"throughputDifferenceThreshold"` + // Consider the msgRate equal if the difference is less than 100 + MsgRateDifferenceThreshold float64 `json:"msgRateDifferenceThreshold"` + // Consider the total topics/producers/consumers equal if the difference is less than 500 + TopicConnectionDifferenceThreshold int64 `json:"topicConnectionDifferenceThreshold"` + // Consider the cache size equal if the difference is less than 100 kb + CacheSizeDifferenceThreshold int64 `json:"cacheSizeDifferenceThreshold"` +} + +func NewNamespaceBundleStats() *NamespaceBundleStats { + return &NamespaceBundleStats{ + ThroughputDifferenceThreshold: 1e5, + MsgRateDifferenceThreshold: 100, + TopicConnectionDifferenceThreshold: 500, + CacheSizeDifferenceThreshold: 100000, + } +} + +type ResourceUsage struct { + Usage float64 `json:"usage"` + Limit float64 `json:"limit"` +} + +func (ru *ResourceUsage) Reset() { + ru.Usage = -1 + ru.Limit = -1 +} + +func (ru *ResourceUsage) CompareTo(o *ResourceUsage) int { + required := o.Limit - o.Usage + available := ru.Limit - ru.Usage + return compare(required, available) +} + +func (ru *ResourceUsage) PercentUsage() float32 { + var proportion float32 + if ru.Limit > 0 { + proportion = float32(ru.Usage) / float32(ru.Limit) + } + return proportion * 100 +} + +func compare(val1, val2 float64) int { + if val1 < val2 { + return -1 + } + + if val1 < val2 { + return 1 + } + + thisBits := math.Float64bits(val1) + anotherBits := math.Float64bits(val2) + + if thisBits == anotherBits { + return 0 + } + + if thisBits < anotherBits { + return -1 + } + return 1 +} diff --git a/pulsaradmin/pkg/utils/long_running_process_status.go b/pulsaradmin/pkg/utils/long_running_process_status.go new file mode 100644 index 000000000..c61919ecc --- /dev/null +++ b/pulsaradmin/pkg/utils/long_running_process_status.go @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type Status string + +const ( + NOTRUN Status = "NOT_RUN" + RUNNING Status = "RUNNING" + SUCCESS Status = "SUCCESS" + ERROR Status = "ERROR" +) + +type LongRunningProcessStatus struct { + Status Status `json:"status"` + LastError string `json:"lastError"` +} + +type OffloadProcessStatus struct { + Status Status `json:"status"` + LastError string `json:"lastError"` + FirstUnOffloadedMessage MessageID `json:"firstUnoffloadedMessage"` +} + +func (s Status) String() string { + return string(s) +} diff --git a/pulsaradmin/pkg/utils/message.go b/pulsaradmin/pkg/utils/message.go new file mode 100644 index 000000000..2f0e4befe --- /dev/null +++ b/pulsaradmin/pkg/utils/message.go @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +//nolint +import ( + "github.com/golang/protobuf/proto" +) + +type Message struct { + MessageID MessageID + Payload []byte + Topic string + Properties map[string]string +} + +func NewMessage(topic string, id MessageID, payload []byte, properties map[string]string) *Message { + return &Message{ + MessageID: id, + Payload: payload, + Topic: topic, + Properties: properties, + } +} + +func (m *Message) GetMessageID() MessageID { + return m.MessageID +} + +func (m *Message) GetProperties() map[string]string { + return m.Properties +} + +func (m *Message) GetPayload() []byte { + return m.Payload +} + +// nolint +type SingleMessageMetadata struct { + Properties []*KeyValue `protobuf:"bytes,1,rep,name=properties" json:"properties,omitempty"` + PartitionKey *string `protobuf:"bytes,2,opt,name=partition_key,json=partitionKey" json:"partition_key,omitempty"` + PayloadSize *int32 `protobuf:"varint,3,req,name=payload_size,json=payloadSize" json:"payload_size,omitempty"` + CompactedOut *bool `protobuf:"varint,4,opt,name=compacted_out,json=compactedOut,def=0" json:"compacted_out,omitempty"` + // the timestamp that this event occurs. it is typically set by applications. + // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. + EventTime *uint64 `protobuf:"varint,5,opt,name=event_time,json=eventTime,def=0" json:"event_time,omitempty"` + PartitionKeyB64Encoded *bool `protobuf:"varint,6,opt,name=partition_key_b64_encoded,json=partitionKeyB64Encoded,def=0" json:"partition_key_b64_encoded,omitempty"` + // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. + OrderingKey []byte `protobuf:"bytes,7,opt,name=ordering_key,json=orderingKey" json:"ordering_key,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SingleMessageMetadata) Reset() { *m = SingleMessageMetadata{} } +func (m *SingleMessageMetadata) String() string { return proto.CompactTextString(m) } +func (*SingleMessageMetadata) ProtoMessage() {} +func (m *SingleMessageMetadata) GetPayloadSize() int32 { + if m != nil && m.PayloadSize != nil { + return *m.PayloadSize + } + return 0 +} + +// nolint +type KeyValue struct { + Key *string `protobuf:"bytes,1,req,name=key" json:"key,omitempty"` + Value *string `protobuf:"bytes,2,req,name=value" json:"value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *KeyValue) Reset() { *m = KeyValue{} } +func (m *KeyValue) String() string { return proto.CompactTextString(m) } +func (*KeyValue) ProtoMessage() {} diff --git a/pulsaradmin/pkg/utils/message_id.go b/pulsaradmin/pkg/utils/message_id.go new file mode 100644 index 000000000..d75b613e1 --- /dev/null +++ b/pulsaradmin/pkg/utils/message_id.go @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "strconv" + "strings" + + "github.com/pkg/errors" +) + +type MessageID struct { + LedgerID int64 `json:"ledgerId"` + EntryID int64 `json:"entryId"` + PartitionIndex int `json:"partitionIndex"` + BatchIndex int `json:"-"` +} + +var Latest = MessageID{0x7fffffffffffffff, 0x7fffffffffffffff, -1, -1} +var Earliest = MessageID{-1, -1, -1, -1} + +func ParseMessageID(str string) (*MessageID, error) { + s := strings.Split(str, ":") + + m := Earliest + + if len(s) < 2 || len(s) > 4 { + return nil, errors.Errorf("invalid message id string. %s", str) + } + + ledgerID, err := strconv.ParseInt(s[0], 10, 64) + if err != nil { + return nil, errors.Errorf("invalid ledger id. %s", str) + } + m.LedgerID = ledgerID + + entryID, err := strconv.ParseInt(s[1], 10, 64) + if err != nil { + return nil, errors.Errorf("invalid entry id. %s", str) + } + m.EntryID = entryID + + if len(s) > 2 { + pi, err := strconv.Atoi(s[2]) + if err != nil { + return nil, errors.Errorf("invalid partition index. %s", str) + } + m.PartitionIndex = pi + } + + if len(s) == 4 { + bi, err := strconv.Atoi(s[3]) + if err != nil { + return nil, errors.Errorf("invalid batch index. %s", str) + } + m.BatchIndex = bi + } + + return &m, nil +} + +func (m MessageID) String() string { + return strconv.FormatInt(m.LedgerID, 10) + ":" + + strconv.FormatInt(m.EntryID, 10) + ":" + + strconv.Itoa(m.PartitionIndex) + ":" + + strconv.Itoa(m.BatchIndex) +} diff --git a/pulsaradmin/pkg/utils/message_id_test.go b/pulsaradmin/pkg/utils/message_id_test.go new file mode 100644 index 000000000..1ce0b5230 --- /dev/null +++ b/pulsaradmin/pkg/utils/message_id_test.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseMessageId(t *testing.T) { + id, err := ParseMessageID("1:1") + assert.Nil(t, err) + assert.Equal(t, MessageID{LedgerID: 1, EntryID: 1, PartitionIndex: -1, BatchIndex: -1}, *id) + + id, err = ParseMessageID("1:2:3") + assert.Nil(t, err) + assert.Equal(t, MessageID{LedgerID: 1, EntryID: 2, PartitionIndex: 3, BatchIndex: -1}, *id) + + id, err = ParseMessageID("1:2:3:4") + assert.Nil(t, err) + assert.Equal(t, MessageID{LedgerID: 1, EntryID: 2, PartitionIndex: 3, BatchIndex: 4}, *id) +} + +func TestParseMessageIdErrors(t *testing.T) { + id, err := ParseMessageID("1;1") + assert.Nil(t, id) + assert.NotNil(t, err) + assert.Equal(t, "invalid message id string. 1;1", err.Error()) + + id, err = ParseMessageID("a:1") + assert.Nil(t, id) + assert.NotNil(t, err) + assert.Equal(t, "invalid ledger id. a:1", err.Error()) + + id, err = ParseMessageID("1:a") + assert.Nil(t, id) + assert.NotNil(t, err) + assert.Equal(t, "invalid entry id. 1:a", err.Error()) + + id, err = ParseMessageID("1:2:a") + assert.Nil(t, id) + assert.NotNil(t, err) + assert.Equal(t, "invalid partition index. 1:2:a", err.Error()) + + id, err = ParseMessageID("1:2:3:a") + assert.Nil(t, id) + assert.NotNil(t, err) + assert.Equal(t, "invalid batch index. 1:2:3:a", err.Error()) +} diff --git a/pulsaradmin/pkg/utils/metrics.go b/pulsaradmin/pkg/utils/metrics.go new file mode 100644 index 000000000..ca1919423 --- /dev/null +++ b/pulsaradmin/pkg/utils/metrics.go @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type Metrics struct { + Metrics map[string]interface{} `json:"metrics"` + Dimensions map[string]string `json:"dimensions"` +} + +func NewMetrics(dimensionMap map[string]string) *Metrics { + return &Metrics{ + Metrics: make(map[string]interface{}), + Dimensions: dimensionMap, + } +} diff --git a/pulsaradmin/pkg/utils/namespace_name.go b/pulsaradmin/pkg/utils/namespace_name.go new file mode 100644 index 000000000..11b743554 --- /dev/null +++ b/pulsaradmin/pkg/utils/namespace_name.go @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "fmt" + "regexp" + "strings" + + "github.com/pkg/errors" +) + +type NameSpaceName struct { + tenant string + nameSpace string +} + +func GetNameSpaceName(tenant, namespace string) (*NameSpaceName, error) { + return GetNamespaceName(fmt.Sprintf("%s/%s", tenant, namespace)) +} + +func GetNamespaceName(completeName string) (*NameSpaceName, error) { + var n NameSpaceName + + if completeName == "" { + return nil, errors.New("the namespace complete name is empty") + } + + parts := strings.Split(completeName, "/") + if len(parts) == 2 { + n.tenant = parts[0] + n.nameSpace = parts[1] + err := validateNamespaceName(n.tenant, n.nameSpace) + if err != nil { + return nil, err + } + } else { + return nil, errors.Errorf("The complete name of namespace is invalid. complete name : [%s]", completeName) + } + + return &n, nil +} + +func (n *NameSpaceName) String() string { + return fmt.Sprintf("%s/%s", n.tenant, n.nameSpace) +} + +func validateNamespaceName(tenant, namespace string) error { + if tenant == "" || namespace == "" { + return errors.Errorf("Invalid tenant or namespace. [%s/%s]", tenant, namespace) + } + + ok := CheckName(tenant) + if !ok { + return errors.Errorf("Tenant name include unsupported special chars. tenant : [%s]", tenant) + } + + ok = CheckName(namespace) + if !ok { + return errors.Errorf("Namespace name include unsupported special chars. namespace : [%s]", namespace) + } + + return nil +} + +// allowed characters for property, namespace, cluster and topic +// names are alphanumeric (a-zA-Z0-9) and these special chars -=:. +// and % is allowed as part of valid URL encoding +var patten = regexp.MustCompile(`^[-=:.\w]*$`) + +func CheckName(name string) bool { + return patten.MatchString(name) +} diff --git a/pulsaradmin/pkg/utils/namespace_name_test.go b/pulsaradmin/pkg/utils/namespace_name_test.go new file mode 100644 index 000000000..bc12070e9 --- /dev/null +++ b/pulsaradmin/pkg/utils/namespace_name_test.go @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetNamespaceName(t *testing.T) { + success, err := GetNamespaceName("public/default") + assert.Nil(t, err) + assert.Equal(t, "public/default", success.String()) + + empty, err := GetNamespaceName("") + assert.NotNil(t, err) + assert.Equal(t, "the namespace complete name is empty", err.Error()) + assert.Nil(t, empty) + + empty, err = GetNamespaceName("/") + assert.NotNil(t, err) + assert.Equal(t, "Invalid tenant or namespace. [/]", err.Error()) + assert.Nil(t, empty) + + invalid, err := GetNamespaceName("public/default/fail") + assert.NotNil(t, err) + assert.Equal(t, "The complete name of namespace is invalid. complete name : [public/default/fail]", err.Error()) + assert.Nil(t, invalid) + + invalid, err = GetNamespaceName("public") + assert.NotNil(t, err) + assert.Equal(t, "The complete name of namespace is invalid. complete name : [public]", err.Error()) + assert.Nil(t, invalid) + + special, err := GetNamespaceName("-=.:/-=.:") + assert.Nil(t, err) + assert.Equal(t, "-=.:/-=.:", special.String()) + + tenantInvalid, err := GetNamespaceName("\"/namespace") + assert.NotNil(t, err) + assert.Equal(t, "Tenant name include unsupported special chars. tenant : [\"]", err.Error()) + assert.Nil(t, tenantInvalid) + + namespaceInvalid, err := GetNamespaceName("tenant/}") + assert.NotNil(t, err) + assert.Equal(t, "Namespace name include unsupported special chars. namespace : [}]", err.Error()) + assert.Nil(t, namespaceInvalid) +} diff --git a/pulsaradmin/pkg/utils/ns_isolation_data.go b/pulsaradmin/pkg/utils/ns_isolation_data.go new file mode 100644 index 000000000..4589eb30c --- /dev/null +++ b/pulsaradmin/pkg/utils/ns_isolation_data.go @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "github.com/pkg/errors" +) + +type NamespaceIsolationData struct { + Namespaces []string `json:"namespaces"` + Primary []string `json:"primary"` + Secondary []string `json:"secondary"` + AutoFailoverPolicy AutoFailoverPolicyData `json:"auto_failover_policy"` +} + +type AutoFailoverPolicyData struct { + PolicyType AutoFailoverPolicyType `json:"policy_type"` + Parameters map[string]string `json:"parameters"` +} + +type AutoFailoverPolicyType string + +const ( + MinAvailable AutoFailoverPolicyType = "min_available" +) + +func fromString(autoFailoverPolicyTypeName string) AutoFailoverPolicyType { + switch autoFailoverPolicyTypeName { + case "min_available": + return MinAvailable + default: + return "" + } +} + +func CreateNamespaceIsolationData(namespaces, primary, secondry []string, autoFailoverPolicyTypeName string, + autoFailoverPolicyParams map[string]string) (*NamespaceIsolationData, error) { + nsIsolationData := new(NamespaceIsolationData) + if len(namespaces) == 0 { + return nil, errors.New("unable to parse namespaces parameter list") + } + + if len(primary) == 0 { + return nil, errors.New("unable to parse primary parameter list") + } + + if len(secondry) == 0 { + return nil, errors.New("unable to parse secondry parameter list") + } + + nsIsolationData.Namespaces = namespaces + nsIsolationData.Primary = primary + nsIsolationData.Secondary = secondry + nsIsolationData.AutoFailoverPolicy.PolicyType = fromString(autoFailoverPolicyTypeName) + nsIsolationData.AutoFailoverPolicy.Parameters = autoFailoverPolicyParams + + // validation if necessary + if nsIsolationData.AutoFailoverPolicy.PolicyType == MinAvailable { + err := true + expectParamKeys := []string{"min_limit", "usage_threshold"} + + if len(autoFailoverPolicyParams) == len(expectParamKeys) { + for _, paramKey := range expectParamKeys { + if _, ok := autoFailoverPolicyParams[paramKey]; !ok { + break + } + } + err = false + } + + if err { + return nil, errors.Errorf("Unknown auto failover policy params specified: %v", autoFailoverPolicyParams) + } + } else { + // either we don't handle the new type or user has specified a bad type + return nil, errors.Errorf("Unknown auto failover policy type specified : %v", autoFailoverPolicyTypeName) + } + + return nsIsolationData, nil +} diff --git a/pulsaradmin/pkg/utils/ns_ownership_status.go b/pulsaradmin/pkg/utils/ns_ownership_status.go new file mode 100644 index 000000000..bb63b704f --- /dev/null +++ b/pulsaradmin/pkg/utils/ns_ownership_status.go @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type NamespaceOwnershipStatus struct { + BrokerAssignment BrokerAssignment `json:"broker_assignment"` + IsControlled bool `json:"is_controlled"` + IsActive bool `json:"is_active"` +} + +type BrokerAssignment string + +const ( + Primary BrokerAssignment = "primary" + Secondary BrokerAssignment = "secondary" + Shared BrokerAssignment = "shared" +) diff --git a/pulsaradmin/pkg/utils/package_metadata.go b/pulsaradmin/pkg/utils/package_metadata.go new file mode 100644 index 000000000..860afd9c8 --- /dev/null +++ b/pulsaradmin/pkg/utils/package_metadata.go @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type PackageMetadata struct { + Description string `json:"description,omitempty" yaml:"description"` + Contact string `json:"contact,omitempty" yaml:"contact"` + CreateTime int64 `json:"createTime,omitempty" yaml:"createTime"` + ModificationTime int64 `json:"modificationTime,omitempty" yaml:"modificationTime"` + Properties map[string]string `json:"properties,omitempty" yaml:"properties"` +} diff --git a/pulsaradmin/pkg/utils/package_name.go b/pulsaradmin/pkg/utils/package_name.go new file mode 100644 index 000000000..afc646d68 --- /dev/null +++ b/pulsaradmin/pkg/utils/package_name.go @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "fmt" + "strings" + + "github.com/pkg/errors" +) + +type PackageName struct { + packageType PackageType + namespace string + tenant string + name string + version string + completePackageName string + completeName string +} + +func invalidPackageNameError(completeName string) error { + return errors.Errorf("Invalid package name '%s', it should be "+ + "in the format of type://tenant/namespace/name@version", completeName) +} + +func GetPackageNameWithComponents(packageType PackageType, + tenant, namespace, name, version string) (*PackageName, error) { + return GetPackageName(fmt.Sprintf("%s://%s/%s/%s@%s", packageType, tenant, namespace, name, version)) +} + +func GetPackageName(completeName string) (*PackageName, error) { + var packageName PackageName + var err error + if !strings.Contains(completeName, "://") { + return nil, invalidPackageNameError(completeName) + } + parts := strings.Split(completeName, "://") + if len(parts) != 2 { + return nil, invalidPackageNameError(completeName) + } + packageName.packageType, err = parsePackageType(parts[0]) + if err != nil { + return nil, err + } + rest := parts[1] + if !strings.Contains(rest, "@") { + // if the package name does not contains '@', that means user does not set the version of package. + // We will set the version to latest. + rest += "@" + } + parts = strings.Split(rest, "@") + if len(parts) != 2 { + return nil, invalidPackageNameError(completeName) + } + partsWithoutVersion := strings.Split(parts[0], "/") + if len(partsWithoutVersion) != 3 { + return nil, invalidPackageNameError(completeName) + } + packageName.tenant = partsWithoutVersion[0] + packageName.namespace = partsWithoutVersion[1] + packageName.name = partsWithoutVersion[2] + packageName.version = "latest" + if parts[1] != "" { + packageName.version = parts[1] + } + packageName.completeName = fmt.Sprintf("%s/%s/%s", + packageName.tenant, packageName.namespace, packageName.name) + packageName.completePackageName = fmt.Sprintf("%s://%s/%s/%s@%s", + packageName.packageType, packageName.tenant, packageName.namespace, packageName.name, packageName.version) + + return &packageName, nil +} + +func (p *PackageName) String() string { + return p.completePackageName +} + +func (p *PackageName) GetType() PackageType { + return p.packageType +} + +func (p *PackageName) GetTenant() string { + return p.tenant +} + +func (p *PackageName) GetNamespace() string { + return p.namespace +} + +func (p *PackageName) GetName() string { + return p.name +} + +func (p *PackageName) GetVersion() string { + return p.version +} + +func (p *PackageName) GetCompleteName() string { + return p.completeName +} diff --git a/pulsaradmin/pkg/utils/package_name_test.go b/pulsaradmin/pkg/utils/package_name_test.go new file mode 100644 index 000000000..8e6edd52d --- /dev/null +++ b/pulsaradmin/pkg/utils/package_name_test.go @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetPackageName(t *testing.T) { + success, err := GetPackageName("function://f-tenant/f-ns/f-name@f-version") + assert.Nil(t, err) + assert.Equal(t, "function://f-tenant/f-ns/f-name@f-version", success.String()) + + success, err = GetPackageName("function://f-tenant/f-ns/f-name") + assert.Nil(t, err) + assert.Equal(t, "function://f-tenant/f-ns/f-name@latest", success.String()) + + success, err = GetPackageName("sink://s-tenant/s-ns/s-name@s-version") + assert.Nil(t, err) + assert.Equal(t, "sink://s-tenant/s-ns/s-name@s-version", success.String()) + + success, err = GetPackageName("sink://s-tenant/s-ns/s-name") + assert.Nil(t, err) + assert.Equal(t, "sink://s-tenant/s-ns/s-name@latest", success.String()) + + success, err = GetPackageName("source://s-tenant/s-ns/s-name@s-version") + assert.Nil(t, err) + assert.Equal(t, "source://s-tenant/s-ns/s-name@s-version", success.String()) + + success, err = GetPackageName("source://s-tenant/s-ns/s-name") + assert.Nil(t, err) + assert.Equal(t, "source://s-tenant/s-ns/s-name@latest", success.String()) + + fail, err := GetPackageName("function:///public/default/test-error@v1") + assert.NotNil(t, err) + assert.Equal(t, "Invalid package name 'function:///public/default/test-error@v1', it should be in the "+ + "format of type://tenant/namespace/name@version", err.Error()) + assert.Nil(t, fail) + + fail, err = GetPackageNameWithComponents("functions", "public", "default", "test-error", "v1") + assert.NotNil(t, err) + assert.Equal(t, "Invalid package type 'functions', it should be function, sink, or source", err.Error()) + assert.Nil(t, fail) + + fail, err = GetPackageNameWithComponents("function", "public/default", "default", "test-error", "v1") + assert.NotNil(t, err) + assert.Equal(t, "Invalid package name 'function://public/default/default/test-error@v1', it should be in the "+ + "format of type://tenant/namespace/name@version", err.Error()) + assert.Nil(t, fail) + + fail, err = GetPackageName("function://public/default/test-error-version/v2") + assert.NotNil(t, err) + assert.Equal(t, "Invalid package name 'function://public/default/test-error-version/v2', it should be in the "+ + "format of type://tenant/namespace/name@version", err.Error()) + assert.Nil(t, fail) +} diff --git a/pulsaradmin/pkg/utils/package_type.go b/pulsaradmin/pkg/utils/package_type.go new file mode 100644 index 000000000..70f455b25 --- /dev/null +++ b/pulsaradmin/pkg/utils/package_type.go @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type PackageType string + +const ( + PackageTypeFunction PackageType = "function" + PackageTypeSink PackageType = "sink" + PackageTypeSource PackageType = "source" +) + +func parsePackageType(packageTypeName string) (PackageType, error) { + switch packageTypeName { + case PackageTypeFunction.String(): + return PackageTypeFunction, nil + case PackageTypeSink.String(): + return PackageTypeSink, nil + case PackageTypeSource.String(): + return PackageTypeSource, nil + default: + return "", errors.Errorf("Invalid package type '%s', it should be "+ + "function, sink, or source", packageTypeName) + } +} + +func (p PackageType) String() string { + return string(p) +} diff --git a/pulsaradmin/pkg/utils/persistence_policies.go b/pulsaradmin/pkg/utils/persistence_policies.go new file mode 100644 index 000000000..d4c8bdb04 --- /dev/null +++ b/pulsaradmin/pkg/utils/persistence_policies.go @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type PersistencePolicies struct { + BookkeeperEnsemble int `json:"bookkeeperEnsemble"` + BookkeeperWriteQuorum int `json:"bookkeeperWriteQuorum"` + BookkeeperAckQuorum int `json:"bookkeeperAckQuorum"` + ManagedLedgerMaxMarkDeleteRate float64 `json:"managedLedgerMaxMarkDeleteRate"` +} + +func NewPersistencePolicies(bookkeeperEnsemble, bookkeeperWriteQuorum, bookkeeperAckQuorum int, + managedLedgerMaxMarkDeleteRate float64) PersistencePolicies { + return PersistencePolicies{ + BookkeeperEnsemble: bookkeeperEnsemble, + BookkeeperWriteQuorum: bookkeeperWriteQuorum, + BookkeeperAckQuorum: bookkeeperAckQuorum, + ManagedLedgerMaxMarkDeleteRate: managedLedgerMaxMarkDeleteRate, + } +} + +type BookieAffinityGroupData struct { + BookkeeperAffinityGroupPrimary string `json:"bookkeeperAffinityGroupPrimary"` + BookkeeperAffinityGroupSecondary string `json:"bookkeeperAffinityGroupSecondary"` +} diff --git a/pulsaradmin/pkg/utils/policies.go b/pulsaradmin/pkg/utils/policies.go new file mode 100644 index 000000000..3d727994c --- /dev/null +++ b/pulsaradmin/pkg/utils/policies.go @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +const ( + FirstBoundary string = "0x00000000" + LastBoundary string = "0xffffffff" +) + +type Policies struct { + Bundles *BundlesData `json:"bundles"` + Persistence *PersistencePolicies `json:"persistence"` + RetentionPolicies *RetentionPolicies `json:"retention_policies"` + SchemaValidationEnforced bool `json:"schema_validation_enforced"` + DeduplicationEnabled *bool `json:"deduplicationEnabled"` + Deleted bool `json:"deleted"` + EncryptionRequired bool `json:"encryption_required"` + MessageTTLInSeconds *int `json:"message_ttl_in_seconds"` + MaxProducersPerTopic *int `json:"max_producers_per_topic"` + MaxConsumersPerTopic *int `json:"max_consumers_per_topic"` + MaxConsumersPerSubscription *int `json:"max_consumers_per_subscription"` + CompactionThreshold *int64 `json:"compaction_threshold"` + OffloadThreshold int64 `json:"offload_threshold"` + OffloadDeletionLagMs *int64 `json:"offload_deletion_lag_ms"` + AntiAffinityGroup string `json:"antiAffinityGroup"` + ReplicationClusters []string `json:"replication_clusters"` + LatencyStatsSampleRate map[string]int `json:"latency_stats_sample_rate"` + BacklogQuotaMap map[BacklogQuotaType]BacklogQuota `json:"backlog_quota_map"` + TopicDispatchRate map[string]DispatchRate `json:"topicDispatchRate"` + SubscriptionDispatchRate map[string]DispatchRate `json:"subscriptionDispatchRate"` + ReplicatorDispatchRate map[string]DispatchRate `json:"replicatorDispatchRate"` + PublishMaxMessageRate map[string]PublishRate `json:"publishMaxMessageRate"` + ClusterSubscribeRate map[string]SubscribeRate `json:"clusterSubscribeRate"` + TopicAutoCreationConfig *TopicAutoCreationConfig `json:"autoTopicCreationOverride"` + SchemaCompatibilityStrategy SchemaCompatibilityStrategy `json:"schema_auto_update_compatibility_strategy"` + AuthPolicies AuthPolicies `json:"auth_policies"` + SubscriptionAuthMode SubscriptionAuthMode `json:"subscription_auth_mode"` + IsAllowAutoUpdateSchema *bool `json:"is_allow_auto_update_schema"` +} + +func NewDefaultPolicies() *Policies { + return &Policies{ + AuthPolicies: *NewAuthPolicies(), + ReplicationClusters: make([]string, 0, 10), + BacklogQuotaMap: make(map[BacklogQuotaType]BacklogQuota), + TopicDispatchRate: make(map[string]DispatchRate), + SubscriptionDispatchRate: make(map[string]DispatchRate), + ReplicatorDispatchRate: make(map[string]DispatchRate), + PublishMaxMessageRate: make(map[string]PublishRate), + ClusterSubscribeRate: make(map[string]SubscribeRate), + LatencyStatsSampleRate: make(map[string]int), + MessageTTLInSeconds: nil, + Deleted: false, + EncryptionRequired: false, + SubscriptionAuthMode: None, + MaxProducersPerTopic: nil, + MaxConsumersPerSubscription: nil, + MaxConsumersPerTopic: nil, + CompactionThreshold: nil, + OffloadThreshold: -1, + SchemaCompatibilityStrategy: Full, + SchemaValidationEnforced: false, + } +} diff --git a/pulsaradmin/pkg/utils/producer_config.go b/pulsaradmin/pkg/utils/producer_config.go new file mode 100644 index 000000000..d4c1c3eb6 --- /dev/null +++ b/pulsaradmin/pkg/utils/producer_config.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type ProducerConfig struct { + MaxPendingMessages int `json:"maxPendingMessages" yaml:"maxPendingMessages"` + //nolint + MaxPendingMessagesAcrossPartitions int `json:"maxPendingMessagesAcrossPartitions" yaml:"maxPendingMessagesAcrossPartitions"` + + UseThreadLocalProducers bool `json:"useThreadLocalProducers" yaml:"useThreadLocalProducers"` + CryptoConfig *CryptoConfig `json:"cryptoConfig" yaml:"cryptoConfig"` + BatchBuilder string `json:"batchBuilder" yaml:"batchBuilder"` + CompressionType string `json:"compressionType" yaml:"compressionType"` +} diff --git a/pulsaradmin/pkg/utils/publish_rate.go b/pulsaradmin/pkg/utils/publish_rate.go new file mode 100644 index 000000000..9dbc2d00d --- /dev/null +++ b/pulsaradmin/pkg/utils/publish_rate.go @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type PublishRate struct { + PublishThrottlingRateInMsg int `json:"publishThrottlingRateInMsg"` + PublishThrottlingRateInByte int64 `json:"publishThrottlingRateInByte"` +} + +func NewPublishRate() *PublishRate { + return &PublishRate{ + PublishThrottlingRateInMsg: -1, + PublishThrottlingRateInByte: -1, + } +} diff --git a/pulsaradmin/pkg/utils/resource_quota.go b/pulsaradmin/pkg/utils/resource_quota.go new file mode 100644 index 000000000..d9ef4f74d --- /dev/null +++ b/pulsaradmin/pkg/utils/resource_quota.go @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type ResourceQuota struct { + // messages published per second + MsgRateIn float64 `json:"msgRateIn"` + // messages consumed per second + MsgRateOut float64 `json:"msgRateOut"` + // incoming bytes per second + BandwidthIn float64 `json:"bandwidthIn"` + // outgoing bytes per second + BandwidthOut float64 `json:"bandwidthOut"` + // used memory in Mbytes + Memory float64 `json:"memory"` + // allow the quota be dynamically re-calculated according to real traffic + Dynamic bool `json:"dynamic"` +} + +func NewResourceQuota() *ResourceQuota { + return &ResourceQuota{ + MsgRateIn: 0.0, + MsgRateOut: 0.0, + BandwidthIn: 0.0, + BandwidthOut: 0.0, + Memory: 0.0, + Dynamic: true, + } +} diff --git a/pulsaradmin/pkg/utils/resources.go b/pulsaradmin/pkg/utils/resources.go new file mode 100644 index 000000000..a4f3ddbcb --- /dev/null +++ b/pulsaradmin/pkg/utils/resources.go @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type Resources struct { + CPU float64 `json:"cpu"` + Disk int64 `json:"disk"` + RAM int64 `json:"ram"` +} + +func NewDefaultResources() *Resources { + resources := &Resources{ + // Default cpu is 1 core + CPU: 1, + // Default memory is 1GB + RAM: 1073741824, + // Default disk is 10GB + Disk: 10737418240, + } + + return resources +} diff --git a/pulsaradmin/pkg/utils/retention_policies.go b/pulsaradmin/pkg/utils/retention_policies.go new file mode 100644 index 000000000..55bf915ed --- /dev/null +++ b/pulsaradmin/pkg/utils/retention_policies.go @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type RetentionPolicies struct { + RetentionTimeInMinutes int `json:"retentionTimeInMinutes"` + RetentionSizeInMB int64 `json:"retentionSizeInMB"` +} + +func NewRetentionPolicies(retentionTimeInMinutes int, retentionSizeInMB int) RetentionPolicies { + return RetentionPolicies{ + RetentionTimeInMinutes: retentionTimeInMinutes, + RetentionSizeInMB: int64(retentionSizeInMB), + } +} diff --git a/pulsaradmin/pkg/utils/schema_strategy.go b/pulsaradmin/pkg/utils/schema_strategy.go new file mode 100644 index 000000000..176f0e0a9 --- /dev/null +++ b/pulsaradmin/pkg/utils/schema_strategy.go @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type SchemaCompatibilityStrategy string + +const ( + AutoUpdateDisabled SchemaCompatibilityStrategy = "AutoUpdateDisabled" + Backward SchemaCompatibilityStrategy = "Backward" + Forward SchemaCompatibilityStrategy = "Forward" + Full SchemaCompatibilityStrategy = "Full" + AlwaysCompatible SchemaCompatibilityStrategy = "AlwaysCompatible" + BackwardTransitive SchemaCompatibilityStrategy = "BackwardTransitive" + ForwardTransitive SchemaCompatibilityStrategy = "ForwardTransitive" + FullTransitive SchemaCompatibilityStrategy = "FullTransitive" +) + +func ParseSchemaAutoUpdateCompatibilityStrategy(str string) (SchemaCompatibilityStrategy, error) { + switch str { + case "AutoUpdateDisabled": + return AutoUpdateDisabled, nil + case "Backward": + return Backward, nil + case "Forward": + return Forward, nil + case "Full": + return Full, nil + case "AlwaysCompatible": + return AlwaysCompatible, nil + case "BackwardTransitive": + return BackwardTransitive, nil + case "ForwardTransitive": + return ForwardTransitive, nil + case "FullTransitive": + return FullTransitive, nil + default: + return "", errors.Errorf("Invalid auth strategy %s", str) + } +} + +func (s SchemaCompatibilityStrategy) String() string { + return string(s) +} diff --git a/pulsaradmin/pkg/utils/schema_util.go b/pulsaradmin/pkg/utils/schema_util.go new file mode 100644 index 000000000..08aaf54ac --- /dev/null +++ b/pulsaradmin/pkg/utils/schema_util.go @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type SchemaInfo struct { + Name string `json:"name"` + Schema []byte `json:"schema"` + Type string `json:"type"` + Properties map[string]string `json:"properties"` +} + +type SchemaInfoWithVersion struct { + Version int64 `json:"version"` + SchemaInfo *SchemaInfo `json:"schemaInfo"` +} + +// Payload with information about a schema +type PostSchemaPayload struct { + SchemaType string `json:"type"` + Schema string `json:"schema"` + Properties map[string]string `json:"properties"` +} + +type GetSchemaResponse struct { + Version int64 `json:"version"` + Type string `json:"type"` + Timestamp int64 `json:"timestamp"` + Data string `json:"data"` + Properties map[string]string `json:"properties"` +} + +func ConvertGetSchemaResponseToSchemaInfo(tn *TopicName, response GetSchemaResponse) *SchemaInfo { + info := new(SchemaInfo) + schema := make([]byte, 0, 10) + if response.Type == "KEY_VALUE" { + // TODO: impl logic + } else { + schema = []byte(response.Data) + } + + info.Schema = schema + info.Type = response.Type + info.Properties = response.Properties + info.Name = tn.GetLocalName() + + return info +} + +func ConvertGetSchemaResponseToSchemaInfoWithVersion(tn *TopicName, response GetSchemaResponse) *SchemaInfoWithVersion { + info := new(SchemaInfoWithVersion) + info.SchemaInfo = ConvertGetSchemaResponseToSchemaInfo(tn, response) + info.Version = response.Version + return info +} diff --git a/pulsaradmin/pkg/utils/sink_config.go b/pulsaradmin/pkg/utils/sink_config.go new file mode 100644 index 000000000..0e9163bf9 --- /dev/null +++ b/pulsaradmin/pkg/utils/sink_config.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type SinkConfig struct { + TopicsPattern *string `json:"topicsPattern,omitempty" yaml:"topicsPattern"` + Resources *Resources `json:"resources,omitempty" yaml:"resources"` + TimeoutMs *int64 `json:"timeoutMs,omitempty" yaml:"timeoutMs"` + + // Whether the subscriptions the functions created/used should be deleted when the functions is deleted + CleanupSubscription bool `json:"cleanupSubscription" yaml:"cleanupSubscription"` + + RetainOrdering bool `json:"retainOrdering" yaml:"retainOrdering"` + RetainKeyOrdering bool `json:"retainKeyOrdering" yaml:"retainKeyOrdering"` + AutoAck bool `json:"autoAck" yaml:"autoAck"` + Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"` + Tenant string `json:"tenant,omitempty" yaml:"tenant"` + Namespace string `json:"namespace,omitempty" yaml:"namespace"` + Name string `json:"name,omitempty" yaml:"name"` + ClassName string `json:"className,omitempty" yaml:"className"` + + SinkType string `json:"sinkType,omitempty" yaml:"sinkType"` + Archive string `json:"archive,omitempty" yaml:"archive"` + ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"` + SourceSubscriptionName string `json:"sourceSubscriptionName,omitempty" yaml:"sourceSubscriptionName"` + SourceSubscriptionPosition string `json:"sourceSubscriptionPosition,omitempty" yaml:"sourceSubscriptionPosition"` + RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"` + + Inputs []string `json:"inputs,omitempty" yaml:"inputs"` + TopicToSerdeClassName map[string]string `json:"topicToSerdeClassName,omitempty" yaml:"topicToSerdeClassName"` + TopicToSchemaType map[string]string `json:"topicToSchemaType,omitempty" yaml:"topicToSchemaType"` + InputSpecs map[string]ConsumerConfig `json:"inputSpecs,omitempty" yaml:"inputSpecs"` + Configs map[string]interface{} `json:"configs,omitempty" yaml:"configs"` + + TopicToSchemaProperties map[string]string `json:"topicToSchemaProperties,omitempty" yaml:"topicToSchemaProperties"` + + CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"` + + // This is a map of secretName(aka how the secret is going to be + // accessed in the function via context) to an object that + // encapsulates how the secret is fetched by the underlying + // secrets provider. The type of an value here can be found by the + // SecretProviderConfigurator.getSecretObjectType() method. + Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"` + + MaxMessageRetries int `json:"maxMessageRetries,omitempty" yaml:"maxMessageRetries"` + DeadLetterTopic string `json:"deadLetterTopic,omitempty" yaml:"deadLetterTopic"` + NegativeAckRedeliveryDelayMs int64 `json:"negativeAckRedeliveryDelayMs,omitempty" yaml:"negativeAckRedeliveryDelayMs"` + TransformFunction string `json:"transformFunction,omitempty" yaml:"transformFunction"` + TransformFunctionClassName string `json:"transformFunctionClassName,omitempty" yaml:"transformFunctionClassName"` + TransformFunctionConfig string `json:"transformFunctionConfig,omitempty" yaml:"transformFunctionConfig"` +} diff --git a/pulsaradmin/pkg/utils/sink_status.go b/pulsaradmin/pkg/utils/sink_status.go new file mode 100644 index 000000000..6cdb091fa --- /dev/null +++ b/pulsaradmin/pkg/utils/sink_status.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type SinkStatus struct { + // The total number of sink instances that ought to be running + NumInstances int `json:"numInstances"` + + // The number of source instances that are actually running + NumRunning int `json:"numRunning"` + + Instances []*SinkInstanceStatus `json:"instances"` +} + +type SinkInstanceStatus struct { + InstanceID int `json:"instanceId"` + Status SourceInstanceStatusData `json:"status"` +} + +type SinkInstanceStatusData struct { + // Is this instance running? + Running bool `json:"running"` + + // Do we have any error while running this instance + Err string `json:"error"` + + // Number of times this instance has restarted + NumRestarts int64 `json:"numRestarts"` + + // Number of messages read from Pulsar + NumReadFromPulsar int64 `json:"numReadFromPulsar"` + + // Number of times there was a system exception handling messages + NumSystemExceptions int64 `json:"numSystemExceptions"` + + // A list of the most recent system exceptions + LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"` + + // Number of times there was a sink exception + NumSinkExceptions int64 `json:"numSinkExceptions"` + + // A list of the most recent sink exceptions + LatestSinkExceptions []ExceptionInformation `json:"latestSinkExceptions"` + + // Number of messages written to sink + NumWrittenToSink int64 `json:"numWrittenToSink"` + + // When was the last time we received a Message from Pulsar + LastReceivedTime int64 `json:"lastReceivedTime"` + + WorkerID string `json:"workerId"` +} diff --git a/pulsaradmin/pkg/utils/source_config.go b/pulsaradmin/pkg/utils/source_config.go new file mode 100644 index 000000000..7b0747610 --- /dev/null +++ b/pulsaradmin/pkg/utils/source_config.go @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type SourceConfig struct { + Tenant string `json:"tenant,omitempty" yaml:"tenant"` + Namespace string `json:"namespace,omitempty" yaml:"namespace"` + Name string `json:"name,omitempty" yaml:"name"` + ClassName string `json:"className,omitempty" yaml:"className"` + + ProducerConfig *ProducerConfig `json:"producerConfig,omitempty" yaml:"producerConfig"` + + TopicName string `json:"topicName,omitempty" yaml:"topicName"` + SerdeClassName string `json:"serdeClassName,omitempty" yaml:"serdeClassName"` + SchemaType string `json:"schemaType,omitempty" yaml:"schemaType"` + + Configs map[string]interface{} `json:"configs,omitempty" yaml:"configs"` + + // This is a map of secretName(aka how the secret is going to be + // accessed in the function via context) to an object that + // encapsulates how the secret is fetched by the underlying + // secrets provider. The type of an value here can be found by the + // SecretProviderConfigurator.getSecretObjectType() method. + Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"` + + Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"` + ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"` + Resources *Resources `json:"resources,omitempty" yaml:"resources"` + Archive string `json:"archive,omitempty" yaml:"archive"` + // Any flags that you want to pass to the runtime. + RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"` + + CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"` + + BatchSourceConfig *BatchSourceConfig `json:"batchSourceConfig,omitempty" yaml:"batchSourceConfig"` + BatchBuilder string `json:"batchBuilder,omitempty" yaml:"batchBuilder"` +} diff --git a/pulsaradmin/pkg/utils/source_status.go b/pulsaradmin/pkg/utils/source_status.go new file mode 100644 index 000000000..71df5a4fa --- /dev/null +++ b/pulsaradmin/pkg/utils/source_status.go @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type SourceStatus struct { + NumInstances int `json:"numInstances"` + NumRunning int `json:"numRunning"` + Instances []*SourceInstanceStatus `json:"instances"` +} + +type SourceInstanceStatus struct { + InstanceID int `json:"instanceId"` + Status SourceInstanceStatusData `json:"status"` +} + +type SourceInstanceStatusData struct { + Running bool `json:"running"` + Err string `json:"error"` + NumRestarts int64 `json:"numRestarts"` + NumReceivedFromSource int64 `json:"numReceivedFromSource"` + NumSystemExceptions int64 `json:"numSystemExceptions"` + LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"` + NumSourceExceptions int64 `json:"numSourceExceptions"` + LatestSourceExceptions []ExceptionInformation `json:"latestSourceExceptions"` + NumWritten int64 `json:"numWritten"` + LastReceivedTime int64 `json:"lastReceivedTime"` + WorkerID string `json:"workerId"` +} diff --git a/pulsaradmin/pkg/utils/subscription_auth_mode.go b/pulsaradmin/pkg/utils/subscription_auth_mode.go new file mode 100644 index 000000000..795b6d0ae --- /dev/null +++ b/pulsaradmin/pkg/utils/subscription_auth_mode.go @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type SubscriptionAuthMode string + +const ( + None SubscriptionAuthMode = "None" + Prefix SubscriptionAuthMode = "Prefix" +) + +func ParseSubscriptionAuthMode(s string) (SubscriptionAuthMode, error) { + switch s { + case "None": + return None, nil + case "Prefix": + return Prefix, nil + default: + return "", errors.New("Invalid subscription auth mode") + } +} + +func (s SubscriptionAuthMode) String() string { + return string(s) +} diff --git a/pulsaradmin/pkg/utils/topic_auto_creation_config.go b/pulsaradmin/pkg/utils/topic_auto_creation_config.go new file mode 100644 index 000000000..666465597 --- /dev/null +++ b/pulsaradmin/pkg/utils/topic_auto_creation_config.go @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type TopicAutoCreationConfig struct { + Allow bool `json:"allowAutoTopicCreation"` + Type TopicType `json:"topicType"` + Partitions int `json:"defaultNumPartitions"` +} diff --git a/pulsaradmin/pkg/utils/topic_domain.go b/pulsaradmin/pkg/utils/topic_domain.go new file mode 100644 index 000000000..98c59a9ce --- /dev/null +++ b/pulsaradmin/pkg/utils/topic_domain.go @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type TopicDomain string + +const ( + persistent TopicDomain = "persistent" + nonPersistent TopicDomain = "non-persistent" +) + +func ParseTopicDomain(domain string) (TopicDomain, error) { + switch domain { + case "persistent": + return persistent, nil + case "non-persistent": + return nonPersistent, nil + default: + return "", errors.Errorf("The domain only can be specified as 'persistent' or "+ + "'non-persistent'. Input domain is '%s'.", domain) + } +} + +func (t TopicDomain) String() string { + return string(t) +} diff --git a/pulsaradmin/pkg/utils/topic_name.go b/pulsaradmin/pkg/utils/topic_name.go new file mode 100644 index 000000000..268abd73d --- /dev/null +++ b/pulsaradmin/pkg/utils/topic_name.go @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +const ( + PUBLICTENANT = "public" + DEFAULTNAMESPACE = "default" + PARTITIONEDTOPICSUFFIX = "-partition-" +) + +type TopicName struct { + domain TopicDomain + tenant string + namespace string + topic string + partitionIndex int + + namespaceName *NameSpaceName +} + +// The topic name can be in two different forms, one is fully qualified topic name, +// the other one is short topic name +func GetTopicName(completeName string) (*TopicName, error) { + var topicName TopicName + // The short topic name can be: + // - + // - // + if !strings.Contains(completeName, "://") { + parts := strings.Split(completeName, "/") + switch len(parts) { + case 3: + completeName = persistent.String() + "://" + completeName + case 1: + completeName = persistent.String() + "://" + PUBLICTENANT + "/" + DEFAULTNAMESPACE + "/" + parts[0] + default: + return nil, errors.Errorf("Invalid short topic name '%s', it should be "+ + "in the format of // or ", completeName) + } + } + + // The fully qualified topic name can be: + // ://// + + parts := strings.SplitN(completeName, "://", 2) + + domain, err := ParseTopicDomain(parts[0]) + if err != nil { + return nil, err + } + topicName.domain = domain + + rest := parts[1] + parts = strings.SplitN(rest, "/", 3) + if len(parts) == 3 { + topicName.tenant = parts[0] + topicName.namespace = parts[1] + topicName.topic = parts[2] + topicName.partitionIndex = getPartitionIndex(completeName) + } else { + return nil, errors.Errorf("invalid topic name '%s', it should be in the format of "+ + "//", rest) + } + + if topicName.topic == "" { + return nil, errors.New("topic name can not be empty") + } + + n, err := GetNameSpaceName(topicName.tenant, topicName.namespace) + if err != nil { + return nil, err + } + topicName.namespaceName = n + + return &topicName, nil +} + +func (t *TopicName) String() string { + return fmt.Sprintf("%s://%s/%s/%s", t.domain, t.tenant, t.namespace, t.topic) +} + +func (t *TopicName) GetDomain() TopicDomain { + return t.domain +} + +func (t *TopicName) GetTenant() string { + return t.tenant +} + +func (t *TopicName) GetNamespace() string { + return t.namespace +} + +func (t *TopicName) IsPersistent() bool { + return t.domain == persistent +} + +func (t *TopicName) GetRestPath() string { + return fmt.Sprintf("%s/%s/%s/%s", t.domain, t.tenant, t.namespace, t.topic) +} + +func (t *TopicName) GetEncodedTopic() string { + return url.QueryEscape(t.topic) +} + +func (t *TopicName) GetLocalName() string { + return t.topic +} + +func (t *TopicName) GetPartition(index int) (*TopicName, error) { + if index < 0 { + return nil, errors.New("invalid partition index number") + } + + if strings.Contains(t.String(), PARTITIONEDTOPICSUFFIX) { + return t, nil + } + + topicNameWithPartition := t.String() + PARTITIONEDTOPICSUFFIX + strconv.Itoa(index) + return GetTopicName(topicNameWithPartition) +} + +func getPartitionIndex(topic string) int { + if strings.Contains(topic, PARTITIONEDTOPICSUFFIX) { + parts := strings.Split(topic, "-") + index, err := strconv.Atoi(parts[len(parts)-1]) + if err == nil { + return index + } + } + return -1 +} diff --git a/pulsaradmin/pkg/utils/topic_name_test.go b/pulsaradmin/pkg/utils/topic_name_test.go new file mode 100644 index 000000000..27e500248 --- /dev/null +++ b/pulsaradmin/pkg/utils/topic_name_test.go @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetTopicName(t *testing.T) { + success, err := GetTopicName("success") + assert.Nil(t, err) + assert.Equal(t, "persistent://public/default/success", success.String()) + + success, err = GetTopicName("tenant/namespace/success") + assert.Nil(t, err) + assert.Equal(t, "persistent://tenant/namespace/success", success.String()) + + success, err = GetTopicName("persistent://tenant/namespace/success") + assert.Nil(t, err) + assert.Equal(t, "persistent://tenant/namespace/success", success.String()) + + success, err = GetTopicName("non-persistent://tenant/namespace/success") + assert.Nil(t, err) + assert.Equal(t, "non-persistent://tenant/namespace/success", success.String()) + + _, err = GetTopicName("://tenant.namespace.topic") + assert.NotNil(t, err) + assert.Equal(t, "The domain only can be specified as 'persistent' or 'non-persistent'."+ + " Input domain is ''.", err.Error()) + + fail, err := GetTopicName("default/fail") + assert.NotNil(t, err) + assert.Equal(t, "Invalid short topic name 'default/fail', it should be in the "+ + "format of // or ", err.Error()) + assert.Nil(t, fail) + + fail, err = GetTopicName("domain://tenant/namespace/fail") + assert.NotNil(t, err) + assert.Equal(t, "The domain only can be specified as 'persistent' or 'non-persistent'. "+ + "Input domain is 'domain'.", err.Error()) + assert.Nil(t, fail) + + fail, err = GetTopicName("persistent:///tenant/namespace/fail") + assert.NotNil(t, err) + assert.Equal(t, "Invalid tenant or namespace. [/tenant]", err.Error()) + assert.Nil(t, fail) + + fail, err = GetTopicName("persistent://tenant/namespace") + assert.NotNil(t, err) + assert.Equal(t, "invalid topic name 'tenant/namespace', it should be in the format "+ + "of //", err.Error()) + assert.Nil(t, fail) + + fail, err = GetTopicName("persistent://tenant/namespace/") + assert.NotNil(t, err) + assert.Equal(t, "topic name can not be empty", err.Error()) + assert.Nil(t, fail) +} diff --git a/pulsaradmin/pkg/utils/topic_type.go b/pulsaradmin/pkg/utils/topic_type.go new file mode 100644 index 000000000..18320594a --- /dev/null +++ b/pulsaradmin/pkg/utils/topic_type.go @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type TopicType string + +const ( + Partitioned TopicType = "partitioned" + NonPartitioned TopicType = "non-partitioned" +) + +func ParseTopicType(topicType string) (TopicType, error) { + switch topicType { + case "partitioned": + return Partitioned, nil + case "non-partitioned": + return NonPartitioned, nil + default: + return "", errors.Errorf("The topic type can only be specified as 'partitioned' or "+ + "'non-partitioned'. Input topic type is '%s'.", topicType) + } +} + +func (t TopicType) String() string { + return string(t) +} diff --git a/pulsaradmin/pkg/utils/topics_stats_stream.go b/pulsaradmin/pkg/utils/topics_stats_stream.go new file mode 100644 index 000000000..7554609e6 --- /dev/null +++ b/pulsaradmin/pkg/utils/topics_stats_stream.go @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +// var TopicsMap map[string]map[string]map[string]TopicStats + +type TopicStatsStream struct { + TopicsMap map[string]map[string]map[string]TopicStats `json:"topicStatsBuf"` +} diff --git a/pulsaradmin/pkg/utils/update_options.go b/pulsaradmin/pkg/utils/update_options.go new file mode 100644 index 000000000..d78fccfce --- /dev/null +++ b/pulsaradmin/pkg/utils/update_options.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +// Options while updating the sink +type UpdateOptions struct { + UpdateAuthData bool +} + +func NewUpdateOptions() *UpdateOptions { + return &UpdateOptions{ + UpdateAuthData: false, + } +} diff --git a/pulsaradmin/pkg/utils/utils.go b/pulsaradmin/pkg/utils/utils.go new file mode 100644 index 000000000..7b23b1565 --- /dev/null +++ b/pulsaradmin/pkg/utils/utils.go @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "fmt" + "reflect" +) + +func MakeHTTPPath(apiVersion string, componentPath string) string { + return fmt.Sprintf("/admin/%s%s", apiVersion, componentPath) +} + +func IsNilFixed(i interface{}) bool { + if i == nil { + return true + } + switch reflect.TypeOf(i).Kind() { + case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice: + return reflect.ValueOf(i).IsNil() + } + return false +} diff --git a/pulsaradmin/pkg/utils/utils_test.go b/pulsaradmin/pkg/utils/utils_test.go new file mode 100644 index 000000000..7a7fe4abe --- /dev/null +++ b/pulsaradmin/pkg/utils/utils_test.go @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type People interface { + MakeSound() string +} + +type Student struct{} + +func (s *Student) MakeSound() string { + return "Student" +} + +type Teacher struct{} + +func (t Teacher) MakeSound() string { + return "Teacher" +} + +// nolint +func TestIsNilFixed(t *testing.T) { + var stu *Student = nil + var people People + people = stu + + var teacher Teacher + people = teacher + + assert.False(t, IsNilFixed(people)) + + var m map[string]string + assert.True(t, IsNilFixed(m)) + + var s []string + assert.True(t, IsNilFixed(s)) + + var ch chan string + assert.True(t, IsNilFixed(ch)) + + var nilInterface People + assert.True(t, IsNilFixed(nilInterface)) + + // pointer to an interface, the IsNilFixed method cannot check this. + assert.False(t, IsNilFixed(&nilInterface)) +} diff --git a/pulsaradmin/pkg/utils/window_confing.go b/pulsaradmin/pkg/utils/window_confing.go new file mode 100644 index 000000000..ccde3b68f --- /dev/null +++ b/pulsaradmin/pkg/utils/window_confing.go @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +const WindowConfigKey = "__WINDOWCONFIGS__" + +type WindowConfig struct { + WindowLengthCount *int `json:"windowLengthCount" yaml:"windowLengthCount"` + WindowLengthDurationMs *int64 `json:"windowLengthDurationMs" yaml:"windowLengthDurationMs"` + SlidingIntervalCount *int `json:"slidingIntervalCount" yaml:"slidingIntervalCount"` + SlidingIntervalDurationMs *int64 `json:"slidingIntervalDurationMs" yaml:"slidingIntervalDurationMs"` + LateDataTopic *string `json:"lateDataTopic" yaml:"lateDataTopic"` + MaxLagMs *int64 `json:"maxLagMs" yaml:"maxLagMs"` + WatermarkEmitIntervalMs *int64 `json:"watermarkEmitIntervalMs" yaml:"watermarkEmitIntervalMs"` + TimestampExtractorClassName *string `json:"timestampExtractorClassName" yaml:"timestampExtractorClassName"` + ActualWindowFunctionClassName *string `json:"actualWindowFunctionClassName" yaml:"actualWindowFunctionClassName"` + ProcessingGuarantees *string `json:"processingGuarantees" yaml:"processingGuarantees"` +} + +func NewDefaultWindowConfing() *WindowConfig { + windowConfig := &WindowConfig{} + + return windowConfig +} diff --git a/pulsaradmin/pkg/utils/worker_info.go b/pulsaradmin/pkg/utils/worker_info.go new file mode 100644 index 000000000..bd0dd806c --- /dev/null +++ b/pulsaradmin/pkg/utils/worker_info.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type WorkerInfo struct { + WorkerID string `json:"workerId"` + WorkerHostname string `json:"workerHostname"` + Port int `json:"port"` +} + +type WorkerFunctionInstanceStats struct { + Name string `json:"name"` + Metrics FunctionInstanceStatsData `json:"metrics"` +}