diff --git a/Makefile b/Makefile index 433db8cb7..4eb590b06 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ GOROOT ?= $(shell go env GOROOT) build: go build ./pulsar + go build ./pulsaradmin go build -o bin/pulsar-perf ./perf lint: bin/golangci-lint diff --git a/go.mod b/go.mod index 69862b208..f88d6ad57 100644 --- a/go.mod +++ b/go.mod @@ -27,9 +27,11 @@ require ( golang.org/x/mod v0.5.1 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.30.0 ) +require github.com/golang/protobuf v1.5.2 + require ( github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/ardielle/ardielle-go v1.5.2 // indirect @@ -39,7 +41,6 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.1 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect diff --git a/go.sum b/go.sum index 3a3987443..00a44917d 100644 --- a/go.sum +++ b/go.sum @@ -551,7 +551,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/pulsaradmin/CONTRIBUTING.md b/pulsaradmin/CONTRIBUTING.md new file mode 100644 index 000000000..662947929 --- /dev/null +++ b/pulsaradmin/CONTRIBUTING.md @@ -0,0 +1,60 @@ +# Contributing guidelines + +## Project structure + +The overall project structure is illustrated below: + +```text +├── pkg/ +│ ├── admin/ +│ │ ├── auth/ +│ │ ├── config/ +│ ├── rest/ +│ └── utils/ +└── alias.go +``` + +- The `alias.go` file in the root defines `pulsaradmin` package scope, which contains shortcuts of some types and functions from the `pkg`. +- The `pkg/admin` package contains all operations for pulsar admin resources. *Note: We should add a new file here if we wanna support a new resource.* + - The `pkg/admin/config` package contains configuration options for constructing a pulsar admin client. + - The `pkg/admin/auth` package contains auth providers which work in transport layer. +- The `pkg/rest` package contains a wrapped HTTP client for requesting pulsar REST API. +- The `pkg/utils` package contains common data structures and functions. + +## Contributing steps +1. Submit an issue describing your proposed change. +2. Discuss and wait for proposal to be accepted. +3. Fork this repo, develop and test your code changes. +4. Submit a pull request. + +## Conventions + +Please read through below conventions before contributions. + +### PullRequest conventions + +- Use [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) to standardize PR title. + +### Code conventions + +- [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) +- [Effective Go](https://golang.org/doc/effective_go.html) +- Know and avoid [Go landmines](https://gist.github.com/lavalamp/4bd23295a9f32706a48f) +- Commenting + - [Go's commenting conventions](http://blog.golang.org/godoc-documenting-go-code) + - If reviewers ask questions about why the code is the way it is, that's a sign that comments might be helpful. +- Naming + - Please consider package name when selecting an interface name, and avoid redundancy. For example, `storage.Interface` is better than `storage.StorageInterface`. + - Do not use uppercase characters, underscores, or dashes in package names. + - Please consider parent directory name when choosing a package name. For example, `pkg/controllers/autoscaler/foo.go` should say `package autoscaler` not `package autoscalercontroller`. + - Unless there's a good reason, the `package foo` line should match the name of the directory in which the `.go` file exists. + - Importers can use a different name if they need to disambiguate. + - Locks should be called `lock` and should never be embedded (always `lock sync.Mutex`). When multiple locks are present, give each lock a distinct name following Go conventions: `stateLock`, `mapLock` etc. + +### Folder and file conventions + +- All filenames should be lowercase. +- Go source files and directories use underscores, not dashes. + - Package directories should generally avoid using separators as much as possible. When package names are multiple words, they usually should be in nested subdirectories. +- Document directories and filenames should use dashes rather than underscores. +- All source files should add a license at the beginning. diff --git a/pulsaradmin/README.md b/pulsaradmin/README.md new file mode 100644 index 000000000..bf5ca739c --- /dev/null +++ b/pulsaradmin/README.md @@ -0,0 +1,114 @@ +# Pulsar Admin Go Library + +Pulsar-Admin-Go is a [Go](https://go.dev) library for [Apache Pulsar](https://pulsar.apache.org/). It provides a unified Go API for managing pulsar resources such as tenants, namespaces and topics, etc. + +## Motivation + +Currently, many projects (e.g, [terraform-provider-pulsar](https://github.com/streamnative/terraform-provider-pulsar) and [pulsar-resources-operator](https://github.com/streamnative/pulsar-resources-operator)) +that need to manipulate the pulsar admin resources rely on the [pulsarctl](https://github.com/streamnative/pulsarctl), +which poses challenges for dependency management and versioning as we have to release a new pulsarctl to get updates. +So we decoupled the pulsar admin related api from pulsarctl and created the [pulsar-admin-go](https://github.com/apache/pulsar-client-go/pulsaradmin) library based on it, +which also provides a clearer perspective and maintainability from an architectural perspective. + +## Quickstart + +### Prerequisite + +- go1.18+ +- pulsar-admin-go in go.mod + + ```shell + go get github.com/apache/pulsar-client-go + ``` + +### Manage pulsar tenants + +- List all tenants + +```go +import ( + "github.com/apache/pulsar-client-go/pulsaradmin" +) + +func main() { + cfg := &pulsaradmin.Config{} + admin, err := pulsaradmin.NewClient(cfg) + if err != nil { + panic(err) + } + + tenants, _ := admin.Tenants().List() +} +``` + +### Manage pulsar namespaces + +- List all namespaces + +```go +import ( + "github.com/apache/pulsar-client-go/pulsaradmin" +) + +func main() { + cfg := &pulsaradmin.Config{} + admin, err := pulsaradmin.NewClient(cfg) + if err != nil { + panic(err) + } + + namespaces, _ := admin.Namespaces().GetNamespaces("public") +} +``` + +- Create a new namespace + +```go +import ( + "github.com/apache/pulsar-client-go/pulsaradmin" +) + +func main() { + cfg := &pulsaradmin.Config{} + admin, err := pulsaradmin.NewClient(cfg) + if err != nil { + panic(err) + } + + admin.Namespaces().CreateNamespace("public/dev") +} +``` + +### Manage pulsar topics + +- Create a topic + +```go + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +func main() { + cfg := &pulsaradmin.Config{} + admin, err := pulsaradmin.NewClient(cfg) + if err != nil { + panic(err) + } + + topic, _ := utils.GetTopicName("public/dev/topic") + + admin.Topics().Create(*topic, 3) +} +``` + +## Contributing + +Contributions are warmly welcomed and greatly appreciated! +The project follows the typical GitHub pull request model. See [CONTRIBUTING.md](CONTRIBUTING.md) for more details. +Before starting any work, please either comment on an existing issue, or file a new one. + +## License + +Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). diff --git a/pulsaradmin/alias.go b/pulsaradmin/alias.go new file mode 100644 index 000000000..53c8d93e0 --- /dev/null +++ b/pulsaradmin/alias.go @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package pulsaradmin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" +) + +// Client contains all admin interfaces for operating pulsar resources +type Client = admin.Client + +// Config are the arguments for creating a new admin Client +type Config = config.Config + +var ( + // NewClient returns a new admin Client for operating pulsar resources + NewClient = admin.New +) diff --git a/pulsaradmin/pkg/admin/admin.go b/pulsaradmin/pkg/admin/admin.go new file mode 100644 index 000000000..a22b22c62 --- /dev/null +++ b/pulsaradmin/pkg/admin/admin.go @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "net/http" + "net/url" + "path" + "time" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/auth" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/rest" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +const ( + DefaultWebServiceURL = "http://localhost:8080" + DefaultHTTPTimeOutDuration = 5 * time.Minute + ReleaseVersion = "None" +) + +type TLSOptions struct { + TrustCertsFilePath string + AllowInsecureConnection bool +} + +// Client provides a client to the Pulsar Restful API +type Client interface { + Clusters() Clusters + Functions() Functions + Tenants() Tenants + Topics() Topics + Subscriptions() Subscriptions + Sources() Sources + Sinks() Sinks + Namespaces() Namespaces + Schemas() Schema + NsIsolationPolicy() NsIsolationPolicy + Brokers() Brokers + BrokerStats() BrokerStats + ResourceQuotas() ResourceQuotas + FunctionsWorker() FunctionsWorker + Packages() Packages +} + +type pulsarClient struct { + Client *rest.Client + APIVersion config.APIVersion +} + +// New returns a new client +func New(config *config.Config) (Client, error) { + authProvider, err := auth.GetAuthProvider(config) + if err != nil { + return nil, err + } + return NewPulsarClientWithAuthProvider(config, authProvider) +} + +// NewWithAuthProvider creates a client with auth provider. +// Deprecated: Use NewPulsarClientWithAuthProvider instead. +func NewWithAuthProvider(config *config.Config, authProvider auth.Provider) Client { + client, err := NewPulsarClientWithAuthProvider(config, authProvider) + if err != nil { + panic(err) + } + return client +} + +// NewPulsarClientWithAuthProvider create a client with auth provider. +func NewPulsarClientWithAuthProvider(config *config.Config, authProvider auth.Provider) (Client, error) { + if len(config.WebServiceURL) == 0 { + config.WebServiceURL = DefaultWebServiceURL + } + + return &pulsarClient{ + APIVersion: config.PulsarAPIVersion, + Client: &rest.Client{ + ServiceURL: config.WebServiceURL, + VersionInfo: ReleaseVersion, + HTTPClient: &http.Client{ + Timeout: DefaultHTTPTimeOutDuration, + Transport: authProvider, + }, + }, + }, nil +} + +func (c *pulsarClient) endpoint(componentPath string, parts ...string) string { + escapedParts := make([]string, len(parts)) + for i, part := range parts { + escapedParts[i] = url.PathEscape(part) + } + return path.Join( + utils.MakeHTTPPath(c.APIVersion.String(), componentPath), + path.Join(escapedParts...), + ) +} diff --git a/pulsaradmin/pkg/admin/admin_test.go b/pulsaradmin/pkg/admin/admin_test.go new file mode 100644 index 000000000..c4fa52956 --- /dev/null +++ b/pulsaradmin/pkg/admin/admin_test.go @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/auth" + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" +) + +func TestPulsarClientEndpointEscapes(t *testing.T) { + client := pulsarClient{Client: nil, APIVersion: config.V2} + actual := client.endpoint("/myendpoint", "abc%? /def", "ghi") + expected := "/admin/v2/myendpoint/abc%25%3F%20%2Fdef/ghi" + assert.Equal(t, expected, actual) +} + +func TestNew(t *testing.T) { + config := &config.Config{} + admin, err := New(config) + require.NoError(t, err) + require.NotNil(t, admin) +} + +func TestNewWithAuthProvider(t *testing.T) { + config := &config.Config{} + + tokenAuth, err := auth.NewAuthenticationToken("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9."+ + "eyJzdWIiOiJhZG1pbiIsImlhdCI6MTUxNjIzOTAyMn0.sVt6cyu3HKd89LcQvZVMNbqT0DTl3FvG9oYbj8hBDqU", nil) + require.NoError(t, err) + require.NotNil(t, tokenAuth) + + admin, err := NewPulsarClientWithAuthProvider(config, tokenAuth) + require.NoError(t, err) + require.NotNil(t, admin) +} + +type customAuthProvider struct { + transport http.RoundTripper +} + +var _ auth.Provider = &customAuthProvider{} + +func (c *customAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { + panic("implement me") +} + +func (c *customAuthProvider) Transport() http.RoundTripper { + return c.transport +} + +func (c *customAuthProvider) WithTransport(transport http.RoundTripper) { + c.transport = transport +} + +func TestNewWithCustomAuthProviderWithTransport(t *testing.T) { + config := &config.Config{} + defaultTransport, err := auth.NewDefaultTransport(config) + require.NoError(t, err) + + customAuthProvider := &customAuthProvider{ + transport: defaultTransport, + } + + admin, err := NewPulsarClientWithAuthProvider(config, customAuthProvider) + require.NoError(t, err) + require.NotNil(t, admin) + + // Expected the customAuthProvider will not be overwritten. + require.Equal(t, customAuthProvider, admin.(*pulsarClient).Client.HTTPClient.Transport) +} + +func TestNewWithTlsAllowInsecure(t *testing.T) { + config := &config.Config{ + TLSAllowInsecureConnection: true, + } + admin, err := New(config) + require.NoError(t, err) + require.NotNil(t, admin) + + pulsarClientS := admin.(*pulsarClient) + require.NotNil(t, pulsarClientS.Client.HTTPClient.Transport) + + ap := pulsarClientS.Client.HTTPClient.Transport.(*auth.DefaultProvider) + tr := ap.Transport().(*http.Transport) + require.NotNil(t, tr) + require.NotNil(t, tr.TLSClientConfig) + require.True(t, tr.TLSClientConfig.InsecureSkipVerify) +} diff --git a/pulsaradmin/pkg/admin/auth/oauth2.go b/pulsaradmin/pkg/admin/auth/oauth2.go new file mode 100644 index 000000000..eab64cace --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/oauth2.go @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "encoding/json" + "net/http" + "path/filepath" + + "github.com/99designs/keyring" + "github.com/apache/pulsar-client-go/oauth2" + "github.com/apache/pulsar-client-go/oauth2/cache" + clock2 "github.com/apache/pulsar-client-go/oauth2/clock" + "github.com/apache/pulsar-client-go/oauth2/store" + "github.com/pkg/errors" + xoauth2 "golang.org/x/oauth2" +) + +const ( + OAuth2PluginName = "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2" + OAuth2PluginShortName = "oauth2" +) + +type OAuth2ClientCredentials struct { + IssuerURL string `json:"issuerUrl,omitempty"` + Audience string `json:"audience,omitempty"` + Scope string `json:"scope,omitempty"` + PrivateKey string `json:"privateKey,omitempty"` + ClientID string `json:"clientId,omitempty"` +} + +type OAuth2Provider struct { + clock clock2.RealClock + issuer oauth2.Issuer + store store.Store + source cache.CachingTokenSource + defaultTransport http.RoundTripper + tokenTransport *transport +} + +func NewAuthenticationOAuth2(issuer oauth2.Issuer, store store.Store) (*OAuth2Provider, error) { + p := &OAuth2Provider{ + clock: clock2.RealClock{}, + issuer: issuer, + store: store, + } + + err := p.loadGrant() + if err != nil { + return nil, err + } + + return p, nil +} + +// NewAuthenticationOAuth2WithDefaultFlow uses memory to save the grant +func NewAuthenticationOAuth2WithDefaultFlow(issuer oauth2.Issuer, keyFile string) (Provider, error) { + st := store.NewMemoryStore() + flow, err := oauth2.NewDefaultClientCredentialsFlow(oauth2.ClientCredentialsFlowOptions{ + KeyFile: keyFile, + }) + if err != nil { + return nil, err + } + + grant, err := flow.Authorize(issuer.Audience) + if err != nil { + return nil, err + } + + err = st.SaveGrant(issuer.Audience, *grant) + if err != nil { + return nil, err + } + + p := &OAuth2Provider{ + clock: clock2.RealClock{}, + issuer: issuer, + store: st, + } + + return p, p.loadGrant() +} + +func NewAuthenticationOAuth2FromAuthParams(encodedAuthParam string, + transport http.RoundTripper) (*OAuth2Provider, error) { + + var paramsJSON OAuth2ClientCredentials + err := json.Unmarshal([]byte(encodedAuthParam), ¶msJSON) + if err != nil { + return nil, err + } + return NewAuthenticationOAuth2WithParams(paramsJSON.IssuerURL, paramsJSON.ClientID, paramsJSON.Audience, + paramsJSON.Scope, transport) +} + +func NewAuthenticationOAuth2WithParams( + issuerEndpoint, + clientID, + audience string, + scope string, + transport http.RoundTripper) (*OAuth2Provider, error) { + + issuer := oauth2.Issuer{ + IssuerEndpoint: issuerEndpoint, + ClientID: clientID, + Audience: audience, + } + + keyringStore, err := MakeKeyringStore() + if err != nil { + return nil, err + } + + p := &OAuth2Provider{ + clock: clock2.RealClock{}, + issuer: issuer, + store: keyringStore, + defaultTransport: transport, + } + + err = p.loadGrant() + if err != nil { + return nil, err + } + + return p, nil +} + +func (o *OAuth2Provider) loadGrant() error { + grant, err := o.store.LoadGrant(o.issuer.Audience) + if err != nil { + if err == store.ErrNoAuthenticationData { + return errors.New("oauth2 login required") + } + return err + } + return o.initCache(grant) +} + +func (o *OAuth2Provider) initCache(grant *oauth2.AuthorizationGrant) error { + refresher, err := o.getRefresher(grant.Type) + if err != nil { + return err + } + + source, err := cache.NewDefaultTokenCache(o.store, o.issuer.Audience, refresher) + if err != nil { + return err + } + o.source = source + o.tokenTransport = &transport{ + source: o.source, + wrapped: &xoauth2.Transport{ + Source: o.source, + Base: o.defaultTransport, + }, + } + return nil +} + +func (o *OAuth2Provider) RoundTrip(req *http.Request) (*http.Response, error) { + return o.tokenTransport.RoundTrip(req) +} + +func (o *OAuth2Provider) WithTransport(tripper http.RoundTripper) { + o.defaultTransport = tripper +} + +func (o *OAuth2Provider) Transport() http.RoundTripper { + return o.tokenTransport +} + +func (o *OAuth2Provider) getRefresher(t oauth2.AuthorizationGrantType) (oauth2.AuthorizationGrantRefresher, error) { + switch t { + case oauth2.GrantTypeClientCredentials: + return oauth2.NewDefaultClientCredentialsGrantRefresher(o.clock) + case oauth2.GrantTypeDeviceCode: + return oauth2.NewDefaultDeviceAuthorizationGrantRefresher(o.clock) + default: + return nil, store.ErrUnsupportedAuthData + } +} + +type transport struct { + source cache.CachingTokenSource + wrapped *xoauth2.Transport +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + if len(req.Header.Get("Authorization")) != 0 { + return t.wrapped.Base.RoundTrip(req) + } + + res, err := t.wrapped.RoundTrip(req) + if err != nil { + return nil, err + } + + if res.StatusCode == 401 { + err := t.source.InvalidateToken() + if err != nil { + return nil, err + } + } + + return res, nil +} + +func (t *transport) WrappedRoundTripper() http.RoundTripper { return t.wrapped.Base } + +const ( + serviceName = "pulsar" + keyChainName = "pulsarctl" +) + +func MakeKeyringStore() (store.Store, error) { + kr, err := makeKeyring() + if err != nil { + return nil, err + } + return store.NewKeyringStore(kr) +} + +func makeKeyring() (keyring.Keyring, error) { + return keyring.Open(keyring.Config{ + AllowedBackends: keyring.AvailableBackends(), + ServiceName: serviceName, + KeychainName: keyChainName, + KeychainTrustApplication: true, + FileDir: filepath.Join("~/.config/pulsar", "credentials"), + FilePasswordFunc: keyringPrompt, + }) +} + +func keyringPrompt(prompt string) (string, error) { + return "", nil +} diff --git a/pulsaradmin/pkg/admin/auth/oauth2_test.go b/pulsaradmin/pkg/admin/auth/oauth2_test.go new file mode 100644 index 000000000..b25e57611 --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/oauth2_test.go @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/apache/pulsar-client-go/oauth2" + "github.com/apache/pulsar-client-go/oauth2/store" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +// mockOAuthServer will mock a oauth service for the tests +func mockOAuthServer() *httptest.Server { + // prepare a port for the mocked server + server := httptest.NewUnstartedServer(http.DefaultServeMux) + + // mock the used REST path for the tests + mockedHandler := http.NewServeMux() + mockedHandler.HandleFunc("/.well-known/openid-configuration", func(writer http.ResponseWriter, request *http.Request) { + s := fmt.Sprintf(`{ + "issuer":"%s", + "authorization_endpoint":"%s/authorize", + "token_endpoint":"%s/oauth/token", + "device_authorization_endpoint":"%s/oauth/device/code" +}`, server.URL, server.URL, server.URL, server.URL) + fmt.Fprintln(writer, s) + }) + mockedHandler.HandleFunc("/oauth/token", func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprintln(writer, "{\n \"access_token\": \"token-content\",\n \"token_type\": \"Bearer\"\n}") + }) + mockedHandler.HandleFunc("/authorize", func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprintln(writer, "true") + }) + + server.Config.Handler = mockedHandler + server.Start() + + return server +} + +// mockKeyFile will mock a temp key file for testing. +func mockKeyFile(server string) (string, error) { + pwd, err := os.Getwd() + if err != nil { + return "", err + } + kf, err := os.CreateTemp(pwd, "test_oauth2") + if err != nil { + return "", err + } + _, err = kf.WriteString(fmt.Sprintf(`{ + "type":"sn_service_account", + "client_id":"client-id", + "client_secret":"client-secret", + "client_email":"oauth@test.org", + "issuer_url":"%s" +}`, server)) + + if err != nil { + return "", err + } + + return kf.Name(), nil +} + +func TestOauth2(t *testing.T) { + server := mockOAuthServer() + defer server.Close() + kf, err := mockKeyFile(server.URL) + defer os.Remove(kf) + if err != nil { + t.Fatal(errors.Wrap(err, "create mocked key file failed")) + } + + issuer := oauth2.Issuer{ + IssuerEndpoint: server.URL, + ClientID: "client-id", + Audience: server.URL, + } + + memoryStore := store.NewMemoryStore() + err = saveGrant(memoryStore, kf, issuer.Audience) + if err != nil { + t.Fatal(err) + } + + auth, err := NewAuthenticationOAuth2(issuer, memoryStore) + if err != nil { + t.Fatal(err) + } + + token, err := auth.source.Token() + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "token-content", token.AccessToken) +} + +func saveGrant(store store.Store, keyFile, audience string) error { + flow, err := oauth2.NewDefaultClientCredentialsFlow(oauth2.ClientCredentialsFlowOptions{ + KeyFile: keyFile, + AdditionalScopes: nil, + }) + if err != nil { + return err + } + + grant, err := flow.Authorize(audience) + if err != nil { + return err + } + + return store.SaveGrant(audience, *grant) +} diff --git a/pulsaradmin/pkg/admin/auth/provider.go b/pulsaradmin/pkg/admin/auth/provider.go new file mode 100644 index 000000000..24f74d265 --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/provider.go @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "net/http" + + "github.com/apache/pulsar-client-go/oauth2" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" +) + +// Provider provide a general method to add auth message +type Provider interface { + RoundTrip(req *http.Request) (*http.Response, error) + Transport() http.RoundTripper + WithTransport(tripper http.RoundTripper) +} + +type DefaultProvider struct { + transport http.RoundTripper +} + +func NewDefaultProvider(t http.RoundTripper) Provider { + return &DefaultProvider{ + transport: t, + } +} + +func (dp *DefaultProvider) RoundTrip(req *http.Request) (*http.Response, error) { + return dp.transport.RoundTrip(req) +} + +func (dp *DefaultProvider) Transport() http.RoundTripper { + return dp.transport +} + +func (dp *DefaultProvider) WithTransport(t http.RoundTripper) { + dp.transport = t +} + +func GetAuthProvider(config *config.Config) (Provider, error) { + var provider Provider + defaultTransport, err := NewDefaultTransport(config) + if err != nil { + return nil, err + } + switch config.AuthPlugin { + case TLSPluginShortName: + fallthrough + case TLSPluginName: + provider, err = NewAuthenticationTLSFromAuthParams(config.AuthParams, defaultTransport) + case TokenPluginName: + fallthrough + case TokePluginShortName: + provider, err = NewAuthenticationTokenFromAuthParams(config.AuthParams, defaultTransport) + case OAuth2PluginName: + fallthrough + case OAuth2PluginShortName: + provider, err = NewAuthenticationOAuth2WithDefaultFlow(oauth2.Issuer{ + IssuerEndpoint: config.IssuerEndpoint, + ClientID: config.ClientID, + Audience: config.Audience, + }, config.KeyFile) + default: + switch { + case len(config.TLSCertFile) > 0 && len(config.TLSKeyFile) > 0: + provider, err = NewAuthenticationTLS(config.TLSCertFile, config.TLSKeyFile, defaultTransport) + case len(config.Token) > 0: + provider, err = NewAuthenticationToken(config.Token, defaultTransport) + case len(config.TokenFile) > 0: + provider, err = NewAuthenticationTokenFromFile(config.TokenFile, defaultTransport) + case len(config.IssuerEndpoint) > 0 || len(config.ClientID) > 0 || len(config.Audience) > 0 || len(config.Scope) > 0: + provider, err = NewAuthenticationOAuth2WithParams( + config.IssuerEndpoint, config.ClientID, config.Audience, config.Scope, defaultTransport) + default: + provider = NewDefaultProvider(defaultTransport) + } + } + return provider, err +} diff --git a/pulsaradmin/pkg/admin/auth/tls.go b/pulsaradmin/pkg/admin/auth/tls.go new file mode 100644 index 000000000..7c11fe27e --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/tls.go @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "crypto/tls" + "encoding/json" + "net/http" + "strings" +) + +const ( + TLSPluginName = "org.apache.pulsar.client.impl.auth.AuthenticationTls" + TLSPluginShortName = "tls" +) + +type TLS struct { + TLSCertFile string `json:"tlsCertFile"` + TLSKeyFile string `json:"tlsKeyFile"` +} + +type TLSAuthProvider struct { + certificatePath string + privateKeyPath string + T http.RoundTripper +} + +// NewAuthenticationTLS initialize the authentication provider +func NewAuthenticationTLS(certificatePath string, privateKeyPath string, + transport http.RoundTripper) (*TLSAuthProvider, error) { + provider := &TLSAuthProvider{ + certificatePath: certificatePath, + privateKeyPath: privateKeyPath, + T: transport, + } + if err := provider.configTLS(); err != nil { + return nil, err + } + return provider, nil +} + +func NewAuthenticationTLSFromAuthParams(encodedAuthParams string, + transport http.RoundTripper) (*TLSAuthProvider, error) { + var certificatePath string + var privateKeyPath string + + var tlsJSON TLS + err := json.Unmarshal([]byte(encodedAuthParams), &tlsJSON) + if err != nil { + parts := strings.Split(encodedAuthParams, ",") + for _, part := range parts { + kv := strings.Split(part, ":") + switch kv[0] { + case "tlsCertFile": + certificatePath = kv[1] + case "tlsKeyFile": + privateKeyPath = kv[1] + } + } + } else { + certificatePath = tlsJSON.TLSCertFile + privateKeyPath = tlsJSON.TLSKeyFile + } + + return NewAuthenticationTLS(certificatePath, privateKeyPath, transport) +} + +func (p *TLSAuthProvider) GetTLSCertificate() (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(p.certificatePath, p.privateKeyPath) + return &cert, err +} + +func (p *TLSAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { + return p.T.RoundTrip(req) +} + +func (p *TLSAuthProvider) Transport() http.RoundTripper { + return p.T +} + +func (p *TLSAuthProvider) configTLS() error { + cert, err := p.GetTLSCertificate() + if err != nil { + return err + } + transport := p.T.(*http.Transport) + transport.TLSClientConfig.Certificates = []tls.Certificate{*cert} + return nil +} + +func (p *TLSAuthProvider) WithTransport(tripper http.RoundTripper) { + p.T = tripper +} diff --git a/pulsaradmin/pkg/admin/auth/token.go b/pulsaradmin/pkg/admin/auth/token.go new file mode 100644 index 000000000..ecf57ecc8 --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/token.go @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + + "github.com/pkg/errors" +) + +const ( + tokenPrefix = "token:" + filePrefix = "file:" + TokenPluginName = "org.apache.pulsar.client.impl.auth.AuthenticationToken" + TokePluginShortName = "token" +) + +type Token struct { + Token string `json:"token"` +} + +type TokenAuthProvider struct { + T http.RoundTripper + token string +} + +// NewAuthenticationToken return a interface of Provider with a string token. +func NewAuthenticationToken(token string, transport http.RoundTripper) (*TokenAuthProvider, error) { + if len(token) == 0 { + return nil, errors.New("No token provided") + } + return &TokenAuthProvider{token: token, T: transport}, nil +} + +// NewAuthenticationTokenFromFile return a interface of a Provider with a string token file path. +func NewAuthenticationTokenFromFile(tokenFilePath string, transport http.RoundTripper) (*TokenAuthProvider, error) { + data, err := os.ReadFile(tokenFilePath) + if err != nil { + return nil, err + } + token := strings.Trim(string(data), " \n") + return NewAuthenticationToken(token, transport) +} + +func NewAuthenticationTokenFromAuthParams(encodedAuthParam string, + transport http.RoundTripper) (*TokenAuthProvider, error) { + var tokenAuthProvider *TokenAuthProvider + var err error + + var tokenJSON Token + err = json.Unmarshal([]byte(encodedAuthParam), &tokenJSON) + if err != nil { + switch { + case strings.HasPrefix(encodedAuthParam, tokenPrefix): + tokenAuthProvider, err = NewAuthenticationToken(strings.TrimPrefix(encodedAuthParam, tokenPrefix), transport) + case strings.HasPrefix(encodedAuthParam, filePrefix): + tokenAuthProvider, err = NewAuthenticationTokenFromFile(strings.TrimPrefix(encodedAuthParam, filePrefix), transport) + default: + tokenAuthProvider, err = NewAuthenticationToken(encodedAuthParam, transport) + } + } else { + tokenAuthProvider, err = NewAuthenticationToken(tokenJSON.Token, transport) + } + return tokenAuthProvider, err +} + +func (p *TokenAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", p.token)) + return p.T.RoundTrip(req) +} + +func (p *TokenAuthProvider) Transport() http.RoundTripper { + return p.T +} + +func (p *TokenAuthProvider) WithTransport(tripper http.RoundTripper) { + p.T = tripper +} diff --git a/pulsaradmin/pkg/admin/auth/transport.go b/pulsaradmin/pkg/admin/auth/transport.go new file mode 100644 index 000000000..d96ab57f5 --- /dev/null +++ b/pulsaradmin/pkg/admin/auth/transport.go @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package auth + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "os" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/admin/config" +) + +type Transport struct { + T http.RoundTripper +} + +// GetDefaultTransport gets a default transport. +// Deprecated: Use NewDefaultTransport instead. +func GetDefaultTransport(config *config.Config) http.RoundTripper { + transport, err := NewDefaultTransport(config) + if err != nil { + panic(err) + } + + return transport +} + +func NewDefaultTransport(config *config.Config) (http.RoundTripper, error) { + transport := http.DefaultTransport.(*http.Transport).Clone() + tlsConfig := &tls.Config{ + InsecureSkipVerify: config.TLSAllowInsecureConnection, + } + if len(config.TLSTrustCertsFilePath) > 0 { + rootCA, err := os.ReadFile(config.TLSTrustCertsFilePath) + if err != nil { + return nil, err + } + tlsConfig.RootCAs = x509.NewCertPool() + tlsConfig.RootCAs.AppendCertsFromPEM(rootCA) + } + transport.MaxIdleConnsPerHost = 10 + transport.TLSClientConfig = tlsConfig + return transport, nil +} diff --git a/pulsaradmin/pkg/admin/broker_stats.go b/pulsaradmin/pkg/admin/broker_stats.go new file mode 100644 index 000000000..c9f9cb01e --- /dev/null +++ b/pulsaradmin/pkg/admin/broker_stats.go @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// BrokerStats is admin interface for broker stats management +type BrokerStats interface { + // GetMetrics returns Monitoring metrics + GetMetrics() ([]utils.Metrics, error) + + // GetMBeans requests JSON string server mbean dump + GetMBeans() ([]utils.Metrics, error) + + // GetTopics returns JSON string topics stats + GetTopics() (string, error) + + // GetLoadReport returns load report of broker + GetLoadReport() (*utils.LocalBrokerData, error) + + // GetAllocatorStats returns stats from broker + GetAllocatorStats(allocatorName string) (*utils.AllocatorStats, error) +} + +type brokerStats struct { + pulsar *pulsarClient + basePath string +} + +// BrokerStats is used to access the broker stats endpoints +func (c *pulsarClient) BrokerStats() BrokerStats { + return &brokerStats{ + pulsar: c, + basePath: "/broker-stats", + } +} + +func (bs *brokerStats) GetMetrics() ([]utils.Metrics, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/metrics") + var response []utils.Metrics + err := bs.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, err + } + + return response, nil +} + +func (bs *brokerStats) GetMBeans() ([]utils.Metrics, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/mbeans") + var response []utils.Metrics + err := bs.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, err + } + + return response, nil +} + +func (bs *brokerStats) GetTopics() (string, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/topics") + buf, err := bs.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + if err != nil { + return "", err + } + + return string(buf), nil +} + +func (bs *brokerStats) GetLoadReport() (*utils.LocalBrokerData, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/load-report") + response := utils.NewLocalBrokerData() + err := bs.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, nil + } + return &response, nil +} + +func (bs *brokerStats) GetAllocatorStats(allocatorName string) (*utils.AllocatorStats, error) { + endpoint := bs.pulsar.endpoint(bs.basePath, "/allocator-stats", allocatorName) + var allocatorStats utils.AllocatorStats + err := bs.pulsar.Client.Get(endpoint, &allocatorStats) + if err != nil { + return nil, err + } + return &allocatorStats, nil +} diff --git a/pulsaradmin/pkg/admin/brokers.go b/pulsaradmin/pkg/admin/brokers.go new file mode 100644 index 000000000..79fcb092e --- /dev/null +++ b/pulsaradmin/pkg/admin/brokers.go @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "fmt" + "net/url" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Brokers is admin interface for brokers management +type Brokers interface { + // GetActiveBrokers returns the list of active brokers in the cluster. + GetActiveBrokers(cluster string) ([]string, error) + + // GetDynamicConfigurationNames returns list of updatable configuration name + GetDynamicConfigurationNames() ([]string, error) + + // GetOwnedNamespaces returns the map of owned namespaces and their status from a single broker in the cluster + GetOwnedNamespaces(cluster, brokerURL string) (map[string]utils.NamespaceOwnershipStatus, error) + + // UpdateDynamicConfiguration updates dynamic configuration value in to Zk that triggers watch on + // brokers and all brokers can update {@link ServiceConfiguration} value locally + UpdateDynamicConfiguration(configName, configValue string) error + + // DeleteDynamicConfiguration deletes dynamic configuration value in to Zk. It will not impact current value + // in broker but next time when broker restarts, it applies value from configuration file only. + DeleteDynamicConfiguration(configName string) error + + // GetRuntimeConfigurations returns values of runtime configuration + GetRuntimeConfigurations() (map[string]string, error) + + // GetInternalConfigurationData returns the internal configuration data + GetInternalConfigurationData() (*utils.InternalConfigurationData, error) + + // GetAllDynamicConfigurations returns values of all overridden dynamic-configs + GetAllDynamicConfigurations() (map[string]string, error) + + // HealthCheck run a health check on the broker + HealthCheck() error +} + +type broker struct { + pulsar *pulsarClient + basePath string +} + +// Brokers is used to access the brokers endpoints +func (c *pulsarClient) Brokers() Brokers { + return &broker{ + pulsar: c, + basePath: "/brokers", + } +} + +func (b *broker) GetActiveBrokers(cluster string) ([]string, error) { + endpoint := b.pulsar.endpoint(b.basePath, cluster) + var res []string + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) GetDynamicConfigurationNames() ([]string, error) { + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/") + var res []string + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) GetOwnedNamespaces(cluster, brokerURL string) (map[string]utils.NamespaceOwnershipStatus, error) { + endpoint := b.pulsar.endpoint(b.basePath, cluster, brokerURL, "ownedNamespaces") + var res map[string]utils.NamespaceOwnershipStatus + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) UpdateDynamicConfiguration(configName, configValue string) error { + value := url.QueryEscape(configValue) + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", configName, value) + return b.pulsar.Client.Post(endpoint, nil) +} + +func (b *broker) DeleteDynamicConfiguration(configName string) error { + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", configName) + return b.pulsar.Client.Delete(endpoint) +} + +func (b *broker) GetRuntimeConfigurations() (map[string]string, error) { + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", "runtime") + var res map[string]string + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) GetInternalConfigurationData() (*utils.InternalConfigurationData, error) { + endpoint := b.pulsar.endpoint(b.basePath, "/internal-configuration") + var res utils.InternalConfigurationData + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return &res, nil +} + +func (b *broker) GetAllDynamicConfigurations() (map[string]string, error) { + endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", "values") + var res map[string]string + err := b.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (b *broker) HealthCheck() error { + endpoint := b.pulsar.endpoint(b.basePath, "/health") + + buf, err := b.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + if err != nil { + return err + } + + if !strings.EqualFold(string(buf), "ok") { + return fmt.Errorf("health check returned unexpected result: %s", string(buf)) + } + return nil +} diff --git a/pulsaradmin/pkg/admin/cluster.go b/pulsaradmin/pkg/admin/cluster.go new file mode 100644 index 000000000..b290e3ef3 --- /dev/null +++ b/pulsaradmin/pkg/admin/cluster.go @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Clusters is admin interface for clusters management +type Clusters interface { + // List returns the list of clusters + List() ([]string, error) + + // Get the configuration data for the specified cluster + Get(string) (utils.ClusterData, error) + + // Create a new cluster + Create(utils.ClusterData) error + + // Delete an existing cluster + Delete(string) error + + // Update the configuration for a cluster + Update(utils.ClusterData) error + + // UpdatePeerClusters updates peer cluster names. + UpdatePeerClusters(string, []string) error + + // GetPeerClusters returns peer-cluster names + GetPeerClusters(string) ([]string, error) + + // CreateFailureDomain creates a domain into cluster + CreateFailureDomain(utils.FailureDomainData) error + + // GetFailureDomain returns the domain registered into a cluster + GetFailureDomain(clusterName, domainName string) (utils.FailureDomainData, error) + + // ListFailureDomains returns all registered domains in cluster + ListFailureDomains(string) (utils.FailureDomainMap, error) + + // DeleteFailureDomain deletes a domain in cluster + DeleteFailureDomain(utils.FailureDomainData) error + + // UpdateFailureDomain updates a domain into cluster + UpdateFailureDomain(utils.FailureDomainData) error +} + +type clusters struct { + pulsar *pulsarClient + basePath string +} + +// Clusters is used to access the cluster endpoints. +func (c *pulsarClient) Clusters() Clusters { + return &clusters{ + pulsar: c, + basePath: "/clusters", + } +} + +func (c *clusters) List() ([]string, error) { + var clusters []string + err := c.pulsar.Client.Get(c.pulsar.endpoint(c.basePath), &clusters) + return clusters, err +} + +func (c *clusters) Get(name string) (utils.ClusterData, error) { + cdata := utils.ClusterData{} + endpoint := c.pulsar.endpoint(c.basePath, name) + err := c.pulsar.Client.Get(endpoint, &cdata) + return cdata, err +} + +func (c *clusters) Create(cdata utils.ClusterData) error { + endpoint := c.pulsar.endpoint(c.basePath, cdata.Name) + return c.pulsar.Client.Put(endpoint, &cdata) +} + +func (c *clusters) Delete(name string) error { + endpoint := c.pulsar.endpoint(c.basePath, name) + return c.pulsar.Client.Delete(endpoint) +} + +func (c *clusters) Update(cdata utils.ClusterData) error { + endpoint := c.pulsar.endpoint(c.basePath, cdata.Name) + return c.pulsar.Client.Post(endpoint, &cdata) +} + +func (c *clusters) GetPeerClusters(name string) ([]string, error) { + var peerClusters []string + endpoint := c.pulsar.endpoint(c.basePath, name, "peers") + err := c.pulsar.Client.Get(endpoint, &peerClusters) + return peerClusters, err +} + +func (c *clusters) UpdatePeerClusters(cluster string, peerClusters []string) error { + endpoint := c.pulsar.endpoint(c.basePath, cluster, "peers") + return c.pulsar.Client.Post(endpoint, peerClusters) +} + +func (c *clusters) CreateFailureDomain(data utils.FailureDomainData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) + return c.pulsar.Client.Post(endpoint, &data) +} + +func (c *clusters) GetFailureDomain(clusterName string, domainName string) (utils.FailureDomainData, error) { + var res utils.FailureDomainData + endpoint := c.pulsar.endpoint(c.basePath, clusterName, "failureDomains", domainName) + err := c.pulsar.Client.Get(endpoint, &res) + return res, err +} + +func (c *clusters) ListFailureDomains(clusterName string) (utils.FailureDomainMap, error) { + var domainData utils.FailureDomainMap + endpoint := c.pulsar.endpoint(c.basePath, clusterName, "failureDomains") + err := c.pulsar.Client.Get(endpoint, &domainData) + return domainData, err +} + +func (c *clusters) DeleteFailureDomain(data utils.FailureDomainData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) + return c.pulsar.Client.Delete(endpoint) +} +func (c *clusters) UpdateFailureDomain(data utils.FailureDomainData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) + return c.pulsar.Client.Post(endpoint, &data) +} diff --git a/pulsaradmin/pkg/admin/config/api_version.go b/pulsaradmin/pkg/admin/config/api_version.go new file mode 100644 index 000000000..95f670458 --- /dev/null +++ b/pulsaradmin/pkg/admin/config/api_version.go @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +type APIVersion int + +const ( + undefined APIVersion = iota + V1 + V2 + V3 +) + +const DefaultAPIVersion = "v2" + +func (v APIVersion) String() string { + switch v { + case undefined: + return DefaultAPIVersion + case V1: + return "" + case V2: + return "v2" + case V3: + return "v3" + } + + return DefaultAPIVersion +} diff --git a/pulsaradmin/pkg/admin/config/api_version_test.go b/pulsaradmin/pkg/admin/config/api_version_test.go new file mode 100644 index 000000000..e1dc7bdb6 --- /dev/null +++ b/pulsaradmin/pkg/admin/config/api_version_test.go @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestApiVersion_String(t *testing.T) { + assert.Equal(t, "", V1.String()) + assert.Equal(t, "v2", V2.String()) + assert.Equal(t, "v3", V3.String()) + var undefinedAPIVersion APIVersion + assert.Equal(t, DefaultAPIVersion, undefinedAPIVersion.String()) +} diff --git a/pulsaradmin/pkg/admin/config/config.go b/pulsaradmin/pkg/admin/config/config.go new file mode 100644 index 000000000..9428f3a90 --- /dev/null +++ b/pulsaradmin/pkg/admin/config/config.go @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +type Config struct { + // the web service url that pulsarctl connects to. Default is http://localhost:8080 + WebServiceURL string + + // the bookkeeper service url that pulsarctl connects to. + BKWebServiceURL string + // Set the path to the trusted TLS certificate file + TLSTrustCertsFilePath string + // Configure whether the Pulsar client accept untrusted TLS certificate from broker (default: false) + TLSAllowInsecureConnection bool + + TLSEnableHostnameVerification bool + + AuthPlugin string + + AuthParams string + + // TLS Cert and Key Files for authentication + TLSCertFile string + TLSKeyFile string + + // Token and TokenFile is used to config the pulsarctl using token to authentication + Token string + TokenFile string + PulsarAPIVersion APIVersion + + // OAuth2 configuration + IssuerEndpoint string + ClientID string + Audience string + KeyFile string + Scope string +} diff --git a/pulsaradmin/pkg/admin/functions.go b/pulsaradmin/pkg/admin/functions.go new file mode 100644 index 000000000..cbaaf6be4 --- /dev/null +++ b/pulsaradmin/pkg/admin/functions.go @@ -0,0 +1,686 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Functions is admin interface for functions management +type Functions interface { + // CreateFunc create a new function. + CreateFunc(data *utils.FunctionConfig, fileName string) error + + // CreateFuncWithURL create a new function by providing url from which fun-pkg can be downloaded. + // supported url: http/file + // eg: + // File: file:/dir/fileName.jar + // Http: http://www.repo.com/fileName.jar + // + // @param functionConfig + // the function configuration object + // @param pkgURL + // url from which pkg can be downloaded + CreateFuncWithURL(data *utils.FunctionConfig, pkgURL string) error + + // StopFunction stop all function instances + StopFunction(tenant, namespace, name string) error + + // StopFunctionWithID stop function instance + StopFunctionWithID(tenant, namespace, name string, instanceID int) error + + // DeleteFunction delete an existing function + DeleteFunction(tenant, namespace, name string) error + + // Download Function Code + // @param destinationFile + // file where data should be downloaded to + // @param path + // Path where data is located + DownloadFunction(path, destinationFile string) error + + // Download Function Code + // @param destinationFile + // file where data should be downloaded to + // @param tenant + // Tenant name + // @param namespace + // Namespace name + // @param function + // Function name + DownloadFunctionByNs(destinationFile, tenant, namespace, function string) error + + // StartFunction start all function instances + StartFunction(tenant, namespace, name string) error + + // StartFunctionWithID start function instance + StartFunctionWithID(tenant, namespace, name string, instanceID int) error + + // RestartFunction restart all function instances + RestartFunction(tenant, namespace, name string) error + + // RestartFunctionWithID restart function instance + RestartFunctionWithID(tenant, namespace, name string, instanceID int) error + + // GetFunctions returns the list of functions + GetFunctions(tenant, namespace string) ([]string, error) + + // GetFunction returns the configuration for the specified function + GetFunction(tenant, namespace, name string) (utils.FunctionConfig, error) + + // GetFunctionStatus returns the current status of a function + GetFunctionStatus(tenant, namespace, name string) (utils.FunctionStatus, error) + + // GetFunctionStatusWithInstanceID returns the current status of a function instance + GetFunctionStatusWithInstanceID(tenant, namespace, name string, instanceID int) ( + utils.FunctionInstanceStatusData, error) + + // GetFunctionStats returns the current stats of a function + GetFunctionStats(tenant, namespace, name string) (utils.FunctionStats, error) + + // GetFunctionStatsWithInstanceID gets the current stats of a function instance + GetFunctionStatsWithInstanceID(tenant, namespace, name string, instanceID int) (utils.FunctionInstanceStatsData, error) + + // GetFunctionState fetch the current state associated with a Pulsar Function + // + // Response Example: + // { "value : 12, version : 2"} + GetFunctionState(tenant, namespace, name, key string) (utils.FunctionState, error) + + // PutFunctionState puts the given state associated with a Pulsar Function + PutFunctionState(tenant, namespace, name string, state utils.FunctionState) error + + // TriggerFunction triggers the function by writing to the input topic + TriggerFunction(tenant, namespace, name, topic, triggerValue, triggerFile string) (string, error) + + // UpdateFunction updates the configuration for a function. + UpdateFunction(functionConfig *utils.FunctionConfig, fileName string, updateOptions *utils.UpdateOptions) error + + // UpdateFunctionWithURL updates the configuration for a function. + // + // Update a function by providing url from which fun-pkg can be downloaded. supported url: http/file + // eg: + // File: file:/dir/fileName.jar + // Http: http://www.repo.com/fileName.jar + UpdateFunctionWithURL(functionConfig *utils.FunctionConfig, pkgURL string, updateOptions *utils.UpdateOptions) error + + // Upload function to Pulsar + Upload(sourceFile, path string) error +} + +type functions struct { + pulsar *pulsarClient + basePath string +} + +// Functions is used to access the functions endpoints +func (c *pulsarClient) Functions() Functions { + return &functions{ + pulsar: c, + basePath: "/functions", + } +} + +func (f *functions) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "application/json") + return w.CreatePart(h) +} + +func (f *functions) createTextFromFiled(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "text/plain") + return w.CreatePart(h) +} + +func (f *functions) CreateFunc(funcConf *utils.FunctionConfig, fileName string) error { + endpoint := f.pulsar.endpoint(f.basePath, funcConf.Tenant, funcConf.Namespace, funcConf.Name) + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + jsonData, err := json.Marshal(funcConf) + if err != nil { + return err + } + + stringWriter, err := f.createStringFromField(multiPartWriter, "functionConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = f.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (f *functions) CreateFuncWithURL(funcConf *utils.FunctionConfig, pkgURL string) error { + endpoint := f.pulsar.endpoint(f.basePath, funcConf.Tenant, funcConf.Namespace, funcConf.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := f.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(funcConf) + if err != nil { + return err + } + + stringWriter, err := f.createStringFromField(multiPartWriter, "functionConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = f.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (f *functions) StopFunction(tenant, namespace, name string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + return f.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (f *functions) StopFunctionWithID(tenant, namespace, name string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + + return f.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (f *functions) DeleteFunction(tenant, namespace, name string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + return f.pulsar.Client.Delete(endpoint) +} + +func (f *functions) DownloadFunction(path, destinationFile string) error { + endpoint := f.pulsar.endpoint(f.basePath, "download") + _, err := os.Open(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("file %s already exists, please delete "+ + "the file first or change the file name", destinationFile) + } + } + file, err := os.Create(destinationFile) + if err != nil { + return err + } + + tmpMap := make(map[string]string) + tmpMap["path"] = path + + _, err = f.pulsar.Client.GetWithOptions(endpoint, nil, tmpMap, false, file) + if err != nil { + return err + } + return nil +} + +func (f *functions) DownloadFunctionByNs(destinationFile, tenant, namespace, function string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, function, "download") + _, err := os.Open(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("file %s already exists, please delete "+ + "the file first or change the file name", destinationFile) + } + } + file, err := os.Create(destinationFile) + if err != nil { + return err + } + + _, err = f.pulsar.Client.GetWithOptions(endpoint, nil, nil, false, file) + if err != nil { + return err + } + + return nil +} + +func (f *functions) StartFunction(tenant, namespace, name string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + return f.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (f *functions) StartFunctionWithID(tenant, namespace, name string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + + return f.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (f *functions) RestartFunction(tenant, namespace, name string) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + return f.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (f *functions) RestartFunctionWithID(tenant, namespace, name string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + + return f.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (f *functions) GetFunctions(tenant, namespace string) ([]string, error) { + var functions []string + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace) + err := f.pulsar.Client.Get(endpoint, &functions) + return functions, err +} + +func (f *functions) GetFunction(tenant, namespace, name string) (utils.FunctionConfig, error) { + var functionConfig utils.FunctionConfig + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + err := f.pulsar.Client.Get(endpoint, &functionConfig) + return functionConfig, err +} + +func (f *functions) UpdateFunction(functionConfig *utils.FunctionConfig, fileName string, + updateOptions *utils.UpdateOptions) error { + endpoint := f.pulsar.endpoint(f.basePath, functionConfig.Tenant, functionConfig.Namespace, functionConfig.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + jsonData, err := json.Marshal(functionConfig) + if err != nil { + return err + } + + stringWriter, err := f.createStringFromField(multiPartWriter, "functionConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := f.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = f.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (f *functions) UpdateFunctionWithURL(functionConfig *utils.FunctionConfig, pkgURL string, + updateOptions *utils.UpdateOptions) error { + endpoint := f.pulsar.endpoint(f.basePath, functionConfig.Tenant, functionConfig.Namespace, functionConfig.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := f.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(functionConfig) + if err != nil { + return err + } + + stringWriter, err := f.createStringFromField(multiPartWriter, "functionConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := f.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = f.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (f *functions) GetFunctionStatus(tenant, namespace, name string) (utils.FunctionStatus, error) { + var functionStatus utils.FunctionStatus + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + err := f.pulsar.Client.Get(endpoint+"/status", &functionStatus) + return functionStatus, err +} + +func (f *functions) GetFunctionStatusWithInstanceID(tenant, namespace, name string, + instanceID int) (utils.FunctionInstanceStatusData, error) { + var functionInstanceStatusData utils.FunctionInstanceStatusData + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + err := f.pulsar.Client.Get(endpoint+"/status", &functionInstanceStatusData) + return functionInstanceStatusData, err +} + +func (f *functions) GetFunctionStats(tenant, namespace, name string) (utils.FunctionStats, error) { + var functionStats utils.FunctionStats + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name) + err := f.pulsar.Client.Get(endpoint+"/stats", &functionStats) + return functionStats, err +} + +func (f *functions) GetFunctionStatsWithInstanceID(tenant, namespace, name string, + instanceID int) (utils.FunctionInstanceStatsData, error) { + var functionInstanceStatsData utils.FunctionInstanceStatsData + id := fmt.Sprintf("%d", instanceID) + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, id) + err := f.pulsar.Client.Get(endpoint+"/stats", &functionInstanceStatsData) + return functionInstanceStatsData, err +} + +func (f *functions) GetFunctionState(tenant, namespace, name, key string) (utils.FunctionState, error) { + var functionState utils.FunctionState + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, "state", key) + err := f.pulsar.Client.Get(endpoint, &functionState) + return functionState, err +} + +func (f *functions) PutFunctionState(tenant, namespace, name string, state utils.FunctionState) error { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, "state", state.Key) + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + stateData, err := json.Marshal(state) + + if err != nil { + return err + } + + stateWriter, err := f.createStringFromField(multiPartWriter, "state") + if err != nil { + return err + } + + _, err = stateWriter.Write(stateData) + + if err != nil { + return err + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + + err = f.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + + if err != nil { + return err + } + + return nil +} + +func (f *functions) TriggerFunction(tenant, namespace, name, topic, triggerValue, triggerFile string) (string, error) { + endpoint := f.pulsar.endpoint(f.basePath, tenant, namespace, name, "trigger") + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + if triggerFile != "" { + file, err := os.Open(triggerFile) + if err != nil { + return "", err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("dataStream", filepath.Base(file.Name())) + + if err != nil { + return "", err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return "", err + } + } + + if triggerValue != "" { + valueWriter, err := f.createTextFromFiled(multiPartWriter, "data") + if err != nil { + return "", err + } + + _, err = valueWriter.Write([]byte(triggerValue)) + if err != nil { + return "", err + } + } + + if topic != "" { + topicWriter, err := f.createTextFromFiled(multiPartWriter, "topic") + if err != nil { + return "", err + } + + _, err = topicWriter.Write([]byte(topic)) + if err != nil { + return "", err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err := multiPartWriter.Close(); err != nil { + return "", err + } + + contentType := multiPartWriter.FormDataContentType() + var str string + err := f.pulsar.Client.PostWithMultiPart(endpoint, &str, bodyBuf, contentType) + if err != nil { + return "", err + } + + return str, nil +} + +func (f *functions) Upload(sourceFile, path string) error { + if strings.TrimSpace(sourceFile) == "" && strings.TrimSpace(path) == "" { + return fmt.Errorf("source file or path is empty") + } + file, err := os.Open(sourceFile) + if err != nil { + return err + } + defer file.Close() + endpoint := f.pulsar.endpoint(f.basePath, "upload") + var b bytes.Buffer + w := multipart.NewWriter(&b) + writer, err := w.CreateFormFile("data", file.Name()) + if err != nil { + return err + } + _, err = io.Copy(writer, file) + if err != nil { + return err + } + if err := w.WriteField("path", path); err != nil { + return err + } + err = w.Close() + if err != nil { + return err + } + return f.pulsar.Client.PostWithMultiPart(endpoint, nil, &b, w.FormDataContentType()) +} diff --git a/pulsaradmin/pkg/admin/functions_worker.go b/pulsaradmin/pkg/admin/functions_worker.go new file mode 100644 index 000000000..3cad65de8 --- /dev/null +++ b/pulsaradmin/pkg/admin/functions_worker.go @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +type FunctionsWorker interface { + // Get all functions stats on a worker + GetFunctionsStats() ([]*utils.WorkerFunctionInstanceStats, error) + + // Get worker metrics + GetMetrics() ([]*utils.Metrics, error) + + // Get List of all workers belonging to this cluster + GetCluster() ([]*utils.WorkerInfo, error) + + // Get the worker who is the leader of the clusterv + GetClusterLeader() (*utils.WorkerInfo, error) + + // Get the function assignment among the cluster + GetAssignments() (map[string][]string, error) +} + +type worker struct { + pulsar *pulsarClient + workerPath string + workerStatsPath string +} + +func (c *pulsarClient) FunctionsWorker() FunctionsWorker { + return &worker{ + pulsar: c, + workerPath: "/worker", + workerStatsPath: "/worker-stats", + } +} + +func (w *worker) GetFunctionsStats() ([]*utils.WorkerFunctionInstanceStats, error) { + endpoint := w.pulsar.endpoint(w.workerStatsPath, "functionsmetrics") + var workerStats []*utils.WorkerFunctionInstanceStats + err := w.pulsar.Client.Get(endpoint, &workerStats) + if err != nil { + return nil, err + } + return workerStats, nil +} + +func (w *worker) GetMetrics() ([]*utils.Metrics, error) { + endpoint := w.pulsar.endpoint(w.workerStatsPath, "metrics") + var metrics []*utils.Metrics + err := w.pulsar.Client.Get(endpoint, &metrics) + if err != nil { + return nil, err + } + return metrics, nil +} + +func (w *worker) GetCluster() ([]*utils.WorkerInfo, error) { + endpoint := w.pulsar.endpoint(w.workerPath, "cluster") + var workersInfo []*utils.WorkerInfo + err := w.pulsar.Client.Get(endpoint, &workersInfo) + if err != nil { + return nil, err + } + return workersInfo, nil +} + +func (w *worker) GetClusterLeader() (*utils.WorkerInfo, error) { + endpoint := w.pulsar.endpoint(w.workerPath, "cluster", "leader") + var workerInfo utils.WorkerInfo + err := w.pulsar.Client.Get(endpoint, &workerInfo) + if err != nil { + return nil, err + } + return &workerInfo, nil +} + +func (w *worker) GetAssignments() (map[string][]string, error) { + endpoint := w.pulsar.endpoint(w.workerPath, "assignments") + var assignments map[string][]string + err := w.pulsar.Client.Get(endpoint, &assignments) + if err != nil { + return nil, err + } + return assignments, nil +} diff --git a/pulsaradmin/pkg/admin/namespace.go b/pulsaradmin/pkg/admin/namespace.go new file mode 100644 index 000000000..732441e8c --- /dev/null +++ b/pulsaradmin/pkg/admin/namespace.go @@ -0,0 +1,875 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "net/url" + "strconv" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Namespaces is admin interface for namespaces management +type Namespaces interface { + // GetNamespaces returns the list of all the namespaces for a certain tenant + GetNamespaces(tenant string) ([]string, error) + + // GetTopics returns the list of all the topics under a certain namespace + GetTopics(namespace string) ([]string, error) + + // GetPolicies returns the dump all the policies specified for a namespace + GetPolicies(namespace string) (*utils.Policies, error) + + // CreateNamespace creates a new empty namespace with no policies attached + CreateNamespace(namespace string) error + + // CreateNsWithNumBundles creates a new empty namespace with no policies attached + CreateNsWithNumBundles(namespace string, numBundles int) error + + // CreateNsWithPolices creates a new namespace with the specified policies + CreateNsWithPolices(namespace string, polices utils.Policies) error + + // CreateNsWithBundlesData creates a new empty namespace with no policies attached + CreateNsWithBundlesData(namespace string, bundleData *utils.BundlesData) error + + // DeleteNamespace deletes an existing namespace + DeleteNamespace(namespace string) error + + // DeleteNamespaceBundle deletes an existing bundle in a namespace + DeleteNamespaceBundle(namespace string, bundleRange string) error + + // SetNamespaceMessageTTL sets the messages Time to Live for all the topics within a namespace + SetNamespaceMessageTTL(namespace string, ttlInSeconds int) error + + // GetNamespaceMessageTTL returns the message TTL for a namespace + GetNamespaceMessageTTL(namespace string) (int, error) + + // GetRetention returns the retention configuration for a namespace + GetRetention(namespace string) (*utils.RetentionPolicies, error) + + // SetRetention sets the retention configuration for all the topics on a namespace + SetRetention(namespace string, policy utils.RetentionPolicies) error + + // GetBacklogQuotaMap returns backlog quota map on a namespace + GetBacklogQuotaMap(namespace string) (map[utils.BacklogQuotaType]utils.BacklogQuota, error) + + // SetBacklogQuota sets a backlog quota for all the topics on a namespace + SetBacklogQuota(namespace string, backlogQuota utils.BacklogQuota, backlogQuotaType utils.BacklogQuotaType) error + + // RemoveBacklogQuota removes a backlog quota policy from a namespace + RemoveBacklogQuota(namespace string) error + + // SetTopicAutoCreation sets topic auto-creation config for a namespace, overriding broker settings + SetTopicAutoCreation(namespace utils.NameSpaceName, config utils.TopicAutoCreationConfig) error + + // RemoveTopicAutoCreation removes topic auto-creation config for a namespace, defaulting to broker settings + RemoveTopicAutoCreation(namespace utils.NameSpaceName) error + + // SetSchemaValidationEnforced sets schema validation enforced for namespace + SetSchemaValidationEnforced(namespace utils.NameSpaceName, schemaValidationEnforced bool) error + + // GetSchemaValidationEnforced returns schema validation enforced for namespace + GetSchemaValidationEnforced(namespace utils.NameSpaceName) (bool, error) + + // SetSchemaAutoUpdateCompatibilityStrategy sets the strategy used to check the a new schema provided + // by a producer is compatible with the current schema before it is installed + SetSchemaAutoUpdateCompatibilityStrategy(namespace utils.NameSpaceName, + strategy utils.SchemaCompatibilityStrategy) error + + // GetSchemaAutoUpdateCompatibilityStrategy returns the strategy used to check the a new schema provided + // by a producer is compatible with the current schema before it is installed + GetSchemaAutoUpdateCompatibilityStrategy(namespace utils.NameSpaceName) (utils.SchemaCompatibilityStrategy, error) + + // ClearOffloadDeleteLag clears the offload deletion lag for a namespace. + ClearOffloadDeleteLag(namespace utils.NameSpaceName) error + + // SetOffloadDeleteLag sets the offload deletion lag for a namespace + SetOffloadDeleteLag(namespace utils.NameSpaceName, timeMs int64) error + + // GetOffloadDeleteLag returns the offload deletion lag for a namespace, in milliseconds + GetOffloadDeleteLag(namespace utils.NameSpaceName) (int64, error) + + // SetOffloadThreshold sets the offloadThreshold for a namespace + SetOffloadThreshold(namespace utils.NameSpaceName, threshold int64) error + + // GetOffloadThreshold returns the offloadThreshold for a namespace + GetOffloadThreshold(namespace utils.NameSpaceName) (int64, error) + + // SetCompactionThreshold sets the compactionThreshold for a namespace + SetCompactionThreshold(namespace utils.NameSpaceName, threshold int64) error + + // GetCompactionThreshold returns the compactionThreshold for a namespace + GetCompactionThreshold(namespace utils.NameSpaceName) (int64, error) + + // SetMaxConsumersPerSubscription sets maxConsumersPerSubscription for a namespace. + SetMaxConsumersPerSubscription(namespace utils.NameSpaceName, max int) error + + // GetMaxConsumersPerSubscription returns the maxConsumersPerSubscription for a namespace. + GetMaxConsumersPerSubscription(namespace utils.NameSpaceName) (int, error) + + // SetMaxConsumersPerTopic sets maxConsumersPerTopic for a namespace. + SetMaxConsumersPerTopic(namespace utils.NameSpaceName, max int) error + + // GetMaxConsumersPerTopic returns the maxProducersPerTopic for a namespace. + GetMaxConsumersPerTopic(namespace utils.NameSpaceName) (int, error) + + // SetMaxProducersPerTopic sets maxProducersPerTopic for a namespace. + SetMaxProducersPerTopic(namespace utils.NameSpaceName, max int) error + + // GetMaxProducersPerTopic returns the maxProducersPerTopic for a namespace. + GetMaxProducersPerTopic(namespace utils.NameSpaceName) (int, error) + + // GetNamespaceReplicationClusters returns the replication clusters for a namespace + GetNamespaceReplicationClusters(namespace string) ([]string, error) + + // SetNamespaceReplicationClusters returns the replication clusters for a namespace + SetNamespaceReplicationClusters(namespace string, clusterIds []string) error + + // SetNamespaceAntiAffinityGroup sets anti-affinity group name for a namespace + SetNamespaceAntiAffinityGroup(namespace string, namespaceAntiAffinityGroup string) error + + // GetAntiAffinityNamespaces returns all namespaces that grouped with given anti-affinity group + GetAntiAffinityNamespaces(tenant, cluster, namespaceAntiAffinityGroup string) ([]string, error) + + // GetNamespaceAntiAffinityGroup returns anti-affinity group name for a namespace + GetNamespaceAntiAffinityGroup(namespace string) (string, error) + + // DeleteNamespaceAntiAffinityGroup deletes anti-affinity group name for a namespace + DeleteNamespaceAntiAffinityGroup(namespace string) error + + // SetDeduplicationStatus sets the deduplication status for all topics within a namespace + // When deduplication is enabled, the broker will prevent to store the same Message multiple times + SetDeduplicationStatus(namespace string, enableDeduplication bool) error + + // SetPersistence sets the persistence configuration for all the topics on a namespace + SetPersistence(namespace string, persistence utils.PersistencePolicies) error + + // GetPersistence returns the persistence configuration for a namespace + GetPersistence(namespace string) (*utils.PersistencePolicies, error) + + // SetBookieAffinityGroup sets bookie affinity group for a namespace to isolate namespace write to bookies that are + // part of given affinity group + SetBookieAffinityGroup(namespace string, bookieAffinityGroup utils.BookieAffinityGroupData) error + + // DeleteBookieAffinityGroup deletes bookie affinity group configured for a namespace + DeleteBookieAffinityGroup(namespace string) error + + // GetBookieAffinityGroup returns bookie affinity group configured for a namespace + GetBookieAffinityGroup(namespace string) (*utils.BookieAffinityGroupData, error) + + // Unload a namespace from the current serving broker + Unload(namespace string) error + + // UnloadNamespaceBundle unloads namespace bundle + UnloadNamespaceBundle(namespace, bundle string) error + + // SplitNamespaceBundle splits namespace bundle + SplitNamespaceBundle(namespace, bundle string, unloadSplitBundles bool) error + + // GetNamespacePermissions returns permissions on a namespace + GetNamespacePermissions(namespace utils.NameSpaceName) (map[string][]utils.AuthAction, error) + + // GrantNamespacePermission grants permission on a namespace. + GrantNamespacePermission(namespace utils.NameSpaceName, role string, action []utils.AuthAction) error + + // RevokeNamespacePermission revokes permissions on a namespace. + RevokeNamespacePermission(namespace utils.NameSpaceName, role string) error + + // GrantSubPermission grants permission to role to access subscription's admin-api + GrantSubPermission(namespace utils.NameSpaceName, sName string, roles []string) error + + // RevokeSubPermission revoke permissions on a subscription's admin-api access + RevokeSubPermission(namespace utils.NameSpaceName, sName, role string) error + + // SetSubscriptionAuthMode sets the given subscription auth mode on all topics on a namespace + SetSubscriptionAuthMode(namespace utils.NameSpaceName, mode utils.SubscriptionAuthMode) error + + // SetEncryptionRequiredStatus sets the encryption required status for all topics within a namespace + SetEncryptionRequiredStatus(namespace utils.NameSpaceName, encrypt bool) error + + // UnsubscribeNamespace unsubscribe the given subscription on all topics on a namespace + UnsubscribeNamespace(namespace utils.NameSpaceName, sName string) error + + // UnsubscribeNamespaceBundle unsubscribe the given subscription on all topics on a namespace bundle + UnsubscribeNamespaceBundle(namespace utils.NameSpaceName, bundle, sName string) error + + // ClearNamespaceBundleBacklogForSubscription clears backlog for a given subscription on all + // topics on a namespace bundle + ClearNamespaceBundleBacklogForSubscription(namespace utils.NameSpaceName, bundle, sName string) error + + // ClearNamespaceBundleBacklog clears backlog for all topics on a namespace bundle + ClearNamespaceBundleBacklog(namespace utils.NameSpaceName, bundle string) error + + // ClearNamespaceBacklogForSubscription clears backlog for a given subscription on all topics on a namespace + ClearNamespaceBacklogForSubscription(namespace utils.NameSpaceName, sName string) error + + // ClearNamespaceBacklog clears backlog for all topics on a namespace + ClearNamespaceBacklog(namespace utils.NameSpaceName) error + + // SetReplicatorDispatchRate sets replicator-Message-dispatch-rate (Replicators under this namespace + // can dispatch this many messages per second) + SetReplicatorDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error + + // Get replicator-Message-dispatch-rate (Replicators under this namespace + // can dispatch this many messages per second) + GetReplicatorDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) + + // SetSubscriptionDispatchRate sets subscription-Message-dispatch-rate (subscriptions under this namespace + // can dispatch this many messages per second) + SetSubscriptionDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error + + // GetSubscriptionDispatchRate returns subscription-Message-dispatch-rate (subscriptions under this namespace + // can dispatch this many messages per second) + GetSubscriptionDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) + + // SetSubscribeRate sets namespace-subscribe-rate (topics under this namespace will limit by subscribeRate) + SetSubscribeRate(namespace utils.NameSpaceName, rate utils.SubscribeRate) error + + // GetSubscribeRate returns namespace-subscribe-rate (topics under this namespace allow subscribe + // times per consumer in a period) + GetSubscribeRate(namespace utils.NameSpaceName) (utils.SubscribeRate, error) + + // SetDispatchRate sets Message-dispatch-rate (topics under this namespace can dispatch + // this many messages per second) + SetDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error + + // GetDispatchRate returns Message-dispatch-rate (topics under this namespace can dispatch + // this many messages per second) + GetDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) + + // SetPublishRate sets the maximum rate or number of messages that producers can publish to topics in this namespace + SetPublishRate(namespace utils.NameSpaceName, pubRate utils.PublishRate) error + + // GetPublishRate gets the maximum rate or number of messages that producer can publish to topics in the namespace + GetPublishRate(namespace utils.NameSpaceName) (utils.PublishRate, error) + + // SetIsAllowAutoUpdateSchema sets whether to allow auto update schema on a namespace + SetIsAllowAutoUpdateSchema(namespace utils.NameSpaceName, isAllowAutoUpdateSchema bool) error + + // GetIsAllowAutoUpdateSchema gets whether to allow auto update schema on a namespace + GetIsAllowAutoUpdateSchema(namespace utils.NameSpaceName) (bool, error) + + // GetInactiveTopicPolicies gets the inactive topic policies on a namespace + GetInactiveTopicPolicies(namespace utils.NameSpaceName) (utils.InactiveTopicPolicies, error) + + // RemoveInactiveTopicPolicies removes inactive topic policies from a namespace + RemoveInactiveTopicPolicies(namespace utils.NameSpaceName) error + + // SetInactiveTopicPolicies sets the inactive topic policies on a namespace + SetInactiveTopicPolicies(namespace utils.NameSpaceName, data utils.InactiveTopicPolicies) error +} + +type namespaces struct { + pulsar *pulsarClient + basePath string +} + +// Namespaces is used to access the namespaces endpoints +func (c *pulsarClient) Namespaces() Namespaces { + return &namespaces{ + pulsar: c, + basePath: "/namespaces", + } +} + +func (n *namespaces) GetNamespaces(tenant string) ([]string, error) { + var namespaces []string + endpoint := n.pulsar.endpoint(n.basePath, tenant) + err := n.pulsar.Client.Get(endpoint, &namespaces) + return namespaces, err +} + +func (n *namespaces) GetTopics(namespace string) ([]string, error) { + var topics []string + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String(), "topics") + err = n.pulsar.Client.Get(endpoint, &topics) + return topics, err +} + +func (n *namespaces) GetPolicies(namespace string) (*utils.Policies, error) { + var police utils.Policies + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + err = n.pulsar.Client.Get(endpoint, &police) + return &police, err +} + +func (n *namespaces) CreateNsWithNumBundles(namespace string, numBundles int) error { + return n.CreateNsWithBundlesData(namespace, utils.NewBundlesDataWithNumBundles(numBundles)) +} + +func (n *namespaces) CreateNsWithPolices(namespace string, policies utils.Policies) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + return n.pulsar.Client.Put(endpoint, &policies) +} + +func (n *namespaces) CreateNsWithBundlesData(namespace string, bundleData *utils.BundlesData) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + polices := new(utils.Policies) + polices.Bundles = bundleData + + return n.pulsar.Client.Put(endpoint, &polices) +} + +func (n *namespaces) CreateNamespace(namespace string) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + return n.pulsar.Client.Put(endpoint, nil) +} + +func (n *namespaces) DeleteNamespace(namespace string) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String()) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) DeleteNamespaceBundle(namespace string, bundleRange string) error { + ns, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, ns.String(), bundleRange) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) GetNamespaceMessageTTL(namespace string) (int, error) { + var ttl int + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return 0, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "messageTTL") + err = n.pulsar.Client.Get(endpoint, &ttl) + return ttl, err +} + +func (n *namespaces) SetNamespaceMessageTTL(namespace string, ttlInSeconds int) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "messageTTL") + return n.pulsar.Client.Post(endpoint, &ttlInSeconds) +} + +func (n *namespaces) SetRetention(namespace string, policy utils.RetentionPolicies) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "retention") + return n.pulsar.Client.Post(endpoint, &policy) +} + +func (n *namespaces) GetRetention(namespace string) (*utils.RetentionPolicies, error) { + var policy utils.RetentionPolicies + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "retention") + err = n.pulsar.Client.Get(endpoint, &policy) + return &policy, err +} + +func (n *namespaces) GetBacklogQuotaMap(namespace string) (map[utils.BacklogQuotaType]utils.BacklogQuota, error) { + var backlogQuotaMap map[utils.BacklogQuotaType]utils.BacklogQuota + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "backlogQuotaMap") + err = n.pulsar.Client.Get(endpoint, &backlogQuotaMap) + return backlogQuotaMap, err +} + +func (n *namespaces) SetBacklogQuota(namespace string, backlogQuota utils.BacklogQuota, + backlogQuotaType utils.BacklogQuotaType) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "backlogQuota") + params := make(map[string]string) + params["backlogQuotaType"] = string(backlogQuotaType) + return n.pulsar.Client.PostWithQueryParams(endpoint, &backlogQuota, params) +} + +func (n *namespaces) RemoveBacklogQuota(namespace string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "backlogQuota") + params := map[string]string{ + "backlogQuotaType": string(utils.DestinationStorage), + } + return n.pulsar.Client.DeleteWithQueryParams(endpoint, params) +} + +func (n *namespaces) SetTopicAutoCreation(namespace utils.NameSpaceName, config utils.TopicAutoCreationConfig) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "autoTopicCreation") + return n.pulsar.Client.Post(endpoint, &config) +} + +func (n *namespaces) RemoveTopicAutoCreation(namespace utils.NameSpaceName) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "autoTopicCreation") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetSchemaValidationEnforced(namespace utils.NameSpaceName, schemaValidationEnforced bool) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "schemaValidationEnforced") + return n.pulsar.Client.Post(endpoint, schemaValidationEnforced) +} + +func (n *namespaces) GetSchemaValidationEnforced(namespace utils.NameSpaceName) (bool, error) { + var result bool + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "schemaValidationEnforced") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetSchemaAutoUpdateCompatibilityStrategy(namespace utils.NameSpaceName, + strategy utils.SchemaCompatibilityStrategy) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "schemaAutoUpdateCompatibilityStrategy") + return n.pulsar.Client.Put(endpoint, strategy.String()) +} + +func (n *namespaces) GetSchemaAutoUpdateCompatibilityStrategy(namespace utils.NameSpaceName) ( + utils.SchemaCompatibilityStrategy, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "schemaAutoUpdateCompatibilityStrategy") + b, err := n.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + if err != nil { + return "", err + } + s, err := utils.ParseSchemaAutoUpdateCompatibilityStrategy(strings.ReplaceAll(string(b), "\"", "")) + if err != nil { + return "", err + } + return s, nil +} + +func (n *namespaces) ClearOffloadDeleteLag(namespace utils.NameSpaceName) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadDeletionLagMs") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetOffloadDeleteLag(namespace utils.NameSpaceName, timeMs int64) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadDeletionLagMs") + return n.pulsar.Client.Put(endpoint, timeMs) +} + +func (n *namespaces) GetOffloadDeleteLag(namespace utils.NameSpaceName) (int64, error) { + var result int64 + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadDeletionLagMs") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetMaxConsumersPerSubscription(namespace utils.NameSpaceName, max int) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxConsumersPerSubscription") + return n.pulsar.Client.Post(endpoint, max) +} + +func (n *namespaces) GetMaxConsumersPerSubscription(namespace utils.NameSpaceName) (int, error) { + var result int + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxConsumersPerSubscription") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetOffloadThreshold(namespace utils.NameSpaceName, threshold int64) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadThreshold") + return n.pulsar.Client.Put(endpoint, threshold) +} + +func (n *namespaces) GetOffloadThreshold(namespace utils.NameSpaceName) (int64, error) { + var result int64 + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "offloadThreshold") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetMaxConsumersPerTopic(namespace utils.NameSpaceName, max int) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxConsumersPerTopic") + return n.pulsar.Client.Post(endpoint, max) +} + +func (n *namespaces) GetMaxConsumersPerTopic(namespace utils.NameSpaceName) (int, error) { + var result int + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxConsumersPerTopic") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetCompactionThreshold(namespace utils.NameSpaceName, threshold int64) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "compactionThreshold") + return n.pulsar.Client.Put(endpoint, threshold) +} + +func (n *namespaces) GetCompactionThreshold(namespace utils.NameSpaceName) (int64, error) { + var result int64 + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "compactionThreshold") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) SetMaxProducersPerTopic(namespace utils.NameSpaceName, max int) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxProducersPerTopic") + return n.pulsar.Client.Post(endpoint, max) +} + +func (n *namespaces) GetMaxProducersPerTopic(namespace utils.NameSpaceName) (int, error) { + var result int + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "maxProducersPerTopic") + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) GetNamespaceReplicationClusters(namespace string) ([]string, error) { + var data []string + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "replication") + err = n.pulsar.Client.Get(endpoint, &data) + return data, err +} + +func (n *namespaces) SetNamespaceReplicationClusters(namespace string, clusterIds []string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "replication") + return n.pulsar.Client.Post(endpoint, &clusterIds) +} + +func (n *namespaces) SetNamespaceAntiAffinityGroup(namespace string, namespaceAntiAffinityGroup string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "antiAffinity") + return n.pulsar.Client.Post(endpoint, namespaceAntiAffinityGroup) +} + +func (n *namespaces) GetAntiAffinityNamespaces(tenant, cluster, namespaceAntiAffinityGroup string) ([]string, error) { + var data []string + endpoint := n.pulsar.endpoint(n.basePath, cluster, "antiAffinity", namespaceAntiAffinityGroup) + params := map[string]string{ + "property": tenant, + } + _, err := n.pulsar.Client.GetWithQueryParams(endpoint, &data, params, false) + return data, err +} + +func (n *namespaces) GetNamespaceAntiAffinityGroup(namespace string) (string, error) { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return "", err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "antiAffinity") + data, err := n.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + return string(data), err +} + +func (n *namespaces) DeleteNamespaceAntiAffinityGroup(namespace string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "antiAffinity") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetDeduplicationStatus(namespace string, enableDeduplication bool) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "deduplication") + return n.pulsar.Client.Post(endpoint, enableDeduplication) +} + +func (n *namespaces) SetPersistence(namespace string, persistence utils.PersistencePolicies) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence") + return n.pulsar.Client.Post(endpoint, &persistence) +} + +func (n *namespaces) SetBookieAffinityGroup(namespace string, bookieAffinityGroup utils.BookieAffinityGroupData) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence", "bookieAffinity") + return n.pulsar.Client.Post(endpoint, &bookieAffinityGroup) +} + +func (n *namespaces) DeleteBookieAffinityGroup(namespace string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence", "bookieAffinity") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) GetBookieAffinityGroup(namespace string) (*utils.BookieAffinityGroupData, error) { + var data utils.BookieAffinityGroupData + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence", "bookieAffinity") + err = n.pulsar.Client.Get(endpoint, &data) + return &data, err +} + +func (n *namespaces) GetPersistence(namespace string) (*utils.PersistencePolicies, error) { + var persistence utils.PersistencePolicies + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return nil, err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "persistence") + err = n.pulsar.Client.Get(endpoint, &persistence) + return &persistence, err +} + +func (n *namespaces) Unload(namespace string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), "unload") + return n.pulsar.Client.Put(endpoint, nil) +} + +func (n *namespaces) UnloadNamespaceBundle(namespace, bundle string) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), bundle, "unload") + return n.pulsar.Client.Put(endpoint, nil) +} + +func (n *namespaces) SplitNamespaceBundle(namespace, bundle string, unloadSplitBundles bool) error { + nsName, err := utils.GetNamespaceName(namespace) + if err != nil { + return err + } + endpoint := n.pulsar.endpoint(n.basePath, nsName.String(), bundle, "split") + params := map[string]string{ + "unload": strconv.FormatBool(unloadSplitBundles), + } + return n.pulsar.Client.PutWithQueryParams(endpoint, nil, nil, params) +} + +func (n *namespaces) GetNamespacePermissions(namespace utils.NameSpaceName) (map[string][]utils.AuthAction, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions") + var permissions map[string][]utils.AuthAction + err := n.pulsar.Client.Get(endpoint, &permissions) + return permissions, err +} + +func (n *namespaces) GrantNamespacePermission(namespace utils.NameSpaceName, role string, + action []utils.AuthAction) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions", role) + s := make([]string, 0) + for _, v := range action { + s = append(s, v.String()) + } + return n.pulsar.Client.Post(endpoint, s) +} + +func (n *namespaces) RevokeNamespacePermission(namespace utils.NameSpaceName, role string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions", role) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) GrantSubPermission(namespace utils.NameSpaceName, sName string, roles []string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions", + "subscription", sName) + return n.pulsar.Client.Post(endpoint, roles) +} + +func (n *namespaces) RevokeSubPermission(namespace utils.NameSpaceName, sName, role string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "permissions", + "subscription", sName, role) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetSubscriptionAuthMode(namespace utils.NameSpaceName, mode utils.SubscriptionAuthMode) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscriptionAuthMode") + return n.pulsar.Client.Post(endpoint, mode.String()) +} + +func (n *namespaces) SetEncryptionRequiredStatus(namespace utils.NameSpaceName, encrypt bool) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "encryptionRequired") + return n.pulsar.Client.Post(endpoint, strconv.FormatBool(encrypt)) +} + +func (n *namespaces) UnsubscribeNamespace(namespace utils.NameSpaceName, sName string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "unsubscribe", url.QueryEscape(sName)) + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) UnsubscribeNamespaceBundle(namespace utils.NameSpaceName, bundle, sName string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), bundle, "unsubscribe", url.QueryEscape(sName)) + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) ClearNamespaceBundleBacklogForSubscription(namespace utils.NameSpaceName, + bundle, sName string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), bundle, "clearBacklog", url.QueryEscape(sName)) + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) ClearNamespaceBundleBacklog(namespace utils.NameSpaceName, bundle string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), bundle, "clearBacklog") + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) ClearNamespaceBacklogForSubscription(namespace utils.NameSpaceName, sName string) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "clearBacklog", url.QueryEscape(sName)) + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) ClearNamespaceBacklog(namespace utils.NameSpaceName) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "clearBacklog") + return n.pulsar.Client.Post(endpoint, nil) +} + +func (n *namespaces) SetReplicatorDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "replicatorDispatchRate") + return n.pulsar.Client.Post(endpoint, rate) +} + +func (n *namespaces) GetReplicatorDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "replicatorDispatchRate") + var rate utils.DispatchRate + err := n.pulsar.Client.Get(endpoint, &rate) + return rate, err +} + +func (n *namespaces) SetSubscriptionDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscriptionDispatchRate") + return n.pulsar.Client.Post(endpoint, rate) +} + +func (n *namespaces) GetSubscriptionDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscriptionDispatchRate") + var rate utils.DispatchRate + err := n.pulsar.Client.Get(endpoint, &rate) + return rate, err +} + +func (n *namespaces) SetSubscribeRate(namespace utils.NameSpaceName, rate utils.SubscribeRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscribeRate") + return n.pulsar.Client.Post(endpoint, rate) +} + +func (n *namespaces) GetSubscribeRate(namespace utils.NameSpaceName) (utils.SubscribeRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "subscribeRate") + var rate utils.SubscribeRate + err := n.pulsar.Client.Get(endpoint, &rate) + return rate, err +} + +func (n *namespaces) SetDispatchRate(namespace utils.NameSpaceName, rate utils.DispatchRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "dispatchRate") + return n.pulsar.Client.Post(endpoint, rate) +} + +func (n *namespaces) GetDispatchRate(namespace utils.NameSpaceName) (utils.DispatchRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "dispatchRate") + var rate utils.DispatchRate + err := n.pulsar.Client.Get(endpoint, &rate) + return rate, err +} + +func (n *namespaces) SetPublishRate(namespace utils.NameSpaceName, pubRate utils.PublishRate) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "publishRate") + return n.pulsar.Client.Post(endpoint, pubRate) +} + +func (n *namespaces) GetPublishRate(namespace utils.NameSpaceName) (utils.PublishRate, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "publishRate") + var pubRate utils.PublishRate + err := n.pulsar.Client.Get(endpoint, &pubRate) + return pubRate, err +} + +func (n *namespaces) SetIsAllowAutoUpdateSchema(namespace utils.NameSpaceName, isAllowAutoUpdateSchema bool) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "isAllowAutoUpdateSchema") + return n.pulsar.Client.Post(endpoint, &isAllowAutoUpdateSchema) +} + +func (n *namespaces) GetIsAllowAutoUpdateSchema(namespace utils.NameSpaceName) (bool, error) { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "isAllowAutoUpdateSchema") + var result bool + err := n.pulsar.Client.Get(endpoint, &result) + return result, err +} + +func (n *namespaces) GetInactiveTopicPolicies(namespace utils.NameSpaceName) (utils.InactiveTopicPolicies, error) { + var out utils.InactiveTopicPolicies + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "inactiveTopicPolicies") + err := n.pulsar.Client.Get(endpoint, &out) + return out, err +} + +func (n *namespaces) RemoveInactiveTopicPolicies(namespace utils.NameSpaceName) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "inactiveTopicPolicies") + return n.pulsar.Client.Delete(endpoint) +} + +func (n *namespaces) SetInactiveTopicPolicies(namespace utils.NameSpaceName, data utils.InactiveTopicPolicies) error { + endpoint := n.pulsar.endpoint(n.basePath, namespace.String(), "inactiveTopicPolicies") + return n.pulsar.Client.Post(endpoint, data) +} diff --git a/pulsaradmin/pkg/admin/ns_isolation_policy.go b/pulsaradmin/pkg/admin/ns_isolation_policy.go new file mode 100644 index 000000000..d8897f9f6 --- /dev/null +++ b/pulsaradmin/pkg/admin/ns_isolation_policy.go @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +type NsIsolationPolicy interface { + // Create a namespace isolation policy for a cluster + CreateNamespaceIsolationPolicy(cluster, policyName string, namespaceIsolationData utils.NamespaceIsolationData) error + + // Delete a namespace isolation policy for a cluster + DeleteNamespaceIsolationPolicy(cluster, policyName string) error + + // Get a single namespace isolation policy for a cluster + GetNamespaceIsolationPolicy(cluster, policyName string) (*utils.NamespaceIsolationData, error) + + // Get the namespace isolation policies of a cluster + GetNamespaceIsolationPolicies(cluster string) (map[string]utils.NamespaceIsolationData, error) + + // Returns list of active brokers with namespace-isolation policies attached to it. + GetBrokersWithNamespaceIsolationPolicy(cluster string) ([]utils.BrokerNamespaceIsolationData, error) + + // Returns active broker with namespace-isolation policies attached to it. + GetBrokerWithNamespaceIsolationPolicy(cluster, broker string) (*utils.BrokerNamespaceIsolationData, error) +} + +type nsIsolationPolicy struct { + pulsar *pulsarClient + basePath string +} + +func (c *pulsarClient) NsIsolationPolicy() NsIsolationPolicy { + return &nsIsolationPolicy{ + pulsar: c, + basePath: "/clusters", + } +} + +func (n *nsIsolationPolicy) CreateNamespaceIsolationPolicy(cluster, policyName string, + namespaceIsolationData utils.NamespaceIsolationData) error { + return n.setNamespaceIsolationPolicy(cluster, policyName, namespaceIsolationData) +} + +func (n *nsIsolationPolicy) setNamespaceIsolationPolicy(cluster, policyName string, + namespaceIsolationData utils.NamespaceIsolationData) error { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) + return n.pulsar.Client.Post(endpoint, &namespaceIsolationData) +} + +func (n *nsIsolationPolicy) DeleteNamespaceIsolationPolicy(cluster, policyName string) error { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) + return n.pulsar.Client.Delete(endpoint) +} + +func (n *nsIsolationPolicy) GetNamespaceIsolationPolicy(cluster, policyName string) ( + *utils.NamespaceIsolationData, error) { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) + var nsIsolationData utils.NamespaceIsolationData + err := n.pulsar.Client.Get(endpoint, &nsIsolationData) + if err != nil { + return nil, err + } + return &nsIsolationData, nil +} + +func (n *nsIsolationPolicy) GetNamespaceIsolationPolicies(cluster string) ( + map[string]utils.NamespaceIsolationData, error) { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies") + var tmpMap map[string]utils.NamespaceIsolationData + err := n.pulsar.Client.Get(endpoint, &tmpMap) + if err != nil { + return nil, err + } + return tmpMap, nil +} + +func (n *nsIsolationPolicy) GetBrokersWithNamespaceIsolationPolicy(cluster string) ( + []utils.BrokerNamespaceIsolationData, error) { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", "brokers") + var res []utils.BrokerNamespaceIsolationData + err := n.pulsar.Client.Get(endpoint, &res) + if err != nil { + return nil, err + } + return res, nil +} + +func (n *nsIsolationPolicy) GetBrokerWithNamespaceIsolationPolicy(cluster, + broker string) (*utils.BrokerNamespaceIsolationData, error) { + endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", "brokers", broker) + var brokerNamespaceIsolationData utils.BrokerNamespaceIsolationData + err := n.pulsar.Client.Get(endpoint, &brokerNamespaceIsolationData) + if err != nil { + return nil, err + } + return &brokerNamespaceIsolationData, nil +} diff --git a/pulsaradmin/pkg/admin/packages.go b/pulsaradmin/pkg/admin/packages.go new file mode 100644 index 000000000..c7a0fd5ff --- /dev/null +++ b/pulsaradmin/pkg/admin/packages.go @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "os" + "path" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Packages is admin interface for functions management +type Packages interface { + // Download Function/Connector Package + // @param destinationFile + // file where data should be downloaded to + // @param packageURL + // the package URL + Download(packageURL, destinationFile string) error + + // Upload Function/Connector Package + // @param filePath + // file where data should be uploaded to + // @param packageURL + // type://tenant/namespace/packageName@version + // @param description + // descriptions of a package + // @param contact + // contact information of a package + // @param properties + // external infromations of a package + Upload(packageURL, filePath, description, contact string, properties map[string]string) error + + // List all the packages with the given type in a namespace + List(typeName, namespace string) ([]string, error) + + // ListVersions list all the versions of a package + ListVersions(packageURL string) ([]string, error) + + // Delete the specified package + Delete(packageURL string) error + + // GetMetadata get a package metadata information + GetMetadata(packageURL string) (utils.PackageMetadata, error) + + // UpdateMetadata update a package metadata information + UpdateMetadata(packageURL, description, contact string, properties map[string]string) error +} + +type packages struct { + pulsar *pulsarClient + basePath string +} + +func (p *packages) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "application/json") + return w.CreatePart(h) +} + +// Packages is used to access the functions endpoints +func (c *pulsarClient) Packages() Packages { + return &packages{ + pulsar: c, + basePath: "/packages", + } +} + +func (p packages) Download(packageURL, destinationFile string) error { + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) + + parent := path.Dir(destinationFile) + if parent != "." { + err = os.MkdirAll(parent, 0755) + if err != nil { + return fmt.Errorf("failed to create parent directory %s: %w", parent, err) + } + } + + _, err = os.Open(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("file %s already exists, please delete "+ + "the file first or change the file name", destinationFile) + } + } + file, err := os.Create(destinationFile) + if err != nil { + return err + } + + _, err = p.pulsar.Client.GetWithOptions(endpoint, nil, nil, false, file) + if err != nil { + return err + } + return nil +} + +func (p packages) Upload(packageURL, filePath, description, contact string, properties map[string]string) error { + if strings.TrimSpace(filePath) == "" { + return errors.New("file path is empty") + } + if strings.TrimSpace(packageURL) == "" { + return errors.New("package URL is empty") + } + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) + metadata := utils.PackageMetadata{ + Description: description, + Contact: contact, + Properties: properties, + } + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + metadataJSON, err := json.Marshal(metadata) + if err != nil { + return err + } + + stringWriter, err := p.createStringFromField(multiPartWriter, "metadata") + if err != nil { + return err + } + + _, err = stringWriter.Write(metadataJSON) + if err != nil { + return err + } + + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("file", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = p.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (p packages) List(typeName, namespace string) ([]string, error) { + var packageList []string + endpoint := p.pulsar.endpoint(p.basePath, typeName, namespace) + err := p.pulsar.Client.Get(endpoint, &packageList) + return packageList, err +} + +func (p packages) ListVersions(packageURL string) ([]string, error) { + var versionList []string + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return versionList, err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName()) + err = p.pulsar.Client.Get(endpoint, &versionList) + return versionList, err +} + +func (p packages) Delete(packageURL string) error { + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) + + return p.pulsar.Client.Delete(endpoint) +} + +func (p packages) GetMetadata(packageURL string) (utils.PackageMetadata, error) { + var metadata utils.PackageMetadata + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return metadata, err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion(), "metadata") + err = p.pulsar.Client.Get(endpoint, &metadata) + return metadata, err +} + +func (p packages) UpdateMetadata(packageURL, description, contact string, properties map[string]string) error { + metadata := utils.PackageMetadata{ + Description: description, + Contact: contact, + Properties: properties, + } + packageName, err := utils.GetPackageName(packageURL) + if err != nil { + return err + } + endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), + packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion(), "metadata") + + return p.pulsar.Client.Put(endpoint, &metadata) +} diff --git a/pulsaradmin/pkg/admin/resource_quotas.go b/pulsaradmin/pkg/admin/resource_quotas.go new file mode 100644 index 000000000..fc5209b54 --- /dev/null +++ b/pulsaradmin/pkg/admin/resource_quotas.go @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +type ResourceQuotas interface { + // Get default resource quota for new resource bundles. + GetDefaultResourceQuota() (*utils.ResourceQuota, error) + + // Set default resource quota for new namespace bundles. + SetDefaultResourceQuota(quota utils.ResourceQuota) error + + // Get resource quota of a namespace bundle. + GetNamespaceBundleResourceQuota(namespace, bundle string) (*utils.ResourceQuota, error) + + // Set resource quota for a namespace bundle. + SetNamespaceBundleResourceQuota(namespace, bundle string, quota utils.ResourceQuota) error + + // Reset resource quota for a namespace bundle to default value. + ResetNamespaceBundleResourceQuota(namespace, bundle string) error +} + +type resource struct { + pulsar *pulsarClient + basePath string +} + +func (c *pulsarClient) ResourceQuotas() ResourceQuotas { + return &resource{ + pulsar: c, + basePath: "/resource-quotas", + } +} + +func (r *resource) GetDefaultResourceQuota() (*utils.ResourceQuota, error) { + endpoint := r.pulsar.endpoint(r.basePath) + var quota utils.ResourceQuota + err := r.pulsar.Client.Get(endpoint, "a) + if err != nil { + return nil, err + } + return "a, nil +} + +func (r *resource) SetDefaultResourceQuota(quota utils.ResourceQuota) error { + endpoint := r.pulsar.endpoint(r.basePath) + return r.pulsar.Client.Post(endpoint, "a) +} + +func (r *resource) GetNamespaceBundleResourceQuota(namespace, bundle string) (*utils.ResourceQuota, error) { + endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) + var quota utils.ResourceQuota + err := r.pulsar.Client.Get(endpoint, "a) + if err != nil { + return nil, err + } + return "a, nil +} + +func (r *resource) SetNamespaceBundleResourceQuota(namespace, bundle string, quota utils.ResourceQuota) error { + endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) + return r.pulsar.Client.Post(endpoint, "a) +} + +func (r *resource) ResetNamespaceBundleResourceQuota(namespace, bundle string) error { + endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) + return r.pulsar.Client.Delete(endpoint) +} diff --git a/pulsaradmin/pkg/admin/schema.go b/pulsaradmin/pkg/admin/schema.go new file mode 100644 index 000000000..146552684 --- /dev/null +++ b/pulsaradmin/pkg/admin/schema.go @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "fmt" + "strconv" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Schema is admin interface for schema management +type Schema interface { + // GetSchemaInfo retrieves the latest schema of a topic + GetSchemaInfo(topic string) (*utils.SchemaInfo, error) + + // GetSchemaInfoWithVersion retrieves the latest schema with version of a topic + GetSchemaInfoWithVersion(topic string) (*utils.SchemaInfoWithVersion, error) + + // GetSchemaInfoByVersion retrieves the schema of a topic at a given version + GetSchemaInfoByVersion(topic string, version int64) (*utils.SchemaInfo, error) + + // DeleteSchema deletes the schema associated with a given topic + DeleteSchema(topic string) error + + // CreateSchemaByPayload creates a schema for a given topic + CreateSchemaByPayload(topic string, schemaPayload utils.PostSchemaPayload) error +} + +type schemas struct { + pulsar *pulsarClient + basePath string +} + +// Schemas is used to access the schemas endpoints +func (c *pulsarClient) Schemas() Schema { + return &schemas{ + pulsar: c, + basePath: "/schemas", + } +} + +func (s *schemas) GetSchemaInfo(topic string) (*utils.SchemaInfo, error) { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return nil, err + } + var response utils.GetSchemaResponse + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), + topicName.GetLocalName(), "schema") + + err = s.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, err + } + + info := utils.ConvertGetSchemaResponseToSchemaInfo(topicName, response) + return info, nil +} + +func (s *schemas) GetSchemaInfoWithVersion(topic string) (*utils.SchemaInfoWithVersion, error) { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return nil, err + } + var response utils.GetSchemaResponse + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), + topicName.GetLocalName(), "schema") + + err = s.pulsar.Client.Get(endpoint, &response) + if err != nil { + fmt.Println("err:", err.Error()) + return nil, err + } + + info := utils.ConvertGetSchemaResponseToSchemaInfoWithVersion(topicName, response) + return info, nil +} + +func (s *schemas) GetSchemaInfoByVersion(topic string, version int64) (*utils.SchemaInfo, error) { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return nil, err + } + + var response utils.GetSchemaResponse + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), topicName.GetLocalName(), + "schema", strconv.FormatInt(version, 10)) + + err = s.pulsar.Client.Get(endpoint, &response) + if err != nil { + return nil, err + } + + info := utils.ConvertGetSchemaResponseToSchemaInfo(topicName, response) + return info, nil +} + +func (s *schemas) DeleteSchema(topic string) error { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return err + } + + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), + topicName.GetLocalName(), "schema") + + fmt.Println(endpoint) + + return s.pulsar.Client.Delete(endpoint) +} + +func (s *schemas) CreateSchemaByPayload(topic string, schemaPayload utils.PostSchemaPayload) error { + topicName, err := utils.GetTopicName(topic) + if err != nil { + return err + } + + endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), + topicName.GetLocalName(), "schema") + + return s.pulsar.Client.Post(endpoint, &schemaPayload) +} diff --git a/pulsaradmin/pkg/admin/sinks.go b/pulsaradmin/pkg/admin/sinks.go new file mode 100644 index 000000000..acbf83113 --- /dev/null +++ b/pulsaradmin/pkg/admin/sinks.go @@ -0,0 +1,437 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Sinks is admin interface for sinks management +type Sinks interface { + // ListSinks returns the list of all the Pulsar Sinks. + ListSinks(tenant, namespace string) ([]string, error) + + // GetSink returns the configuration for the specified sink + GetSink(tenant, namespace, Sink string) (utils.SinkConfig, error) + + // CreateSink creates a new sink + CreateSink(config *utils.SinkConfig, fileName string) error + + // CreateSinkWithURL creates a new sink by providing url from which fun-pkg can be downloaded. supported url: http/file + CreateSinkWithURL(config *utils.SinkConfig, pkgURL string) error + + // UpdateSink updates the configuration for a sink. + UpdateSink(config *utils.SinkConfig, fileName string, options *utils.UpdateOptions) error + + // UpdateSinkWithURL updates a sink by providing url from which fun-pkg can be downloaded. supported url: http/file + UpdateSinkWithURL(config *utils.SinkConfig, pkgURL string, options *utils.UpdateOptions) error + + // DeleteSink deletes an existing sink + DeleteSink(tenant, namespace, Sink string) error + + // GetSinkStatus returns the current status of a sink. + GetSinkStatus(tenant, namespace, Sink string) (utils.SinkStatus, error) + + // GetSinkStatusWithID returns the current status of a sink instance. + GetSinkStatusWithID(tenant, namespace, Sink string, id int) (utils.SinkInstanceStatusData, error) + + // RestartSink restarts all sink instances + RestartSink(tenant, namespace, Sink string) error + + // RestartSinkWithID restarts sink instance + RestartSinkWithID(tenant, namespace, Sink string, id int) error + + // StopSink stops all sink instances + StopSink(tenant, namespace, Sink string) error + + // StopSinkWithID stops sink instance + StopSinkWithID(tenant, namespace, Sink string, id int) error + + // StartSink starts all sink instances + StartSink(tenant, namespace, Sink string) error + + // StartSinkWithID starts sink instance + StartSinkWithID(tenant, namespace, Sink string, id int) error + + // GetBuiltInSinks fetches a list of supported Pulsar IO sinks currently running in cluster mode + GetBuiltInSinks() ([]*utils.ConnectorDefinition, error) + + // ReloadBuiltInSinks reload the available built-in connectors, include Source and Sink + ReloadBuiltInSinks() error +} + +type sinks struct { + pulsar *pulsarClient + basePath string +} + +// Sinks is used to access the sinks endpoints +func (c *pulsarClient) Sinks() Sinks { + return &sinks{ + pulsar: c, + basePath: "/sinks", + } +} + +func (s *sinks) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "application/json") + return w.CreatePart(h) +} + +func (s *sinks) createTextFromFiled(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "text/plain") + return w.CreatePart(h) +} + +func (s *sinks) ListSinks(tenant, namespace string) ([]string, error) { + var sinks []string + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace) + err := s.pulsar.Client.Get(endpoint, &sinks) + return sinks, err +} + +func (s *sinks) GetSink(tenant, namespace, sink string) (utils.SinkConfig, error) { + var sinkConfig utils.SinkConfig + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + err := s.pulsar.Client.Get(endpoint, &sinkConfig) + return sinkConfig, err +} + +func (s *sinks) CreateSink(config *utils.SinkConfig, fileName string) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sinkConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sinks) CreateSinkWithURL(config *utils.SinkConfig, pkgURL string) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := s.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sinkConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sinks) UpdateSink(config *utils.SinkConfig, fileName string, updateOptions *utils.UpdateOptions) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sinkConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := s.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sinks) UpdateSinkWithURL(config *utils.SinkConfig, pkgURL string, updateOptions *utils.UpdateOptions) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := s.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sinkConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := s.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sinks) DeleteSink(tenant, namespace, sink string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + return s.pulsar.Client.Delete(endpoint) +} + +func (s *sinks) GetSinkStatus(tenant, namespace, sink string) (utils.SinkStatus, error) { + var sinkStatus utils.SinkStatus + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + err := s.pulsar.Client.Get(endpoint+"/status", &sinkStatus) + return sinkStatus, err +} + +func (s *sinks) GetSinkStatusWithID(tenant, namespace, sink string, id int) (utils.SinkInstanceStatusData, error) { + var sinkInstanceStatusData utils.SinkInstanceStatusData + instanceID := fmt.Sprintf("%d", id) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink, instanceID) + err := s.pulsar.Client.Get(endpoint+"/status", &sinkInstanceStatusData) + return sinkInstanceStatusData, err +} + +func (s *sinks) RestartSink(tenant, namespace, sink string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + return s.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (s *sinks) RestartSinkWithID(tenant, namespace, sink string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink, id) + + return s.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (s *sinks) StopSink(tenant, namespace, sink string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + return s.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (s *sinks) StopSinkWithID(tenant, namespace, sink string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink, id) + + return s.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (s *sinks) StartSink(tenant, namespace, sink string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink) + return s.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (s *sinks) StartSinkWithID(tenant, namespace, sink string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, sink, id) + + return s.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (s *sinks) GetBuiltInSinks() ([]*utils.ConnectorDefinition, error) { + var connectorDefinition []*utils.ConnectorDefinition + endpoint := s.pulsar.endpoint(s.basePath, "builtinsinks") + err := s.pulsar.Client.Get(endpoint, &connectorDefinition) + return connectorDefinition, err +} + +func (s *sinks) ReloadBuiltInSinks() error { + endpoint := s.pulsar.endpoint(s.basePath, "reloadBuiltInSinks") + return s.pulsar.Client.Post(endpoint, nil) +} diff --git a/pulsaradmin/pkg/admin/sources.go b/pulsaradmin/pkg/admin/sources.go new file mode 100644 index 000000000..e10d1da06 --- /dev/null +++ b/pulsaradmin/pkg/admin/sources.go @@ -0,0 +1,440 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "os" + "path/filepath" + "strings" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Sources is admin interface for sources management +type Sources interface { + // ListSources returns the list of all the Pulsar Sources. + ListSources(tenant, namespace string) ([]string, error) + + // GetSource return the configuration for the specified source + GetSource(tenant, namespace, source string) (utils.SourceConfig, error) + + // CreateSource creates a new source + CreateSource(config *utils.SourceConfig, fileName string) error + + // CreateSourceWithURL creates a new source by providing url from which fun-pkg can be downloaded. + // supported url: http/file + CreateSourceWithURL(config *utils.SourceConfig, pkgURL string) error + + // UpdateSource updates the configuration for a source. + UpdateSource(config *utils.SourceConfig, fileName string, options *utils.UpdateOptions) error + + // UpdateSourceWithURL updates a source by providing url from which fun-pkg can be downloaded. supported url: http/file + UpdateSourceWithURL(config *utils.SourceConfig, pkgURL string, options *utils.UpdateOptions) error + + // DeleteSource deletes an existing source + DeleteSource(tenant, namespace, source string) error + + // GetSourceStatus returns the current status of a source. + GetSourceStatus(tenant, namespace, source string) (utils.SourceStatus, error) + + // GetSourceStatusWithID returns the current status of a source instance. + GetSourceStatusWithID(tenant, namespace, source string, id int) (utils.SourceInstanceStatusData, error) + + // RestartSource restarts all source instances + RestartSource(tenant, namespace, source string) error + + // RestartSourceWithID restarts source instance + RestartSourceWithID(tenant, namespace, source string, id int) error + + // StopSource stops all source instances + StopSource(tenant, namespace, source string) error + + // StopSourceWithID stops source instance + StopSourceWithID(tenant, namespace, source string, id int) error + + // StartSource starts all source instances + StartSource(tenant, namespace, source string) error + + // StartSourceWithID starts source instance + StartSourceWithID(tenant, namespace, source string, id int) error + + // GetBuiltInSources fetches a list of supported Pulsar IO sources currently running in cluster mode + GetBuiltInSources() ([]*utils.ConnectorDefinition, error) + + // ReloadBuiltInSources reloads the available built-in connectors, include Source and Sink + ReloadBuiltInSources() error +} + +type sources struct { + pulsar *pulsarClient + basePath string +} + +// Sources is used to access the sources endpoints +func (c *pulsarClient) Sources() Sources { + return &sources{ + pulsar: c, + basePath: "/sources", + } +} + +func (s *sources) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "application/json") + return w.CreatePart(h) +} + +func (s *sources) createTextFromFiled(w *multipart.Writer, value string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) + h.Set("Content-Type", "text/plain") + return w.CreatePart(h) +} + +func (s *sources) ListSources(tenant, namespace string) ([]string, error) { + var sources []string + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace) + err := s.pulsar.Client.Get(endpoint, &sources) + return sources, err +} + +func (s *sources) GetSource(tenant, namespace, source string) (utils.SourceConfig, error) { + var sourceConfig utils.SourceConfig + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + err := s.pulsar.Client.Get(endpoint, &sourceConfig) + return sourceConfig, err +} + +func (s *sources) CreateSource(config *utils.SourceConfig, fileName string) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sourceConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sources) CreateSourceWithURL(config *utils.SourceConfig, pkgURL string) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := s.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sourceConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sources) UpdateSource(config *utils.SourceConfig, fileName string, updateOptions *utils.UpdateOptions) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sourceConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := s.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + if fileName != "" && !strings.HasPrefix(fileName, "builtin://") { + // If the function code is built in, we don't need to submit here + file, err := os.Open(fileName) + if err != nil { + return err + } + defer file.Close() + + part, err := multiPartWriter.CreateFormFile("data", filepath.Base(file.Name())) + + if err != nil { + return err + } + + // copy the actual file content to the filed's writer + _, err = io.Copy(part, file) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sources) UpdateSourceWithURL(config *utils.SourceConfig, pkgURL string, + updateOptions *utils.UpdateOptions) error { + endpoint := s.pulsar.endpoint(s.basePath, config.Tenant, config.Namespace, config.Name) + // buffer to store our request as bytes + bodyBuf := bytes.NewBufferString("") + + multiPartWriter := multipart.NewWriter(bodyBuf) + + textWriter, err := s.createTextFromFiled(multiPartWriter, "url") + if err != nil { + return err + } + + _, err = textWriter.Write([]byte(pkgURL)) + if err != nil { + return err + } + + jsonData, err := json.Marshal(config) + if err != nil { + return err + } + + stringWriter, err := s.createStringFromField(multiPartWriter, "sourceConfig") + if err != nil { + return err + } + + _, err = stringWriter.Write(jsonData) + if err != nil { + return err + } + + if updateOptions != nil { + updateData, err := json.Marshal(updateOptions) + if err != nil { + return err + } + + updateStrWriter, err := s.createStringFromField(multiPartWriter, "updateOptions") + if err != nil { + return err + } + + _, err = updateStrWriter.Write(updateData) + if err != nil { + return err + } + } + + // In here, we completed adding the file and the fields, let's close the multipart writer + // So it writes the ending boundary + if err = multiPartWriter.Close(); err != nil { + return err + } + + contentType := multiPartWriter.FormDataContentType() + err = s.pulsar.Client.PutWithMultiPart(endpoint, bodyBuf, contentType) + if err != nil { + return err + } + + return nil +} + +func (s *sources) DeleteSource(tenant, namespace, source string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + return s.pulsar.Client.Delete(endpoint) +} + +func (s *sources) GetSourceStatus(tenant, namespace, source string) (utils.SourceStatus, error) { + var sourceStatus utils.SourceStatus + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + err := s.pulsar.Client.Get(endpoint+"/status", &sourceStatus) + return sourceStatus, err +} + +func (s *sources) GetSourceStatusWithID(tenant, namespace, source string, id int) ( + utils.SourceInstanceStatusData, error) { + var sourceInstanceStatusData utils.SourceInstanceStatusData + instanceID := fmt.Sprintf("%d", id) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source, instanceID) + err := s.pulsar.Client.Get(endpoint+"/status", &sourceInstanceStatusData) + return sourceInstanceStatusData, err +} + +func (s *sources) RestartSource(tenant, namespace, source string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + return s.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (s *sources) RestartSourceWithID(tenant, namespace, source string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source, id) + + return s.pulsar.Client.Post(endpoint+"/restart", nil) +} + +func (s *sources) StopSource(tenant, namespace, source string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + return s.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (s *sources) StopSourceWithID(tenant, namespace, source string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source, id) + + return s.pulsar.Client.Post(endpoint+"/stop", nil) +} + +func (s *sources) StartSource(tenant, namespace, source string) error { + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source) + return s.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (s *sources) StartSourceWithID(tenant, namespace, source string, instanceID int) error { + id := fmt.Sprintf("%d", instanceID) + endpoint := s.pulsar.endpoint(s.basePath, tenant, namespace, source, id) + + return s.pulsar.Client.Post(endpoint+"/start", nil) +} + +func (s *sources) GetBuiltInSources() ([]*utils.ConnectorDefinition, error) { + var connectorDefinition []*utils.ConnectorDefinition + endpoint := s.pulsar.endpoint(s.basePath, "builtinsources") + err := s.pulsar.Client.Get(endpoint, &connectorDefinition) + return connectorDefinition, err +} + +func (s *sources) ReloadBuiltInSources() error { + endpoint := s.pulsar.endpoint(s.basePath, "reloadBuiltInSources") + return s.pulsar.Client.Post(endpoint, nil) +} diff --git a/pulsaradmin/pkg/admin/subscription.go b/pulsaradmin/pkg/admin/subscription.go new file mode 100644 index 000000000..456de46cd --- /dev/null +++ b/pulsaradmin/pkg/admin/subscription.go @@ -0,0 +1,316 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "bytes" + "encoding/binary" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/golang/protobuf/proto" //nolint:staticcheck + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Subscriptions is admin interface for subscriptions management +type Subscriptions interface { + // Create a new subscription on a topic + Create(utils.TopicName, string, utils.MessageID) error + + // Delete a subscription. + // Delete a persistent subscription from a topic. There should not be any active consumers on the subscription + Delete(utils.TopicName, string) error + + // ForceDelete deletes a subscription forcefully + ForceDelete(utils.TopicName, string) error + + // List returns the list of subscriptions + List(utils.TopicName) ([]string, error) + + // ResetCursorToMessageID resets cursor position on a topic subscription + // @param + // messageID reset subscription to messageId (or previous nearest messageId if given messageId is not valid) + ResetCursorToMessageID(utils.TopicName, string, utils.MessageID) error + + // ResetCursorToTimestamp resets cursor position on a topic subscription + // @param + // time reset subscription to position closest to time in ms since epoch + ResetCursorToTimestamp(utils.TopicName, string, int64) error + + // ClearBacklog skips all messages on a topic subscription + ClearBacklog(utils.TopicName, string) error + + // SkipMessages skips messages on a topic subscription + SkipMessages(utils.TopicName, string, int64) error + + // ExpireMessages expires all messages older than given N (expireTimeInSeconds) seconds for a given subscription + ExpireMessages(utils.TopicName, string, int64) error + + // ExpireAllMessages expires all messages older than given N (expireTimeInSeconds) seconds for all + // subscriptions of the persistent-topic + ExpireAllMessages(utils.TopicName, int64) error + + // PeekMessages peeks messages from a topic subscription + PeekMessages(utils.TopicName, string, int) ([]*utils.Message, error) + + // GetMessageByID gets message by its ledgerID and entryID + GetMessageByID(topic utils.TopicName, ledgerID, entryID int64) (*utils.Message, error) +} + +type subscriptions struct { + pulsar *pulsarClient + basePath string + SubPath string +} + +// Subscriptions is used to access the subscriptions endpoints +func (c *pulsarClient) Subscriptions() Subscriptions { + return &subscriptions{ + pulsar: c, + basePath: "", + SubPath: "subscription", + } +} + +func (s *subscriptions) Create(topic utils.TopicName, sName string, messageID utils.MessageID) error { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName)) + return s.pulsar.Client.Put(endpoint, messageID) +} + +func (s *subscriptions) delete(topic utils.TopicName, subName string, force bool) error { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(subName)) + queryParams := make(map[string]string) + queryParams["force"] = strconv.FormatBool(force) + return s.pulsar.Client.DeleteWithQueryParams(endpoint, queryParams) +} + +func (s *subscriptions) Delete(topic utils.TopicName, sName string) error { + return s.delete(topic, sName, false) +} + +func (s *subscriptions) ForceDelete(topic utils.TopicName, sName string) error { + return s.delete(topic, sName, true) +} + +func (s *subscriptions) List(topic utils.TopicName) ([]string, error) { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), "subscriptions") + var list []string + return list, s.pulsar.Client.Get(endpoint, &list) +} + +func (s *subscriptions) ResetCursorToMessageID(topic utils.TopicName, sName string, id utils.MessageID) error { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), "resetcursor") + return s.pulsar.Client.Post(endpoint, id) +} + +func (s *subscriptions) ResetCursorToTimestamp(topic utils.TopicName, sName string, timestamp int64) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), + "resetcursor", strconv.FormatInt(timestamp, 10)) + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) ClearBacklog(topic utils.TopicName, sName string) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), "skip_all") + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) SkipMessages(topic utils.TopicName, sName string, n int64) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), + "skip", strconv.FormatInt(n, 10)) + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) ExpireMessages(topic utils.TopicName, sName string, expire int64) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), s.SubPath, url.PathEscape(sName), + "expireMessages", strconv.FormatInt(expire, 10)) + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) ExpireAllMessages(topic utils.TopicName, expire int64) error { + endpoint := s.pulsar.endpoint( + s.basePath, topic.GetRestPath(), "all_subscription", + "expireMessages", strconv.FormatInt(expire, 10)) + return s.pulsar.Client.Post(endpoint, nil) +} + +func (s *subscriptions) PeekMessages(topic utils.TopicName, sName string, n int) ([]*utils.Message, error) { + var msgs []*utils.Message + + count := 1 + for n > 0 { + m, err := s.peekNthMessage(topic, sName, count) + if err != nil { + return nil, err + } + msgs = append(msgs, m...) + n -= len(m) + count++ + } + + return msgs, nil +} + +func (s *subscriptions) peekNthMessage(topic utils.TopicName, sName string, pos int) ([]*utils.Message, error) { + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), "subscription", url.PathEscape(sName), + "position", strconv.Itoa(pos)) + + resp, err := s.pulsar.Client.MakeRequest(http.MethodGet, endpoint) + if err != nil { + return nil, err + } + defer safeRespClose(resp) + + return handleResp(topic, resp) +} + +func (s *subscriptions) GetMessageByID(topic utils.TopicName, ledgerID, entryID int64) (*utils.Message, error) { + ledgerIDStr := strconv.FormatInt(ledgerID, 10) + entryIDStr := strconv.FormatInt(entryID, 10) + + endpoint := s.pulsar.endpoint(s.basePath, topic.GetRestPath(), "ledger", ledgerIDStr, "entry", entryIDStr) + resp, err := s.pulsar.Client.MakeRequest(http.MethodGet, endpoint) + if err != nil { + return nil, err + } + defer safeRespClose(resp) + + messages, err := handleResp(topic, resp) + if err != nil { + return nil, err + } + + if len(messages) == 0 { + return nil, nil + } + return messages[0], nil +} + +// safeRespClose is used to close a response body +func safeRespClose(resp *http.Response) { + if resp != nil { + // ignore error since it is closing a response body + _ = resp.Body.Close() + } +} + +const ( + PublishTimeHeader = "X-Pulsar-Publish-Time" + BatchHeader = "X-Pulsar-Num-Batch-Message" + PropertyPrefix = "X-Pulsar-Property-" +) + +func handleResp(topic utils.TopicName, resp *http.Response) ([]*utils.Message, error) { + msgID := resp.Header.Get("X-Pulsar-Message-ID") + ID, err := utils.ParseMessageID(msgID) + if err != nil { + return nil, err + } + + // read data + payload, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + properties := make(map[string]string) + for k := range resp.Header { + switch { + case k == PublishTimeHeader: + h := resp.Header.Get(k) + if h != "" { + properties["publish-time"] = h + } + case k == BatchHeader: + h := resp.Header.Get(k) + if h != "" { + properties[BatchHeader] = h + } + return getIndividualMsgsFromBatch(topic, ID, payload, properties) + case strings.Contains(k, PropertyPrefix): + key := strings.TrimPrefix(k, PropertyPrefix) + properties[key] = resp.Header.Get(k) + } + } + + return []*utils.Message{utils.NewMessage(topic.String(), *ID, payload, properties)}, nil +} + +func getIndividualMsgsFromBatch(topic utils.TopicName, msgID *utils.MessageID, data []byte, + properties map[string]string) ([]*utils.Message, error) { + + batchSize, err := strconv.Atoi(properties[BatchHeader]) + if err != nil { + return nil, nil + } + + msgs := make([]*utils.Message, 0, batchSize) + + // read all messages in batch + buf32 := make([]byte, 4) + rdBuf := bytes.NewReader(data) + for i := 0; i < batchSize; i++ { + msgID.BatchIndex = i + // singleMetaSize + if _, err := io.ReadFull(rdBuf, buf32); err != nil { + return nil, err + } + singleMetaSize := binary.BigEndian.Uint32(buf32) + + // singleMeta + singleMetaBuf := make([]byte, singleMetaSize) + if _, err := io.ReadFull(rdBuf, singleMetaBuf); err != nil { + return nil, err + } + + singleMeta := new(utils.SingleMessageMetadata) + if err := proto.Unmarshal(singleMetaBuf, singleMeta); err != nil { + return nil, err + } + + if len(singleMeta.Properties) > 0 { + for _, v := range singleMeta.Properties { + k := *v.Key + property := *v.Value + properties[k] = property + } + } + + // payload + singlePayload := make([]byte, singleMeta.GetPayloadSize()) + if _, err := io.ReadFull(rdBuf, singlePayload); err != nil { + return nil, err + } + + msgs = append(msgs, &utils.Message{ + Topic: topic.String(), + MessageID: *msgID, + Payload: singlePayload, + Properties: properties, + }) + } + + return msgs, nil +} diff --git a/pulsaradmin/pkg/admin/tenant.go b/pulsaradmin/pkg/admin/tenant.go new file mode 100644 index 000000000..62e176bcb --- /dev/null +++ b/pulsaradmin/pkg/admin/tenant.go @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Tenants is admin interface for tenants management +type Tenants interface { + // Create a new tenant + Create(utils.TenantData) error + + // Delete an existing tenant + Delete(string) error + + // Update the admins for a tenant + Update(utils.TenantData) error + + // List returns the list of tenants + List() ([]string, error) + + // Get returns the config of the tenant. + Get(string) (utils.TenantData, error) +} + +type tenants struct { + pulsar *pulsarClient + basePath string +} + +// Tenants is used to access the tenants endpoints +func (c *pulsarClient) Tenants() Tenants { + return &tenants{ + pulsar: c, + basePath: "/tenants", + } +} + +func (c *tenants) Create(data utils.TenantData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.Name) + return c.pulsar.Client.Put(endpoint, &data) +} + +func (c *tenants) Delete(name string) error { + endpoint := c.pulsar.endpoint(c.basePath, name) + return c.pulsar.Client.Delete(endpoint) +} + +func (c *tenants) Update(data utils.TenantData) error { + endpoint := c.pulsar.endpoint(c.basePath, data.Name) + return c.pulsar.Client.Post(endpoint, &data) +} + +func (c *tenants) List() ([]string, error) { + var tenantList []string + endpoint := c.pulsar.endpoint(c.basePath, "") + err := c.pulsar.Client.Get(endpoint, &tenantList) + return tenantList, err +} + +func (c *tenants) Get(name string) (utils.TenantData, error) { + var data utils.TenantData + endpoint := c.pulsar.endpoint(c.basePath, name) + err := c.pulsar.Client.Get(endpoint, &data) + return data, err +} diff --git a/pulsaradmin/pkg/admin/topic.go b/pulsaradmin/pkg/admin/topic.go new file mode 100644 index 000000000..c888827bf --- /dev/null +++ b/pulsaradmin/pkg/admin/topic.go @@ -0,0 +1,725 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package admin + +import ( + "fmt" + "strconv" + + "github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils" +) + +// Topics is admin interface for topics management +type Topics interface { + // Create a topic + Create(utils.TopicName, int) error + + // Delete a topic + Delete(utils.TopicName, bool, bool) error + + // Update number of partitions of a non-global partitioned topic + // It requires partitioned-topic to be already exist and number of new partitions must be greater than existing + // number of partitions. Decrementing number of partitions requires deletion of topic which is not supported. + Update(utils.TopicName, int) error + + // GetMetadata returns metadata of a partitioned topic + GetMetadata(utils.TopicName) (utils.PartitionedTopicMetadata, error) + + // List returns the list of topics under a namespace + List(utils.NameSpaceName) ([]string, []string, error) + + // GetInternalInfo returns the internal metadata info for the topic + GetInternalInfo(utils.TopicName) (utils.ManagedLedgerInfo, error) + + // GetPermissions returns permissions on a topic + // Retrieve the effective permissions for a topic. These permissions are defined by the permissions set at the + // namespace level combined (union) with any eventual specific permission set on the topic. + GetPermissions(utils.TopicName) (map[string][]utils.AuthAction, error) + + // GrantPermission grants a new permission to a client role on a single topic + GrantPermission(utils.TopicName, string, []utils.AuthAction) error + + // RevokePermission revokes permissions to a client role on a single topic. If the permission + // was not set at the topic level, but rather at the namespace level, this operation will + // return an error (HTTP status code 412). + RevokePermission(utils.TopicName, string) error + + // Lookup a topic returns the broker URL that serves the topic + Lookup(utils.TopicName) (utils.LookupData, error) + + // GetBundleRange returns a bundle range of a topic + GetBundleRange(utils.TopicName) (string, error) + + // GetLastMessageID returns the last commit message Id of a topic + GetLastMessageID(utils.TopicName) (utils.MessageID, error) + + // GetMessageID returns the message Id by timestamp(ms) of a topic + GetMessageID(utils.TopicName, int64) (utils.MessageID, error) + + // GetStats returns the stats for the topic + // All the rates are computed over a 1 minute window and are relative the last completed 1 minute period + GetStats(utils.TopicName) (utils.TopicStats, error) + + // GetInternalStats returns the internal stats for the topic. + GetInternalStats(utils.TopicName) (utils.PersistentTopicInternalStats, error) + + // GetPartitionedStats returns the stats for the partitioned topic + // All the rates are computed over a 1 minute window and are relative the last completed 1 minute period + GetPartitionedStats(utils.TopicName, bool) (utils.PartitionedTopicStats, error) + + // Terminate the topic and prevent any more messages being published on it + Terminate(utils.TopicName) (utils.MessageID, error) + + // Offload triggers offloading messages in topic to longterm storage + Offload(utils.TopicName, utils.MessageID) error + + // OffloadStatus checks the status of an ongoing offloading operation for a topic + OffloadStatus(utils.TopicName) (utils.OffloadProcessStatus, error) + + // Unload a topic + Unload(utils.TopicName) error + + // Compact triggers compaction to run for a topic. A single topic can only have one instance of compaction + // running at any time. Any attempt to trigger another will be met with a ConflictException. + Compact(utils.TopicName) error + + // CompactStatus checks the status of an ongoing compaction for a topic + CompactStatus(utils.TopicName) (utils.LongRunningProcessStatus, error) + + // GetMessageTTL Get the message TTL for a topic + GetMessageTTL(utils.TopicName) (int, error) + + // SetMessageTTL Set the message TTL for a topic + SetMessageTTL(utils.TopicName, int) error + + // RemoveMessageTTL Remove the message TTL for a topic + RemoveMessageTTL(utils.TopicName) error + + // GetMaxProducers Get max number of producers for a topic + GetMaxProducers(utils.TopicName) (int, error) + + // SetMaxProducers Set max number of producers for a topic + SetMaxProducers(utils.TopicName, int) error + + // RemoveMaxProducers Remove max number of producers for a topic + RemoveMaxProducers(utils.TopicName) error + + // GetMaxConsumers Get max number of consumers for a topic + GetMaxConsumers(utils.TopicName) (int, error) + + // SetMaxConsumers Set max number of consumers for a topic + SetMaxConsumers(utils.TopicName, int) error + + // RemoveMaxConsumers Remove max number of consumers for a topic + RemoveMaxConsumers(utils.TopicName) error + + // GetMaxUnackMessagesPerConsumer Get max unacked messages policy on consumer for a topic + GetMaxUnackMessagesPerConsumer(utils.TopicName) (int, error) + + // SetMaxUnackMessagesPerConsumer Set max unacked messages policy on consumer for a topic + SetMaxUnackMessagesPerConsumer(utils.TopicName, int) error + + // RemoveMaxUnackMessagesPerConsumer Remove max unacked messages policy on consumer for a topic + RemoveMaxUnackMessagesPerConsumer(utils.TopicName) error + + // GetMaxUnackMessagesPerSubscription Get max unacked messages policy on subscription for a topic + GetMaxUnackMessagesPerSubscription(utils.TopicName) (int, error) + + // SetMaxUnackMessagesPerSubscription Set max unacked messages policy on subscription for a topic + SetMaxUnackMessagesPerSubscription(utils.TopicName, int) error + + // RemoveMaxUnackMessagesPerSubscription Remove max unacked messages policy on subscription for a topic + RemoveMaxUnackMessagesPerSubscription(utils.TopicName) error + + // GetPersistence Get the persistence policies for a topic + GetPersistence(utils.TopicName) (*utils.PersistenceData, error) + + // SetPersistence Set the persistence policies for a topic + SetPersistence(utils.TopicName, utils.PersistenceData) error + + // RemovePersistence Remove the persistence policies for a topic + RemovePersistence(utils.TopicName) error + + // GetDelayedDelivery Get the delayed delivery policy for a topic + GetDelayedDelivery(utils.TopicName) (*utils.DelayedDeliveryData, error) + + // SetDelayedDelivery Set the delayed delivery policy on a topic + SetDelayedDelivery(utils.TopicName, utils.DelayedDeliveryData) error + + // RemoveDelayedDelivery Remove the delayed delivery policy on a topic + RemoveDelayedDelivery(utils.TopicName) error + + // GetDispatchRate Get message dispatch rate for a topic + GetDispatchRate(utils.TopicName) (*utils.DispatchRateData, error) + + // SetDispatchRate Set message dispatch rate for a topic + SetDispatchRate(utils.TopicName, utils.DispatchRateData) error + + // RemoveDispatchRate Remove message dispatch rate for a topic + RemoveDispatchRate(utils.TopicName) error + + // GetPublishRate Get message publish rate for a topic + GetPublishRate(utils.TopicName) (*utils.PublishRateData, error) + + // SetPublishRate Set message publish rate for a topic + SetPublishRate(utils.TopicName, utils.PublishRateData) error + + // RemovePublishRate Remove message publish rate for a topic + RemovePublishRate(utils.TopicName) error + + // GetDeduplicationStatus Get the deduplication policy for a topic + GetDeduplicationStatus(utils.TopicName) (bool, error) + + // SetDeduplicationStatus Set the deduplication policy for a topic + SetDeduplicationStatus(utils.TopicName, bool) error + + // RemoveDeduplicationStatus Remove the deduplication policy for a topic + RemoveDeduplicationStatus(utils.TopicName) error + + // GetRetention returns the retention configuration for a topic + GetRetention(utils.TopicName, bool) (*utils.RetentionPolicies, error) + + // RemoveRetention removes the retention configuration on a topic + RemoveRetention(utils.TopicName) error + + // SetRetention sets the retention policy for a topic + SetRetention(utils.TopicName, utils.RetentionPolicies) error + + // Get the compaction threshold for a topic + GetCompactionThreshold(topic utils.TopicName, applied bool) (int64, error) + + // Set the compaction threshold for a topic + SetCompactionThreshold(topic utils.TopicName, threshold int64) error + + // Remove compaction threshold for a topic + RemoveCompactionThreshold(utils.TopicName) error + + // GetBacklogQuotaMap returns backlog quota map for a topic + GetBacklogQuotaMap(topic utils.TopicName, applied bool) (map[utils.BacklogQuotaType]utils.BacklogQuota, error) + + // SetBacklogQuota sets a backlog quota for a topic + SetBacklogQuota(utils.TopicName, utils.BacklogQuota, utils.BacklogQuotaType) error + + // RemoveBacklogQuota removes a backlog quota policy from a topic + RemoveBacklogQuota(utils.TopicName, utils.BacklogQuotaType) error + + // GetInactiveTopicPolicies gets the inactive topic policies on a topic + GetInactiveTopicPolicies(topic utils.TopicName, applied bool) (utils.InactiveTopicPolicies, error) + + // RemoveInactiveTopicPolicies removes inactive topic policies from a topic + RemoveInactiveTopicPolicies(utils.TopicName) error + + // SetInactiveTopicPolicies sets the inactive topic policies on a topic + SetInactiveTopicPolicies(topic utils.TopicName, data utils.InactiveTopicPolicies) error + + // GetReplicationClusters get the replication clusters of a topic + GetReplicationClusters(topic utils.TopicName) ([]string, error) + + // SetReplicationClusters sets the replication clusters on a topic + SetReplicationClusters(topic utils.TopicName, data []string) error +} + +type topics struct { + pulsar *pulsarClient + basePath string + persistentPath string + nonPersistentPath string + lookupPath string +} + +// Check whether the topics struct implements the Topics interface. +var _ Topics = &topics{} + +// Topics is used to access the topics endpoints +func (c *pulsarClient) Topics() Topics { + return &topics{ + pulsar: c, + basePath: "", + persistentPath: "/persistent", + nonPersistentPath: "/non-persistent", + lookupPath: "/lookup/v2/topic", + } +} + +func (t *topics) Create(topic utils.TopicName, partitions int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitions") + data := &partitions + if partitions == 0 { + endpoint = t.pulsar.endpoint(t.basePath, topic.GetRestPath()) + data = nil + } + + return t.pulsar.Client.Put(endpoint, data) +} + +func (t *topics) Delete(topic utils.TopicName, force bool, nonPartitioned bool) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitions") + if nonPartitioned { + endpoint = t.pulsar.endpoint(t.basePath, topic.GetRestPath()) + } + params := map[string]string{ + "force": strconv.FormatBool(force), + } + return t.pulsar.Client.DeleteWithQueryParams(endpoint, params) +} + +func (t *topics) Update(topic utils.TopicName, partitions int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitions") + return t.pulsar.Client.Post(endpoint, partitions) +} + +func (t *topics) GetMetadata(topic utils.TopicName) (utils.PartitionedTopicMetadata, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitions") + var partitionedMeta utils.PartitionedTopicMetadata + err := t.pulsar.Client.Get(endpoint, &partitionedMeta) + return partitionedMeta, err +} + +func (t *topics) List(namespace utils.NameSpaceName) ([]string, []string, error) { + var partitionedTopics, nonPartitionedTopics []string + partitionedTopicsChan := make(chan []string) + nonPartitionedTopicsChan := make(chan []string) + errChan := make(chan error) + + pp := t.pulsar.endpoint(t.persistentPath, namespace.String(), "partitioned") + np := t.pulsar.endpoint(t.nonPersistentPath, namespace.String(), "partitioned") + p := t.pulsar.endpoint(t.persistentPath, namespace.String()) + n := t.pulsar.endpoint(t.nonPersistentPath, namespace.String()) + + go t.getTopics(pp, partitionedTopicsChan, errChan) + go t.getTopics(np, partitionedTopicsChan, errChan) + go t.getTopics(p, nonPartitionedTopicsChan, errChan) + go t.getTopics(n, nonPartitionedTopicsChan, errChan) + + requestCount := 4 + for { + select { + case err := <-errChan: + if err != nil { + return nil, nil, err + } + continue + case pTopic := <-partitionedTopicsChan: + requestCount-- + partitionedTopics = append(partitionedTopics, pTopic...) + case npTopic := <-nonPartitionedTopicsChan: + requestCount-- + nonPartitionedTopics = append(nonPartitionedTopics, npTopic...) + } + if requestCount == 0 { + break + } + } + return partitionedTopics, nonPartitionedTopics, nil +} + +func (t *topics) getTopics(endpoint string, out chan<- []string, err chan<- error) { + var topics []string + err <- t.pulsar.Client.Get(endpoint, &topics) + out <- topics +} + +func (t *topics) GetInternalInfo(topic utils.TopicName) (utils.ManagedLedgerInfo, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "internal-info") + var info utils.ManagedLedgerInfo + err := t.pulsar.Client.Get(endpoint, &info) + return info, err +} + +func (t *topics) GetPermissions(topic utils.TopicName) (map[string][]utils.AuthAction, error) { + var permissions map[string][]utils.AuthAction + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "permissions") + err := t.pulsar.Client.Get(endpoint, &permissions) + return permissions, err +} + +func (t *topics) GrantPermission(topic utils.TopicName, role string, action []utils.AuthAction) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "permissions", role) + s := []string{} + for _, v := range action { + s = append(s, v.String()) + } + return t.pulsar.Client.Post(endpoint, s) +} + +func (t *topics) RevokePermission(topic utils.TopicName, role string) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "permissions", role) + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) Lookup(topic utils.TopicName) (utils.LookupData, error) { + var lookup utils.LookupData + endpoint := fmt.Sprintf("%s/%s", t.lookupPath, topic.GetRestPath()) + err := t.pulsar.Client.Get(endpoint, &lookup) + return lookup, err +} + +func (t *topics) GetBundleRange(topic utils.TopicName) (string, error) { + endpoint := fmt.Sprintf("%s/%s/%s", t.lookupPath, topic.GetRestPath(), "bundle") + data, err := t.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) + return string(data), err +} + +func (t *topics) GetLastMessageID(topic utils.TopicName) (utils.MessageID, error) { + var messageID utils.MessageID + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "lastMessageId") + err := t.pulsar.Client.Get(endpoint, &messageID) + return messageID, err +} + +func (t *topics) GetMessageID(topic utils.TopicName, timestamp int64) (utils.MessageID, error) { + var messageID utils.MessageID + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "messageid", strconv.FormatInt(timestamp, 10)) + err := t.pulsar.Client.Get(endpoint, &messageID) + return messageID, err +} + +func (t *topics) GetStats(topic utils.TopicName) (utils.TopicStats, error) { + var stats utils.TopicStats + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "stats") + err := t.pulsar.Client.Get(endpoint, &stats) + return stats, err +} + +func (t *topics) GetInternalStats(topic utils.TopicName) (utils.PersistentTopicInternalStats, error) { + var stats utils.PersistentTopicInternalStats + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "internalStats") + err := t.pulsar.Client.Get(endpoint, &stats) + return stats, err +} + +func (t *topics) GetPartitionedStats(topic utils.TopicName, perPartition bool) (utils.PartitionedTopicStats, error) { + var stats utils.PartitionedTopicStats + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "partitioned-stats") + params := map[string]string{ + "perPartition": strconv.FormatBool(perPartition), + } + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &stats, params, true) + return stats, err +} + +func (t *topics) Terminate(topic utils.TopicName) (utils.MessageID, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "terminate") + var messageID utils.MessageID + err := t.pulsar.Client.PostWithObj(endpoint, nil, &messageID) + return messageID, err +} + +func (t *topics) Offload(topic utils.TopicName, messageID utils.MessageID) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "offload") + return t.pulsar.Client.Put(endpoint, messageID) +} + +func (t *topics) OffloadStatus(topic utils.TopicName) (utils.OffloadProcessStatus, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "offload") + var status utils.OffloadProcessStatus + err := t.pulsar.Client.Get(endpoint, &status) + return status, err +} + +func (t *topics) Unload(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "unload") + return t.pulsar.Client.Put(endpoint, nil) +} + +func (t *topics) Compact(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compaction") + return t.pulsar.Client.Put(endpoint, nil) +} + +func (t *topics) CompactStatus(topic utils.TopicName) (utils.LongRunningProcessStatus, error) { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compaction") + var status utils.LongRunningProcessStatus + err := t.pulsar.Client.Get(endpoint, &status) + return status, err +} + +func (t *topics) GetMessageTTL(topic utils.TopicName) (int, error) { + var ttl int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "messageTTL") + err := t.pulsar.Client.Get(endpoint, &ttl) + return ttl, err +} + +func (t *topics) SetMessageTTL(topic utils.TopicName, messageTTL int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "messageTTL") + var params = make(map[string]string) + params["messageTTL"] = strconv.Itoa(messageTTL) + err := t.pulsar.Client.PostWithQueryParams(endpoint, nil, params) + return err +} + +func (t *topics) RemoveMessageTTL(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "messageTTL") + var params = make(map[string]string) + params["messageTTL"] = strconv.Itoa(0) + err := t.pulsar.Client.DeleteWithQueryParams(endpoint, params) + return err +} + +func (t *topics) GetMaxProducers(topic utils.TopicName) (int, error) { + var maxProducers int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxProducers") + err := t.pulsar.Client.Get(endpoint, &maxProducers) + return maxProducers, err +} + +func (t *topics) SetMaxProducers(topic utils.TopicName, maxProducers int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxProducers") + err := t.pulsar.Client.Post(endpoint, &maxProducers) + return err +} + +func (t *topics) RemoveMaxProducers(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxProducers") + err := t.pulsar.Client.Delete(endpoint) + return err +} + +func (t *topics) GetMaxConsumers(topic utils.TopicName) (int, error) { + var maxConsumers int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxConsumers") + err := t.pulsar.Client.Get(endpoint, &maxConsumers) + return maxConsumers, err +} + +func (t *topics) SetMaxConsumers(topic utils.TopicName, maxConsumers int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxConsumers") + err := t.pulsar.Client.Post(endpoint, &maxConsumers) + return err +} + +func (t *topics) RemoveMaxConsumers(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxConsumers") + err := t.pulsar.Client.Delete(endpoint) + return err +} + +func (t *topics) GetMaxUnackMessagesPerConsumer(topic utils.TopicName) (int, error) { + var maxNum int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnConsumer") + err := t.pulsar.Client.Get(endpoint, &maxNum) + return maxNum, err +} + +func (t *topics) SetMaxUnackMessagesPerConsumer(topic utils.TopicName, maxUnackedNum int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnConsumer") + return t.pulsar.Client.Post(endpoint, &maxUnackedNum) +} + +func (t *topics) RemoveMaxUnackMessagesPerConsumer(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnConsumer") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetMaxUnackMessagesPerSubscription(topic utils.TopicName) (int, error) { + var maxNum int + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnSubscription") + err := t.pulsar.Client.Get(endpoint, &maxNum) + return maxNum, err +} + +func (t *topics) SetMaxUnackMessagesPerSubscription(topic utils.TopicName, maxUnackedNum int) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnSubscription") + return t.pulsar.Client.Post(endpoint, &maxUnackedNum) +} + +func (t *topics) RemoveMaxUnackMessagesPerSubscription(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "maxUnackedMessagesOnSubscription") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetPersistence(topic utils.TopicName) (*utils.PersistenceData, error) { + var persistenceData utils.PersistenceData + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "persistence") + err := t.pulsar.Client.Get(endpoint, &persistenceData) + return &persistenceData, err +} + +func (t *topics) SetPersistence(topic utils.TopicName, persistenceData utils.PersistenceData) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "persistence") + return t.pulsar.Client.Post(endpoint, &persistenceData) +} + +func (t *topics) RemovePersistence(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "persistence") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetDelayedDelivery(topic utils.TopicName) (*utils.DelayedDeliveryData, error) { + var delayedDeliveryData utils.DelayedDeliveryData + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "delayedDelivery") + err := t.pulsar.Client.Get(endpoint, &delayedDeliveryData) + return &delayedDeliveryData, err +} + +func (t *topics) SetDelayedDelivery(topic utils.TopicName, delayedDeliveryData utils.DelayedDeliveryData) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "delayedDelivery") + return t.pulsar.Client.Post(endpoint, &delayedDeliveryData) +} + +func (t *topics) RemoveDelayedDelivery(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "delayedDelivery") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetDispatchRate(topic utils.TopicName) (*utils.DispatchRateData, error) { + var dispatchRateData utils.DispatchRateData + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "dispatchRate") + err := t.pulsar.Client.Get(endpoint, &dispatchRateData) + return &dispatchRateData, err +} + +func (t *topics) SetDispatchRate(topic utils.TopicName, dispatchRateData utils.DispatchRateData) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "dispatchRate") + return t.pulsar.Client.Post(endpoint, &dispatchRateData) +} + +func (t *topics) RemoveDispatchRate(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "dispatchRate") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetPublishRate(topic utils.TopicName) (*utils.PublishRateData, error) { + var publishRateData utils.PublishRateData + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "publishRate") + err := t.pulsar.Client.Get(endpoint, &publishRateData) + return &publishRateData, err +} + +func (t *topics) SetPublishRate(topic utils.TopicName, publishRateData utils.PublishRateData) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "publishRate") + return t.pulsar.Client.Post(endpoint, &publishRateData) +} + +func (t *topics) RemovePublishRate(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "publishRate") + return t.pulsar.Client.Delete(endpoint) +} +func (t *topics) GetDeduplicationStatus(topic utils.TopicName) (bool, error) { + var enabled bool + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "deduplicationEnabled") + err := t.pulsar.Client.Get(endpoint, &enabled) + return enabled, err +} + +func (t *topics) SetDeduplicationStatus(topic utils.TopicName, enabled bool) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "deduplicationEnabled") + return t.pulsar.Client.Post(endpoint, enabled) +} +func (t *topics) RemoveDeduplicationStatus(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "deduplicationEnabled") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) GetRetention(topic utils.TopicName, applied bool) (*utils.RetentionPolicies, error) { + var policy utils.RetentionPolicies + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "retention") + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &policy, map[string]string{ + "applied": strconv.FormatBool(applied), + }, true) + return &policy, err +} + +func (t *topics) RemoveRetention(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "retention") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) SetRetention(topic utils.TopicName, data utils.RetentionPolicies) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "retention") + return t.pulsar.Client.Post(endpoint, data) +} + +func (t *topics) GetCompactionThreshold(topic utils.TopicName, applied bool) (int64, error) { + var threshold int64 + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compactionThreshold") + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &threshold, map[string]string{ + "applied": strconv.FormatBool(applied), + }, true) + return threshold, err +} + +func (t *topics) SetCompactionThreshold(topic utils.TopicName, threshold int64) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compactionThreshold") + err := t.pulsar.Client.Post(endpoint, threshold) + return err +} + +func (t *topics) RemoveCompactionThreshold(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "compactionThreshold") + err := t.pulsar.Client.Delete(endpoint) + return err +} + +func (t *topics) GetBacklogQuotaMap(topic utils.TopicName, applied bool) (map[utils.BacklogQuotaType]utils.BacklogQuota, + error) { + var backlogQuotaMap map[utils.BacklogQuotaType]utils.BacklogQuota + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "backlogQuotaMap") + + queryParams := map[string]string{"applied": strconv.FormatBool(applied)} + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &backlogQuotaMap, queryParams, true) + + return backlogQuotaMap, err +} + +func (t *topics) SetBacklogQuota(topic utils.TopicName, backlogQuota utils.BacklogQuota, + backlogQuotaType utils.BacklogQuotaType) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "backlogQuota") + params := make(map[string]string) + params["backlogQuotaType"] = string(backlogQuotaType) + return t.pulsar.Client.PostWithQueryParams(endpoint, &backlogQuota, params) +} + +func (t *topics) RemoveBacklogQuota(topic utils.TopicName, backlogQuotaType utils.BacklogQuotaType) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "backlogQuota") + return t.pulsar.Client.DeleteWithQueryParams(endpoint, map[string]string{ + "backlogQuotaType": string(backlogQuotaType), + }) +} + +func (t *topics) GetInactiveTopicPolicies(topic utils.TopicName, applied bool) (utils.InactiveTopicPolicies, error) { + var out utils.InactiveTopicPolicies + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "inactiveTopicPolicies") + _, err := t.pulsar.Client.GetWithQueryParams(endpoint, &out, map[string]string{ + "applied": strconv.FormatBool(applied), + }, true) + return out, err +} + +func (t *topics) RemoveInactiveTopicPolicies(topic utils.TopicName) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "inactiveTopicPolicies") + return t.pulsar.Client.Delete(endpoint) +} + +func (t *topics) SetInactiveTopicPolicies(topic utils.TopicName, data utils.InactiveTopicPolicies) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "inactiveTopicPolicies") + return t.pulsar.Client.Post(endpoint, data) +} + +func (t *topics) SetReplicationClusters(topic utils.TopicName, data []string) error { + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "replication") + return t.pulsar.Client.Post(endpoint, data) +} + +func (t *topics) GetReplicationClusters(topic utils.TopicName) ([]string, error) { + var data []string + endpoint := t.pulsar.endpoint(t.basePath, topic.GetRestPath(), "replication") + err := t.pulsar.Client.Get(endpoint, &data) + return data, err +} diff --git a/pulsaradmin/pkg/rest/client.go b/pulsaradmin/pkg/rest/client.go new file mode 100644 index 000000000..a1a79737c --- /dev/null +++ b/pulsaradmin/pkg/rest/client.go @@ -0,0 +1,414 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rest + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/url" + "path" +) + +// Client is a base client that is used to make http request to the ServiceURL +type Client struct { + ServiceURL string + HTTPClient *http.Client + VersionInfo string +} + +func (c *Client) newRequest(method, path string) (*request, error) { + base, err := url.Parse(c.ServiceURL) + if err != nil { + return nil, err + } + u, err := url.Parse(path) + if err != nil { + return nil, err + } + + req := &request{ + method: method, + url: &url.URL{ + Scheme: base.Scheme, + User: base.User, + Host: base.Host, + Path: endpoint(base.Path, u.Path), + }, + params: make(url.Values), + } + return req, nil +} + +func (c *Client) doRequest(r *request) (*http.Response, error) { + req, err := r.toHTTP() + if err != nil { + return nil, err + } + + if r.contentType != "" { + req.Header.Set("Content-Type", r.contentType) + } else if req.Body != nil { + req.Header.Set("Content-Type", "application/json") + } + + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", c.useragent()) + hc := c.HTTPClient + if hc == nil { + hc = http.DefaultClient + } + + return hc.Do(req) +} + +// MakeRequest can make a simple request and handle the response by yourself +func (c *Client) MakeRequest(method, endpoint string) (*http.Response, error) { + req, err := c.newRequest(method, endpoint) + if err != nil { + return nil, err + } + + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *Client) Get(endpoint string, obj interface{}) error { + _, err := c.GetWithQueryParams(endpoint, obj, nil, true) + return err +} + +func (c *Client) GetWithQueryParams(endpoint string, obj interface{}, params map[string]string, + decode bool) ([]byte, error) { + return c.GetWithOptions(endpoint, obj, params, decode, nil) +} + +func (c *Client) GetWithOptions(endpoint string, obj interface{}, params map[string]string, + decode bool, file io.Writer) ([]byte, error) { + + req, err := c.newRequest(http.MethodGet, endpoint) + if err != nil { + return nil, err + } + + if params != nil { + query := req.url.Query() + for k, v := range params { + query.Add(k, v) + } + req.params = query + } + + //nolint:bodyclose + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return nil, err + } + defer safeRespClose(resp) + + if obj != nil { + if err := decodeJSONBody(resp, &obj); err != nil { + if err == io.EOF { + return nil, nil + } + return nil, err + } + } else if !decode { + if file != nil { + _, err := io.Copy(file, resp.Body) + if err != nil { + return nil, err + } + } else { + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return body, err + } + } + + return nil, err +} + +func (c *Client) useragent() string { + return c.VersionInfo +} + +func (c *Client) Put(endpoint string, in interface{}) error { + return c.PutWithQueryParams(endpoint, in, nil, nil) +} + +func (c *Client) PutWithQueryParams(endpoint string, in, obj interface{}, params map[string]string) error { + req, err := c.newRequest(http.MethodPut, endpoint) + if err != nil { + return err + } + req.obj = in + + if params != nil { + query := req.url.Query() + for k, v := range params { + query.Add(k, v) + } + req.params = query + } + + //nolint:bodyclose + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + if obj != nil { + if err := decodeJSONBody(resp, &obj); err != nil { + return err + } + } + + return nil +} + +func (c *Client) PutWithMultiPart(endpoint string, body io.Reader, contentType string) error { + req, err := c.newRequest(http.MethodPut, endpoint) + if err != nil { + return err + } + req.body = body + req.contentType = contentType + + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + return nil +} + +func (c *Client) Delete(endpoint string) error { + return c.DeleteWithQueryParams(endpoint, nil) +} + +func (c *Client) DeleteWithQueryParams(endpoint string, params map[string]string) error { + req, err := c.newRequest(http.MethodDelete, endpoint) + if err != nil { + return err + } + + if params != nil { + query := req.url.Query() + for k, v := range params { + query.Add(k, v) + } + req.params = query + } + + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + return nil +} + +func (c *Client) Post(endpoint string, in interface{}) error { + return c.PostWithObj(endpoint, in, nil) +} + +func (c *Client) PostWithObj(endpoint string, in, obj interface{}) error { + req, err := c.newRequest(http.MethodPost, endpoint) + if err != nil { + return err + } + req.obj = in + + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + if obj != nil { + if err := decodeJSONBody(resp, &obj); err != nil { + return err + } + } + + return nil +} + +func (c *Client) PostWithMultiPart(endpoint string, in interface{}, body io.Reader, contentType string) error { + req, err := c.newRequest(http.MethodPost, endpoint) + if err != nil { + return err + } + req.obj = in + req.body = body + req.contentType = contentType + + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + return nil +} + +func (c *Client) PostWithQueryParams(endpoint string, in interface{}, params map[string]string) error { + req, err := c.newRequest(http.MethodPost, endpoint) + if err != nil { + return err + } + if in != nil { + req.obj = in + } + if params != nil { + query := req.url.Query() + for k, v := range params { + query.Add(k, v) + } + req.params = query + } + //nolint + resp, err := checkSuccessful(c.doRequest(req)) + if err != nil { + return err + } + defer safeRespClose(resp) + + return nil +} + +type request struct { + method string + contentType string + url *url.URL + params url.Values + + obj interface{} + body io.Reader +} + +func (r *request) toHTTP() (*http.Request, error) { + r.url.RawQuery = r.params.Encode() + + // add a request body if there is one + if r.body == nil && r.obj != nil { + body, err := encodeJSONBody(r.obj) + if err != nil { + return nil, err + } + r.body = body + } + + req, err := http.NewRequest(r.method, r.url.RequestURI(), r.body) + if err != nil { + return nil, err + } + + req.URL.Host = r.url.Host + req.URL.Scheme = r.url.Scheme + req.Host = r.url.Host + return req, nil +} + +// respIsOk is used to validate a successful http status code +func respIsOk(resp *http.Response) bool { + return resp.StatusCode >= http.StatusOK && resp.StatusCode <= http.StatusNoContent +} + +// checkSuccessful checks for a valid response and parses an error +func checkSuccessful(resp *http.Response, err error) (*http.Response, error) { + if err != nil { + safeRespClose(resp) + return nil, err + } + + if !respIsOk(resp) { + defer safeRespClose(resp) + return nil, responseError(resp) + } + + return resp, nil +} + +func endpoint(parts ...string) string { + return path.Join(parts...) +} + +// encodeJSONBody is used to JSON encode a body +func encodeJSONBody(obj interface{}) (io.Reader, error) { + b, err := json.Marshal(obj) + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil +} + +// decodeJSONBody is used to JSON decode a body +func decodeJSONBody(resp *http.Response, out interface{}) error { + if resp.ContentLength == 0 { + return nil + } + dec := json.NewDecoder(resp.Body) + return dec.Decode(out) +} + +// safeRespClose is used to close a response body +func safeRespClose(resp *http.Response) { + if resp != nil { + // ignore error since it is closing a response body + _ = resp.Body.Close() + } +} + +// responseError is used to parse a response into a client error +func responseError(resp *http.Response) error { + e := Error{ + Code: resp.StatusCode, + Reason: resp.Status, + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + e.Reason = err.Error() + return e + } + + err = json.Unmarshal(body, &e) + if err != nil { + if len(body) != 0 { + e.Reason = string(body) + } + return e + } + + return e +} diff --git a/pulsaradmin/pkg/rest/client_test.go b/pulsaradmin/pkg/rest/client_test.go new file mode 100644 index 000000000..50a7c5a84 --- /dev/null +++ b/pulsaradmin/pkg/rest/client_test.go @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rest + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEncodeJSONBody(t *testing.T) { + testcases := []struct { + obj interface{} + expected int + }{ + {obj: "1", expected: 3}, + {obj: "12", expected: 4}, + {obj: 1, expected: 1}, + {obj: 12, expected: 2}, + } + + for _, testcase := range testcases { + r, err := encodeJSONBody(testcase.obj) + require.NoError(t, err) + + b, err := io.ReadAll(r) + require.NoError(t, err) + + require.Equal(t, testcase.expected, len(b)) + } +} diff --git a/pulsaradmin/pkg/rest/errors.go b/pulsaradmin/pkg/rest/errors.go new file mode 100644 index 000000000..dc611ec38 --- /dev/null +++ b/pulsaradmin/pkg/rest/errors.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package rest + +import "fmt" + +// Error is a admin error type +type Error struct { + Reason string `json:"reason"` + Code int +} + +func (e Error) Error() string { + return fmt.Sprintf("code: %d reason: %s", e.Code, e.Reason) +} + +func IsAdminError(err error) bool { + _, ok := err.(Error) + return ok +} diff --git a/pulsaradmin/pkg/utils/allocator_stats.go b/pulsaradmin/pkg/utils/allocator_stats.go new file mode 100644 index 000000000..4c2a7fb52 --- /dev/null +++ b/pulsaradmin/pkg/utils/allocator_stats.go @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type AllocatorStats struct { + NumDirectArenas int `json:"numDirectArenas"` + NumHeapArenas int `json:"numHeapArenas"` + NumThreadLocalCaches int `json:"numThreadLocalCaches"` + NormalCacheSize int `json:"normalCacheSize"` + SmallCacheSize int `json:"smallCacheSize"` + TinyCacheSize int `json:"tinyCacheSize"` + DirectArenas []PoolArenaStats `json:"directArenas"` + HeapArenas []PoolArenaStats `json:"heapArenas"` +} + +type PoolArenaStats struct { + NumTinySubpages int `json:"numTinySubpages"` + NumSmallSubpages int `json:"numSmallSubpages"` + NumChunkLists int `json:"numChunkLists"` + TinySubpages []PoolSubpageStats `json:"tinySubpages"` + SmallSubpages []PoolSubpageStats `json:"smallSubpages"` + ChunkLists []PoolChunkListStats `json:"chunkLists"` + NumAllocations int64 `json:"numAllocations"` + NumTinyAllocations int64 `json:"numTinyAllocations"` + NumSmallAllocations int64 `json:"numSmallAllocations"` + NumNormalAllocations int64 `json:"numNormalAllocations"` + NumHugeAllocations int64 `json:"numHugeAllocations"` + NumDeallocations int64 `json:"numDeallocations"` + NumTinyDeallocations int64 `json:"numTinyDeallocations"` + NumSmallDeallocations int64 `json:"numSmallDeallocations"` + NumNormalDeallocations int64 `json:"numNormalDeallocations"` + NumHugeDeallocations int64 `json:"numHugeDeallocations"` + NumActiveAllocations int64 `json:"numActiveAllocations"` + NumActiveTinyAllocations int64 `json:"numActiveTinyAllocations"` + NumActiveSmallAllocations int64 `json:"numActiveSmallAllocations"` + NumActiveNormalAllocations int64 `json:"numActiveNormalAllocations"` + NumActiveHugeAllocations int64 `json:"numActiveHugeAllocations"` +} + +type PoolSubpageStats struct { + MaxNumElements int `json:"maxNumElements"` + NumAvailable int `json:"numAvailable"` + ElementSize int `json:"elementSize"` + PageSize int `json:"pageSize"` +} + +type PoolChunkListStats struct { + MinUsage int `json:"minUsage"` + MaxUsage int `json:"maxUsage"` + Chunks []PoolChunkStats `json:"chunks"` +} + +type PoolChunkStats struct { + Usage int `json:"usage"` + ChunkSize int `json:"chunkSize"` + FreeBytes int `json:"freeBytes"` +} diff --git a/pulsaradmin/pkg/utils/auth_action.go b/pulsaradmin/pkg/utils/auth_action.go new file mode 100644 index 000000000..7f2bf2573 --- /dev/null +++ b/pulsaradmin/pkg/utils/auth_action.go @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type AuthAction string + +const ( + produce AuthAction = "produce" + consume AuthAction = "consume" + functionsAuth AuthAction = "functions" + packages AuthAction = "packages" + sinks AuthAction = "sinks" + sources AuthAction = "sources" +) + +func ParseAuthAction(action string) (AuthAction, error) { + switch action { + case "produce": + return produce, nil + case "consume": + return consume, nil + case "functions": + return functionsAuth, nil + case "packages": + return packages, nil + case "sinks": + return sinks, nil + case "sources": + return sources, nil + default: + return "", errors.Errorf("The auth action only can be specified as 'produce', "+ + "'consume', 'sources', 'sinks', 'packages', or 'functions'. Invalid auth action '%s'", action) + } +} + +func (a AuthAction) String() string { + return string(a) +} diff --git a/pulsaradmin/pkg/utils/auth_polices.go b/pulsaradmin/pkg/utils/auth_polices.go new file mode 100644 index 000000000..065b3c434 --- /dev/null +++ b/pulsaradmin/pkg/utils/auth_polices.go @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type AuthPolicies struct { + NamespaceAuth map[string][]AuthAction `json:"namespace_auth"` + DestinationAuth map[string]map[string][]AuthAction `json:"destination_auth"` + SubscriptionAuthRoles map[string][]string `json:"subscription_auth_roles"` +} + +func NewAuthPolicies() *AuthPolicies { + return &AuthPolicies{ + NamespaceAuth: make(map[string][]AuthAction), + DestinationAuth: make(map[string]map[string][]AuthAction), + SubscriptionAuthRoles: make(map[string][]string), + } +} diff --git a/pulsaradmin/pkg/utils/auth_polices_test.go b/pulsaradmin/pkg/utils/auth_polices_test.go new file mode 100644 index 000000000..014f594bd --- /dev/null +++ b/pulsaradmin/pkg/utils/auth_polices_test.go @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAuthPolicies(t *testing.T) { + testData := "{\n" + + " \"namespace_auth\": {\n" + + " \"persistent://public/default/ns_auth\": [\n" + + " \"produce\",\n" + + " \"consume\",\n" + + " \"function\"\n" + + " ]\n" + + " },\n" + + " \"destination_auth\": {\n" + + " \"persistent://public/default/dest_auth\": {\n" + + " \"admin-role\": [\n" + + " \"produce\",\n" + + " \"consume\",\n" + + " \"function\"\n" + + " ]\n" + + " },\n" + + " \"persistent://public/default/dest_auth_1\": {\n" + + " \"grant-partitioned-role\": [\n" + + " \"produce\",\n" + + " \"consume\"\n" + + " ]\n" + + " },\n" + + " \"persistent://public/default/test-revoke-partitioned-topic\": {},\n" + + " \"persistent://public/default/test-revoke-non-partitioned-topic\": {}\n },\n" + + " \"subscription_auth_roles\": {}\n" + + "}" + + policies := &AuthPolicies{} + err := json.Unmarshal([]byte(testData), policies) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, 3, len(policies.NamespaceAuth["persistent://public/default/ns_auth"])) + assert.Equal(t, 4, len(policies.DestinationAuth)) + assert.Equal(t, 3, len(policies.DestinationAuth["persistent://public/default/dest_auth"]["admin-role"])) + assert.Equal(t, 0, len(policies.SubscriptionAuthRoles)) +} diff --git a/pulsaradmin/pkg/utils/backlog_quota.go b/pulsaradmin/pkg/utils/backlog_quota.go new file mode 100644 index 000000000..1930cad8c --- /dev/null +++ b/pulsaradmin/pkg/utils/backlog_quota.go @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "github.com/pkg/errors" + +type BacklogQuota struct { + LimitTime int64 `json:"limitTime"` + LimitSize int64 `json:"limitSize"` + Policy RetentionPolicy `json:"policy"` +} + +func NewBacklogQuota(limitSize int64, limitTime int64, policy RetentionPolicy) BacklogQuota { + return BacklogQuota{ + LimitSize: limitSize, + LimitTime: limitTime, + Policy: policy, + } +} + +type RetentionPolicy string + +const ( + ProducerRequestHold RetentionPolicy = "producer_request_hold" + ProducerException RetentionPolicy = "producer_exception" + ConsumerBacklogEviction RetentionPolicy = "consumer_backlog_eviction" +) + +func ParseRetentionPolicy(str string) (RetentionPolicy, error) { + switch str { + case ProducerRequestHold.String(): + return ProducerRequestHold, nil + case ProducerException.String(): + return ProducerException, nil + case ConsumerBacklogEviction.String(): + return ConsumerBacklogEviction, nil + default: + return "", errors.Errorf("Invalid retention policy %s", str) + } +} + +func (s RetentionPolicy) String() string { + return string(s) +} + +type BacklogQuotaType string + +const ( + DestinationStorage BacklogQuotaType = "destination_storage" + MessageAge BacklogQuotaType = "message_age" +) + +func ParseBacklogQuotaType(str string) (BacklogQuotaType, error) { + switch str { + case "": + fallthrough + case DestinationStorage.String(): + return DestinationStorage, nil + case MessageAge.String(): + return MessageAge, nil + default: + return "", errors.Errorf("Invalid backlog quota type: %s", str) + } +} + +func (b BacklogQuotaType) String() string { + return string(b) +} diff --git a/pulsaradmin/pkg/utils/batch_source_config.go b/pulsaradmin/pkg/utils/batch_source_config.go new file mode 100644 index 000000000..3db2db697 --- /dev/null +++ b/pulsaradmin/pkg/utils/batch_source_config.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +const ( + BatchsourceConfigKey string = "__BATCHSOURCECONFIGS__" + BatchsourceClassnameKey string = "__BATCHSOURCECLASSNAME__" +) + +type BatchSourceConfig struct { + DiscoveryTriggererClassName string `json:"discoveryTriggererClassName" yaml:"discoveryTriggererClassName"` + + DiscoveryTriggererConfig map[string]interface{} `json:"discoveryTriggererConfig" yaml:"discoveryTriggererConfig"` +} diff --git a/pulsaradmin/pkg/utils/broker_ns_isolation_data.go b/pulsaradmin/pkg/utils/broker_ns_isolation_data.go new file mode 100644 index 000000000..1381647b6 --- /dev/null +++ b/pulsaradmin/pkg/utils/broker_ns_isolation_data.go @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type BrokerNamespaceIsolationData struct { + BrokerName string `json:"brokerName"` + PolicyName string `json:"policyName"` + IsPrimary bool `json:"isPrimary"` + NamespaceRegex []string `json:"namespaceRegex"` +} diff --git a/pulsaradmin/pkg/utils/bundles_data.go b/pulsaradmin/pkg/utils/bundles_data.go new file mode 100644 index 000000000..2bba7ae49 --- /dev/null +++ b/pulsaradmin/pkg/utils/bundles_data.go @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +type BundlesData struct { + Boundaries []string `json:"boundaries"` + NumBundles int `json:"numBundles"` +} + +func NewBundlesData(boundaries []string) BundlesData { + return BundlesData{ + Boundaries: boundaries, + NumBundles: len(boundaries) - 1, + } +} + +func NewBundlesDataWithNumBundles(numBundles int) *BundlesData { + return &BundlesData{ + Boundaries: nil, + NumBundles: numBundles, + } +} + +func NewDefaultBoundle() *BundlesData { + bundleData := NewBundlesDataWithNumBundles(1) + bundleData.Boundaries = append(bundleData.Boundaries, FirstBoundary, LastBoundary) + return bundleData +} diff --git a/pulsaradmin/pkg/utils/connector_definition.go b/pulsaradmin/pkg/utils/connector_definition.go new file mode 100644 index 000000000..72b12893c --- /dev/null +++ b/pulsaradmin/pkg/utils/connector_definition.go @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +// Basic information about a Pulsar connector +type ConnectorDefinition struct { + // The name of the connector type + Name string `json:"name"` + + // Description to be used for user help + Description string `json:"description"` + + // The class name for the connector source implementation + //
If not defined, it will be assumed this connector cannot act as a data source + SourceClass string `json:"sourceClass"` + + // The class name for the connector sink implementation + //
If not defined, it will be assumed this connector cannot act as a data sink
+ SinkClass string `json:"sinkClass"`
+}
diff --git a/pulsaradmin/pkg/utils/consumer_config.go b/pulsaradmin/pkg/utils/consumer_config.go
new file mode 100644
index 000000000..f609ae48d
--- /dev/null
+++ b/pulsaradmin/pkg/utils/consumer_config.go
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type ConsumerConfig struct {
+ SchemaType string `json:"schemaType,omitempty" yaml:"schemaType"`
+ SerdeClassName string `json:"serdeClassName,omitempty" yaml:"serdeClassName"`
+ RegexPattern bool `json:"regexPattern,omitempty" yaml:"regexPattern"`
+ ReceiverQueueSize int `json:"receiverQueueSize,omitempty" yaml:"receiverQueueSize"`
+ SchemaProperties map[string]string `json:"schemaProperties,omitempty" yaml:"schemaProperties"`
+ ConsumerProperties map[string]string `json:"consumerProperties,omitempty" yaml:"consumerProperties"`
+ CryptoConfig *CryptoConfig `json:"cryptoConfig,omitempty" yaml:"cryptoConfig"`
+ PoolMessages bool `json:"poolMessages,omitempty" yaml:"poolMessages"`
+}
diff --git a/pulsaradmin/pkg/utils/crypto_config.go b/pulsaradmin/pkg/utils/crypto_config.go
new file mode 100644
index 000000000..e411bb800
--- /dev/null
+++ b/pulsaradmin/pkg/utils/crypto_config.go
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type CryptoConfig struct {
+ CryptoKeyReaderClassName string `json:"cryptoKeyReaderClassName" yaml:"cryptoKeyReaderClassName"`
+ CryptoKeyReaderConfig map[string]interface{} `json:"cryptoKeyReaderConfig" yaml:"cryptoKeyReaderConfig"`
+
+ EncryptionKeys []string `json:"encryptionKeys" yaml:"encryptionKeys"`
+ ProducerCryptoFailureAction string `json:"producerCryptoFailureAction" yaml:"producerCryptoFailureAction"`
+ ConsumerCryptoFailureAction string `json:"consumerCryptoFailureAction" yaml:"consumerCryptoFailureAction"`
+}
diff --git a/pulsaradmin/pkg/utils/data.go b/pulsaradmin/pkg/utils/data.go
new file mode 100644
index 000000000..55888aab2
--- /dev/null
+++ b/pulsaradmin/pkg/utils/data.go
@@ -0,0 +1,467 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+// ClusterData information on a cluster
+type ClusterData struct {
+ Name string `json:"-"`
+ ServiceURL string `json:"serviceUrl"`
+ ServiceURLTls string `json:"serviceUrlTls"`
+ BrokerServiceURL string `json:"brokerServiceUrl"`
+ BrokerServiceURLTls string `json:"brokerServiceUrlTls"`
+ PeerClusterNames []string `json:"peerClusterNames"`
+ AuthenticationPlugin string `json:"authenticationPlugin"`
+ AuthenticationParameters string `json:"authenticationParameters"`
+ BrokerClientTrustCertsFilePath string `json:"brokerClientTrustCertsFilePath"`
+ BrokerClientTLSEnabled bool `json:"brokerClientTlsEnabled"`
+}
+
+// FunctionData information for a Pulsar Function
+type FunctionData struct {
+ UpdateAuthData bool `json:"updateAuthData"`
+ RetainOrdering bool `json:"retainOrdering"`
+ Watch bool `json:"watch"`
+ AutoAck bool `json:"autoAck"`
+ Parallelism int `json:"parallelism"`
+ WindowLengthCount int `json:"windowLengthCount"`
+ SlidingIntervalCount int `json:"slidingIntervalCount"`
+ MaxMessageRetries int `json:"maxMessageRetries"`
+
+ TimeoutMs int64 `json:"timeoutMs"`
+ SlidingIntervalDurationMs int64 `json:"slidingIntervalDurationMs"`
+ WindowLengthDurationMs int64 `json:"windowLengthDurationMs"`
+ RAM int64 `json:"ram"`
+ Disk int64 `json:"disk"`
+ CPU float64 `json:"cpu"`
+ SubsName string `json:"subsName"`
+ DeadLetterTopic string `json:"deadLetterTopic"`
+ Key string `json:"key"`
+ State string `json:"state"`
+ TriggerValue string `json:"triggerValue"`
+ TriggerFile string `json:"triggerFile"`
+ Topic string `json:"topic"`
+
+ UserCodeFile string `json:"-"`
+ FQFN string `json:"fqfn"`
+ Tenant string `json:"tenant"`
+ Namespace string `json:"namespace"`
+ FuncName string `json:"functionName"`
+ InstanceID string `json:"instance_id"`
+ ClassName string `json:"className"`
+ FunctionType string `json:"functionType"`
+ CleanupSubscription bool `json:"cleanupSubscription"`
+ Jar string `json:"jarFile"`
+ Py string `json:"pyFile"`
+ Go string `json:"goFile"`
+ Inputs string `json:"inputs"`
+ TopicsPattern string `json:"topicsPattern"`
+ Output string `json:"output"`
+ ProducerConfig string `json:"producerConfig"`
+ LogTopic string `json:"logTopic"`
+ SchemaType string `json:"schemaType"`
+ CustomSerDeInputs string `json:"customSerdeInputString"`
+ CustomSchemaInput string `json:"customSchemaInputString"`
+ CustomSchemaOutput string `json:"customSchemaOutputString"`
+ InputSpecs string `json:"inputSpecs"`
+ InputTypeClassName string `json:"inputTypeClassName"`
+ OutputSerDeClassName string `json:"outputSerdeClassName"`
+ OutputTypeClassName string `json:"outputTypeClassName"`
+ FunctionConfigFile string `json:"fnConfigFile"`
+ ProcessingGuarantees string `json:"processingGuarantees"`
+ UserConfig string `json:"userConfigString"`
+ RetainKeyOrdering bool `json:"retainKeyOrdering"`
+ BatchBuilder string `json:"batchBuilder"`
+ ForwardSourceMessageProperty bool `json:"forwardSourceMessageProperty"`
+ SubsPosition string `json:"subsPosition"`
+ SkipToLatest bool `json:"skipToLatest"`
+ CustomRuntimeOptions string `json:"customRuntimeOptions"`
+ Secrets string `json:"secretsString"`
+ DestinationFile string `json:"destinationFile"`
+ Path string `json:"path"`
+ FuncConf *FunctionConfig `json:"-"`
+}
+
+// Failure Domain information
+type FailureDomainData struct {
+ ClusterName string `json:"-"`
+ DomainName string `json:"-"`
+ BrokerList []string `json:"brokers"`
+}
+
+type FailureDomainMap map[string]FailureDomainData
+
+// Tenant args
+type TenantData struct {
+ Name string `json:"-"`
+ AdminRoles []string `json:"adminRoles"`
+ AllowedClusters []string `json:"allowedClusters"`
+}
+
+type SourceData struct {
+ Tenant string `json:"tenant,omitempty"`
+ Namespace string `json:"namespace,omitempty"`
+ Name string `json:"name,omitempty"`
+ SourceType string `json:"sourceType,omitempty"`
+ ProcessingGuarantees string `json:"processingGuarantees,omitempty"`
+ DestinationTopicName string `json:"destinationTopicName,omitempty"`
+ ProducerConfig string `json:"producerConfig,omitempty"`
+ BatchBuilder string `json:"batchBuilder,omitempty"`
+ DeserializationClassName string `json:"deserializationClassName,omitempty"`
+ SchemaType string `json:"schemaType,omitempty"`
+ Parallelism int `json:"parallelism,omitempty"`
+ Archive string `json:"archive,omitempty"`
+ ClassName string `json:"className,omitempty"`
+ SourceConfigFile string `json:"sourceConfigFile,omitempty"`
+ CPU float64 `json:"cpu,omitempty"`
+ RAM int64 `json:"ram,omitempty"`
+ Disk int64 `json:"disk,omitempty"`
+ SourceConfigString string `json:"sourceConfigString,omitempty"`
+ BatchSourceConfigString string `json:"batchSourceConfigString,omitempty"`
+ CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty"`
+ Secrets string `json:"secretsString,omitempty"`
+
+ SourceConf *SourceConfig `json:"-,omitempty"`
+ InstanceID string `json:"instanceId,omitempty"`
+
+ UpdateAuthData bool `json:"updateAuthData,omitempty"`
+}
+
+type SinkData struct {
+ UpdateAuthData bool `json:"updateAuthData,omitempty"`
+ RetainOrdering bool `json:"retainOrdering,omitempty"`
+ AutoAck bool `json:"autoAck,omitempty"`
+ Parallelism int `json:"parallelism,omitempty"`
+ RAM int64 `json:"ram,omitempty"`
+ Disk int64 `json:"disk,omitempty"`
+ TimeoutMs int64 `json:"timeoutMs,omitempty"`
+ CPU float64 `json:"cpu,omitempty"`
+ Tenant string `json:"tenant,omitempty"`
+ Namespace string `json:"namespace,omitempty"`
+ Name string `json:"name,omitempty"`
+ SinkType string `json:"sinkType,omitempty"`
+ CleanupSubscription bool `json:"cleanupSubscription"`
+ Inputs string `json:"inputs,omitempty"`
+ TopicsPattern string `json:"topicsPattern,omitempty"`
+ SubsName string `json:"subsName,omitempty"`
+ SubsPosition string `json:"subsPosition,omitempty"`
+ CustomSerdeInputString string `json:"customSerdeInputString,omitempty"`
+ CustomSchemaInputString string `json:"customSchemaInputString,omitempty"`
+ InputSpecs string `json:"inputSpecs,omitempty"`
+ MaxMessageRetries int `json:"maxMessageRetries,omitempty"`
+ DeadLetterTopic string `json:"deadLetterTopic,omitempty"`
+ ProcessingGuarantees string `json:"processingGuarantees,omitempty"`
+ RetainKeyOrdering bool `json:"retainKeyOrdering,omitempty"`
+ Archive string `json:"archive,omitempty"`
+ ClassName string `json:"className,omitempty"`
+ SinkConfigFile string `json:"sinkConfigFile,omitempty"`
+ SinkConfigString string `json:"sinkConfigString,omitempty"`
+ NegativeAckRedeliveryDelayMs int64 `json:"negativeAckRedeliveryDelayMs,omitempty"`
+ CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty"`
+ Secrets string `json:"secretsString,omitempty"`
+ InstanceID string `json:"instanceId,omitempty"`
+ TransformFunction string `json:"transformFunction,omitempty"`
+ TransformFunctionClassName string `json:"transformFunctionClassName,omitempty"`
+ TransformFunctionConfig string `json:"transformFunctionConfig,omitempty"`
+ SinkConf *SinkConfig `json:"-,omitempty"`
+}
+
+// Topic data
+type PartitionedTopicMetadata struct {
+ Partitions int `json:"partitions"`
+}
+
+type ManagedLedgerInfoLedgerInfo struct {
+ LedgerID int64 `json:"ledgerId"`
+ Entries int64 `json:"entries"`
+ Size int64 `json:"size"`
+ Timestamp int64 `json:"timestamp"`
+ Offloaded bool `json:"isOffloaded"`
+ OffloadedContextUUID string `json:"offloadedContextUuid"`
+}
+
+type ManagedLedgerInfo struct {
+ Version int `json:"version"`
+ CreationDate string `json:"creationDate"`
+ ModificationData string `json:"modificationData"`
+ Ledgers []ManagedLedgerInfoLedgerInfo `json:"ledgers"`
+ TerminatedPosition PositionInfo `json:"terminatedPosition"`
+ Cursors map[string]CursorInfo `json:"cursors"`
+}
+
+type NamespacesData struct {
+ Enable bool `json:"enable"`
+ Unload bool `json:"unload"`
+ NumBundles int `json:"numBundles"`
+ BookkeeperEnsemble int `json:"bookkeeperEnsemble"`
+ BookkeeperWriteQuorum int `json:"bookkeeperWriteQuorum"`
+ MessageTTL int `json:"messageTTL"`
+ BookkeeperAckQuorum int `json:"bookkeeperAckQuorum"`
+ ManagedLedgerMaxMarkDeleteRate float64 `json:"managedLedgerMaxMarkDeleteRate"`
+ ClusterIds string `json:"clusterIds"`
+ RetentionTimeStr string `json:"retentionTimeStr"`
+ LimitStr string `json:"limitStr"`
+ LimitTime int64 `json:"limitTime"`
+ PolicyStr string `json:"policyStr"`
+ BacklogQuotaType string `json:"backlogQuotaType"`
+ AntiAffinityGroup string `json:"antiAffinityGroup"`
+ Tenant string `json:"tenant"`
+ Cluster string `json:"cluster"`
+ Bundle string `json:"bundle"`
+ Clusters []string `json:"clusters"`
+}
+
+type TopicStats struct {
+ BacklogSize int64 `json:"backlogSize"`
+ MsgCounterIn int64 `json:"msgInCounter"`
+ MsgCounterOut int64 `json:"msgOutCounter"`
+ MsgRateIn float64 `json:"msgRateIn"`
+ MsgRateOut float64 `json:"msgRateOut"`
+ MsgThroughputIn float64 `json:"msgThroughputIn"`
+ MsgThroughputOut float64 `json:"msgThroughputOut"`
+ AverageMsgSize float64 `json:"averageMsgSize"`
+ StorageSize int64 `json:"storageSize"`
+ Publishers []PublisherStats `json:"publishers"`
+ Subscriptions map[string]SubscriptionStats `json:"subscriptions"`
+ Replication map[string]ReplicatorStats `json:"replication"`
+ DeDuplicationStatus string `json:"deduplicationStatus"`
+}
+
+type PublisherStats struct {
+ ProducerID int64 `json:"producerId"`
+ MsgRateIn float64 `json:"msgRateIn"`
+ MsgThroughputIn float64 `json:"msgThroughputIn"`
+ AverageMsgSize float64 `json:"averageMsgSize"`
+ Metadata map[string]string `json:"metadata"`
+}
+
+type SubscriptionStats struct {
+ BlockedSubscriptionOnUnackedMsgs bool `json:"blockedSubscriptionOnUnackedMsgs"`
+ IsReplicated bool `json:"isReplicated"`
+ LastConsumedFlowTimestamp int64 `json:"lastConsumedFlowTimestamp"`
+ LastConsumedTimestamp int64 `json:"lastConsumedTimestamp"`
+ LastAckedTimestamp int64 `json:"lastAckedTimestamp"`
+ MsgRateOut float64 `json:"msgRateOut"`
+ MsgThroughputOut float64 `json:"msgThroughputOut"`
+ MsgRateRedeliver float64 `json:"msgRateRedeliver"`
+ MsgRateExpired float64 `json:"msgRateExpired"`
+ MsgBacklog int64 `json:"msgBacklog"`
+ MsgBacklogNoDelayed int64 `json:"msgBacklogNoDelayed"`
+ MsgDelayed int64 `json:"msgDelayed"`
+ UnAckedMessages int64 `json:"unackedMessages"`
+ SubType string `json:"type"`
+ ActiveConsumerName string `json:"activeConsumerName"`
+ Consumers []ConsumerStats `json:"consumers"`
+}
+
+type ConsumerStats struct {
+ BlockedConsumerOnUnAckedMsgs bool `json:"blockedConsumerOnUnackedMsgs"`
+ AvailablePermits int `json:"availablePermits"`
+ UnAckedMessages int `json:"unackedMessages"`
+ MsgRateOut float64 `json:"msgRateOut"`
+ MsgThroughputOut float64 `json:"msgThroughputOut"`
+ MsgRateRedeliver float64 `json:"msgRateRedeliver"`
+ ConsumerName string `json:"consumerName"`
+ Metadata map[string]string `json:"metadata"`
+}
+
+type ReplicatorStats struct {
+ Connected bool `json:"connected"`
+ MsgRateIn float64 `json:"msgRateIn"`
+ MsgRateOut float64 `json:"msgRateOut"`
+ MsgThroughputIn float64 `json:"msgThroughputIn"`
+ MsgThroughputOut float64 `json:"msgThroughputOut"`
+ MsgRateExpired float64 `json:"msgRateExpired"`
+ ReplicationBacklog int64 `json:"replicationBacklog"`
+ ReplicationDelayInSeconds int64 `json:"replicationDelayInSeconds"`
+ InboundConnection string `json:"inboundConnection"`
+ InboundConnectedSince string `json:"inboundConnectedSince"`
+ OutboundConnection string `json:"outboundConnection"`
+ OutboundConnectedSince string `json:"outboundConnectedSince"`
+}
+
+type PersistentTopicInternalStats struct {
+ WaitingCursorsCount int `json:"waitingCursorsCount"`
+ PendingAddEntriesCount int `json:"pendingAddEntriesCount"`
+ EntriesAddedCounter int64 `json:"entriesAddedCounter"`
+ NumberOfEntries int64 `json:"numberOfEntries"`
+ TotalSize int64 `json:"totalSize"`
+ CurrentLedgerEntries int64 `json:"currentLedgerEntries"`
+ CurrentLedgerSize int64 `json:"currentLedgerSize"`
+ LastLedgerCreatedTimestamp string `json:"lastLedgerCreatedTimestamp"`
+ LastLedgerCreationFailureTimestamp string `json:"lastLedgerCreationFailureTimestamp"`
+ LastConfirmedEntry string `json:"lastConfirmedEntry"`
+ State string `json:"state"`
+ Ledgers []LedgerInfo `json:"ledgers"`
+ Cursors map[string]CursorStats `json:"cursors"`
+ SchemaLedgers []SchemaLedger `json:"schemaLedgers"`
+ CompactedLedger CompactedLedger `json:"compactedLedger"`
+}
+
+type LedgerInfo struct {
+ LedgerID int64 `json:"ledgerId"`
+ Entries int64 `json:"entries"`
+ Size int64 `json:"size"`
+ Timestamp int64 `json:"timestamp"`
+ Offloaded bool `json:"offloaded"`
+ MetaData string `json:"metadata"`
+ UnderReplicated bool `json:"underReplicated"`
+}
+
+type CursorInfo struct {
+ Version int `json:"version"`
+ CreationDate string `json:"creationDate"`
+ ModificationDate string `json:"modificationDate"`
+ CursorsLedgerID int64 `json:"cursorsLedgerId"`
+ MarkDelete PositionInfo `json:"markDelete"`
+ IndividualDeletedMessages []MessageRangeInfo `json:"individualDeletedMessages"`
+ Properties map[string]int64
+}
+
+type PositionInfo struct {
+ LedgerID int64 `json:"ledgerId"`
+ EntryID int64 `json:"entryId"`
+}
+
+type MessageRangeInfo struct {
+ From PositionInfo `json:"from"`
+ To PositionInfo `json:"to"`
+ Offloaded bool `json:"offloaded"`
+}
+
+type CursorStats struct {
+ MarkDeletePosition string `json:"markDeletePosition"`
+ ReadPosition string `json:"readPosition"`
+ WaitingReadOp bool `json:"waitingReadOp"`
+ PendingReadOps int `json:"pendingReadOps"`
+ MessagesConsumedCounter int64 `json:"messagesConsumedCounter"`
+ CursorLedger int64 `json:"cursorLedger"`
+ CursorLedgerLastEntry int64 `json:"cursorLedgerLastEntry"`
+ IndividuallyDeletedMessages string `json:"individuallyDeletedMessages"`
+ LastLedgerWitchTimestamp string `json:"lastLedgerWitchTimestamp"`
+ State string `json:"state"`
+ NumberOfEntriesSinceFirstNotAckedMessage int64 `json:"numberOfEntriesSinceFirstNotAckedMessage"`
+ TotalNonContiguousDeletedMessagesRange int `json:"totalNonContiguousDeletedMessagesRange"`
+ Properties map[string]int64 `json:"properties"`
+}
+
+type PartitionedTopicStats struct {
+ MsgRateIn float64 `json:"msgRateIn"`
+ MsgRateOut float64 `json:"msgRateOut"`
+ MsgThroughputIn float64 `json:"msgThroughputIn"`
+ MsgThroughputOut float64 `json:"msgThroughputOut"`
+ AverageMsgSize float64 `json:"averageMsgSize"`
+ StorageSize int64 `json:"storageSize"`
+ Publishers []PublisherStats `json:"publishers"`
+ Subscriptions map[string]SubscriptionStats `json:"subscriptions"`
+ Replication map[string]ReplicatorStats `json:"replication"`
+ DeDuplicationStatus string `json:"deduplicationStatus"`
+ Metadata PartitionedTopicMetadata `json:"metadata"`
+ Partitions map[string]TopicStats `json:"partitions"`
+}
+
+type SchemaData struct {
+ Version int64 `json:"version"`
+ Filename string `json:"filename"`
+ Jar string `json:"jar"`
+ Type string `json:"type"`
+ Classname string `json:"classname"`
+ AlwaysAllowNull bool `json:"alwaysAllowNull"`
+ DryRun bool `json:"dryRun"`
+}
+
+type LookupData struct {
+ BrokerURL string `json:"brokerUrl"`
+ BrokerURLTLS string `json:"brokerUrlTls"`
+ HTTPURL string `json:"httpUrl"`
+ HTTPURLTLS string `json:"httpUrlTls"`
+}
+
+type NsIsolationPoliciesData struct {
+ Namespaces []string `json:"namespaces"`
+ Primary []string `json:"primary"`
+ Secondary []string `json:"secondary"`
+ AutoFailoverPolicyTypeName string `json:"autoFailoverPolicyTypeName"`
+ AutoFailoverPolicyParams string `json:"autoFailoverPolicyParams"`
+}
+
+type BrokerData struct {
+ URL string `json:"brokerUrl"`
+ ConfigName string `json:"configName"`
+ ConfigValue string `json:"configValue"`
+}
+
+type BrokerStatsData struct {
+ Indent bool `json:"indent"`
+}
+
+type ResourceQuotaData struct {
+ Names string `json:"names"`
+ Bundle string `json:"bundle"`
+ MsgRateIn int64 `json:"msgRateIn"`
+ MsgRateOut int64 `json:"msgRateOut"`
+ BandwidthIn int64 `json:"bandwidthIn"`
+ BandwidthOut int64 `json:"bandwidthOut"`
+ Memory int64 `json:"memory"`
+ Dynamic bool `json:"dynamic"`
+}
+
+type PersistenceData struct {
+ BookkeeperEnsemble int64 `json:"bookkeeperEnsemble"`
+ BookkeeperWriteQuorum int64 `json:"bookkeeperWriteQuorum"`
+ BookkeeperAckQuorum int64 `json:"bookkeeperAckQuorum"`
+ ManagedLedgerMaxMarkDeleteRate float64 `json:"managedLedgerMaxMarkDeleteRate"`
+}
+
+type DelayedDeliveryCmdData struct {
+ Enable bool `json:"enable"`
+ Disable bool `json:"disable"`
+ DelayedDeliveryTimeStr string `json:"delayedDeliveryTimeStr"`
+}
+
+type DelayedDeliveryData struct {
+ TickTime float64 `json:"tickTime"`
+ Active bool `json:"active"`
+}
+
+type DispatchRateData struct {
+ DispatchThrottlingRateInMsg int64 `json:"dispatchThrottlingRateInMsg"`
+ DispatchThrottlingRateInByte int64 `json:"dispatchThrottlingRateInByte"`
+ RatePeriodInSecond int64 `json:"ratePeriodInSecond"`
+ RelativeToPublishRate bool `json:"relativeToPublishRate"`
+}
+
+type PublishRateData struct {
+ PublishThrottlingRateInMsg int64 `json:"publishThrottlingRateInMsg"`
+ PublishThrottlingRateInByte int64 `json:"publishThrottlingRateInByte"`
+}
+
+type SchemaLedger struct {
+ LedgerID int64 `json:"ledgerId"`
+ Entries int64 `json:"entries"`
+ Size int64 `json:"size"`
+ Timestamp int64 `json:"timestamp"`
+ IsOffloaded bool `json:"isOffloaded"`
+}
+
+type CompactedLedger struct {
+ LedgerID int64 `json:"ledgerId"`
+ Entries int64 `json:"entries"`
+ Size int64 `json:"size"`
+ Offloaded bool `json:"offloaded"`
+ UnderReplicated bool `json:"underReplicated"`
+}
diff --git a/pulsaradmin/pkg/utils/dispatch_rate.go b/pulsaradmin/pkg/utils/dispatch_rate.go
new file mode 100644
index 000000000..9c6ccce88
--- /dev/null
+++ b/pulsaradmin/pkg/utils/dispatch_rate.go
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type DispatchRate struct {
+ DispatchThrottlingRateInMsg int `json:"dispatchThrottlingRateInMsg"`
+ DispatchThrottlingRateInByte int64 `json:"dispatchThrottlingRateInByte"`
+ RatePeriodInSecond int `json:"ratePeriodInSecond"`
+}
+
+func NewDispatchRate() *DispatchRate {
+ return &DispatchRate{
+ DispatchThrottlingRateInMsg: -1,
+ DispatchThrottlingRateInByte: -1,
+ RatePeriodInSecond: 1,
+ }
+}
+
+type SubscribeRate struct {
+ SubscribeThrottlingRatePerConsumer int `json:"subscribeThrottlingRatePerConsumer"`
+ RatePeriodInSecond int `json:"ratePeriodInSecond"`
+}
+
+func NewSubscribeRate() *SubscribeRate {
+ return &SubscribeRate{
+ SubscribeThrottlingRatePerConsumer: -1,
+ RatePeriodInSecond: 30,
+ }
+}
diff --git a/pulsaradmin/pkg/utils/function_confg.go b/pulsaradmin/pkg/utils/function_confg.go
new file mode 100644
index 000000000..02a00db3b
--- /dev/null
+++ b/pulsaradmin/pkg/utils/function_confg.go
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+const (
+ JavaRuntime = "JAVA"
+ PythonRuntime = "PYTHON"
+ GoRuntime = "GO"
+)
+
+type FunctionConfig struct {
+ TimeoutMs *int64 `json:"timeoutMs,omitempty" yaml:"timeoutMs"`
+ TopicsPattern *string `json:"topicsPattern,omitempty" yaml:"topicsPattern"`
+ // Whether the subscriptions the functions created/used should be deleted when the functions is deleted
+ CleanupSubscription bool `json:"cleanupSubscription" yaml:"cleanupSubscription"`
+ RetainOrdering bool `json:"retainOrdering" yaml:"retainOrdering"`
+ RetainKeyOrdering bool `json:"retainKeyOrdering" yaml:"retainKeyOrdering"`
+ BatchBuilder string `json:"batchBuilder,omitempty" yaml:"batchBuilder"`
+ ForwardSourceMessageProperty bool `json:"forwardSourceMessageProperty" yaml:"forwardSourceMessageProperty"`
+ AutoAck bool `json:"autoAck" yaml:"autoAck"`
+ Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"`
+ MaxMessageRetries *int `json:"maxMessageRetries,omitempty" yaml:"maxMessageRetries"`
+
+ Output string `json:"output,omitempty" yaml:"output"`
+
+ ProducerConfig *ProducerConfig `json:"producerConfig,omitempty" yaml:"producerConfig"`
+ CustomSchemaOutputs map[string]string `json:"customSchemaOutputs,omitempty" yaml:"customSchemaOutputs"`
+
+ OutputSerdeClassName string `json:"outputSerdeClassName,omitempty" yaml:"outputSerdeClassName"`
+ LogTopic string `json:"logTopic,omitempty" yaml:"logTopic"`
+ ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"`
+
+ // Represents either a builtin schema type (eg: 'avro', 'json', etc) or the class name for a Schema implementation
+ OutputSchemaType string `json:"outputSchemaType,omitempty" yaml:"outputSchemaType"`
+ OutputTypeClassName string `json:"outputTypeClassName,omitempty" yaml:"outputTypeClassName"`
+
+ Runtime string `json:"runtime,omitempty" yaml:"runtime"`
+ DeadLetterTopic string `json:"deadLetterTopic,omitempty" yaml:"deadLetterTopic"`
+ SubName string `json:"subName,omitempty" yaml:"subName"`
+ FQFN string `json:"fqfn,omitempty" yaml:"fqfn"`
+ Jar *string `json:"jar,omitempty" yaml:"jar"`
+ Py *string `json:"py,omitempty" yaml:"py"`
+ Go *string `json:"go,omitempty" yaml:"go"`
+ FunctionType *string `json:"functionType,omitempty" yaml:"functionType"`
+ // Any flags that you want to pass to the runtime.
+ // note that in thread mode, these flags will have no impact
+ RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"`
+
+ Tenant string `json:"tenant,omitempty" yaml:"tenant"`
+ Namespace string `json:"namespace,omitempty" yaml:"namespace"`
+ Name string `json:"name,omitempty" yaml:"name"`
+ ClassName string `json:"className,omitempty" yaml:"className"`
+
+ Resources *Resources `json:"resources,omitempty" yaml:"resources"`
+ WindowConfig *WindowConfig `json:"windowConfig,omitempty" yaml:"windowConfig"`
+ Inputs []string `json:"inputs,omitempty" yaml:"inputs"`
+ UserConfig map[string]interface{} `json:"userConfig,omitempty" yaml:"userConfig"`
+ CustomSerdeInputs map[string]string `json:"customSerdeInputs,omitempty" yaml:"customSerdeInputs"`
+ CustomSchemaInputs map[string]string `json:"customSchemaInputs,omitempty" yaml:"customSchemaInputs"`
+
+ // A generalized way of specifying inputs
+ InputSpecs map[string]ConsumerConfig `json:"inputSpecs,omitempty" yaml:"inputSpecs"`
+ InputTypeClassName string `json:"inputTypeClassName,omitempty" yaml:"inputTypeClassName"`
+
+ CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"`
+
+ // This is a map of secretName(aka how the secret is going to be
+ // accessed in the function via context) to an object that
+ // encapsulates how the secret is fetched by the underlying
+ // secrets provider. The type of an value here can be found by the
+ // SecretProviderConfigurator.getSecretObjectType() method.
+ Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"`
+
+ MaxPendingAsyncRequests int `json:"maxPendingAsyncRequests,omitempty" yaml:"maxPendingAsyncRequests"`
+ //nolint
+ ExposePulsarAdminClientEnabled bool `json:"exposePulsarAdminClientEnabled" yaml:"exposePulsarAdminClientEnabled"`
+ SkipToLatest bool `json:"skipToLatest" yaml:"skipToLatest"`
+ SubscriptionPosition string `json:"subscriptionPosition,omitempty" yaml:"subscriptionPosition"`
+}
diff --git a/pulsaradmin/pkg/utils/function_state.go b/pulsaradmin/pkg/utils/function_state.go
new file mode 100644
index 000000000..63fa15057
--- /dev/null
+++ b/pulsaradmin/pkg/utils/function_state.go
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type FunctionState struct {
+ Key string `json:"key"`
+ StringValue string `json:"stringValue"`
+ ByteValue []byte `json:"byteValue"`
+ NumValue int64 `json:"numberValue"`
+ Version int64 `json:"version"`
+}
diff --git a/pulsaradmin/pkg/utils/function_status.go b/pulsaradmin/pkg/utils/function_status.go
new file mode 100644
index 000000000..360e61b07
--- /dev/null
+++ b/pulsaradmin/pkg/utils/function_status.go
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type FunctionStatus struct {
+ NumInstances int `json:"numInstances"`
+ NumRunning int `json:"numRunning"`
+ Instances []FunctionInstanceStatus `json:"instances"`
+}
+
+type FunctionInstanceStatus struct {
+ InstanceID int `json:"instanceId"`
+ Status FunctionInstanceStatusData `json:"status"`
+}
+
+type FunctionInstanceStatusData struct {
+ Running bool `json:"running"`
+ Err string `json:"error"`
+ NumRestarts int64 `json:"numRestarts"`
+ NumReceived int64 `json:"numReceived"`
+ NumSuccessfullyProcessed int64 `json:"numSuccessfullyProcessed"`
+ NumUserExceptions int64 `json:"numUserExceptions"`
+ LatestUserExceptions []ExceptionInformation `json:"latestUserExceptions"`
+ NumSystemExceptions int64 `json:"numSystemExceptions"`
+ LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"`
+ AverageLatency float64 `json:"averageLatency"`
+ LastInvocationTime int64 `json:"lastInvocationTime"`
+ WorkerID string `json:"workerId"`
+}
+
+type ExceptionInformation struct {
+ ExceptionString string `json:"exceptionString"`
+ TimestampMs int64 `json:"timestampMs"`
+}
diff --git a/pulsaradmin/pkg/utils/functions_stats.go b/pulsaradmin/pkg/utils/functions_stats.go
new file mode 100644
index 000000000..82761afe5
--- /dev/null
+++ b/pulsaradmin/pkg/utils/functions_stats.go
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type FunctionStats struct {
+ // Overall total number of records function received from source
+ ReceivedTotal int64 `json:"receivedTotal"`
+
+ // Overall total number of records successfully processed by user function
+ ProcessedSuccessfullyTotal int64 `json:"processedSuccessfullyTotal"`
+
+ // Overall total number of system exceptions thrown
+ SystemExceptionsTotal int64 `json:"systemExceptionsTotal"`
+
+ // Overall total number of user exceptions thrown
+ UserExceptionsTotal int64 `json:"userExceptionsTotal"`
+
+ // Average process latency for function
+ AvgProcessLatency float64 `json:"avgProcessLatency"`
+
+ // Timestamp of when the function was last invoked by any instance
+ LastInvocation int64 `json:"lastInvocation"`
+
+ OneMin FunctionInstanceStatsDataBase `json:"oneMin"`
+
+ Instances []FunctionInstanceStats `json:"instances"`
+
+ FunctionInstanceStats
+}
+
+type FunctionInstanceStats struct {
+ FunctionInstanceStatsDataBase
+
+ InstanceID int64 `json:"instanceId"`
+
+ Metrics FunctionInstanceStatsData `json:"metrics"`
+}
+
+type FunctionInstanceStatsDataBase struct {
+ // Total number of records function received from source for instance
+ ReceivedTotal int64 `json:"receivedTotal"`
+
+ // Total number of records successfully processed by user function for instance
+ ProcessedSuccessfullyTotal int64 `json:"processedSuccessfullyTotal"`
+
+ // Total number of system exceptions thrown for instance
+ SystemExceptionsTotal int64 `json:"systemExceptionsTotal"`
+
+ // Total number of user exceptions thrown for instance
+ UserExceptionsTotal int64 `json:"userExceptionsTotal"`
+
+ // Average process latency for function for instance
+ AvgProcessLatency float64 `json:"avgProcessLatency"`
+}
+
+type FunctionInstanceStatsData struct {
+ OneMin FunctionInstanceStatsDataBase `json:"oneMin"`
+
+ // Timestamp of when the function was last invoked for instance
+ LastInvocation int64 `json:"lastInvocation"`
+
+ // Map of user defined metrics
+ UserMetrics map[string]float64 `json:"userMetrics"`
+
+ FunctionInstanceStatsDataBase
+}
+
+func (fs *FunctionStats) AddInstance(functionInstanceStats FunctionInstanceStats) {
+ fs.Instances = append(fs.Instances, functionInstanceStats)
+}
+
+func (fs *FunctionStats) CalculateOverall() *FunctionStats {
+ var (
+ nonNullInstances int
+ nonNullInstancesOneMin int
+ )
+
+ for _, functionInstanceStats := range fs.Instances {
+ functionInstanceStatsData := functionInstanceStats.Metrics
+ fs.ReceivedTotal += functionInstanceStatsData.ReceivedTotal
+ fs.ProcessedSuccessfullyTotal += functionInstanceStatsData.ProcessedSuccessfullyTotal
+ fs.SystemExceptionsTotal += functionInstanceStatsData.SystemExceptionsTotal
+ fs.UserExceptionsTotal += functionInstanceStatsData.UserExceptionsTotal
+
+ if functionInstanceStatsData.AvgProcessLatency != 0 {
+ if fs.AvgProcessLatency == 0 {
+ fs.AvgProcessLatency = 0.0
+ }
+
+ fs.AvgProcessLatency += functionInstanceStatsData.AvgProcessLatency
+ nonNullInstances++
+ }
+
+ fs.OneMin.ReceivedTotal += functionInstanceStatsData.OneMin.ReceivedTotal
+ fs.OneMin.ProcessedSuccessfullyTotal += functionInstanceStatsData.OneMin.ProcessedSuccessfullyTotal
+ fs.OneMin.SystemExceptionsTotal += functionInstanceStatsData.OneMin.SystemExceptionsTotal
+ fs.OneMin.UserExceptionsTotal += functionInstanceStatsData.OneMin.UserExceptionsTotal
+
+ if functionInstanceStatsData.OneMin.AvgProcessLatency != 0 {
+ if fs.OneMin.AvgProcessLatency == 0 {
+ fs.OneMin.AvgProcessLatency = 0.0
+ }
+
+ fs.OneMin.AvgProcessLatency += functionInstanceStatsData.OneMin.AvgProcessLatency
+ nonNullInstancesOneMin++
+ }
+
+ if functionInstanceStatsData.LastInvocation != 0 {
+ if fs.LastInvocation == 0 || functionInstanceStatsData.LastInvocation > fs.LastInvocation {
+ fs.LastInvocation = functionInstanceStatsData.LastInvocation
+ }
+ }
+ }
+
+ // calculate average from sum
+ if nonNullInstances > 0 {
+ fs.AvgProcessLatency /= float64(nonNullInstances)
+ } else {
+ fs.AvgProcessLatency = 0
+ }
+
+ // calculate 1min average from sum
+ if nonNullInstancesOneMin > 0 {
+ fs.OneMin.AvgProcessLatency /= float64(nonNullInstancesOneMin)
+ } else {
+ fs.AvgProcessLatency = 0
+ }
+
+ return fs
+}
diff --git a/pulsaradmin/pkg/utils/home_dir.go b/pulsaradmin/pkg/utils/home_dir.go
new file mode 100644
index 000000000..330831c40
--- /dev/null
+++ b/pulsaradmin/pkg/utils/home_dir.go
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "os"
+ "path/filepath"
+ "runtime"
+)
+
+// HomeDir returns the home directory for the current user.
+// On Windows:
+// 1. the first of %HOME%, %HOMEDRIVE%%HOMEPATH%, %USERPROFILE% containing a `.pulsar\config` file is returned.
+// 2. if none of those locations contain a `.pulsar\config` file, the first of
+// %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH%
+// that exists and is writeable is returned.
+// 3. if none of those locations are writeable, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH%
+// that exists is returned.
+// 4. if none of those locations exists, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH%
+// that is set is returned.
+func HomeDir() string {
+ if runtime.GOOS == "windows" {
+ home := os.Getenv("HOME")
+ homeDriveHomePath := ""
+ if homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH"); len(homeDrive) > 0 && len(homePath) > 0 {
+ homeDriveHomePath = homeDrive + homePath
+ }
+ userProfile := os.Getenv("USERPROFILE")
+
+ // Return first of %HOME%, %HOMEDRIVE%/%HOMEPATH%, %USERPROFILE% that contains a `.pulsar\config` file.
+ // %HOMEDRIVE%/%HOMEPATH% is preferred over %USERPROFILE% for backwards-compatibility.
+ for _, p := range []string{home, homeDriveHomePath, userProfile} {
+ if len(p) == 0 {
+ continue
+ }
+ if _, err := os.Stat(filepath.Join(p, ".pulsar", "config")); err != nil {
+ continue
+ }
+ return p
+ }
+
+ firstSetPath := ""
+ firstExistingPath := ""
+
+ // Prefer %USERPROFILE% over %HOMEDRIVE%/%HOMEPATH% for compatibility with other auth-writing tools
+ for _, p := range []string{home, userProfile, homeDriveHomePath} {
+ if len(p) == 0 {
+ continue
+ }
+ if len(firstSetPath) == 0 {
+ // remember the first path that is set
+ firstSetPath = p
+ }
+ info, err := os.Stat(p)
+ if err != nil {
+ continue
+ }
+ if len(firstExistingPath) == 0 {
+ // remember the first path that exists
+ firstExistingPath = p
+ }
+ if info.IsDir() && info.Mode().Perm()&(1<<(uint(7))) != 0 {
+ // return first path that is writeable
+ return p
+ }
+ }
+
+ // If none are writeable, return first location that exists
+ if len(firstExistingPath) > 0 {
+ return firstExistingPath
+ }
+
+ // If none exist, return first location that is set
+ if len(firstSetPath) > 0 {
+ return firstSetPath
+ }
+
+ // We've got nothing
+ return ""
+ }
+ return os.Getenv("HOME")
+}
diff --git a/pulsaradmin/pkg/utils/inactive_topic_policies.go b/pulsaradmin/pkg/utils/inactive_topic_policies.go
new file mode 100644
index 000000000..05f81b664
--- /dev/null
+++ b/pulsaradmin/pkg/utils/inactive_topic_policies.go
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import "github.com/pkg/errors"
+
+type InactiveTopicDeleteMode string
+
+const (
+ // The topic can be deleted when no subscriptions and no active producers.
+ DeleteWhenNoSubscriptions InactiveTopicDeleteMode = "delete_when_no_subscriptions"
+ // The topic can be deleted when all subscriptions catchup and no active producers/consumers.
+ DeleteWhenSubscriptionsCaughtUp InactiveTopicDeleteMode = "delete_when_subscriptions_caught_up"
+)
+
+func (i InactiveTopicDeleteMode) String() string {
+ return string(i)
+}
+
+func ParseInactiveTopicDeleteMode(str string) (InactiveTopicDeleteMode, error) {
+ switch str {
+ case DeleteWhenNoSubscriptions.String():
+ return DeleteWhenNoSubscriptions, nil
+ case DeleteWhenSubscriptionsCaughtUp.String():
+ return DeleteWhenSubscriptionsCaughtUp, nil
+ default:
+ return "", errors.Errorf("cannot parse %s to InactiveTopicDeleteMode type", str)
+ }
+}
+
+type InactiveTopicPolicies struct {
+ InactiveTopicDeleteMode *InactiveTopicDeleteMode `json:"inactiveTopicDeleteMode"`
+ MaxInactiveDurationSeconds int `json:"maxInactiveDurationSeconds"`
+ DeleteWhileInactive bool `json:"deleteWhileInactive"`
+}
+
+func NewInactiveTopicPolicies(inactiveTopicDeleteMode *InactiveTopicDeleteMode, maxInactiveDurationSeconds int,
+ deleteWhileInactive bool) InactiveTopicPolicies {
+ return InactiveTopicPolicies{
+ InactiveTopicDeleteMode: inactiveTopicDeleteMode,
+ MaxInactiveDurationSeconds: maxInactiveDurationSeconds,
+ DeleteWhileInactive: deleteWhileInactive,
+ }
+}
diff --git a/pulsaradmin/pkg/utils/internal_configuration_data.go b/pulsaradmin/pkg/utils/internal_configuration_data.go
new file mode 100644
index 000000000..75cad0e24
--- /dev/null
+++ b/pulsaradmin/pkg/utils/internal_configuration_data.go
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type InternalConfigurationData struct {
+ ZookeeperServers string `json:"zookeeperServers"`
+ ConfigurationStoreServers string `json:"configurationStoreServers"`
+ LedgersRootPath string `json:"ledgersRootPath"`
+ StateStorageServiceURL string `json:"stateStorageServiceUrl"`
+}
diff --git a/pulsaradmin/pkg/utils/load_manage_report.go b/pulsaradmin/pkg/utils/load_manage_report.go
new file mode 100644
index 000000000..5196da9b5
--- /dev/null
+++ b/pulsaradmin/pkg/utils/load_manage_report.go
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "math"
+)
+
+type LocalBrokerData struct {
+ // URLs to satisfy contract of ServiceLookupData (used by NamespaceService).
+ WebServiceURL string `json:"webServiceUrl"`
+ WebServiceURLTLS string `json:"webServiceUrlTls"`
+ PulsarServiceURL string `json:"pulsarServiceUrl"`
+ PulsarServiceURLTLS string `json:"pulsarServiceUrlTls"`
+ PersistentTopicsEnabled bool `json:"persistentTopicsEnabled"`
+ NonPersistentTopicsEnabled bool `json:"nonPersistentTopicsEnabled"`
+
+ // Most recently available system resource usage.
+ CPU ResourceUsage `json:"cpu"`
+ Memory ResourceUsage `json:"memory"`
+ DirectMemory ResourceUsage `json:"directMemory"`
+ BandwidthIn ResourceUsage `json:"bandwidthIn"`
+ BandwidthOut ResourceUsage `json:"bandwidthOut"`
+
+ // Message data from the most recent namespace bundle stats.
+ MsgThroughputIn float64 `json:"msgThroughputIn"`
+ MsgThroughputOut float64 `json:"msgThroughputOut"`
+ MsgRateIn float64 `json:"msgRateIn"`
+ MsgRateOut float64 `json:"msgRateOut"`
+
+ // Timestamp of last update.
+ LastUpdate int64 `json:"lastUpdate"`
+
+ // The stats given in the most recent invocation of update.
+ LastStats map[string]*NamespaceBundleStats `json:"lastStats"`
+ NumTopics int `json:"numTopics"`
+ NumBundles int `json:"numBundles"`
+ NumConsumers int `json:"numConsumers"`
+ NumProducers int `json:"numProducers"`
+
+ // All bundles belonging to this broker.
+ Bundles []string `json:"bundles"`
+
+ // The bundles gained since the last invocation of update.
+ LastBundleGains []string `json:"lastBundleGains"`
+
+ // The bundles lost since the last invocation of update.
+ LastBundleLosses []string `json:"lastBundleLosses"`
+
+ // The version string that this broker is running, obtained from the Maven build artifact in the POM
+ BrokerVersionString string `json:"brokerVersionString"`
+
+ // This place-holder requires to identify correct LoadManagerReport type while deserializing
+ LoadReportType string `json:"loadReportType"`
+
+ // the external protocol data advertised by protocol handlers.
+ Protocols map[string]string `json:"protocols"`
+}
+
+func NewLocalBrokerData() LocalBrokerData {
+ lastStats := make(map[string]*NamespaceBundleStats)
+ lastStats[""] = NewNamespaceBundleStats()
+ return LocalBrokerData{
+ LastStats: lastStats,
+ }
+}
+
+type NamespaceBundleStats struct {
+ MsgRateIn float64 `json:"msgRateIn"`
+ MsgThroughputIn float64 `json:"msgThroughputIn"`
+ MsgRateOut float64 `json:"msgRateOut"`
+ MsgThroughputOut float64 `json:"msgThroughputOut"`
+ ConsumerCount int `json:"consumerCount"`
+ ProducerCount int `json:"producerCount"`
+ TopicsNum int64 `json:"topics"`
+ CacheSize int64 `json:"cacheSize"`
+
+ // Consider the throughput equal if difference is less than 100 KB/s
+ ThroughputDifferenceThreshold float64 `json:"throughputDifferenceThreshold"`
+ // Consider the msgRate equal if the difference is less than 100
+ MsgRateDifferenceThreshold float64 `json:"msgRateDifferenceThreshold"`
+ // Consider the total topics/producers/consumers equal if the difference is less than 500
+ TopicConnectionDifferenceThreshold int64 `json:"topicConnectionDifferenceThreshold"`
+ // Consider the cache size equal if the difference is less than 100 kb
+ CacheSizeDifferenceThreshold int64 `json:"cacheSizeDifferenceThreshold"`
+}
+
+func NewNamespaceBundleStats() *NamespaceBundleStats {
+ return &NamespaceBundleStats{
+ ThroughputDifferenceThreshold: 1e5,
+ MsgRateDifferenceThreshold: 100,
+ TopicConnectionDifferenceThreshold: 500,
+ CacheSizeDifferenceThreshold: 100000,
+ }
+}
+
+type ResourceUsage struct {
+ Usage float64 `json:"usage"`
+ Limit float64 `json:"limit"`
+}
+
+func (ru *ResourceUsage) Reset() {
+ ru.Usage = -1
+ ru.Limit = -1
+}
+
+func (ru *ResourceUsage) CompareTo(o *ResourceUsage) int {
+ required := o.Limit - o.Usage
+ available := ru.Limit - ru.Usage
+ return compare(required, available)
+}
+
+func (ru *ResourceUsage) PercentUsage() float32 {
+ var proportion float32
+ if ru.Limit > 0 {
+ proportion = float32(ru.Usage) / float32(ru.Limit)
+ }
+ return proportion * 100
+}
+
+func compare(val1, val2 float64) int {
+ if val1 < val2 {
+ return -1
+ }
+
+ if val1 < val2 {
+ return 1
+ }
+
+ thisBits := math.Float64bits(val1)
+ anotherBits := math.Float64bits(val2)
+
+ if thisBits == anotherBits {
+ return 0
+ }
+
+ if thisBits < anotherBits {
+ return -1
+ }
+ return 1
+}
diff --git a/pulsaradmin/pkg/utils/long_running_process_status.go b/pulsaradmin/pkg/utils/long_running_process_status.go
new file mode 100644
index 000000000..c61919ecc
--- /dev/null
+++ b/pulsaradmin/pkg/utils/long_running_process_status.go
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type Status string
+
+const (
+ NOTRUN Status = "NOT_RUN"
+ RUNNING Status = "RUNNING"
+ SUCCESS Status = "SUCCESS"
+ ERROR Status = "ERROR"
+)
+
+type LongRunningProcessStatus struct {
+ Status Status `json:"status"`
+ LastError string `json:"lastError"`
+}
+
+type OffloadProcessStatus struct {
+ Status Status `json:"status"`
+ LastError string `json:"lastError"`
+ FirstUnOffloadedMessage MessageID `json:"firstUnoffloadedMessage"`
+}
+
+func (s Status) String() string {
+ return string(s)
+}
diff --git a/pulsaradmin/pkg/utils/message.go b/pulsaradmin/pkg/utils/message.go
new file mode 100644
index 000000000..2f0e4befe
--- /dev/null
+++ b/pulsaradmin/pkg/utils/message.go
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+//nolint
+import (
+ "github.com/golang/protobuf/proto"
+)
+
+type Message struct {
+ MessageID MessageID
+ Payload []byte
+ Topic string
+ Properties map[string]string
+}
+
+func NewMessage(topic string, id MessageID, payload []byte, properties map[string]string) *Message {
+ return &Message{
+ MessageID: id,
+ Payload: payload,
+ Topic: topic,
+ Properties: properties,
+ }
+}
+
+func (m *Message) GetMessageID() MessageID {
+ return m.MessageID
+}
+
+func (m *Message) GetProperties() map[string]string {
+ return m.Properties
+}
+
+func (m *Message) GetPayload() []byte {
+ return m.Payload
+}
+
+// nolint
+type SingleMessageMetadata struct {
+ Properties []*KeyValue `protobuf:"bytes,1,rep,name=properties" json:"properties,omitempty"`
+ PartitionKey *string `protobuf:"bytes,2,opt,name=partition_key,json=partitionKey" json:"partition_key,omitempty"`
+ PayloadSize *int32 `protobuf:"varint,3,req,name=payload_size,json=payloadSize" json:"payload_size,omitempty"`
+ CompactedOut *bool `protobuf:"varint,4,opt,name=compacted_out,json=compactedOut,def=0" json:"compacted_out,omitempty"`
+ // the timestamp that this event occurs. it is typically set by applications.
+ // if this field is omitted, `publish_time` can be used for the purpose of `event_time`.
+ EventTime *uint64 `protobuf:"varint,5,opt,name=event_time,json=eventTime,def=0" json:"event_time,omitempty"`
+ PartitionKeyB64Encoded *bool `protobuf:"varint,6,opt,name=partition_key_b64_encoded,json=partitionKeyB64Encoded,def=0" json:"partition_key_b64_encoded,omitempty"`
+ // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode.
+ OrderingKey []byte `protobuf:"bytes,7,opt,name=ordering_key,json=orderingKey" json:"ordering_key,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *SingleMessageMetadata) Reset() { *m = SingleMessageMetadata{} }
+func (m *SingleMessageMetadata) String() string { return proto.CompactTextString(m) }
+func (*SingleMessageMetadata) ProtoMessage() {}
+func (m *SingleMessageMetadata) GetPayloadSize() int32 {
+ if m != nil && m.PayloadSize != nil {
+ return *m.PayloadSize
+ }
+ return 0
+}
+
+// nolint
+type KeyValue struct {
+ Key *string `protobuf:"bytes,1,req,name=key" json:"key,omitempty"`
+ Value *string `protobuf:"bytes,2,req,name=value" json:"value,omitempty"`
+ XXX_NoUnkeyedLiteral struct{} `json:"-"`
+ XXX_unrecognized []byte `json:"-"`
+ XXX_sizecache int32 `json:"-"`
+}
+
+func (m *KeyValue) Reset() { *m = KeyValue{} }
+func (m *KeyValue) String() string { return proto.CompactTextString(m) }
+func (*KeyValue) ProtoMessage() {}
diff --git a/pulsaradmin/pkg/utils/message_id.go b/pulsaradmin/pkg/utils/message_id.go
new file mode 100644
index 000000000..d75b613e1
--- /dev/null
+++ b/pulsaradmin/pkg/utils/message_id.go
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "strconv"
+ "strings"
+
+ "github.com/pkg/errors"
+)
+
+type MessageID struct {
+ LedgerID int64 `json:"ledgerId"`
+ EntryID int64 `json:"entryId"`
+ PartitionIndex int `json:"partitionIndex"`
+ BatchIndex int `json:"-"`
+}
+
+var Latest = MessageID{0x7fffffffffffffff, 0x7fffffffffffffff, -1, -1}
+var Earliest = MessageID{-1, -1, -1, -1}
+
+func ParseMessageID(str string) (*MessageID, error) {
+ s := strings.Split(str, ":")
+
+ m := Earliest
+
+ if len(s) < 2 || len(s) > 4 {
+ return nil, errors.Errorf("invalid message id string. %s", str)
+ }
+
+ ledgerID, err := strconv.ParseInt(s[0], 10, 64)
+ if err != nil {
+ return nil, errors.Errorf("invalid ledger id. %s", str)
+ }
+ m.LedgerID = ledgerID
+
+ entryID, err := strconv.ParseInt(s[1], 10, 64)
+ if err != nil {
+ return nil, errors.Errorf("invalid entry id. %s", str)
+ }
+ m.EntryID = entryID
+
+ if len(s) > 2 {
+ pi, err := strconv.Atoi(s[2])
+ if err != nil {
+ return nil, errors.Errorf("invalid partition index. %s", str)
+ }
+ m.PartitionIndex = pi
+ }
+
+ if len(s) == 4 {
+ bi, err := strconv.Atoi(s[3])
+ if err != nil {
+ return nil, errors.Errorf("invalid batch index. %s", str)
+ }
+ m.BatchIndex = bi
+ }
+
+ return &m, nil
+}
+
+func (m MessageID) String() string {
+ return strconv.FormatInt(m.LedgerID, 10) + ":" +
+ strconv.FormatInt(m.EntryID, 10) + ":" +
+ strconv.Itoa(m.PartitionIndex) + ":" +
+ strconv.Itoa(m.BatchIndex)
+}
diff --git a/pulsaradmin/pkg/utils/message_id_test.go b/pulsaradmin/pkg/utils/message_id_test.go
new file mode 100644
index 000000000..1ce0b5230
--- /dev/null
+++ b/pulsaradmin/pkg/utils/message_id_test.go
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestParseMessageId(t *testing.T) {
+ id, err := ParseMessageID("1:1")
+ assert.Nil(t, err)
+ assert.Equal(t, MessageID{LedgerID: 1, EntryID: 1, PartitionIndex: -1, BatchIndex: -1}, *id)
+
+ id, err = ParseMessageID("1:2:3")
+ assert.Nil(t, err)
+ assert.Equal(t, MessageID{LedgerID: 1, EntryID: 2, PartitionIndex: 3, BatchIndex: -1}, *id)
+
+ id, err = ParseMessageID("1:2:3:4")
+ assert.Nil(t, err)
+ assert.Equal(t, MessageID{LedgerID: 1, EntryID: 2, PartitionIndex: 3, BatchIndex: 4}, *id)
+}
+
+func TestParseMessageIdErrors(t *testing.T) {
+ id, err := ParseMessageID("1;1")
+ assert.Nil(t, id)
+ assert.NotNil(t, err)
+ assert.Equal(t, "invalid message id string. 1;1", err.Error())
+
+ id, err = ParseMessageID("a:1")
+ assert.Nil(t, id)
+ assert.NotNil(t, err)
+ assert.Equal(t, "invalid ledger id. a:1", err.Error())
+
+ id, err = ParseMessageID("1:a")
+ assert.Nil(t, id)
+ assert.NotNil(t, err)
+ assert.Equal(t, "invalid entry id. 1:a", err.Error())
+
+ id, err = ParseMessageID("1:2:a")
+ assert.Nil(t, id)
+ assert.NotNil(t, err)
+ assert.Equal(t, "invalid partition index. 1:2:a", err.Error())
+
+ id, err = ParseMessageID("1:2:3:a")
+ assert.Nil(t, id)
+ assert.NotNil(t, err)
+ assert.Equal(t, "invalid batch index. 1:2:3:a", err.Error())
+}
diff --git a/pulsaradmin/pkg/utils/metrics.go b/pulsaradmin/pkg/utils/metrics.go
new file mode 100644
index 000000000..ca1919423
--- /dev/null
+++ b/pulsaradmin/pkg/utils/metrics.go
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type Metrics struct {
+ Metrics map[string]interface{} `json:"metrics"`
+ Dimensions map[string]string `json:"dimensions"`
+}
+
+func NewMetrics(dimensionMap map[string]string) *Metrics {
+ return &Metrics{
+ Metrics: make(map[string]interface{}),
+ Dimensions: dimensionMap,
+ }
+}
diff --git a/pulsaradmin/pkg/utils/namespace_name.go b/pulsaradmin/pkg/utils/namespace_name.go
new file mode 100644
index 000000000..11b743554
--- /dev/null
+++ b/pulsaradmin/pkg/utils/namespace_name.go
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "fmt"
+ "regexp"
+ "strings"
+
+ "github.com/pkg/errors"
+)
+
+type NameSpaceName struct {
+ tenant string
+ nameSpace string
+}
+
+func GetNameSpaceName(tenant, namespace string) (*NameSpaceName, error) {
+ return GetNamespaceName(fmt.Sprintf("%s/%s", tenant, namespace))
+}
+
+func GetNamespaceName(completeName string) (*NameSpaceName, error) {
+ var n NameSpaceName
+
+ if completeName == "" {
+ return nil, errors.New("the namespace complete name is empty")
+ }
+
+ parts := strings.Split(completeName, "/")
+ if len(parts) == 2 {
+ n.tenant = parts[0]
+ n.nameSpace = parts[1]
+ err := validateNamespaceName(n.tenant, n.nameSpace)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ return nil, errors.Errorf("The complete name of namespace is invalid. complete name : [%s]", completeName)
+ }
+
+ return &n, nil
+}
+
+func (n *NameSpaceName) String() string {
+ return fmt.Sprintf("%s/%s", n.tenant, n.nameSpace)
+}
+
+func validateNamespaceName(tenant, namespace string) error {
+ if tenant == "" || namespace == "" {
+ return errors.Errorf("Invalid tenant or namespace. [%s/%s]", tenant, namespace)
+ }
+
+ ok := CheckName(tenant)
+ if !ok {
+ return errors.Errorf("Tenant name include unsupported special chars. tenant : [%s]", tenant)
+ }
+
+ ok = CheckName(namespace)
+ if !ok {
+ return errors.Errorf("Namespace name include unsupported special chars. namespace : [%s]", namespace)
+ }
+
+ return nil
+}
+
+// allowed characters for property, namespace, cluster and topic
+// names are alphanumeric (a-zA-Z0-9) and these special chars -=:.
+// and % is allowed as part of valid URL encoding
+var patten = regexp.MustCompile(`^[-=:.\w]*$`)
+
+func CheckName(name string) bool {
+ return patten.MatchString(name)
+}
diff --git a/pulsaradmin/pkg/utils/namespace_name_test.go b/pulsaradmin/pkg/utils/namespace_name_test.go
new file mode 100644
index 000000000..bc12070e9
--- /dev/null
+++ b/pulsaradmin/pkg/utils/namespace_name_test.go
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGetNamespaceName(t *testing.T) {
+ success, err := GetNamespaceName("public/default")
+ assert.Nil(t, err)
+ assert.Equal(t, "public/default", success.String())
+
+ empty, err := GetNamespaceName("")
+ assert.NotNil(t, err)
+ assert.Equal(t, "the namespace complete name is empty", err.Error())
+ assert.Nil(t, empty)
+
+ empty, err = GetNamespaceName("/")
+ assert.NotNil(t, err)
+ assert.Equal(t, "Invalid tenant or namespace. [/]", err.Error())
+ assert.Nil(t, empty)
+
+ invalid, err := GetNamespaceName("public/default/fail")
+ assert.NotNil(t, err)
+ assert.Equal(t, "The complete name of namespace is invalid. complete name : [public/default/fail]", err.Error())
+ assert.Nil(t, invalid)
+
+ invalid, err = GetNamespaceName("public")
+ assert.NotNil(t, err)
+ assert.Equal(t, "The complete name of namespace is invalid. complete name : [public]", err.Error())
+ assert.Nil(t, invalid)
+
+ special, err := GetNamespaceName("-=.:/-=.:")
+ assert.Nil(t, err)
+ assert.Equal(t, "-=.:/-=.:", special.String())
+
+ tenantInvalid, err := GetNamespaceName("\"/namespace")
+ assert.NotNil(t, err)
+ assert.Equal(t, "Tenant name include unsupported special chars. tenant : [\"]", err.Error())
+ assert.Nil(t, tenantInvalid)
+
+ namespaceInvalid, err := GetNamespaceName("tenant/}")
+ assert.NotNil(t, err)
+ assert.Equal(t, "Namespace name include unsupported special chars. namespace : [}]", err.Error())
+ assert.Nil(t, namespaceInvalid)
+}
diff --git a/pulsaradmin/pkg/utils/ns_isolation_data.go b/pulsaradmin/pkg/utils/ns_isolation_data.go
new file mode 100644
index 000000000..4589eb30c
--- /dev/null
+++ b/pulsaradmin/pkg/utils/ns_isolation_data.go
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "github.com/pkg/errors"
+)
+
+type NamespaceIsolationData struct {
+ Namespaces []string `json:"namespaces"`
+ Primary []string `json:"primary"`
+ Secondary []string `json:"secondary"`
+ AutoFailoverPolicy AutoFailoverPolicyData `json:"auto_failover_policy"`
+}
+
+type AutoFailoverPolicyData struct {
+ PolicyType AutoFailoverPolicyType `json:"policy_type"`
+ Parameters map[string]string `json:"parameters"`
+}
+
+type AutoFailoverPolicyType string
+
+const (
+ MinAvailable AutoFailoverPolicyType = "min_available"
+)
+
+func fromString(autoFailoverPolicyTypeName string) AutoFailoverPolicyType {
+ switch autoFailoverPolicyTypeName {
+ case "min_available":
+ return MinAvailable
+ default:
+ return ""
+ }
+}
+
+func CreateNamespaceIsolationData(namespaces, primary, secondry []string, autoFailoverPolicyTypeName string,
+ autoFailoverPolicyParams map[string]string) (*NamespaceIsolationData, error) {
+ nsIsolationData := new(NamespaceIsolationData)
+ if len(namespaces) == 0 {
+ return nil, errors.New("unable to parse namespaces parameter list")
+ }
+
+ if len(primary) == 0 {
+ return nil, errors.New("unable to parse primary parameter list")
+ }
+
+ if len(secondry) == 0 {
+ return nil, errors.New("unable to parse secondry parameter list")
+ }
+
+ nsIsolationData.Namespaces = namespaces
+ nsIsolationData.Primary = primary
+ nsIsolationData.Secondary = secondry
+ nsIsolationData.AutoFailoverPolicy.PolicyType = fromString(autoFailoverPolicyTypeName)
+ nsIsolationData.AutoFailoverPolicy.Parameters = autoFailoverPolicyParams
+
+ // validation if necessary
+ if nsIsolationData.AutoFailoverPolicy.PolicyType == MinAvailable {
+ err := true
+ expectParamKeys := []string{"min_limit", "usage_threshold"}
+
+ if len(autoFailoverPolicyParams) == len(expectParamKeys) {
+ for _, paramKey := range expectParamKeys {
+ if _, ok := autoFailoverPolicyParams[paramKey]; !ok {
+ break
+ }
+ }
+ err = false
+ }
+
+ if err {
+ return nil, errors.Errorf("Unknown auto failover policy params specified: %v", autoFailoverPolicyParams)
+ }
+ } else {
+ // either we don't handle the new type or user has specified a bad type
+ return nil, errors.Errorf("Unknown auto failover policy type specified : %v", autoFailoverPolicyTypeName)
+ }
+
+ return nsIsolationData, nil
+}
diff --git a/pulsaradmin/pkg/utils/ns_ownership_status.go b/pulsaradmin/pkg/utils/ns_ownership_status.go
new file mode 100644
index 000000000..bb63b704f
--- /dev/null
+++ b/pulsaradmin/pkg/utils/ns_ownership_status.go
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type NamespaceOwnershipStatus struct {
+ BrokerAssignment BrokerAssignment `json:"broker_assignment"`
+ IsControlled bool `json:"is_controlled"`
+ IsActive bool `json:"is_active"`
+}
+
+type BrokerAssignment string
+
+const (
+ Primary BrokerAssignment = "primary"
+ Secondary BrokerAssignment = "secondary"
+ Shared BrokerAssignment = "shared"
+)
diff --git a/pulsaradmin/pkg/utils/package_metadata.go b/pulsaradmin/pkg/utils/package_metadata.go
new file mode 100644
index 000000000..860afd9c8
--- /dev/null
+++ b/pulsaradmin/pkg/utils/package_metadata.go
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type PackageMetadata struct {
+ Description string `json:"description,omitempty" yaml:"description"`
+ Contact string `json:"contact,omitempty" yaml:"contact"`
+ CreateTime int64 `json:"createTime,omitempty" yaml:"createTime"`
+ ModificationTime int64 `json:"modificationTime,omitempty" yaml:"modificationTime"`
+ Properties map[string]string `json:"properties,omitempty" yaml:"properties"`
+}
diff --git a/pulsaradmin/pkg/utils/package_name.go b/pulsaradmin/pkg/utils/package_name.go
new file mode 100644
index 000000000..afc646d68
--- /dev/null
+++ b/pulsaradmin/pkg/utils/package_name.go
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/pkg/errors"
+)
+
+type PackageName struct {
+ packageType PackageType
+ namespace string
+ tenant string
+ name string
+ version string
+ completePackageName string
+ completeName string
+}
+
+func invalidPackageNameError(completeName string) error {
+ return errors.Errorf("Invalid package name '%s', it should be "+
+ "in the format of type://tenant/namespace/name@version", completeName)
+}
+
+func GetPackageNameWithComponents(packageType PackageType,
+ tenant, namespace, name, version string) (*PackageName, error) {
+ return GetPackageName(fmt.Sprintf("%s://%s/%s/%s@%s", packageType, tenant, namespace, name, version))
+}
+
+func GetPackageName(completeName string) (*PackageName, error) {
+ var packageName PackageName
+ var err error
+ if !strings.Contains(completeName, "://") {
+ return nil, invalidPackageNameError(completeName)
+ }
+ parts := strings.Split(completeName, "://")
+ if len(parts) != 2 {
+ return nil, invalidPackageNameError(completeName)
+ }
+ packageName.packageType, err = parsePackageType(parts[0])
+ if err != nil {
+ return nil, err
+ }
+ rest := parts[1]
+ if !strings.Contains(rest, "@") {
+ // if the package name does not contains '@', that means user does not set the version of package.
+ // We will set the version to latest.
+ rest += "@"
+ }
+ parts = strings.Split(rest, "@")
+ if len(parts) != 2 {
+ return nil, invalidPackageNameError(completeName)
+ }
+ partsWithoutVersion := strings.Split(parts[0], "/")
+ if len(partsWithoutVersion) != 3 {
+ return nil, invalidPackageNameError(completeName)
+ }
+ packageName.tenant = partsWithoutVersion[0]
+ packageName.namespace = partsWithoutVersion[1]
+ packageName.name = partsWithoutVersion[2]
+ packageName.version = "latest"
+ if parts[1] != "" {
+ packageName.version = parts[1]
+ }
+ packageName.completeName = fmt.Sprintf("%s/%s/%s",
+ packageName.tenant, packageName.namespace, packageName.name)
+ packageName.completePackageName = fmt.Sprintf("%s://%s/%s/%s@%s",
+ packageName.packageType, packageName.tenant, packageName.namespace, packageName.name, packageName.version)
+
+ return &packageName, nil
+}
+
+func (p *PackageName) String() string {
+ return p.completePackageName
+}
+
+func (p *PackageName) GetType() PackageType {
+ return p.packageType
+}
+
+func (p *PackageName) GetTenant() string {
+ return p.tenant
+}
+
+func (p *PackageName) GetNamespace() string {
+ return p.namespace
+}
+
+func (p *PackageName) GetName() string {
+ return p.name
+}
+
+func (p *PackageName) GetVersion() string {
+ return p.version
+}
+
+func (p *PackageName) GetCompleteName() string {
+ return p.completeName
+}
diff --git a/pulsaradmin/pkg/utils/package_name_test.go b/pulsaradmin/pkg/utils/package_name_test.go
new file mode 100644
index 000000000..8e6edd52d
--- /dev/null
+++ b/pulsaradmin/pkg/utils/package_name_test.go
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGetPackageName(t *testing.T) {
+ success, err := GetPackageName("function://f-tenant/f-ns/f-name@f-version")
+ assert.Nil(t, err)
+ assert.Equal(t, "function://f-tenant/f-ns/f-name@f-version", success.String())
+
+ success, err = GetPackageName("function://f-tenant/f-ns/f-name")
+ assert.Nil(t, err)
+ assert.Equal(t, "function://f-tenant/f-ns/f-name@latest", success.String())
+
+ success, err = GetPackageName("sink://s-tenant/s-ns/s-name@s-version")
+ assert.Nil(t, err)
+ assert.Equal(t, "sink://s-tenant/s-ns/s-name@s-version", success.String())
+
+ success, err = GetPackageName("sink://s-tenant/s-ns/s-name")
+ assert.Nil(t, err)
+ assert.Equal(t, "sink://s-tenant/s-ns/s-name@latest", success.String())
+
+ success, err = GetPackageName("source://s-tenant/s-ns/s-name@s-version")
+ assert.Nil(t, err)
+ assert.Equal(t, "source://s-tenant/s-ns/s-name@s-version", success.String())
+
+ success, err = GetPackageName("source://s-tenant/s-ns/s-name")
+ assert.Nil(t, err)
+ assert.Equal(t, "source://s-tenant/s-ns/s-name@latest", success.String())
+
+ fail, err := GetPackageName("function:///public/default/test-error@v1")
+ assert.NotNil(t, err)
+ assert.Equal(t, "Invalid package name 'function:///public/default/test-error@v1', it should be in the "+
+ "format of type://tenant/namespace/name@version", err.Error())
+ assert.Nil(t, fail)
+
+ fail, err = GetPackageNameWithComponents("functions", "public", "default", "test-error", "v1")
+ assert.NotNil(t, err)
+ assert.Equal(t, "Invalid package type 'functions', it should be function, sink, or source", err.Error())
+ assert.Nil(t, fail)
+
+ fail, err = GetPackageNameWithComponents("function", "public/default", "default", "test-error", "v1")
+ assert.NotNil(t, err)
+ assert.Equal(t, "Invalid package name 'function://public/default/default/test-error@v1', it should be in the "+
+ "format of type://tenant/namespace/name@version", err.Error())
+ assert.Nil(t, fail)
+
+ fail, err = GetPackageName("function://public/default/test-error-version/v2")
+ assert.NotNil(t, err)
+ assert.Equal(t, "Invalid package name 'function://public/default/test-error-version/v2', it should be in the "+
+ "format of type://tenant/namespace/name@version", err.Error())
+ assert.Nil(t, fail)
+}
diff --git a/pulsaradmin/pkg/utils/package_type.go b/pulsaradmin/pkg/utils/package_type.go
new file mode 100644
index 000000000..70f455b25
--- /dev/null
+++ b/pulsaradmin/pkg/utils/package_type.go
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import "github.com/pkg/errors"
+
+type PackageType string
+
+const (
+ PackageTypeFunction PackageType = "function"
+ PackageTypeSink PackageType = "sink"
+ PackageTypeSource PackageType = "source"
+)
+
+func parsePackageType(packageTypeName string) (PackageType, error) {
+ switch packageTypeName {
+ case PackageTypeFunction.String():
+ return PackageTypeFunction, nil
+ case PackageTypeSink.String():
+ return PackageTypeSink, nil
+ case PackageTypeSource.String():
+ return PackageTypeSource, nil
+ default:
+ return "", errors.Errorf("Invalid package type '%s', it should be "+
+ "function, sink, or source", packageTypeName)
+ }
+}
+
+func (p PackageType) String() string {
+ return string(p)
+}
diff --git a/pulsaradmin/pkg/utils/persistence_policies.go b/pulsaradmin/pkg/utils/persistence_policies.go
new file mode 100644
index 000000000..d4c8bdb04
--- /dev/null
+++ b/pulsaradmin/pkg/utils/persistence_policies.go
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type PersistencePolicies struct {
+ BookkeeperEnsemble int `json:"bookkeeperEnsemble"`
+ BookkeeperWriteQuorum int `json:"bookkeeperWriteQuorum"`
+ BookkeeperAckQuorum int `json:"bookkeeperAckQuorum"`
+ ManagedLedgerMaxMarkDeleteRate float64 `json:"managedLedgerMaxMarkDeleteRate"`
+}
+
+func NewPersistencePolicies(bookkeeperEnsemble, bookkeeperWriteQuorum, bookkeeperAckQuorum int,
+ managedLedgerMaxMarkDeleteRate float64) PersistencePolicies {
+ return PersistencePolicies{
+ BookkeeperEnsemble: bookkeeperEnsemble,
+ BookkeeperWriteQuorum: bookkeeperWriteQuorum,
+ BookkeeperAckQuorum: bookkeeperAckQuorum,
+ ManagedLedgerMaxMarkDeleteRate: managedLedgerMaxMarkDeleteRate,
+ }
+}
+
+type BookieAffinityGroupData struct {
+ BookkeeperAffinityGroupPrimary string `json:"bookkeeperAffinityGroupPrimary"`
+ BookkeeperAffinityGroupSecondary string `json:"bookkeeperAffinityGroupSecondary"`
+}
diff --git a/pulsaradmin/pkg/utils/policies.go b/pulsaradmin/pkg/utils/policies.go
new file mode 100644
index 000000000..3d727994c
--- /dev/null
+++ b/pulsaradmin/pkg/utils/policies.go
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+const (
+ FirstBoundary string = "0x00000000"
+ LastBoundary string = "0xffffffff"
+)
+
+type Policies struct {
+ Bundles *BundlesData `json:"bundles"`
+ Persistence *PersistencePolicies `json:"persistence"`
+ RetentionPolicies *RetentionPolicies `json:"retention_policies"`
+ SchemaValidationEnforced bool `json:"schema_validation_enforced"`
+ DeduplicationEnabled *bool `json:"deduplicationEnabled"`
+ Deleted bool `json:"deleted"`
+ EncryptionRequired bool `json:"encryption_required"`
+ MessageTTLInSeconds *int `json:"message_ttl_in_seconds"`
+ MaxProducersPerTopic *int `json:"max_producers_per_topic"`
+ MaxConsumersPerTopic *int `json:"max_consumers_per_topic"`
+ MaxConsumersPerSubscription *int `json:"max_consumers_per_subscription"`
+ CompactionThreshold *int64 `json:"compaction_threshold"`
+ OffloadThreshold int64 `json:"offload_threshold"`
+ OffloadDeletionLagMs *int64 `json:"offload_deletion_lag_ms"`
+ AntiAffinityGroup string `json:"antiAffinityGroup"`
+ ReplicationClusters []string `json:"replication_clusters"`
+ LatencyStatsSampleRate map[string]int `json:"latency_stats_sample_rate"`
+ BacklogQuotaMap map[BacklogQuotaType]BacklogQuota `json:"backlog_quota_map"`
+ TopicDispatchRate map[string]DispatchRate `json:"topicDispatchRate"`
+ SubscriptionDispatchRate map[string]DispatchRate `json:"subscriptionDispatchRate"`
+ ReplicatorDispatchRate map[string]DispatchRate `json:"replicatorDispatchRate"`
+ PublishMaxMessageRate map[string]PublishRate `json:"publishMaxMessageRate"`
+ ClusterSubscribeRate map[string]SubscribeRate `json:"clusterSubscribeRate"`
+ TopicAutoCreationConfig *TopicAutoCreationConfig `json:"autoTopicCreationOverride"`
+ SchemaCompatibilityStrategy SchemaCompatibilityStrategy `json:"schema_auto_update_compatibility_strategy"`
+ AuthPolicies AuthPolicies `json:"auth_policies"`
+ SubscriptionAuthMode SubscriptionAuthMode `json:"subscription_auth_mode"`
+ IsAllowAutoUpdateSchema *bool `json:"is_allow_auto_update_schema"`
+}
+
+func NewDefaultPolicies() *Policies {
+ return &Policies{
+ AuthPolicies: *NewAuthPolicies(),
+ ReplicationClusters: make([]string, 0, 10),
+ BacklogQuotaMap: make(map[BacklogQuotaType]BacklogQuota),
+ TopicDispatchRate: make(map[string]DispatchRate),
+ SubscriptionDispatchRate: make(map[string]DispatchRate),
+ ReplicatorDispatchRate: make(map[string]DispatchRate),
+ PublishMaxMessageRate: make(map[string]PublishRate),
+ ClusterSubscribeRate: make(map[string]SubscribeRate),
+ LatencyStatsSampleRate: make(map[string]int),
+ MessageTTLInSeconds: nil,
+ Deleted: false,
+ EncryptionRequired: false,
+ SubscriptionAuthMode: None,
+ MaxProducersPerTopic: nil,
+ MaxConsumersPerSubscription: nil,
+ MaxConsumersPerTopic: nil,
+ CompactionThreshold: nil,
+ OffloadThreshold: -1,
+ SchemaCompatibilityStrategy: Full,
+ SchemaValidationEnforced: false,
+ }
+}
diff --git a/pulsaradmin/pkg/utils/producer_config.go b/pulsaradmin/pkg/utils/producer_config.go
new file mode 100644
index 000000000..d4c1c3eb6
--- /dev/null
+++ b/pulsaradmin/pkg/utils/producer_config.go
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type ProducerConfig struct {
+ MaxPendingMessages int `json:"maxPendingMessages" yaml:"maxPendingMessages"`
+ //nolint
+ MaxPendingMessagesAcrossPartitions int `json:"maxPendingMessagesAcrossPartitions" yaml:"maxPendingMessagesAcrossPartitions"`
+
+ UseThreadLocalProducers bool `json:"useThreadLocalProducers" yaml:"useThreadLocalProducers"`
+ CryptoConfig *CryptoConfig `json:"cryptoConfig" yaml:"cryptoConfig"`
+ BatchBuilder string `json:"batchBuilder" yaml:"batchBuilder"`
+ CompressionType string `json:"compressionType" yaml:"compressionType"`
+}
diff --git a/pulsaradmin/pkg/utils/publish_rate.go b/pulsaradmin/pkg/utils/publish_rate.go
new file mode 100644
index 000000000..9dbc2d00d
--- /dev/null
+++ b/pulsaradmin/pkg/utils/publish_rate.go
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type PublishRate struct {
+ PublishThrottlingRateInMsg int `json:"publishThrottlingRateInMsg"`
+ PublishThrottlingRateInByte int64 `json:"publishThrottlingRateInByte"`
+}
+
+func NewPublishRate() *PublishRate {
+ return &PublishRate{
+ PublishThrottlingRateInMsg: -1,
+ PublishThrottlingRateInByte: -1,
+ }
+}
diff --git a/pulsaradmin/pkg/utils/resource_quota.go b/pulsaradmin/pkg/utils/resource_quota.go
new file mode 100644
index 000000000..d9ef4f74d
--- /dev/null
+++ b/pulsaradmin/pkg/utils/resource_quota.go
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type ResourceQuota struct {
+ // messages published per second
+ MsgRateIn float64 `json:"msgRateIn"`
+ // messages consumed per second
+ MsgRateOut float64 `json:"msgRateOut"`
+ // incoming bytes per second
+ BandwidthIn float64 `json:"bandwidthIn"`
+ // outgoing bytes per second
+ BandwidthOut float64 `json:"bandwidthOut"`
+ // used memory in Mbytes
+ Memory float64 `json:"memory"`
+ // allow the quota be dynamically re-calculated according to real traffic
+ Dynamic bool `json:"dynamic"`
+}
+
+func NewResourceQuota() *ResourceQuota {
+ return &ResourceQuota{
+ MsgRateIn: 0.0,
+ MsgRateOut: 0.0,
+ BandwidthIn: 0.0,
+ BandwidthOut: 0.0,
+ Memory: 0.0,
+ Dynamic: true,
+ }
+}
diff --git a/pulsaradmin/pkg/utils/resources.go b/pulsaradmin/pkg/utils/resources.go
new file mode 100644
index 000000000..a4f3ddbcb
--- /dev/null
+++ b/pulsaradmin/pkg/utils/resources.go
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type Resources struct {
+ CPU float64 `json:"cpu"`
+ Disk int64 `json:"disk"`
+ RAM int64 `json:"ram"`
+}
+
+func NewDefaultResources() *Resources {
+ resources := &Resources{
+ // Default cpu is 1 core
+ CPU: 1,
+ // Default memory is 1GB
+ RAM: 1073741824,
+ // Default disk is 10GB
+ Disk: 10737418240,
+ }
+
+ return resources
+}
diff --git a/pulsaradmin/pkg/utils/retention_policies.go b/pulsaradmin/pkg/utils/retention_policies.go
new file mode 100644
index 000000000..55bf915ed
--- /dev/null
+++ b/pulsaradmin/pkg/utils/retention_policies.go
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type RetentionPolicies struct {
+ RetentionTimeInMinutes int `json:"retentionTimeInMinutes"`
+ RetentionSizeInMB int64 `json:"retentionSizeInMB"`
+}
+
+func NewRetentionPolicies(retentionTimeInMinutes int, retentionSizeInMB int) RetentionPolicies {
+ return RetentionPolicies{
+ RetentionTimeInMinutes: retentionTimeInMinutes,
+ RetentionSizeInMB: int64(retentionSizeInMB),
+ }
+}
diff --git a/pulsaradmin/pkg/utils/schema_strategy.go b/pulsaradmin/pkg/utils/schema_strategy.go
new file mode 100644
index 000000000..176f0e0a9
--- /dev/null
+++ b/pulsaradmin/pkg/utils/schema_strategy.go
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import "github.com/pkg/errors"
+
+type SchemaCompatibilityStrategy string
+
+const (
+ AutoUpdateDisabled SchemaCompatibilityStrategy = "AutoUpdateDisabled"
+ Backward SchemaCompatibilityStrategy = "Backward"
+ Forward SchemaCompatibilityStrategy = "Forward"
+ Full SchemaCompatibilityStrategy = "Full"
+ AlwaysCompatible SchemaCompatibilityStrategy = "AlwaysCompatible"
+ BackwardTransitive SchemaCompatibilityStrategy = "BackwardTransitive"
+ ForwardTransitive SchemaCompatibilityStrategy = "ForwardTransitive"
+ FullTransitive SchemaCompatibilityStrategy = "FullTransitive"
+)
+
+func ParseSchemaAutoUpdateCompatibilityStrategy(str string) (SchemaCompatibilityStrategy, error) {
+ switch str {
+ case "AutoUpdateDisabled":
+ return AutoUpdateDisabled, nil
+ case "Backward":
+ return Backward, nil
+ case "Forward":
+ return Forward, nil
+ case "Full":
+ return Full, nil
+ case "AlwaysCompatible":
+ return AlwaysCompatible, nil
+ case "BackwardTransitive":
+ return BackwardTransitive, nil
+ case "ForwardTransitive":
+ return ForwardTransitive, nil
+ case "FullTransitive":
+ return FullTransitive, nil
+ default:
+ return "", errors.Errorf("Invalid auth strategy %s", str)
+ }
+}
+
+func (s SchemaCompatibilityStrategy) String() string {
+ return string(s)
+}
diff --git a/pulsaradmin/pkg/utils/schema_util.go b/pulsaradmin/pkg/utils/schema_util.go
new file mode 100644
index 000000000..08aaf54ac
--- /dev/null
+++ b/pulsaradmin/pkg/utils/schema_util.go
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type SchemaInfo struct {
+ Name string `json:"name"`
+ Schema []byte `json:"schema"`
+ Type string `json:"type"`
+ Properties map[string]string `json:"properties"`
+}
+
+type SchemaInfoWithVersion struct {
+ Version int64 `json:"version"`
+ SchemaInfo *SchemaInfo `json:"schemaInfo"`
+}
+
+// Payload with information about a schema
+type PostSchemaPayload struct {
+ SchemaType string `json:"type"`
+ Schema string `json:"schema"`
+ Properties map[string]string `json:"properties"`
+}
+
+type GetSchemaResponse struct {
+ Version int64 `json:"version"`
+ Type string `json:"type"`
+ Timestamp int64 `json:"timestamp"`
+ Data string `json:"data"`
+ Properties map[string]string `json:"properties"`
+}
+
+func ConvertGetSchemaResponseToSchemaInfo(tn *TopicName, response GetSchemaResponse) *SchemaInfo {
+ info := new(SchemaInfo)
+ schema := make([]byte, 0, 10)
+ if response.Type == "KEY_VALUE" {
+ // TODO: impl logic
+ } else {
+ schema = []byte(response.Data)
+ }
+
+ info.Schema = schema
+ info.Type = response.Type
+ info.Properties = response.Properties
+ info.Name = tn.GetLocalName()
+
+ return info
+}
+
+func ConvertGetSchemaResponseToSchemaInfoWithVersion(tn *TopicName, response GetSchemaResponse) *SchemaInfoWithVersion {
+ info := new(SchemaInfoWithVersion)
+ info.SchemaInfo = ConvertGetSchemaResponseToSchemaInfo(tn, response)
+ info.Version = response.Version
+ return info
+}
diff --git a/pulsaradmin/pkg/utils/sink_config.go b/pulsaradmin/pkg/utils/sink_config.go
new file mode 100644
index 000000000..0e9163bf9
--- /dev/null
+++ b/pulsaradmin/pkg/utils/sink_config.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type SinkConfig struct {
+ TopicsPattern *string `json:"topicsPattern,omitempty" yaml:"topicsPattern"`
+ Resources *Resources `json:"resources,omitempty" yaml:"resources"`
+ TimeoutMs *int64 `json:"timeoutMs,omitempty" yaml:"timeoutMs"`
+
+ // Whether the subscriptions the functions created/used should be deleted when the functions is deleted
+ CleanupSubscription bool `json:"cleanupSubscription" yaml:"cleanupSubscription"`
+
+ RetainOrdering bool `json:"retainOrdering" yaml:"retainOrdering"`
+ RetainKeyOrdering bool `json:"retainKeyOrdering" yaml:"retainKeyOrdering"`
+ AutoAck bool `json:"autoAck" yaml:"autoAck"`
+ Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"`
+ Tenant string `json:"tenant,omitempty" yaml:"tenant"`
+ Namespace string `json:"namespace,omitempty" yaml:"namespace"`
+ Name string `json:"name,omitempty" yaml:"name"`
+ ClassName string `json:"className,omitempty" yaml:"className"`
+
+ SinkType string `json:"sinkType,omitempty" yaml:"sinkType"`
+ Archive string `json:"archive,omitempty" yaml:"archive"`
+ ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"`
+ SourceSubscriptionName string `json:"sourceSubscriptionName,omitempty" yaml:"sourceSubscriptionName"`
+ SourceSubscriptionPosition string `json:"sourceSubscriptionPosition,omitempty" yaml:"sourceSubscriptionPosition"`
+ RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"`
+
+ Inputs []string `json:"inputs,omitempty" yaml:"inputs"`
+ TopicToSerdeClassName map[string]string `json:"topicToSerdeClassName,omitempty" yaml:"topicToSerdeClassName"`
+ TopicToSchemaType map[string]string `json:"topicToSchemaType,omitempty" yaml:"topicToSchemaType"`
+ InputSpecs map[string]ConsumerConfig `json:"inputSpecs,omitempty" yaml:"inputSpecs"`
+ Configs map[string]interface{} `json:"configs,omitempty" yaml:"configs"`
+
+ TopicToSchemaProperties map[string]string `json:"topicToSchemaProperties,omitempty" yaml:"topicToSchemaProperties"`
+
+ CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"`
+
+ // This is a map of secretName(aka how the secret is going to be
+ // accessed in the function via context) to an object that
+ // encapsulates how the secret is fetched by the underlying
+ // secrets provider. The type of an value here can be found by the
+ // SecretProviderConfigurator.getSecretObjectType() method.
+ Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"`
+
+ MaxMessageRetries int `json:"maxMessageRetries,omitempty" yaml:"maxMessageRetries"`
+ DeadLetterTopic string `json:"deadLetterTopic,omitempty" yaml:"deadLetterTopic"`
+ NegativeAckRedeliveryDelayMs int64 `json:"negativeAckRedeliveryDelayMs,omitempty" yaml:"negativeAckRedeliveryDelayMs"`
+ TransformFunction string `json:"transformFunction,omitempty" yaml:"transformFunction"`
+ TransformFunctionClassName string `json:"transformFunctionClassName,omitempty" yaml:"transformFunctionClassName"`
+ TransformFunctionConfig string `json:"transformFunctionConfig,omitempty" yaml:"transformFunctionConfig"`
+}
diff --git a/pulsaradmin/pkg/utils/sink_status.go b/pulsaradmin/pkg/utils/sink_status.go
new file mode 100644
index 000000000..6cdb091fa
--- /dev/null
+++ b/pulsaradmin/pkg/utils/sink_status.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type SinkStatus struct {
+ // The total number of sink instances that ought to be running
+ NumInstances int `json:"numInstances"`
+
+ // The number of source instances that are actually running
+ NumRunning int `json:"numRunning"`
+
+ Instances []*SinkInstanceStatus `json:"instances"`
+}
+
+type SinkInstanceStatus struct {
+ InstanceID int `json:"instanceId"`
+ Status SourceInstanceStatusData `json:"status"`
+}
+
+type SinkInstanceStatusData struct {
+ // Is this instance running?
+ Running bool `json:"running"`
+
+ // Do we have any error while running this instance
+ Err string `json:"error"`
+
+ // Number of times this instance has restarted
+ NumRestarts int64 `json:"numRestarts"`
+
+ // Number of messages read from Pulsar
+ NumReadFromPulsar int64 `json:"numReadFromPulsar"`
+
+ // Number of times there was a system exception handling messages
+ NumSystemExceptions int64 `json:"numSystemExceptions"`
+
+ // A list of the most recent system exceptions
+ LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"`
+
+ // Number of times there was a sink exception
+ NumSinkExceptions int64 `json:"numSinkExceptions"`
+
+ // A list of the most recent sink exceptions
+ LatestSinkExceptions []ExceptionInformation `json:"latestSinkExceptions"`
+
+ // Number of messages written to sink
+ NumWrittenToSink int64 `json:"numWrittenToSink"`
+
+ // When was the last time we received a Message from Pulsar
+ LastReceivedTime int64 `json:"lastReceivedTime"`
+
+ WorkerID string `json:"workerId"`
+}
diff --git a/pulsaradmin/pkg/utils/source_config.go b/pulsaradmin/pkg/utils/source_config.go
new file mode 100644
index 000000000..7b0747610
--- /dev/null
+++ b/pulsaradmin/pkg/utils/source_config.go
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type SourceConfig struct {
+ Tenant string `json:"tenant,omitempty" yaml:"tenant"`
+ Namespace string `json:"namespace,omitempty" yaml:"namespace"`
+ Name string `json:"name,omitempty" yaml:"name"`
+ ClassName string `json:"className,omitempty" yaml:"className"`
+
+ ProducerConfig *ProducerConfig `json:"producerConfig,omitempty" yaml:"producerConfig"`
+
+ TopicName string `json:"topicName,omitempty" yaml:"topicName"`
+ SerdeClassName string `json:"serdeClassName,omitempty" yaml:"serdeClassName"`
+ SchemaType string `json:"schemaType,omitempty" yaml:"schemaType"`
+
+ Configs map[string]interface{} `json:"configs,omitempty" yaml:"configs"`
+
+ // This is a map of secretName(aka how the secret is going to be
+ // accessed in the function via context) to an object that
+ // encapsulates how the secret is fetched by the underlying
+ // secrets provider. The type of an value here can be found by the
+ // SecretProviderConfigurator.getSecretObjectType() method.
+ Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"`
+
+ Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"`
+ ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"`
+ Resources *Resources `json:"resources,omitempty" yaml:"resources"`
+ Archive string `json:"archive,omitempty" yaml:"archive"`
+ // Any flags that you want to pass to the runtime.
+ RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"`
+
+ CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"`
+
+ BatchSourceConfig *BatchSourceConfig `json:"batchSourceConfig,omitempty" yaml:"batchSourceConfig"`
+ BatchBuilder string `json:"batchBuilder,omitempty" yaml:"batchBuilder"`
+}
diff --git a/pulsaradmin/pkg/utils/source_status.go b/pulsaradmin/pkg/utils/source_status.go
new file mode 100644
index 000000000..71df5a4fa
--- /dev/null
+++ b/pulsaradmin/pkg/utils/source_status.go
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type SourceStatus struct {
+ NumInstances int `json:"numInstances"`
+ NumRunning int `json:"numRunning"`
+ Instances []*SourceInstanceStatus `json:"instances"`
+}
+
+type SourceInstanceStatus struct {
+ InstanceID int `json:"instanceId"`
+ Status SourceInstanceStatusData `json:"status"`
+}
+
+type SourceInstanceStatusData struct {
+ Running bool `json:"running"`
+ Err string `json:"error"`
+ NumRestarts int64 `json:"numRestarts"`
+ NumReceivedFromSource int64 `json:"numReceivedFromSource"`
+ NumSystemExceptions int64 `json:"numSystemExceptions"`
+ LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"`
+ NumSourceExceptions int64 `json:"numSourceExceptions"`
+ LatestSourceExceptions []ExceptionInformation `json:"latestSourceExceptions"`
+ NumWritten int64 `json:"numWritten"`
+ LastReceivedTime int64 `json:"lastReceivedTime"`
+ WorkerID string `json:"workerId"`
+}
diff --git a/pulsaradmin/pkg/utils/subscription_auth_mode.go b/pulsaradmin/pkg/utils/subscription_auth_mode.go
new file mode 100644
index 000000000..795b6d0ae
--- /dev/null
+++ b/pulsaradmin/pkg/utils/subscription_auth_mode.go
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import "github.com/pkg/errors"
+
+type SubscriptionAuthMode string
+
+const (
+ None SubscriptionAuthMode = "None"
+ Prefix SubscriptionAuthMode = "Prefix"
+)
+
+func ParseSubscriptionAuthMode(s string) (SubscriptionAuthMode, error) {
+ switch s {
+ case "None":
+ return None, nil
+ case "Prefix":
+ return Prefix, nil
+ default:
+ return "", errors.New("Invalid subscription auth mode")
+ }
+}
+
+func (s SubscriptionAuthMode) String() string {
+ return string(s)
+}
diff --git a/pulsaradmin/pkg/utils/topic_auto_creation_config.go b/pulsaradmin/pkg/utils/topic_auto_creation_config.go
new file mode 100644
index 000000000..666465597
--- /dev/null
+++ b/pulsaradmin/pkg/utils/topic_auto_creation_config.go
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+type TopicAutoCreationConfig struct {
+ Allow bool `json:"allowAutoTopicCreation"`
+ Type TopicType `json:"topicType"`
+ Partitions int `json:"defaultNumPartitions"`
+}
diff --git a/pulsaradmin/pkg/utils/topic_domain.go b/pulsaradmin/pkg/utils/topic_domain.go
new file mode 100644
index 000000000..98c59a9ce
--- /dev/null
+++ b/pulsaradmin/pkg/utils/topic_domain.go
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import "github.com/pkg/errors"
+
+type TopicDomain string
+
+const (
+ persistent TopicDomain = "persistent"
+ nonPersistent TopicDomain = "non-persistent"
+)
+
+func ParseTopicDomain(domain string) (TopicDomain, error) {
+ switch domain {
+ case "persistent":
+ return persistent, nil
+ case "non-persistent":
+ return nonPersistent, nil
+ default:
+ return "", errors.Errorf("The domain only can be specified as 'persistent' or "+
+ "'non-persistent'. Input domain is '%s'.", domain)
+ }
+}
+
+func (t TopicDomain) String() string {
+ return string(t)
+}
diff --git a/pulsaradmin/pkg/utils/topic_name.go b/pulsaradmin/pkg/utils/topic_name.go
new file mode 100644
index 000000000..268abd73d
--- /dev/null
+++ b/pulsaradmin/pkg/utils/topic_name.go
@@ -0,0 +1,155 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package utils
+
+import (
+ "fmt"
+ "net/url"
+ "strconv"
+ "strings"
+
+ "github.com/pkg/errors"
+)
+
+const (
+ PUBLICTENANT = "public"
+ DEFAULTNAMESPACE = "default"
+ PARTITIONEDTOPICSUFFIX = "-partition-"
+)
+
+type TopicName struct {
+ domain TopicDomain
+ tenant string
+ namespace string
+ topic string
+ partitionIndex int
+
+ namespaceName *NameSpaceName
+}
+
+// The topic name can be in two different forms, one is fully qualified topic name,
+// the other one is short topic name
+func GetTopicName(completeName string) (*TopicName, error) {
+ var topicName TopicName
+ // The short topic name can be:
+ // -