Merge branch 'master' into broker_client_tls

apache · Feb 16, 2020 · 6c3c06b · 6c3c06b
2 parents a90d03a + 5a8f420
commit 6c3c06b
Show file tree

Hide file tree

Showing 100 changed files with 1,576 additions and 888 deletions.
diff --git a/.github/workflows/ci-pulsarbot.yaml b/.github/workflows/ci-pulsarbot.yaml
@@ -35,5 +35,5 @@ jobs:
       - name: Execute pulsarbot command
         id:   pulsarbot
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.PULSARBOT_TOKEN }}
         uses: apache/pulsar-test-infra/pulsarbot@master
diff --git a/conf/broker.conf b/conf/broker.conf
@@ -442,6 +442,11 @@ saslJaasClientAllowedIds=
 # Default value `SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME`, which is "Broker".
 saslJaasBrokerSectionName=
 
+### --- HTTP Server config --- ###
+
+# If >0, it will reject all HTTP requests with bodies larged than the configured limit
+httpMaxRequestSize=-1
+
 ### --- BookKeeper Client --- ###
 
 # Authentication plugin to use when connecting to bookies
@@ -533,7 +538,7 @@ bookkeeperDiskWeightBasedPlacementEnabled=false
 
 # Set the interval to check the need for sending an explicit LAC
 # A value of '0' disables sending any explicit LACs. Default is 0.
-bookkeeperExplicitLacIntervalInMills=0;
+bookkeeperExplicitLacIntervalInMills=0
 
 ### --- Managed Ledger --- ###
 
@@ -700,6 +705,14 @@ loadBalancerOverrideBrokerNicSpeedGbps=
 # Name of load manager to use
 loadManagerClassName=org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl
 
+# Supported algorithms name for namespace bundle split.
+# "range_equally_divide" divides the bundle into two parts with the same hash range size.
+# "topic_count_equally_divide" divides the bundle into two parts with the same topics count.
+supportedNamespaceBundleSplitAlgorithms=[range_equally_divide,topic_count_equally_divide]
+
+# Default algorithm name for namespace bundle split
+defaultNamespaceBundleSplitAlgorithm=range_equally_divide
+
 ### --- Replication --- ###
 
 # Enable replication metrics

diff --git a/conf/standalone.conf b/conf/standalone.conf
@@ -336,7 +336,7 @@ bookkeeperDiskWeightBasedPlacementEnabled=false
 
 # Set the interval to check the need for sending an explicit LAC
 # A value of '0' disables sending any explicit LACs. Default is 0.
-bookkeeperExplicitLacIntervalInMills=0;
+bookkeeperExplicitLacIntervalInMills=0
 
 ### --- Managed Ledger --- ###
 

diff --git a/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml b/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml
@@ -39,4 +39,9 @@ data:
   clusterName: {{ template "pulsar.fullname" . }}
   functionsWorkerEnabled: "true"
   PF_pulsarFunctionsCluster: {{ template "pulsar.fullname" . }}
-{{ toYaml .Values.broker.configData | indent 2 }}
+  {{- if .Values.extra.functionsAsPods }}
+  PF_functionRuntimeFactoryClassName: "org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory"
+  PF_functionRuntimeFactoryConfigs_submittingInsidePod: "true"
+  PF_functionRuntimeFactoryConfigs_jobNamespace: {{ .Values.namespace }}
+  {{- end }}
+  {{ toYaml .Values.broker.configData | indent 2 }}
diff --git a/deployment/kubernetes/helm/pulsar/templates/broker-deployment.yaml b/deployment/kubernetes/helm/pulsar/templates/broker-deployment.yaml
@@ -55,6 +55,9 @@ spec:
       tolerations:
 {{ toYaml .Values.broker.tolerations | indent 8 }}
     {{- end }}
+      {{- if .Values.extra.functionsAsPods }}
+      serviceAccount: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.functions.component }}"
+      {{- end }}
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:

diff --git a/deployment/kubernetes/helm/pulsar/templates/broker-rbac.yaml b/deployment/kubernetes/helm/pulsar/templates/broker-rbac.yaml
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+  {{- if .Values.extra.functionsAsPods }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.functions.component }}"
+rules:
+- apiGroups: [""]
+  resources:
+  - services
+  - configmaps
+  - pods
+  verbs:
+  - '*'
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  verbs:
+  - '*'
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.functions.component }}"
+  namespace: {{ .Values.namespace }}
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.functions.component }}"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.functions.component }}"
+subjects:
+- kind: ServiceAccount
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.broker.functions.component }}"
+  namespace: {{ .Values.namespace }}
+  {{- end }}
diff --git a/deployment/kubernetes/helm/pulsar/templates/grafana-service.yaml b/deployment/kubernetes/helm/pulsar/templates/grafana-service.yaml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-  {{- if .Values.extra.monitoring }}
+{{- if .Values.extra.monitoring }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -31,14 +31,14 @@ metadata:
     component: {{ .Values.grafana.component }}
     cluster: {{ template "pulsar.fullname" . }}
   annotations:
-  {{ toYaml .Values.grafana.service.annotations | indent 4 }}
+{{ toYaml .Values.grafana.service.annotations | indent 4 }}
 spec:
   ports:
-  {{ toYaml .Values.grafana.service.ports | indent 2 }}
+{{ toYaml .Values.grafana.service.ports | indent 2 }}
   selector:
     app: {{ template "pulsar.name" . }}
     release: {{ .Release.Name }}
     component: {{ .Values.grafana.component }}
   type: ClusterIP
   sessionAffinity: None
-  {{- end }}
+{{- end }}
diff --git a/deployment/kubernetes/helm/pulsar/templates/pulsar-manager-configmap.yaml b/deployment/kubernetes/helm/pulsar/templates/pulsar-manager-configmap.yaml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}"
+  namespace: {{ .Values.namespace }}
+  labels:
+    app: {{ template "pulsar.name" . }}
+    chart: {{ template "pulsar.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+    component: {{ .Values.pulsar_manager.component }}
+    cluster: {{ template "pulsar.fullname" . }}
+data:
+{{ toYaml .Values.pulsar_manager.configData | indent 2 }}
diff --git a/deployment/kubernetes/helm/pulsar/templates/pulsar-manager-deployment.yaml b/deployment/kubernetes/helm/pulsar/templates/pulsar-manager-deployment.yaml
@@ -69,18 +69,12 @@ spec:
           volumeMounts:
           - name: pulsar-manager-data
             mountPath: /data
+          envFrom:
+          - configMapRef:
+              name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}"
           env:
-          # for supporting apachepulsar/pulsar-manager
           - name: PULSAR_CLUSTER
             value: {{ template "pulsar.fullname" . }}
-          - name: REDIRECT_HOST
-            value: http://127.0.0.1
-          - name: REDIRECT_PORT
-            value: "9527"
-          - name: DRIVER_CLASS_NAME
-            value: org.postgresql.Driver
-          - name: URL
-            value: jdbc:postgresql://127.0.0.1:5432/pulsar_manager
           - name: USERNAME
             valueFrom:
               secretKeyRef:
@@ -91,8 +85,6 @@ spec:
               secretKeyRef:
                 key: PULSAR_MANAGER_ADMIN_PASSWORD
                 name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_manager.component }}-secret"
-          - name: LOG_LEVEL
-            value: DEBUG
       volumes:
         - name: pulsar-manager-data
           emptyDir: {}

diff --git a/deployment/kubernetes/helm/pulsar/values-mini.yaml b/deployment/kubernetes/helm/pulsar/values-mini.yaml
@@ -49,6 +49,8 @@ extra:
   bastion: yes
   # Monitoring stack (prometheus and grafana)
   monitoring: yes
+  # Configure Kubernetes runtime for Functions
+  functionsAsPods: no
 
 ## Which pulsar image to use
 image:
@@ -245,6 +247,11 @@ broker:
   pdb:
     usePolicy: yes
     maxUnavailable: 1
+  ## Broker rbac
+  ## templates/broker-rbac.yaml
+  ##
+  functions:
+    component: functions-worker
 
 ## Pulsar Extra: Proxy
 ## templates/proxy-deployment.yaml
@@ -497,6 +504,15 @@ pulsar_manager:
     requests:
       memory: 250Mi
       cpu: 0.1
+  configData:
+    REDIRECT_HOST: "http://127.0.0.1"
+    REDIRECT_PORT: "9527"
+    DRIVER_CLASS_NAME: org.postgresql.Driver
+    URL: jdbc:postgresql://127.0.0.1:5432/pulsar_manager
+    LOG_LEVEL: DEBUG
+    ## If you enabled authentication support
+    #JWT_TOKEN: <token>
+    #SECRET_KEY: data:base64,<secret key>
   ## Pulsar manager service
   ## templates/pulsar-manager-service.yaml
   ##

diff --git a/deployment/kubernetes/helm/pulsar/values.yaml b/deployment/kubernetes/helm/pulsar/values.yaml
@@ -49,6 +49,8 @@ extra:
   bastion: yes
   # Monitoring stack (prometheus and grafana)
   monitoring: yes
+  # Configure Kubernetes runtime for Functions
+  functionsAsPods: no
 
 ## Which pulsar image to use
 image:
@@ -247,6 +249,11 @@ broker:
   pdb:
     usePolicy: yes
     maxUnavailable: 1
+  ## Broker rbac
+  ## templates/broker-rbac.yaml
+  ##
+  functions:
+    component: functions-worker
 
 ## Pulsar Extra: Proxy
 ## templates/proxy-deployment.yaml
@@ -493,6 +500,15 @@ pulsar_manager:
     requests:
       memory: 250Mi
       cpu: 0.1
+  configData:
+    REDIRECT_HOST: "http://127.0.0.1"
+    REDIRECT_PORT: "9527"
+    DRIVER_CLASS_NAME: org.postgresql.Driver
+    URL: jdbc:postgresql://127.0.0.1:5432/pulsar_manager
+    LOG_LEVEL: DEBUG
+    ## If you enabled authentication support
+    #JWT_TOKEN: <token>
+    #SECRET_KEY: data:base64,<secret key>
   ## Pulsar manager service
   ## templates/pulsar-manager-service.yaml
   ##

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -497,7 +497,7 @@ The Apache Software License, Version 2.0
     - io.vertx-vertx-core-3.4.1.jar
     - io.vertx-vertx-web-3.4.1.jar
   * Apache ZooKeeper
-    - org.apache.zookeeper-zookeeper-jute-3.5.6.jar
+    - org.apache.zookeeper-zookeeper-jute-3.5.7.jar
 
 BSD 3-clause "New" or "Revised" License
  * Google auth library

diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
@@ -60,6 +60,12 @@
       <artifactId>pulsar-common</artifactId>
       <version>${project.version}</version>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.pulsar</groupId>
+      <artifactId>pulsar-metadata</artifactId>
+      <version>${project.version}</version>
+    </dependency>
 
     <dependency>
       <groupId>com.google.guava</groupId>

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerException.java
@@ -39,21 +39,33 @@ public static ManagedLedgerException getManagedLedgerException(Throwable e) {
     }
 
     public static class MetaStoreException extends ManagedLedgerException {
-        public MetaStoreException(Exception e) {
-            super(e);
+        public MetaStoreException(Throwable t) {
+            super(t);
+        }
+
+        public MetaStoreException(String msg) {
+            super(msg);
         }
     }
 
     public static class BadVersionException extends MetaStoreException {
         public BadVersionException(Exception e) {
             super(e);
         }
+
+        public BadVersionException(String msg) {
+            super(msg);
+        }
     }
 
     public static class MetadataNotFoundException extends MetaStoreException {
         public MetadataNotFoundException(Exception e) {
             super(e);
         }
+
+        public MetadataNotFoundException(String msg) {
+            super(msg);
+        }
     }
 
 

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerInfo.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerInfo.java
@@ -24,7 +24,7 @@
 @SuppressWarnings("checkstyle:javadoctype")
 public class ManagedLedgerInfo {
     /** Z-Node version. */
-    public int version;
+    public long version;
     public String creationDate;
     public String modificationDate;
 
@@ -42,7 +42,7 @@ public static class LedgerInfo {
 
     public static class CursorInfo {
         /** Z-Node version. */
-        public int version;
+        public long version;
         public String creationDate;
         public String modificationDate;
 

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -83,7 +83,6 @@
 import org.apache.bookkeeper.mledger.Position;
 import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound;
 import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
-import org.apache.bookkeeper.mledger.impl.MetaStore.Stat;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
@@ -93,6 +92,7 @@
 import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet;
 import org.apache.pulsar.common.util.collections.LongPairRangeSet;
 import org.apache.pulsar.common.util.collections.LongPairRangeSet.LongPairConsumer;
+import org.apache.pulsar.metadata.api.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;