Failure to install with default values, pso-db pods missing

[grandich]

On a healthy cluster with prereqs fulfilled and default values used, pso-db-* pods don't get created, even though pso-db-cockroach-operator doesn't log errors apparently.

Any help will be appreciated.

$ kubectl version
Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.9", GitCommit:"9dd794e454ac32d97cde41ae10be801ae98f75df", GitTreeState:"clean", BuildDate:"2021-03-18T01:09:28Z", GoVersion:"go1.15.8", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.9", GitCommit:"9dd794e454ac32d97cde41ae10be801ae98f75df", GitTreeState:"clean", BuildDate:"2021-03-18T01:00:06Z", GoVersion:"go1.15.8", Compiler:"gc", Platform:"linux/amd64"}

$ helm ls -n cloud-system
pure-pso                cloud-system    1               2021-10-21 17:18:17.102638878 -0400 EDT deployed        pure-pso-6.2.0                  6.2.0

$ helm get values --namespace=cloud-system pure-pso
USER-SUPPLIED VALUES:
DBTopology:
  enforce: false
affinity: {}
app:
  debug: false
arrays:
  FlashArrays:
  - APIToken: xxxxxxxxxx
    MgmtEndPoint: xxxxxx
  FlashBlades: []
clusterID: xxxxxxxxx
clusterrolebinding:
  serviceAccount:
    name: pure
controllerServer:
  affinity: {}
  nodeSelector: {}
  tolerations: []
database:
  affinity: {}
  maxStartupSeconds: 600
  maxSuspectSeconds: 3600
  nodeSelector: {}
  resources:
    limits:
      memory: 1Gi
  tolerations: []
flasharray:
  defaultFSOpt: -q
  defaultFSType: xfs
  defaultMountOpt:
  - discard
  iSCSIAllowedCIDR: ""
  iSCSILoginTimeout: 20
  preemptAttachments: "true"
  sanType: ISCSI
flashblade:
  exportRules: '*(rw,no_root_squash)'
  snapshotDirectoryEnabled: "false"
images:
  csi:
    attacher:
      name: quay.io/k8scsi/csi-attacher
      pullPolicy: Always
    livenessProbe:
      name: quay.io/k8scsi/livenessprobe
      pullPolicy: Always
    nodeDriverRegistrar:
      name: quay.io/k8scsi/csi-node-driver-registrar
      pullPolicy: Always
    provisioner:
      name: quay.io/k8scsi/csi-provisioner
      pullPolicy: Always
    resizer:
      name: quay.io/k8scsi/csi-resizer
      pullPolicy: Always
    snapshotter:
      name: quay.io/k8scsi/csi-snapshotter
      pullPolicy: Always
  database:
    cockroachOperator:
      name: purestorage/cockroach-operator
      pullPolicy: Always
      tag: v1.2.0
    cockroachdb:
      name: cockroachdb/cockroach
      tag: v20.2.6
    deployer:
      name: purestorage/dbdeployer
      pullPolicy: Always
      tag: v1.2.0
    psctl:
      name: purestorage/psctl
      tag: v1.1.0
  plugin:
    name: purestorage/k8s
    pullPolicy: Always
    tag: v6.2.0
nodeSelector: {}
nodeServer:
  affinity: {}
  nodeSelector: {}
  tolerations: []
orchestrator:
  basePath: /var/lib/kubelet
  name: k8s
storagetopology:
  enable: false
  strictTopology: false
tolerations: []
upgrade: false

$ kubectl get pod -n cloud-system | grep pso
pso-csi-controller-0                                     5/6     Running   0          3m26s
pso-csi-node-4rtll                                       2/3     Running   0          3m26s
pso-csi-node-87mgj                                       2/3     Running   0          3m26s
pso-csi-node-9mrzs                                       2/3     Running   0          3m26s
pso-csi-node-pgkz9                                       2/3     Running   0          3m26s
pso-csi-node-wr24z                                       2/3     Running   0          3m26s
pso-db-cockroach-operator-7544d54f88-rddx2               1/1     Running   0          3m26s
pso-db-deployer-6748ddc94-zrclh                          1/1     Running   0          3m26s

$ kubectl logs -n cloud-system pso-db-cockroach-operator-7544d54f88-rddx2
{"level":"info","ts":1634851114.5586638,"logger":"cmd","msg":"","is_debug_mode":false}
{"level":"info","ts":1634851114.5587416,"logger":"cmd","msg":"Go Version: go1.15.12"}
{"level":"info","ts":1634851114.5587597,"logger":"cmd","msg":"Go OS/Arch: linux/amd64"}
{"level":"info","ts":1634851114.558776,"logger":"cmd","msg":"Version of operator-sdk: v0.6.0"}
{"level":"info","ts":1634851114.5588267,"logger":"cmd","msg":"Setting max startup duration to 600 seconds"}
{"level":"info","ts":1634851114.5588372,"logger":"cmd","msg":"Setting max suspect duration to 3600 seconds"}
{"level":"info","ts":1634851114.5588984,"logger":"cmd","msg":"Did not find config file for affinity"}
{"level":"info","ts":1634851114.5589225,"logger":"cmd","msg":"Successfully parsed config file","file":"affinity","value":"&Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:nil,}"}
{"level":"info","ts":1634851114.5635457,"logger":"cmd","msg":"Did not find config file for tolerations"}
{"level":"info","ts":1634851114.5637438,"logger":"cmd","msg":"Successfully parsed config file","file":"tolerations","value":null}
{"level":"info","ts":1634851114.564393,"logger":"cmd","msg":"Successfully parsed config file","file":"resources","value":"&ResourceRequirements{Limits:ResourceList{memory: {{1073741824 0} {<nil>} 1Gi BinarySI},},Requests:ResourceList{},}"}
{"level":"info","ts":1634851114.5646725,"logger":"cmd","msg":"Did not find config file for nodeSelector"}
{"level":"info","ts":1634851114.5646825,"logger":"cmd","msg":"Successfully parsed config file","file":"nodeSelector","value":{}}
{"level":"info","ts":1634851114.565181,"logger":"leader","msg":"Trying to become the leader."}
{"level":"info","ts":1634851115.1553597,"logger":"leader","msg":"No pre-existing lock was found."}
{"level":"info","ts":1634851115.1731136,"logger":"leader","msg":"Became the leader."}
{"level":"info","ts":1634851115.4093795,"logger":"cmd","msg":"Registering Components."}
{"level":"info","ts":1634851115.4163942,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1634851115.4169033,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1634851115.4232342,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1634851115.424368,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1634851115.42497,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1634851115.4254565,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"intrusion-controller","source":"kind source: /, Kind="}
time="2021-10-21T21:18:35Z" level=info msg="Successfully started watches." reconciler_name=controller_intrusion
{"level":"info","ts":1634851115.8073902,"logger":"metrics","msg":"Metrics Service object created","Service.Name":"cockroach-operator","Service.Namespace":"cloud-system"}
{"level":"info","ts":1634851115.8074362,"logger":"cmd","msg":"Starting the Cmd."}
{"level":"info","ts":1634851116.115943,"logger":"kubebuilder.controller","msg":"Starting Controller","controller":"intrusion-controller"}
{"level":"info","ts":1634851116.2169046,"logger":"kubebuilder.controller","msg":"Starting workers","controller":"intrusion-controller","worker count":1}

$ kubectl get intrusion --all-namespaces
No resources found

$ kubectl logs -n cloud-system pso-db-deployer-6748ddc94-zrclh 
time="2021-10-21T21:18:35Z" level=info is_debug_mode=false name=dbdeployer_cmd
time="2021-10-21T21:18:35Z" level=info msg="version info" OS=linux arch=amd64 go=go1.15.12 name=dbdeployer_cmd
{"level":"info","ts":1634851115.0486882,"logger":"leader","msg":"Trying to become the leader."}
{"level":"info","ts":1634851115.4552867,"logger":"leader","msg":"No pre-existing lock was found."}
{"level":"info","ts":1634851115.4713528,"logger":"leader","msg":"Became the leader."}
time="2021-10-21T21:18:35Z" level=info msg="Registering Components." name=dbdeployer_cmd
{"level":"info","ts":1634851115.8281531,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"dbdeployer-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1634851115.8284357,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"dbdeployer-controller","source":"kind source: /, Kind="}
time="2021-10-21T21:18:35Z" level=info msg="Starting the Cmd." name=dbdeployer_cmd
{"level":"info","ts":1634851116.0308127,"logger":"kubebuilder.controller","msg":"Starting Controller","controller":"dbdeployer-controller"}
{"level":"info","ts":1634851116.1313999,"logger":"kubebuilder.controller","msg":"Starting workers","controller":"dbdeployer-controller","worker count":1}
time="2021-10-21T21:18:36Z" level=info msg="--------------------- Begin reconcile ---------------------" reconciler_name=dbdeployer_controller request=cloud-system/sh.helm.release.v1.topolvm.v2
time="2021-10-21T21:18:36Z" level=info msg="--------------------- Begin reconcile ---------------------" reconciler_name=dbdeployer_controller request=cloud-system/sh.helm.release.v1.topolvm.v5
time="2021-10-21T21:18:36Z" level=info msg="--------------------- Begin reconcile ---------------------" reconciler_name=dbdeployer_controller request=cloud-system/pure-provisioner-secret
time="2021-10-21T21:18:36Z" level=info msg="Discovery json validated successfully"
time="2021-10-21T21:18:36Z" level=warning msg="could not get intrusion" error="Intrusion.pso.purestorage.com \"pso-db\" not found" reconciler_name=dbdeployer_controller
time="2021-10-21T21:18:36Z" level=warning msg="could not get intrusion" error="Intrusion.pso.purestorage.com \"pso-db\" not found" reconciler_name=dbdeployer_controller
time="2021-10-21T21:18:36Z" level=error msg="unable to create initial intrusion: Intrusion.pso.purestorage.com \"pso-db\" is invalid: [status.underreplicatedRanges: Invalid value: \"null\": status.underreplicatedRanges in body must be of type integer: \"null\", status.asOf: Invalid value: \"null\": status.asOf in body must be of type string: \"null\", status.initialized: Invalid value: \"null\": status.initialized in body must be of type boolean: \"null\", status.readyNodes: Invalid value: \"null\": status.readyNodes in body must be of type string: \"null\", status.replicas: Invalid value: \"null\": status.replicas in body must be of type array: \"null\", status.status: Invalid value: \"null\": status.status in body must be of type string: \"null\", status.totalRanges: Invalid value: \"null\": status.totalRanges in body must be of type integer: \"null\", status.unavailableRanges: Invalid value: \"null\": status.unavailableRanges in body must be of type integer: \"null\"]" reconciler_name=dbdeployer_controller
time="2021-10-21T21:18:36Z" level=error msg="Failed applying replicas" error="Intrusion.pso.purestorage.com \"pso-db\" is invalid: [status.underreplicatedRanges: Invalid value: \"null\": status.underreplicatedRanges in body must be of type integer: \"null\", status.asOf: Invalid value: \"null\": status.asOf in body must be of type string: \"null\", status.initialized: Invalid value: \"null\": status.initialized in body must be of type boolean: \"null\", status.readyNodes: Invalid value: \"null\": status.readyNodes in body must be of type string: \"null\", status.replicas: Invalid value: \"null\": status.replicas in body must be of type array: \"null\", status.status: Invalid value: \"null\": status.status in body must be of type string: \"null\", status.totalRanges: Invalid value: \"null\": status.totalRanges in body must be of type integer: \"null\", status.unavailableRanges: Invalid value: \"null\": status.unavailableRanges in body must be of type integer: \"null\"]"