@johscheuer after upgrade the cluster is failed to connect, and all the connectionstring are old. But if I delete foundationdbcluster and recreate a new one in the same environment with the same yaml file, the new created foundationdbcluster can be connected. So I think it should be foundationdb cluster upgrade issue, not operator upgrade issue. The following is the pure-install output foundationdbcluster yaml file:
...
apiVersion: apps.foundationdb.org/v1beta2
kind: FoundationDBCluster
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"foundationdb.opencontent.ibm.com/v1","kind":"FdbCluster","metadata":{"annotations":{},"name":"mdm-foundationdb-ibm","namespace":"testoperator1"},"spec":{"backup_agents_config":{"deploymentSa":"fdb-controller-manager","pvcCapacity":"10G","pvcStorageClass":""},"backup_agents_template":{"metadata":{"labels":{"backup-custom-label":"backup-custom-value"}},"spec":{"containers":[{"env":[{"name":"FDB_TLS_CERTIFICATE_FILE","value":"/var/fdb-certs/tls.crt"},{"name":"FDB_TLS_KEY_FILE","value":"/var/fdb-certs/tls.key"},{"name":"FDB_TLS_CA_FILE","value":"/var/fdb-certs/ca.crt"}],"imagePullPolicy":"Always","name":"backup-agent","resources":{"limits":{"cpu":"500m","ephemeral-storage":"150Mi","memory":"1024Mi"},"requests":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"1024Mi"}},"volumeMounts":[{"mountPath":"/var/fdb-certs","name":"fdb-certs"}]}],"serviceAccountName":"ibm-fdb-controller-manager","volumes":[{"name":"fdb-certs","secret":{"secretName":"internal-tls"}}]}},"foundationdb_cluster_spec":{"databaseConfiguration":null,"mainContainer":{"enableTls":true},"processCounts":{"proxy":2,"stateless":1},"processes":{"general":{"allowTagOverride":true,"podTemplate":{"metadata":{"labels":{"customlabel":"customvalue"}},"spec":{"containers":[{"env":[{"name":"FDB_TLS_CERTIFICATE_FILE","value":"/var/fdb-certs/tls.crt"},{"name":"FDB_TLS_KEY_FILE","value":"/var/fdb-certs/tls.key"},{"name":"FDB_TLS_CA_FILE","value":"/var/fdb-certs/ca.crt"}],"imagePullPolicy":"Always","livenessProbe":{"exec":{"command":["/bin/sh","-c","ps -ef | grep fdbmonitor"]},"initialDelaySeconds":60,"periodSeconds":30},"name":"foundationdb","resources":{"requests":{"cpu":"150m","memory":"256Mi"}},"volumeMounts":[{"mountPath":"/var/fdb-certs","name":"fdb-certs"}]},{"env":[{"name":"FDB_TLS_CERTIFICATE_FILE","value":"/var/fdb-certs/tls.crt"},{"name":"FDB_TLS_KEY_FILE","value":"/var/fdb-certs/tls.key"},{"name":"FDB_TLS_CA_FILE","value":"/var/fdb-certs/ca.crt"}],"imagePullPolicy":"Always","name":"foundationdb-kubernetes-sidecar","resources":{"requests":{"cpu":"150m","memory":"256Mi"}},"volumeMounts":[{"mountPath":"/var/fdb-certs","name":"fdb-certs"}]}],"initContainers":[{"env":[{"name":"FDB_TLS_CERTIFICATE_FILE","value":"/var/fdb-certs/tls.crt"},{"name":"FDB_TLS_KEY_FILE","value":"/var/fdb-certs/tls.key"},{"name":"FDB_TLS_CA_FILE","value":"/var/fdb-certs/ca.crt"}],"imagePullPolicy":"Always","name":"foundationdb-kubernetes-init","resources":{"requests":{"cpu":"150m","memory":"256Mi"}},"volumeMounts":[{"mountPath":"/var/fdb-certs","name":"fdb-certs"}]}],"securityContext":{"allowPrivilegeEscalation":false,"privileged":false,"readOnlyRootFilesystem":true},"volumes":[{"name":"fdb-certs","secret":{"secretName":"internal-tls"}}]}},"volumeClaimTemplate":{"spec":{"resources":{"requests":{"storage":"2Gi"}},"storageClassName":""}}}},"sidecarContainer":{"enableTls":true},"skip":false,"version":"6.3.24"},"ignoreForMaintenance":false,"restore_job_config":{"sa":"restore-job-sa"},"restore_job_template":{"metadata":{"labels":{"restore-custom-label":"restore-custom-value"}},"spec":{"containers":[{"env":[{"name":"FDB_TLS_CERTIFICATE_FILE","value":"/var/fdb-certs/tls.crt"},{"name":"FDB_TLS_KEY_FILE","value":"/var/fdb-certs/tls.key"},{"name":"FDB_TLS_CA_FILE","value":"/var/fdb-certs/ca.crt"}],"name":"restore","volumeMounts":[{"mountPath":"/var/fdb-certs","name":"fdb-certs"}]}],"serviceAccountName":"ibm-fdb-controller-manager","volumes":[{"name":"fdb-certs","secret":{"secretName":"internal-tls"}}]}},"shutdown":"false","size":"small"}}
creationTimestamp: "2022-09-09T14:01:25Z"
generation: 1
labels:
fdb-cluster: mdm-foundationdb-ibm
name: mdm-foundationdb-ibm
namespace: testoperator1
ownerReferences:
- apiVersion: foundationdb.opencontent.ibm.com/v1
controller: true
kind: FdbCluster
name: mdm-foundationdb-ibm
uid: 5b5d4879-ecd5-44c8-97a2-8efcdde261c2
resourceVersion: "42692956"
uid: 55eed2b7-57e1-44bb-9fe6-02e128a3bca3
spec:
automationOptions:
deletionMode: Zone
podUpdateStrategy: ReplaceTransactionSystem
removalMode: Zone
replacements:
maxConcurrentReplacements: 1
buggify: {}
databaseConfiguration:
storage_engine: ssd-2
faultDomain: {}
labels: {}
lockOptions: {}
mainContainer:
enableTls: true
imageConfigs:
- baseImage: cp.stg.icr.io/cp/cpd/fdb
tag: 6.3.24
minimumUptimeSecondsForBounce: 600
partialConnectionString: {}
processCounts:
proxy: 2
stateless: 1
processes:
general:
podTemplate:
metadata:
labels:
customlabel: customvalue
spec:
automountServiceAccountToken: false
containers:
- env:
- name: FDB_TLS_CERTIFICATE_FILE
value: /var/fdb-certs/tls.crt
- name: FDB_TLS_KEY_FILE
value: /var/fdb-certs/tls.key
- name: FDB_TLS_CA_FILE
value: /var/fdb-certs/ca.crt
imagePullPolicy: Always
livenessProbe:
exec:
command:
- /bin/sh
- -c
- ps -ef | grep fdbmonitor
initialDelaySeconds: 60
periodSeconds: 30
name: foundationdb
resources:
requests:
cpu: 150m
memory: 256Mi
volumeMounts:
- mountPath: /var/fdb-certs
name: fdb-certs
- env:
- name: FDB_TLS_CERTIFICATE_FILE
value: /var/fdb-certs/tls.crt
- name: FDB_TLS_KEY_FILE
value: /var/fdb-certs/tls.key
- name: FDB_TLS_CA_FILE
value: /var/fdb-certs/ca.crt
imagePullPolicy: Always
name: foundationdb-kubernetes-sidecar
resources:
requests:
cpu: 150m
memory: 256Mi
volumeMounts:
- mountPath: /var/fdb-certs
name: fdb-certs
initContainers:
- env:
- name: FDB_TLS_CERTIFICATE_FILE
value: /var/fdb-certs/tls.crt
- name: FDB_TLS_KEY_FILE
value: /var/fdb-certs/tls.key
- name: FDB_TLS_CA_FILE
value: /var/fdb-certs/ca.crt
imagePullPolicy: Always
name: foundationdb-kubernetes-init
resources:
requests:
cpu: 150m
memory: 256Mi
volumeMounts:
- mountPath: /var/fdb-certs
name: fdb-certs
securityContext: {}
volumes:
- name: fdb-certs
secret:
secretName: internal-tls
volumeClaimTemplate:
metadata: {}
spec:
resources:
requests:
storage: 2Gi
storageClassName: managed-nfs-storage
status: {}
replaceInstancesWhenResourcesChange: false
routing: {}
sidecarContainer:
enableTls: true
imageConfigs:
- baseImage: cp.stg.icr.io/cp/cpd/fdb-sidecar
tag: 6.3.24-1
skip: false
version: 6.3.24
status:
configured: true
connectionString: mdm_foundationdb_ibm:yTnSIZXwGH6YgVT8sZHy4vLAcX3zkscn@10.254.16.114:4500:tls,10.254.13.255:4500:tls,10.254.22.27:4500:tls
databaseConfiguration:
log_routers: -1
logs: 3
proxies: 3
redundancy_mode: double
remote_logs: -1
resolvers: 1
storage_engine: ssd-2
usable_regions: 1
generations:
reconciled: 1
hasListenIPsForAllPods: true
health:
available: true
fullReplication: true
healthy: true
imageTypes:
- split
locks: {}
processGroups:
- addresses:
- 10.254.16.115
processClass: log
processGroupID: log-1
- addresses:
- 10.254.22.28
processClass: log
processGroupID: log-2
- addresses:
- 10.254.14.0
processClass: log
processGroupID: log-3
- addresses:
- 10.254.16.116
processClass: log
processGroupID: log-4
- addresses:
- 10.254.13.253
processClass: proxy
processGroupID: proxy-1
- addresses:
- 10.254.22.26
processClass: proxy
processGroupID: proxy-2
- addresses:
- 10.254.13.254
processClass: stateless
processGroupID: stateless-1
- addresses:
- 10.254.16.114
processClass: storage
processGroupID: storage-1
- addresses:
- 10.254.13.255
processClass: storage
processGroupID: storage-2
- addresses:
- 10.254.22.27
processClass: storage
processGroupID: storage-3
requiredAddresses:
tls: true
runningVersion: 6.3.24
storageServersPerDisk:
- 1
...