I am running foundation 6.1.12 on Ubuntu 16.04.6 in one of my single node test setup.
After shutting down the node (with foundationdb running) and starting it I am observing that database is not available. Can someone let me know what could be happening here?
Below is the output of status details
fdb> status details
Using cluster file `/etc/foundationdb/fdb.cluster'.
Locking coordination state. Verify that a majority of coordination server
processes are active.
127.0.0.1:4500 (reachable)
Unable to locate the data distributor worker.
Unable to locate the ratekeeper worker.
The output of status json is below
{
"client" : {
"cluster_file" : {
"path" : "/etc/foundationdb/fdb.cluster",
"up_to_date" : true
},
"coordinators" : {
"coordinators" : [
{
"address" : "127.0.0.1:4500",
"reachable" : true
}
],
"quorum_reachable" : true
},
"database_status" : {
"available" : false,
"healthy" : false
},
"messages" : [
],
"timestamp" : 1589212043
},
"cluster" : {
"clients" : {
"count" : 7,
"supported_versions" : [
{
"client_version" : "6.1.12",
"connected_clients" : [
{
"address" : "127.0.0.1:55764",
"connected_coordinators" : 1,
"log_group" : "default"
},
{
"address" : "127.0.0.1:55900",
"connected_coordinators" : 1,
"log_group" : "default"
},
{
"address" : "127.0.0.1:55914",
"connected_coordinators" : 1,
"log_group" : "default"
},
{
"address" : "127.0.0.1:56078",
"connected_coordinators" : 1,
"log_group" : "default"
},
{
"address" : "127.0.0.1:56148",
"connected_coordinators" : 1,
"log_group" : "default"
},
{
"address" : "127.0.0.1:56314",
"connected_coordinators" : 1,
"log_group" : "default"
},
{
"address" : "127.0.0.1:56450",
"connected_coordinators" : 1,
"log_group" : "default"
}
],
"count" : 7,
"protocol_version" : "fdb00b061060001",
"source_version" : "90ba203c166c62ae9173b2523081a0c6afaa1604"
}
]
},
"cluster_controller_timestamp" : 1589212048,
"connection_string" : "IqgQyLoo:XrIlR1QY@127.0.0.1:4500",
"datacenter_version_difference" : 0,
"degraded_processes" : 0,
"incompatible_connections" : [
],
"layers" : {
"_error" : "configurationMissing",
"_valid" : false
},
"machines" : {
"3b2ef843117d3dbdfe93ffc023fbf374" : {
"address" : "127.0.0.1",
"contributing_workers" : 2,
"cpu" : {
"logical_core_utilization" : 0.0027520600000000002
},
"excluded" : false,
"locality" : {
"machineid" : "3b2ef843117d3dbdfe93ffc023fbf374",
"processid" : "4e9fa38cd41ba4893090e422144b30b6",
"zoneid" : "3b2ef843117d3dbdfe93ffc023fbf374"
},
"machine_id" : "3b2ef843117d3dbdfe93ffc023fbf374",
"memory" : {
"committed_bytes" : 1567232000,
"free_bytes" : 15258861568,
"total_bytes" : 16826093568
},
"network" : {
"megabits_received" : {
"hz" : 0.25043499999999996
},
"megabits_sent" : {
"hz" : 0.25043499999999996
},
"tcp_segments_retransmitted" : {
"hz" : 0
}
}
}
},
"messages" : [
{
"description" : "Unable to locate the data distributor worker.",
"name" : "unreachable_dataDistributor_worker"
},
{
"description" : "Unable to locate the ratekeeper worker.",
"name" : "unreachable_ratekeeper_worker"
},
{
"description" : "Unable to read database configuration.",
"name" : "unreadable_configuration"
}
],
"processes" : {
"4e9fa38cd41ba4893090e422144b30b6" : {
"address" : "127.0.0.1:4500",
"class_source" : "command_line",
"class_type" : "unset",
"command_line" : "/usr/bin/fdbserver --cluster_file=/etc/foundationdb/fdb.cluster --datadir=/var/lib/foundationdb/data/4500 --listen_address=public --logdir=/var/log/foundationdb --public_address=auto:4500",
"cpu" : {
"usage_cores" : 0.0031999800000000003
},
"disk" : {
"busy" : 0,
"free_bytes" : 111117496320,
"reads" : {
"counter" : 52138,
"hz" : 0,
"sectors" : 0
},
"total_bytes" : 117510316032,
"writes" : {
"counter" : 208806,
"hz" : 4.3999699999999997,
"sectors" : 176
}
},
"fault_domain" : "3b2ef843117d3dbdfe93ffc023fbf374",
"locality" : {
"machineid" : "3b2ef843117d3dbdfe93ffc023fbf374",
"processid" : "4e9fa38cd41ba4893090e422144b30b6",
"zoneid" : "3b2ef843117d3dbdfe93ffc023fbf374"
},
"machine_id" : "3b2ef843117d3dbdfe93ffc023fbf374",
"memory" : {
"available_bytes" : 7881066496,
"limit_bytes" : 8589934592,
"unused_allocated_memory" : 0,
"used_bytes" : 361541632
},
"messages" : [
],
"network" : {
"connection_errors" : {
"hz" : 0
},
"connections_closed" : {
"hz" : 0.19999899999999998
},
"connections_established" : {
"hz" : 0
},
"current_connections" : 5,
"megabits_received" : {
"hz" : 0.0218783
},
"megabits_sent" : {
"hz" : 0.011631899999999999
}
},
"roles" : [
],
"uptime_seconds" : 90985.699999999997,
"version" : "6.1.12"
},
"c9c79c494b819706b0eadafdbc15dfd0" : {
"address" : "127.0.0.1:4501",
"class_source" : "command_line",
"class_type" : "stateless",
"command_line" : "/usr/bin/fdbserver --class=stateless --cluster_file=/etc/foundationdb/fdb.cluster --datadir=/var/lib/foundationdb/data/4501 --listen_address=public --logdir=/var/log/foundationdb --public_address=auto:4501",
"cpu" : {
"usage_cores" : 0.0047999799999999997
},
"disk" : {
"busy" : 0,
"free_bytes" : 111117496320,
"reads" : {
"counter" : 52138,
"hz" : 0,
"sectors" : 0
},
"total_bytes" : 117510316032,
"writes" : {
"counter" : 208806,
"hz" : 4.3999800000000002,
"sectors" : 176
}
},
"fault_domain" : "3b2ef843117d3dbdfe93ffc023fbf374",
"locality" : {
"machineid" : "3b2ef843117d3dbdfe93ffc023fbf374",
"processid" : "c9c79c494b819706b0eadafdbc15dfd0",
"zoneid" : "3b2ef843117d3dbdfe93ffc023fbf374"
},
"machine_id" : "3b2ef843117d3dbdfe93ffc023fbf374",
"memory" : {
"available_bytes" : 7881066496,
"limit_bytes" : 8589934592,
"unused_allocated_memory" : 0,
"used_bytes" : 141729792
},
"messages" : [
],
"network" : {
"connection_errors" : {
"hz" : 0
},
"connections_closed" : {
"hz" : 0.19999899999999998
},
"connections_established" : {
"hz" : 0
},
"current_connections" : 5,
"megabits_received" : {
"hz" : 0.034758199999999996
},
"megabits_sent" : {
"hz" : 0.047089399999999997
}
},
"roles" : [
{
"id" : "c7a0a4b137cf2015",
"role" : "master"
},
{
"id" : "956b61b1e846510d",
"role" : "cluster_controller"
}
],
"uptime_seconds" : 90985.699999999997,
"version" : "6.1.12"
}
},
"protocol_version" : "fdb00b061060001",
"recovery_state" : {
"description" : "Locking coordination state. Verify that a majority of coordination server processes are active.",
"name" : "locking_coordinated_state"
}
}
}
I am not seeing any trace with Severity 30 and Severity 40 in the fdb traces log. I have placed the logs here
My foundationdb.conf looks like below
[fdbmonitor]
user = ubuntu
group = ubuntu
[general]
restart_delay = 60
cluster_file = /etc/foundationdb/fdb.cluster
[fdbserver]
command = /usr/bin/fdbserver
public_address = auto:$ID
listen_address = public
datadir = /var/lib/foundationdb/data/$ID
logdir = /var/log/foundationdb
[fdbserver.4500]
[fdbserver.4501]
class = stateless
[backup_agent]
command = /usr/lib/foundationdb/backup_agent/backup_agent
logdir = /var/log/foundationdb
[backup_agent.1]