riak-repl status fails with 'badarg' when leader is undefined [JIRA: RIAK-2435]

[ian-mi]

The following is an example seen at REDACTED of a failed riak-repl status command called shortly after a node was revived from a bad state:

{'EXIT',
                                    {badarg,
                                     [{erlang,'++',
                                       [[{realtime_enabled,"REDACTED, REDACTED"},
                                         {realtime_started,"REDACTED, REDACTED"},
                                         {fullsync_enabled,[]},
                                         {fullsync_running,[]},
                                         {proxy_get_enabled,[]},
                                         {server_bytes_sent,0},
                                         {server_bytes_recv,0},
                                         {server_connects,0},
                                         {server_connect_errors,0},
                                         {server_fullsyncs,0},
                                         {client_bytes_sent,0},
                                         {client_bytes_recv,0},
                                         {client_connects,0},
                                         {client_connect_errors,0},
                                         {client_redirect,0},
                                         {objects_dropped_no_clients,0},
                                         {objects_dropped_no_leader,67924},
                                         {objects_sent,135415},
                                         {objects_forwarded,0},
                                         {elections_elected,0},
                                         {elections_leader_changed,0},
                                         {client_rx_kbps,[0,0,0,0,0,0,0,0]},
                                         {client_tx_kbps,[0,0,0,0,0,0,0,0]},
                                         {server_rx_kbps,[0,0,0,0,0,0,0,0]},
                                         {server_tx_kbps,[0,0,0,0,0,0,0,0]},
                                         {rt_source_errors,0},
                                         {rt_sink_errors,1},
                                         {rt_dirty,2},
                                         {leader,undefined},
                                         {"local_leader_message_queue_len",0},
                                         {"local_leader_heap_size",987},
                                         {client_stats,[]},
                                         {sinks,
                                          [{sink_stats,
                                            [{pid,"<0.18178.284>"},
                                             {message_queue_len,0},
                                             {rt_sink_connected_to,
                                              [{source,"REDACTED"},
                                               {pid,"<0.18178.284>"},
                                               {connected,true},
                                               {transport,ranch_ssl},
                                               {socket,
                                                [{peername,
                                                  "REDACTED:42870"},
                                                 {sockname,
                                                  "REDACTED:9080"},
                                                 {recv_avg,
                                                  "[1416, 1416, 1416, 1416, 1416, 1416, 1416]"},
                                                 {recv_cnt,
                                                  "[0, 0, 0, 0, 0, 0]"},
                                                 {recv_dvi,
                                                  "[99, 99, 99, 99, 99, 99, 99]"},
                                                 {recv_kbps,
                                                  "[0, 0, 0, 0, 0, 0]"},
                                                 {recv_max,
                                                  "[1460, 1460, 1460, 1460, 1460, 1460, 1460]"},
                                                 {send_cnt,
                                                  "[0, 0, 0, 0, 0, 0]"},
                                                 {send_kbps,
                                                  "[0, 0, 0, 0, 0, 0]"},
                                                 {send_pend,
                                                  "[0, 0, 0, 0, 0, 0, 0]"}]},
                                               {hb_last,{1457,638177,514130}},
                                               {helper,"<0.8828.284>"},
                                               {helper_msgq_len,0},
                                               {active,{false,scheduled}},
                                               {deactivated,4},
                                               {source_drops,0},
                                               {expect_seq,3361024},
                                               {acked_seq,3360852},
                                               {pending,101}]}]}]},
                                         {sources,
                                          [{source_stats,
                                            [{pid,"<0.1935.0>"},
                                             {message_queue_len,0},
                                             {rt_source_connected_to,
                                              [{sink,"REDACTED"},
                                               {pid,"<0.1935.0>"},
                                               {connected,true},
                                               {transport,ranch_ssl},
                                               {socket,
                                                [{peername,
                                                  "REDACTED:9080"},
                                                 {sockname,
                                                  "REDACTED:55804"},
                                                 {recv_avg,
                                                  "[69, 69, 69, 69, 69, 69, 69]"},
                                                 {recv_cnt,
                                                  "[17, 18, 18, 20, 14, 13]"},
                                                 {recv_dvi,
                                                  "[0, 0, 0, 0, 0, 0, 0]"},
                                                 {recv_kbps,
                                                  "[0, 0, 0, 1, 0, 0]"},
                                                 {recv_max,
                                                  "[1460, 1460, 1460, 1460, 1460, 1460, 1460]"},
                                                 {send_cnt,
                                                  "[17, 18, 18, 20, 14, 13]"},
                                                 {send_kbps,
                                                  "[10, 12, 11, 12, 9, 10]"},
                                                 {send_pend,
                                                  "[0, 0, 0, 0, 0, 0, 0]"}]},
                                               {helper_pid,"<0.1983.0>"},
                                               {hb_rtt,51},
                                               {sent_seq,67925},
                                               {objects,67925}]}]},
                                           {source_stats,
                                            [{pid,"<0.16838.284>"},
                                             {message_queue_len,0},
                                             {rt_source_connected_to,
                                              [{sink,"REDACTED"},
                                               {pid,"<0.16838.284>"},
                                               {connected,true},
                                               {transport,ranch_ssl},
                                               {socket,
                                                [{peername,
                                                  "REDACTED:9080"},
                                                 {sockname,
                                                  "REDACTED:44428"},
                                                 {recv_avg,
                                                  "[103, 106, 112, 118, 129, 140, 153]"},
                                                 {recv_cnt,
                                                  "[17, 18, 18, 20, 14, 13]"},
                                                 {recv_dvi,
                                                  "[99, 106, 115, 126, 142, 155, 171]"},
                                                 {recv_kbps,
                                                  "[0, 0, 0, 1, 0, 0]"},
                                                 {recv_max,
                                                  "[1460, 1460, 1460, 1460, 1460, 1460, 1460]"},
                                                 {send_cnt,
                                                  "[17, 18, 18, 20, 14, 13]"},
                                                 {send_kbps,
                                                  "[10, 12, 11, 12, 9, 10]"},
                                                 {send_pend,
                                                  "[0, 0, 0, 0, 0, 0, 0]"}]},
                                               {helper_pid,"<0.17950.284>"},
                                               {hb_rtt,27},
                                               {sent_seq,67925},
                                               {objects,184}]}]}]},
                                         {fullsync_coordinator,[]},
                                         {fullsync_coordinator_srv,[]},
                                         {cluster_name,<<"REDACTED">>},
                                         {cluster_leader,
                                          'riak@REDACTED'},
                                         {connected_clusters,
                                          [<<"REDACTED">>,<<"REDACTED">>]},
                                         {realtime_queue_stats,
                                          [{percent_bytes_used,0.001},
                                           {bytes,712},
                                           {max_bytes,104857600},
                                           {consumers,
                                            [{"REDACTED",
                                              [{pending,0},
                                               {unacked,0},
                                               {drops,0},
                                               {errs,0}]},
                                             {"REDACTED",
                                              [{pending,0},
                                               {unacked,0},
                                               {drops,0},
                                               {errs,0}]}]},
                                           {overload_drops,0}]}|
                                         {[],[]}],
                                        [{realtime_send_kbps,20},
                                         {realtime_recv_kbps,0},
                                         {fullsync_send_kbps,0},
                                         {fullsync_recv_kbps,0}]],
                                       []},
                                      {riak_repl_console,status2,1,
                                       [{file,"src/riak_repl_console.erl"},
                                        {line,156}]},
                                      {rpc,'-handle_call_call/6-fun-0-',5,
                                       [{file,"rpc.erl"},{line,205}]}]}}

It appears that the blame is with riak_repl_pg:status returning {[],[]} instead of a list when the leader node is undefined.