From 02de657b695a78b55e2747ee44981f042287922a Mon Sep 17 00:00:00 2001 From: Klaus Wenninger Date: Mon, 9 Dec 2019 15:13:11 +0100 Subject: [PATCH] Fix: sbd-integration: sync pacemakerd with sbd Make pacemakerd wait to be pinged by sbd before starting sub-daemons. Pings further reply health-state and timestamp of last successful check. On shutdown bring down all the sub-daemons and wait to be polled for state by sbd before finally exiting pacemakerd. --- daemons/pacemakerd/pacemakerd.c | 140 +++++++++++++++++++++++++------- include/crm/msg_xml.h | 7 ++ tools/crmadmin.c | 94 ++++++++++++++++----- 3 files changed, 188 insertions(+), 53 deletions(-) diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c index 96e02094f4d..8b02dff21a5 100644 --- a/daemons/pacemakerd/pacemakerd.c +++ b/daemons/pacemakerd/pacemakerd.c @@ -48,8 +48,14 @@ static bool global_keep_tracking = false; static const char *local_name = NULL; static uint32_t local_nodeid = 0; static crm_trigger_t *shutdown_trigger = NULL; +static crm_trigger_t *startup_trigger = NULL; static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid"; +static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT; +static gboolean running_with_sbd = FALSE; +static uint shutdown_complete_state_reported_to = 0; +static gboolean shutdown_complete_state_reported_client_closed = FALSE; + typedef struct pcmk_child_s { int pid; long flag; @@ -435,21 +441,20 @@ escalate_shutdown(gpointer data) static gboolean pcmk_shutdown_worker(gpointer user_data) { - static int phase = 0; + static int phase = SIZEOF(pcmk_children); static time_t next_log = 0; - static int max = SIZEOF(pcmk_children); int lpc = 0; - if (phase == 0) { + if (phase == SIZEOF(pcmk_children)) { crm_notice("Shutting down Pacemaker"); - phase = max; + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN; } for (; phase > 0; phase--) { /* Don't stop anything with start_seq < 1 */ - for (lpc = max - 1; lpc >= 0; lpc--) { + for (lpc = SIZEOF(pcmk_children) - 1; lpc >= 0; lpc--) { pcmk_child_t *child = &(pcmk_children[lpc]); if (phase != child->start_seq) { @@ -497,6 +502,11 @@ pcmk_shutdown_worker(gpointer user_data) /* send_cluster_id(); */ crm_notice("Shutdown complete"); + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE; + if (!fatal_error && running_with_sbd && + !shutdown_complete_state_reported_client_closed) { + return TRUE; + } { const char *delay = daemon_option("shutdown_delay"); @@ -553,6 +563,50 @@ pcmk_ipc_created(qb_ipcs_connection_t * c) crm_trace("Connection %p", c); } +static void +pcmk_handle_ping_request( crm_client_t *c, xmlNode *msg, uint32_t id) +{ + const char *value = NULL; + xmlNode *ping = NULL; + xmlNode *reply = NULL; + time_t pinged = time(NULL); + const char *from = crm_element_value(msg, F_CRM_SYS_FROM); + + /* Pinged for status */ + crm_trace("Pinged from %s.%s", + crm_element_value(msg, F_CRM_ORIGIN), + from?from:"unknown"); + ping = create_xml_node(NULL, XML_CRM_TAG_PING); + value = crm_element_value(msg, F_CRM_SYS_TO); + crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value); + crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state); + crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged); + crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok"); + reply = create_reply(msg, ping); + free_xml(ping); + if (reply) { + if (crm_ipcs_send(c, id, reply, crm_ipc_server_event) <= 0) { + crm_err("Failed sending ping-reply"); + } + free_xml(reply); + } else { + crm_err("Failed building ping-reply"); + } + /* just proceed state on sbd pinging us */ + if (from && strstr(from, "sbd")) { + if (crm_str_eq(pacemakerd_state, + XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, + TRUE)) { + shutdown_complete_state_reported_to = c->pid; + } else if (crm_str_eq(pacemakerd_state, + XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, + TRUE)) { + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; + mainloop_set_trigger(startup_trigger); + } + } +} + /* Exit code means? */ static int32_t pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) @@ -563,35 +617,44 @@ pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) crm_client_t *c = crm_client_get(qbc); xmlNode *msg = crm_ipcs_recv(c, data, size, &id, &flags); - crm_ipcs_send_ack(c, id, flags, "ack", __FUNCTION__, __LINE__); - if (msg == NULL) { - return 0; + if (msg != NULL) { + task = crm_element_value(msg, F_CRM_TASK); } - task = crm_element_value(msg, F_CRM_TASK); - if (crm_str_eq(task, CRM_OP_QUIT, TRUE)) { - /* Time to quit */ - crm_notice("Shutting down in response to ticket %s (%s)", - crm_element_value(msg, F_CRM_REFERENCE), crm_element_value(msg, F_CRM_ORIGIN)); - pcmk_shutdown(15); + if (crm_str_eq(task, CRM_OP_PING, TRUE)) { + pcmk_handle_ping_request(c, msg, id); + } else { + crm_ipcs_send_ack(c, id, flags, "ack", __FUNCTION__, __LINE__); - } else if (crm_str_eq(task, CRM_OP_RM_NODE_CACHE, TRUE)) { - /* Send to everyone */ - struct iovec *iov; - int id = 0; - const char *name = NULL; + if (msg == NULL) { + return 0; + } - crm_element_value_int(msg, XML_ATTR_ID, &id); - name = crm_element_value(msg, XML_ATTR_UNAME); - crm_notice("Instructing peers to remove references to node %s/%u", name, id); + if (crm_str_eq(task, CRM_OP_QUIT, TRUE)) { + /* Time to quit */ + crm_notice("Shutting down in response to ticket %s (%s)", + crm_element_value(msg, F_CRM_REFERENCE), + crm_element_value(msg, F_CRM_ORIGIN)); + pcmk_shutdown(15); - iov = calloc(1, sizeof(struct iovec)); - iov->iov_base = dump_xml_unformatted(msg); - iov->iov_len = 1 + strlen(iov->iov_base); - send_cpg_iov(iov); + } else if (crm_str_eq(task, CRM_OP_RM_NODE_CACHE, TRUE)) { + /* Send to everyone */ + struct iovec *iov; + int id = 0; + const char *name = NULL; - } else { - update_process_clients(c); + crm_element_value_int(msg, XML_ATTR_ID, &id); + name = crm_element_value(msg, XML_ATTR_UNAME); + crm_notice("Instructing peers to remove references to node %s/%u", name, id); + + iov = calloc(1, sizeof(struct iovec)); + iov->iov_base = dump_xml_unformatted(msg); + iov->iov_len = 1 + strlen(iov->iov_base); + send_cpg_iov(iov); + + } else { + update_process_clients(c); + } } free_xml(msg); @@ -608,6 +671,12 @@ pcmk_ipc_closed(qb_ipcs_connection_t * c) return 0; } crm_trace("Connection %p", c); + if (shutdown_complete_state_reported_to == client->pid) { + shutdown_complete_state_reported_client_closed = TRUE; + if (shutdown_trigger) { + mainloop_set_trigger(shutdown_trigger); + } + } crm_client_destroy(client); return 0; } @@ -1051,8 +1120,8 @@ find_and_track_existing_processes(void) return (tracking > INT_MAX) ? INT_MAX : tracking; } -static void -init_children_processes(void) +static gboolean +init_children_processes(gpointer user_data) { int start_seq = 1, lpc = 0; static int max = SIZEOF(pcmk_children); @@ -1078,6 +1147,8 @@ init_children_processes(void) * This may be useful for the daemons to know */ setenv("PCMK_respawned", "true", 1); + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING; + return TRUE; } static void @@ -1356,6 +1427,7 @@ main(int argc, char **argv) if(pcmk_locate_sbd() > 0) { setenv("PCMK_watchdog", "true", 1); + running_with_sbd = TRUE; } else { setenv("PCMK_watchdog", "false", 1); } @@ -1394,7 +1466,13 @@ main(int argc, char **argv) mainloop_add_signal(SIGTERM, pcmk_shutdown); mainloop_add_signal(SIGINT, pcmk_shutdown); - init_children_processes(); + if (running_with_sbd) { + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING; + startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL); + } else { + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; + init_children_processes(NULL); + } crm_notice("Pacemaker daemon successfully started and accepting connections"); g_main_loop_run(mainloop); diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h index d56e40c6379..24696016dd7 100644 --- a/include/crm/msg_xml.h +++ b/include/crm/msg_xml.h @@ -123,6 +123,13 @@ extern "C" { # define XML_PING_ATTR_STATUS "result" # define XML_PING_ATTR_SYSFROM "crm_subsystem" # define XML_PING_ATTR_CRMDSTATE "crmd_state" +# define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state" +# define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init" +# define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons" +# define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping" +# define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running" +# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down" +# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete" # define XML_TAG_FRAGMENT "cib_fragment" diff --git a/tools/crmadmin.c b/tools/crmadmin.c index 41bbe24f4ba..7f0929fbb27 100644 --- a/tools/crmadmin.c +++ b/tools/crmadmin.c @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -32,8 +33,9 @@ static int message_timer_id = -1; static int message_timeout_ms = 30 * 1000; static GMainLoop *mainloop = NULL; -static crm_ipc_t *crmd_channel = NULL; +static crm_ipc_t *ipc_channel = NULL; static char *admin_uuid = NULL; +static char *ipc_name = NULL; gboolean do_init(void); int do_work(void); @@ -46,6 +48,7 @@ static gboolean BE_VERBOSE = FALSE; static int expected_responses = 1; static gboolean BASH_EXPORT = FALSE; static gboolean DO_HEALTH = FALSE; +static gboolean DO_PACEMAKERD_HEALTH = FALSE; static gboolean DO_RESET = FALSE; static gboolean DO_RESOURCE = FALSE; static gboolean DO_ELECT_DC = FALSE; @@ -70,18 +73,17 @@ static struct crm_option long_options[] = { /* daemon options */ {"status", 1, 0, 'S', "Display the status of the specified node." }, {"-spacer-", 1, 0, '-', "\n\tResult is the node's internal FSM state which can be useful for debugging\n"}, + {"pacemakerd",0, 0, 'P', "Display the status of local pacemakerd."}, + {"-spacer-", 1, 0, '-', "\n\tResult is the state of the sub-daemons watched by pacemakerd\n"}, {"dc_lookup", 0, 0, 'D', "Display the uname of the node co-ordinating the cluster."}, {"-spacer-", 1, 0, '-', "\n\tThis is an internal detail and is rarely useful to administrators except when deciding on which node to examine the logs.\n"}, {"nodes", 0, 0, 'N', "\tDisplay the uname of all member nodes"}, {"election", 0, 0, 'E', "(Advanced) Start an election for the cluster co-ordinator"}, - { - "kill", 1, 0, 'K', - "(Advanced) Stop the controller (not the rest of the cluster stack) on specified node" - }, + {"kill", 1, 0, 'K', "(Advanced) Stop the controller (not the rest of the cluster stack) on specified node"}, {"health", 0, 0, 'H', NULL, 1}, - - {"-spacer-", 1, 0, '-', "\nAdditional Options:"}, + {"-spacer-", 1, 0, '-', "\nAdditional Options:"}, {XML_ATTR_TIMEOUT, 1, 0, 't', "Time (in milliseconds) to wait before declaring the operation failed"}, + {"ipc-name", 1, 0, 'i', "Name to use for ipc instead of 'crmadmin'"}, {"bash-export", 0, 0, 'B', "Create Bash export entries of the form 'export uname=uuid'\n"}, {"-spacer-", 1, 0, '-', "Notes:"}, @@ -122,7 +124,9 @@ main(int argc, char **argv) message_timeout_ms = 30 * 1000; } break; - + case 'i': + ipc_name = strdup(optarg); + break; case '$': case '?': crm_help(flag, CRM_EX_OK); @@ -142,6 +146,9 @@ main(int argc, char **argv) case 'q': BE_SILENT = TRUE; break; + case 'P': + DO_PACEMAKERD_HEALTH = TRUE; + break; case 'S': DO_HEALTH = TRUE; crm_trace("Option %c => %s", flag, optarg); @@ -215,7 +222,7 @@ do_work(void) xmlNode *msg_data = NULL; gboolean all_is_good = TRUE; - if (DO_HEALTH == TRUE) { + if (DO_HEALTH) { crm_trace("Querying the system"); sys_to = CRM_SYSTEM_DC; @@ -233,6 +240,16 @@ do_work(void) all_is_good = FALSE; } + } else if (DO_PACEMAKERD_HEALTH) { + crm_trace("Querying pacemakerd state"); + + sys_to = CRM_SYSTEM_MCP; + crmd_operation = CRM_OP_PING; + + if (BE_VERBOSE) { + expected_responses = 1; + } + } else if (DO_ELECT_DC) { /* tell the local node to initiate an election */ @@ -251,7 +268,8 @@ do_work(void) cib_t *the_cib = cib_new(); xmlNode *output = NULL; - int rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command); + int rc = the_cib->cmds->signon(the_cib, + ipc_name?ipc_name:crm_system_name, cib_command); if (rc != pcmk_ok) { return -1; @@ -286,7 +304,7 @@ do_work(void) } /* send it */ - if (crmd_channel == NULL) { + if (ipc_channel == NULL) { crm_err("The IPC connection is not valid, cannot send anything"); return -1; } @@ -300,10 +318,10 @@ do_work(void) } { - xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to, - crm_system_name, admin_uuid); + xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, + sys_to, ipc_name?ipc_name:crm_system_name, admin_uuid); - crm_ipc_send(crmd_channel, cmd, 0, 0, NULL); + crm_ipc_send(ipc_channel, cmd, 0, 0, NULL); free_xml(cmd); } @@ -329,20 +347,23 @@ struct ipc_client_callbacks crm_callbacks = { gboolean do_init(void) { - mainloop_io_t *source = - mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks); + mainloop_io_t *ipc_source = + mainloop_add_ipc_client( + DO_PACEMAKERD_HEALTH?CRM_SYSTEM_MCP:CRM_SYSTEM_CRMD, + G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks); admin_uuid = crm_getpid_s(); - crmd_channel = mainloop_get_ipc_client(source); + ipc_channel = mainloop_get_ipc_client(ipc_source); - if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST) { + if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST || DO_PACEMAKERD_HEALTH) { return TRUE; - } else if (crmd_channel != NULL) { - xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1"); + } else if (ipc_channel != NULL) { + xmlNode *xml = create_hello_message(admin_uuid, + ipc_name?ipc_name:crm_system_name, "0", "1"); - crm_ipc_send(crmd_channel, xml, 0, 0, NULL); + crm_ipc_send(ipc_channel, xml, 0, 0, NULL); return TRUE; } return FALSE; @@ -392,8 +413,10 @@ admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata) if (xml == NULL) { crm_info("XML in IPC message was not valid... " "discarding."); - } else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) { + } else if (validate_crm_message(xml, ipc_name?ipc_name:crm_system_name, + admin_uuid, XML_ATTR_RESPONSE) == FALSE) { crm_trace("Message was not a CRM response. Discarding."); + printf("Validation of response failed\n"); } else if (DO_HEALTH) { xmlNode *data = get_message_xml(xml, F_CRM_DATA); @@ -408,6 +431,33 @@ admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata) fprintf(stderr, "%s\n", state); } + } else if (DO_PACEMAKERD_HEALTH) { + xmlNode *data = get_message_xml(xml, F_CRM_DATA); + const char *state = + crm_element_value(data, XML_PING_ATTR_PACEMAKERDSTATE); + time_t pinged = (time_t) 0; + long long value_ll = 0; + crm_time_t *crm_when = crm_time_new(NULL); + char *pinged_buf = NULL; + + crm_element_value_ll(data, XML_ATTR_TSTAMP, &value_ll); + pinged = (time_t) value_ll; + crm_time_set_timet(crm_when, &pinged); + pinged_buf = crm_time_as_string(crm_when, + crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone); + printf("Status of %s: %s (%s%s%s)\n", + crm_element_value(data, XML_PING_ATTR_SYSFROM), + state, crm_element_value(data, XML_PING_ATTR_STATUS), + (pinged != (time_t) 0)?" @ ":"", + (pinged != (time_t) 0)?pinged_buf:""); + + free(pinged_buf); + crm_time_free(crm_when); + + if (BE_SILENT && state != NULL) { + fprintf(stderr, "%s\n", state); + } + } else if (DO_WHOIS_DC) { const char *dc = crm_element_value(xml, F_CRM_HOST_FROM);