From 02de657b695a78b55e2747ee44981f042287922a Mon Sep 17 00:00:00 2001 From: Klaus Wenninger Date: Mon, 9 Dec 2019 15:13:11 +0100 Subject: [PATCH 1/2] Fix: sbd-integration: sync pacemakerd with sbd Make pacemakerd wait to be pinged by sbd before starting sub-daemons. Pings further reply health-state and timestamp of last successful check. On shutdown bring down all the sub-daemons and wait to be polled for state by sbd before finally exiting pacemakerd. --- daemons/pacemakerd/pacemakerd.c | 140 +++++++++++++++++++++++++------- include/crm/msg_xml.h | 7 ++ tools/crmadmin.c | 94 ++++++++++++++++----- 3 files changed, 188 insertions(+), 53 deletions(-) diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c index 96e02094f4d..8b02dff21a5 100644 --- a/daemons/pacemakerd/pacemakerd.c +++ b/daemons/pacemakerd/pacemakerd.c @@ -48,8 +48,14 @@ static bool global_keep_tracking = false; static const char *local_name = NULL; static uint32_t local_nodeid = 0; static crm_trigger_t *shutdown_trigger = NULL; +static crm_trigger_t *startup_trigger = NULL; static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid"; +static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT; +static gboolean running_with_sbd = FALSE; +static uint shutdown_complete_state_reported_to = 0; +static gboolean shutdown_complete_state_reported_client_closed = FALSE; + typedef struct pcmk_child_s { int pid; long flag; @@ -435,21 +441,20 @@ escalate_shutdown(gpointer data) static gboolean pcmk_shutdown_worker(gpointer user_data) { - static int phase = 0; + static int phase = SIZEOF(pcmk_children); static time_t next_log = 0; - static int max = SIZEOF(pcmk_children); int lpc = 0; - if (phase == 0) { + if (phase == SIZEOF(pcmk_children)) { crm_notice("Shutting down Pacemaker"); - phase = max; + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN; } for (; phase > 0; phase--) { /* Don't stop anything with start_seq < 1 */ - for (lpc = max - 1; lpc >= 0; lpc--) { + for (lpc = SIZEOF(pcmk_children) - 1; lpc >= 0; lpc--) { pcmk_child_t *child = &(pcmk_children[lpc]); if (phase != child->start_seq) { @@ -497,6 +502,11 @@ pcmk_shutdown_worker(gpointer user_data) /* send_cluster_id(); */ crm_notice("Shutdown complete"); + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE; + if (!fatal_error && running_with_sbd && + !shutdown_complete_state_reported_client_closed) { + return TRUE; + } { const char *delay = daemon_option("shutdown_delay"); @@ -553,6 +563,50 @@ pcmk_ipc_created(qb_ipcs_connection_t * c) crm_trace("Connection %p", c); } +static void +pcmk_handle_ping_request( crm_client_t *c, xmlNode *msg, uint32_t id) +{ + const char *value = NULL; + xmlNode *ping = NULL; + xmlNode *reply = NULL; + time_t pinged = time(NULL); + const char *from = crm_element_value(msg, F_CRM_SYS_FROM); + + /* Pinged for status */ + crm_trace("Pinged from %s.%s", + crm_element_value(msg, F_CRM_ORIGIN), + from?from:"unknown"); + ping = create_xml_node(NULL, XML_CRM_TAG_PING); + value = crm_element_value(msg, F_CRM_SYS_TO); + crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value); + crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state); + crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged); + crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok"); + reply = create_reply(msg, ping); + free_xml(ping); + if (reply) { + if (crm_ipcs_send(c, id, reply, crm_ipc_server_event) <= 0) { + crm_err("Failed sending ping-reply"); + } + free_xml(reply); + } else { + crm_err("Failed building ping-reply"); + } + /* just proceed state on sbd pinging us */ + if (from && strstr(from, "sbd")) { + if (crm_str_eq(pacemakerd_state, + XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, + TRUE)) { + shutdown_complete_state_reported_to = c->pid; + } else if (crm_str_eq(pacemakerd_state, + XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, + TRUE)) { + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; + mainloop_set_trigger(startup_trigger); + } + } +} + /* Exit code means? */ static int32_t pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) @@ -563,35 +617,44 @@ pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size) crm_client_t *c = crm_client_get(qbc); xmlNode *msg = crm_ipcs_recv(c, data, size, &id, &flags); - crm_ipcs_send_ack(c, id, flags, "ack", __FUNCTION__, __LINE__); - if (msg == NULL) { - return 0; + if (msg != NULL) { + task = crm_element_value(msg, F_CRM_TASK); } - task = crm_element_value(msg, F_CRM_TASK); - if (crm_str_eq(task, CRM_OP_QUIT, TRUE)) { - /* Time to quit */ - crm_notice("Shutting down in response to ticket %s (%s)", - crm_element_value(msg, F_CRM_REFERENCE), crm_element_value(msg, F_CRM_ORIGIN)); - pcmk_shutdown(15); + if (crm_str_eq(task, CRM_OP_PING, TRUE)) { + pcmk_handle_ping_request(c, msg, id); + } else { + crm_ipcs_send_ack(c, id, flags, "ack", __FUNCTION__, __LINE__); - } else if (crm_str_eq(task, CRM_OP_RM_NODE_CACHE, TRUE)) { - /* Send to everyone */ - struct iovec *iov; - int id = 0; - const char *name = NULL; + if (msg == NULL) { + return 0; + } - crm_element_value_int(msg, XML_ATTR_ID, &id); - name = crm_element_value(msg, XML_ATTR_UNAME); - crm_notice("Instructing peers to remove references to node %s/%u", name, id); + if (crm_str_eq(task, CRM_OP_QUIT, TRUE)) { + /* Time to quit */ + crm_notice("Shutting down in response to ticket %s (%s)", + crm_element_value(msg, F_CRM_REFERENCE), + crm_element_value(msg, F_CRM_ORIGIN)); + pcmk_shutdown(15); - iov = calloc(1, sizeof(struct iovec)); - iov->iov_base = dump_xml_unformatted(msg); - iov->iov_len = 1 + strlen(iov->iov_base); - send_cpg_iov(iov); + } else if (crm_str_eq(task, CRM_OP_RM_NODE_CACHE, TRUE)) { + /* Send to everyone */ + struct iovec *iov; + int id = 0; + const char *name = NULL; - } else { - update_process_clients(c); + crm_element_value_int(msg, XML_ATTR_ID, &id); + name = crm_element_value(msg, XML_ATTR_UNAME); + crm_notice("Instructing peers to remove references to node %s/%u", name, id); + + iov = calloc(1, sizeof(struct iovec)); + iov->iov_base = dump_xml_unformatted(msg); + iov->iov_len = 1 + strlen(iov->iov_base); + send_cpg_iov(iov); + + } else { + update_process_clients(c); + } } free_xml(msg); @@ -608,6 +671,12 @@ pcmk_ipc_closed(qb_ipcs_connection_t * c) return 0; } crm_trace("Connection %p", c); + if (shutdown_complete_state_reported_to == client->pid) { + shutdown_complete_state_reported_client_closed = TRUE; + if (shutdown_trigger) { + mainloop_set_trigger(shutdown_trigger); + } + } crm_client_destroy(client); return 0; } @@ -1051,8 +1120,8 @@ find_and_track_existing_processes(void) return (tracking > INT_MAX) ? INT_MAX : tracking; } -static void -init_children_processes(void) +static gboolean +init_children_processes(gpointer user_data) { int start_seq = 1, lpc = 0; static int max = SIZEOF(pcmk_children); @@ -1078,6 +1147,8 @@ init_children_processes(void) * This may be useful for the daemons to know */ setenv("PCMK_respawned", "true", 1); + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING; + return TRUE; } static void @@ -1356,6 +1427,7 @@ main(int argc, char **argv) if(pcmk_locate_sbd() > 0) { setenv("PCMK_watchdog", "true", 1); + running_with_sbd = TRUE; } else { setenv("PCMK_watchdog", "false", 1); } @@ -1394,7 +1466,13 @@ main(int argc, char **argv) mainloop_add_signal(SIGTERM, pcmk_shutdown); mainloop_add_signal(SIGINT, pcmk_shutdown); - init_children_processes(); + if (running_with_sbd) { + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING; + startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL); + } else { + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; + init_children_processes(NULL); + } crm_notice("Pacemaker daemon successfully started and accepting connections"); g_main_loop_run(mainloop); diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h index d56e40c6379..24696016dd7 100644 --- a/include/crm/msg_xml.h +++ b/include/crm/msg_xml.h @@ -123,6 +123,13 @@ extern "C" { # define XML_PING_ATTR_STATUS "result" # define XML_PING_ATTR_SYSFROM "crm_subsystem" # define XML_PING_ATTR_CRMDSTATE "crmd_state" +# define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state" +# define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init" +# define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons" +# define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping" +# define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running" +# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down" +# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete" # define XML_TAG_FRAGMENT "cib_fragment" diff --git a/tools/crmadmin.c b/tools/crmadmin.c index 41bbe24f4ba..7f0929fbb27 100644 --- a/tools/crmadmin.c +++ b/tools/crmadmin.c @@ -23,6 +23,7 @@ #include #include #include +#include #include @@ -32,8 +33,9 @@ static int message_timer_id = -1; static int message_timeout_ms = 30 * 1000; static GMainLoop *mainloop = NULL; -static crm_ipc_t *crmd_channel = NULL; +static crm_ipc_t *ipc_channel = NULL; static char *admin_uuid = NULL; +static char *ipc_name = NULL; gboolean do_init(void); int do_work(void); @@ -46,6 +48,7 @@ static gboolean BE_VERBOSE = FALSE; static int expected_responses = 1; static gboolean BASH_EXPORT = FALSE; static gboolean DO_HEALTH = FALSE; +static gboolean DO_PACEMAKERD_HEALTH = FALSE; static gboolean DO_RESET = FALSE; static gboolean DO_RESOURCE = FALSE; static gboolean DO_ELECT_DC = FALSE; @@ -70,18 +73,17 @@ static struct crm_option long_options[] = { /* daemon options */ {"status", 1, 0, 'S', "Display the status of the specified node." }, {"-spacer-", 1, 0, '-', "\n\tResult is the node's internal FSM state which can be useful for debugging\n"}, + {"pacemakerd",0, 0, 'P', "Display the status of local pacemakerd."}, + {"-spacer-", 1, 0, '-', "\n\tResult is the state of the sub-daemons watched by pacemakerd\n"}, {"dc_lookup", 0, 0, 'D', "Display the uname of the node co-ordinating the cluster."}, {"-spacer-", 1, 0, '-', "\n\tThis is an internal detail and is rarely useful to administrators except when deciding on which node to examine the logs.\n"}, {"nodes", 0, 0, 'N', "\tDisplay the uname of all member nodes"}, {"election", 0, 0, 'E', "(Advanced) Start an election for the cluster co-ordinator"}, - { - "kill", 1, 0, 'K', - "(Advanced) Stop the controller (not the rest of the cluster stack) on specified node" - }, + {"kill", 1, 0, 'K', "(Advanced) Stop the controller (not the rest of the cluster stack) on specified node"}, {"health", 0, 0, 'H', NULL, 1}, - - {"-spacer-", 1, 0, '-', "\nAdditional Options:"}, + {"-spacer-", 1, 0, '-', "\nAdditional Options:"}, {XML_ATTR_TIMEOUT, 1, 0, 't', "Time (in milliseconds) to wait before declaring the operation failed"}, + {"ipc-name", 1, 0, 'i', "Name to use for ipc instead of 'crmadmin'"}, {"bash-export", 0, 0, 'B', "Create Bash export entries of the form 'export uname=uuid'\n"}, {"-spacer-", 1, 0, '-', "Notes:"}, @@ -122,7 +124,9 @@ main(int argc, char **argv) message_timeout_ms = 30 * 1000; } break; - + case 'i': + ipc_name = strdup(optarg); + break; case '$': case '?': crm_help(flag, CRM_EX_OK); @@ -142,6 +146,9 @@ main(int argc, char **argv) case 'q': BE_SILENT = TRUE; break; + case 'P': + DO_PACEMAKERD_HEALTH = TRUE; + break; case 'S': DO_HEALTH = TRUE; crm_trace("Option %c => %s", flag, optarg); @@ -215,7 +222,7 @@ do_work(void) xmlNode *msg_data = NULL; gboolean all_is_good = TRUE; - if (DO_HEALTH == TRUE) { + if (DO_HEALTH) { crm_trace("Querying the system"); sys_to = CRM_SYSTEM_DC; @@ -233,6 +240,16 @@ do_work(void) all_is_good = FALSE; } + } else if (DO_PACEMAKERD_HEALTH) { + crm_trace("Querying pacemakerd state"); + + sys_to = CRM_SYSTEM_MCP; + crmd_operation = CRM_OP_PING; + + if (BE_VERBOSE) { + expected_responses = 1; + } + } else if (DO_ELECT_DC) { /* tell the local node to initiate an election */ @@ -251,7 +268,8 @@ do_work(void) cib_t *the_cib = cib_new(); xmlNode *output = NULL; - int rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command); + int rc = the_cib->cmds->signon(the_cib, + ipc_name?ipc_name:crm_system_name, cib_command); if (rc != pcmk_ok) { return -1; @@ -286,7 +304,7 @@ do_work(void) } /* send it */ - if (crmd_channel == NULL) { + if (ipc_channel == NULL) { crm_err("The IPC connection is not valid, cannot send anything"); return -1; } @@ -300,10 +318,10 @@ do_work(void) } { - xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to, - crm_system_name, admin_uuid); + xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, + sys_to, ipc_name?ipc_name:crm_system_name, admin_uuid); - crm_ipc_send(crmd_channel, cmd, 0, 0, NULL); + crm_ipc_send(ipc_channel, cmd, 0, 0, NULL); free_xml(cmd); } @@ -329,20 +347,23 @@ struct ipc_client_callbacks crm_callbacks = { gboolean do_init(void) { - mainloop_io_t *source = - mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks); + mainloop_io_t *ipc_source = + mainloop_add_ipc_client( + DO_PACEMAKERD_HEALTH?CRM_SYSTEM_MCP:CRM_SYSTEM_CRMD, + G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks); admin_uuid = crm_getpid_s(); - crmd_channel = mainloop_get_ipc_client(source); + ipc_channel = mainloop_get_ipc_client(ipc_source); - if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST) { + if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST || DO_PACEMAKERD_HEALTH) { return TRUE; - } else if (crmd_channel != NULL) { - xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1"); + } else if (ipc_channel != NULL) { + xmlNode *xml = create_hello_message(admin_uuid, + ipc_name?ipc_name:crm_system_name, "0", "1"); - crm_ipc_send(crmd_channel, xml, 0, 0, NULL); + crm_ipc_send(ipc_channel, xml, 0, 0, NULL); return TRUE; } return FALSE; @@ -392,8 +413,10 @@ admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata) if (xml == NULL) { crm_info("XML in IPC message was not valid... " "discarding."); - } else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) { + } else if (validate_crm_message(xml, ipc_name?ipc_name:crm_system_name, + admin_uuid, XML_ATTR_RESPONSE) == FALSE) { crm_trace("Message was not a CRM response. Discarding."); + printf("Validation of response failed\n"); } else if (DO_HEALTH) { xmlNode *data = get_message_xml(xml, F_CRM_DATA); @@ -408,6 +431,33 @@ admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata) fprintf(stderr, "%s\n", state); } + } else if (DO_PACEMAKERD_HEALTH) { + xmlNode *data = get_message_xml(xml, F_CRM_DATA); + const char *state = + crm_element_value(data, XML_PING_ATTR_PACEMAKERDSTATE); + time_t pinged = (time_t) 0; + long long value_ll = 0; + crm_time_t *crm_when = crm_time_new(NULL); + char *pinged_buf = NULL; + + crm_element_value_ll(data, XML_ATTR_TSTAMP, &value_ll); + pinged = (time_t) value_ll; + crm_time_set_timet(crm_when, &pinged); + pinged_buf = crm_time_as_string(crm_when, + crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone); + printf("Status of %s: %s (%s%s%s)\n", + crm_element_value(data, XML_PING_ATTR_SYSFROM), + state, crm_element_value(data, XML_PING_ATTR_STATUS), + (pinged != (time_t) 0)?" @ ":"", + (pinged != (time_t) 0)?pinged_buf:""); + + free(pinged_buf); + crm_time_free(crm_when); + + if (BE_SILENT && state != NULL) { + fprintf(stderr, "%s\n", state); + } + } else if (DO_WHOIS_DC) { const char *dc = crm_element_value(xml, F_CRM_HOST_FROM); From f9f74c30335d0636dbcd79095cf1e573aa46d356 Mon Sep 17 00:00:00 2001 From: Klaus Wenninger Date: Tue, 17 Dec 2019 01:08:43 +0100 Subject: [PATCH 2/2] Fix: sbd-integration: sync pacemakerd with sbd Add new api as not to make the xml-structure an external interface. --- daemons/pacemakerd/pacemakerd.c | 76 +++---- include/crm/common/Makefile.am | 2 +- include/crm/common/pacemakerd_types.h | 70 ++++++ lib/common/Makefile.am | 1 + lib/common/pacemakerd_client.c | 299 ++++++++++++++++++++++++++ tools/crmadmin.c | 92 +++++--- 6 files changed, 470 insertions(+), 70 deletions(-) create mode 100644 include/crm/common/pacemakerd_types.h create mode 100644 lib/common/pacemakerd_client.c diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c index 8b02dff21a5..2f200aa365a 100644 --- a/daemons/pacemakerd/pacemakerd.c +++ b/daemons/pacemakerd/pacemakerd.c @@ -564,47 +564,47 @@ pcmk_ipc_created(qb_ipcs_connection_t * c) } static void -pcmk_handle_ping_request( crm_client_t *c, xmlNode *msg, uint32_t id) +pcmk_handle_ping_request(crm_client_t *c, xmlNode *msg, uint32_t id) { - const char *value = NULL; - xmlNode *ping = NULL; - xmlNode *reply = NULL; - time_t pinged = time(NULL); - const char *from = crm_element_value(msg, F_CRM_SYS_FROM); - - /* Pinged for status */ - crm_trace("Pinged from %s.%s", - crm_element_value(msg, F_CRM_ORIGIN), - from?from:"unknown"); - ping = create_xml_node(NULL, XML_CRM_TAG_PING); - value = crm_element_value(msg, F_CRM_SYS_TO); - crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value); - crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state); - crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged); - crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok"); - reply = create_reply(msg, ping); - free_xml(ping); - if (reply) { - if (crm_ipcs_send(c, id, reply, crm_ipc_server_event) <= 0) { - crm_err("Failed sending ping-reply"); - } - free_xml(reply); - } else { - crm_err("Failed building ping-reply"); + const char *value = NULL; + xmlNode *ping = NULL; + xmlNode *reply = NULL; + time_t pinged = time(NULL); + const char *from = crm_element_value(msg, F_CRM_SYS_FROM); + + /* Pinged for status */ + crm_trace("Pinged from %s.%s", + crm_element_value(msg, F_CRM_ORIGIN), + from?from:"unknown"); + ping = create_xml_node(NULL, XML_CRM_TAG_PING); + value = crm_element_value(msg, F_CRM_SYS_TO); + crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value); + crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state); + crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged); + crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok"); + reply = create_reply(msg, ping); + free_xml(ping); + if (reply) { + if (crm_ipcs_send(c, id, reply, crm_ipc_server_event) <= 0) { + crm_err("Failed sending ping-reply"); } - /* just proceed state on sbd pinging us */ - if (from && strstr(from, "sbd")) { - if (crm_str_eq(pacemakerd_state, - XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, - TRUE)) { - shutdown_complete_state_reported_to = c->pid; - } else if (crm_str_eq(pacemakerd_state, - XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, - TRUE)) { - pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; - mainloop_set_trigger(startup_trigger); - } + free_xml(reply); + } else { + crm_err("Failed building ping-reply"); + } + /* just proceed state on sbd pinging us */ + if (from && strstr(from, "sbd")) { + if (crm_str_eq(pacemakerd_state, + XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE, + TRUE)) { + shutdown_complete_state_reported_to = c->pid; + } else if (crm_str_eq(pacemakerd_state, + XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, + TRUE)) { + pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS; + mainloop_set_trigger(startup_trigger); } + } } /* Exit code means? */ diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am index 7c1c0901292..68b7a71aa87 100644 --- a/include/crm/common/Makefile.am +++ b/include/crm/common/Makefile.am @@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in headerdir=$(pkgincludedir)/crm/common header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \ - nvpair.h + nvpair.h pacemakerd_types.h noinst_HEADERS = cib_secrets.h ipcs.h internal.h alerts_internal.h \ iso8601_internal.h remote_internal.h xml_internal.h \ ipc_internal.h output.h cmdline_internal.h curses_internal.h diff --git a/include/crm/common/pacemakerd_types.h b/include/crm/common/pacemakerd_types.h new file mode 100644 index 00000000000..dba39a2c8dd --- /dev/null +++ b/include/crm/common/pacemakerd_types.h @@ -0,0 +1,70 @@ +/* + * Copyright 2004-2019 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + +#ifndef PACEMAKERD_TYPES__H +# define PACEMAKERD_TYPES__H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +enum pacemakerd_conn_state { + pacemakerd_conn_connected, + pacemakerd_conn_disconnected +}; + +enum pacemakerd_state { + pacemakerd_state_invalid = -1, + pacemakerd_state_init = 0, + pacemakerd_state_starting_daemons, + pacemakerd_state_wait_for_ping, + pacemakerd_state_running, + pacemakerd_state_shutting_down, + pacemakerd_state_shutdown_complete, + pacemakerd_state_max = pacemakerd_state_shutdown_complete, +}; + +typedef struct pacemakerd_s pacemakerd_t; + +typedef struct pacemakerd_api_operations_s { + int (*connect) (pacemakerd_t *pacemakerd, const char *name); + int (*disconnect) (pacemakerd_t *pacemakerd); + void (*free) (pacemakerd_t *pacemakerd); + int (*set_ping_callback) (pacemakerd_t *pacemakerd, + void (*callback) (pacemakerd_t *pacemakerd, + time_t last_good, + enum pacemakerd_state state, + int rc, gpointer userdata), + gpointer userdata); + int (*set_disconnect_callback) (pacemakerd_t *pacemakerd, + void (*callback) (gpointer userdata), + gpointer userdata + ); + int (*ping) (pacemakerd_t *pacemakerd, const char *name, + const char *admin_uuid, int call_options); +} pacemakerd_api_operations_t; + +struct pacemakerd_s { + enum pacemakerd_conn_state conn_state; + void *pacemakerd_private; + pacemakerd_api_operations_t *cmds; +}; + +pacemakerd_t * pacemakerd_api_new(void); +void pacemakerd_api_delete(pacemakerd_t * pacemakerd); +enum pacemakerd_state pacemakerd_state_text2enum(const char *state); +const char *pacemakerd_state_enum2text(enum pacemakerd_state state); + +#ifdef __cplusplus +} +#endif + +#endif // PACEMAKERD_TYPES__H \ No newline at end of file diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am index 7e14da944e3..e5fc4bb7bca 100644 --- a/lib/common/Makefile.am +++ b/lib/common/Makefile.am @@ -68,6 +68,7 @@ libcrmcommon_la_SOURCES += utils.c libcrmcommon_la_SOURCES += watchdog.c libcrmcommon_la_SOURCES += xml.c libcrmcommon_la_SOURCES += xpath.c +libcrmcommon_la_SOURCES += pacemakerd_client.c # It's possible to build the library adding ../gnu/md5.c directly to SOURCES, # but distclean chokes on that because it tries to include the source's .Plo diff --git a/lib/common/pacemakerd_client.c b/lib/common/pacemakerd_client.c new file mode 100644 index 00000000000..7309cd5a25d --- /dev/null +++ b/lib/common/pacemakerd_client.c @@ -0,0 +1,299 @@ +/* + * Copyright 2004-2019 the Pacemaker project contributors + * + * The version control history for this file may have further details. + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + +#include +#include +#include + +#include +#include +#include + +CRM_TRACE_INIT_DATA(pacemakerd); + +typedef struct pacemakerd_api_private_s { + mainloop_io_t *source; + void (*ping_callback) (pacemakerd_t *pacemakerd, time_t last_good, + enum pacemakerd_state state, int rc, + gpointer userdata); + gpointer ping_userdata; + void (*disconnect_callback) (gpointer userdata); + gpointer disconnect_userdata; +} pacemakerd_api_private_t; + +const char *pacemakerd_state_str[] = { + XML_PING_ATTR_PACEMAKERDSTATE_INIT, + XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS, + XML_PING_ATTR_PACEMAKERDSTATE_WAITPING, + XML_PING_ATTR_PACEMAKERDSTATE_RUNNING, + XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN, + XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE +}; + +enum pacemakerd_state pacemakerd_state_text2enum(const char *state) +{ + int i; + + if (state == NULL) { + return pacemakerd_state_invalid; + } + for (i=pacemakerd_state_init; i <= pacemakerd_state_max; + i++) { + if (crm_str_eq(state, pacemakerd_state_str[i], TRUE)) { + return i; + } + } + return pacemakerd_state_invalid; +} + +const char *pacemakerd_state_enum2text(enum pacemakerd_state state) +{ + if ((state >= pacemakerd_state_init) && + (state <= pacemakerd_state_max)) { + return pacemakerd_state_str[state]; + } + return NULL; +} + +static int +pacemakerd_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) +{ + pacemakerd_t *pacemakerd = userdata; + pacemakerd_api_private_t *native; + const char *type = NULL; + const char *crm_msg_reference = NULL; + xmlNode *xml; + int rc = 0; + + CRM_CHECK(pacemakerd != NULL, return rc); + native = pacemakerd->pacemakerd_private; + CRM_ASSERT(native != NULL); + CRM_ASSERT(native->ping_callback != NULL); + + xml = string2xml(buffer); + if (xml == NULL) { + return rc; + } + + type = crm_element_value(xml, F_CRM_MSG_TYPE); + crm_msg_reference = crm_element_value(xml, XML_ATTR_REFERENCE); + + if (type == NULL) { + crm_info("No message type defined."); + } else if (strcasecmp(XML_ATTR_RESPONSE, type) != 0) { + crm_info("Expecting a (%s) message but received a (%s).", + XML_ATTR_RESPONSE, type); + } else if (crm_msg_reference == NULL) { + crm_info("No message crm_msg_reference defined."); + } else { + xmlNode *data = get_message_xml(xml, F_CRM_DATA); + const char *state = + crm_element_value(data, XML_PING_ATTR_PACEMAKERDSTATE); + const char *status = + crm_element_value(data, XML_PING_ATTR_STATUS); + time_t pinged = (time_t) 0; + long long value_ll = 0; + + crm_element_value_ll(data, XML_ATTR_TSTAMP, &value_ll); + pinged = (time_t) value_ll; + native->ping_callback(pacemakerd, pinged, + pacemakerd_state_text2enum(state), + crm_str_eq(status, "ok", FALSE)?pcmk_ok:pcmk_err_generic, + userdata); + rc = 1; + } + + free_xml(xml); + return rc; +} + +static void +pacemakerd_connection_destroy(gpointer userdata) +{ + pacemakerd_t *pacemakerd = userdata; + pacemakerd_api_private_t *native; + + CRM_CHECK(pacemakerd != NULL, return); + native = pacemakerd->pacemakerd_private; + CRM_ASSERT(native != NULL); + CRM_ASSERT(native->disconnect_callback != NULL); + + crm_trace("Sending destroyed notification"); + native->source = NULL; + pacemakerd->conn_state = pacemakerd_conn_disconnected; + native->disconnect_callback(native->disconnect_userdata); +} + +static int +pacemakerd_api_connect(pacemakerd_t *pacemakerd, const char *name) +{ + pacemakerd_api_private_t *native; + const char *display_name = name? name : "client"; + static struct ipc_client_callbacks pacemakerd_callbacks = { + .dispatch = pacemakerd_dispatch_internal, + .destroy = pacemakerd_connection_destroy + }; + + CRM_CHECK(pacemakerd != NULL, return -EINVAL); + native = pacemakerd->pacemakerd_private; + CRM_ASSERT(native != NULL); + + crm_debug("Attempting pacemakerd connection by %s", display_name); + if (pacemakerd->conn_state != pacemakerd_conn_connected) { + native->source = + mainloop_add_ipc_client(CRM_SYSTEM_MCP, G_PRIORITY_DEFAULT, 0, + (gpointer) pacemakerd, + &pacemakerd_callbacks); + if (native->source) { + pacemakerd->conn_state = pacemakerd_conn_connected; + } + } + + return (pacemakerd->conn_state == pacemakerd_conn_connected)? + pcmk_ok:-ENOTCONN; +} + +static int +pacemakerd_api_disconnect(pacemakerd_t *pacemakerd) +{ + pacemakerd_api_private_t *native; + + CRM_CHECK(pacemakerd != NULL, return -EINVAL); + native = pacemakerd->pacemakerd_private; + CRM_ASSERT(native != NULL); + + crm_debug("Disconnecting from pacemakerd"); + + if (native->source) { + mainloop_del_ipc_client(native->source); + native->source = NULL; + } + + pacemakerd->conn_state = pacemakerd_conn_disconnected; + return pcmk_ok; +} + +static void +pacemakerd_api_free(pacemakerd_t *pacemakerd) +{ + if (pacemakerd) { + if (pacemakerd->conn_state == pacemakerd_conn_connected) { + pacemakerd_api_disconnect(pacemakerd); + } + free(pacemakerd->pacemakerd_private); + free(pacemakerd->cmds); + free(pacemakerd); + } +} + +static int +pacemakerd_api_set_ping_callback(pacemakerd_t *pacemakerd, + void (*callback) (pacemakerd_t *pacemakerd, + time_t last_good, + enum pacemakerd_state state, + int rc, gpointer userdata), + gpointer userdata) +{ + pacemakerd_api_private_t *native; + + CRM_CHECK(pacemakerd != NULL, return -EINVAL); + native = pacemakerd->pacemakerd_private; + CRM_ASSERT(native != NULL); + native->ping_callback = callback; + native->ping_userdata = userdata; + + return pcmk_ok; +} + +static int +pacemakerd_api_set_disconnect_callback(pacemakerd_t *pacemakerd, + void (*callback) (gpointer userdata), + gpointer userdata) +{ + pacemakerd_api_private_t *native; + + CRM_CHECK(pacemakerd != NULL, return -EINVAL); + native = pacemakerd->pacemakerd_private; + CRM_ASSERT(native != NULL); + native->disconnect_callback = callback; + native->disconnect_userdata = userdata; + + return pcmk_ok; +} + +static int +pacemakerd_api_ping(pacemakerd_t *pacemakerd, const char *name, + const char *admin_uuid, int call_options) +{ + pacemakerd_api_private_t *native; + xmlNode *cmd; + int rc; + + CRM_CHECK(pacemakerd != NULL, return -EINVAL); + native = pacemakerd->pacemakerd_private; + CRM_ASSERT(native != NULL); + + cmd = create_request(CRM_OP_PING, NULL, NULL, + CRM_SYSTEM_MCP, name, admin_uuid); + + if (cmd) { + rc = crm_ipc_send(mainloop_get_ipc_client(native->source), + cmd, 0, 0, NULL); + if (rc < 0) { + crm_debug("Couldn't register with the pacemakerd: %s " + CRM_XS " rc=%d", pcmk_strerror(rc), rc); + rc = -ECOMM; + } + free_xml(cmd); + } else { + rc = -ENOMSG; + } + + return rc; +} + +pacemakerd_t * +pacemakerd_api_new(void) +{ + pacemakerd_t *api = calloc(1, sizeof(pacemakerd_t)); + pacemakerd_api_operations_t *cmds = + calloc(1, sizeof(pacemakerd_api_operations_t)); + pacemakerd_api_private_t *private = + calloc(1, sizeof(pacemakerd_api_private_t)); + + if (api && cmds && private) { + api->pacemakerd_private = private; + api->cmds = cmds; + api->conn_state = pacemakerd_conn_disconnected; + + cmds->connect = pacemakerd_api_connect; + cmds->disconnect = pacemakerd_api_disconnect; + cmds->free = pacemakerd_api_free; + cmds->set_ping_callback = pacemakerd_api_set_ping_callback; + cmds->set_disconnect_callback = + pacemakerd_api_set_disconnect_callback; + cmds->ping = pacemakerd_api_ping; + + } else { + free(cmds); + free(api); + free(private); + api = NULL; + } + return api; +} + +void +pacemakerd_api_delete(pacemakerd_t * pacemakerd) +{ + if ((pacemakerd) && (pacemakerd->cmds) && + (pacemakerd->cmds->free)) { + pacemakerd->cmds->free(pacemakerd); + } +} \ No newline at end of file diff --git a/tools/crmadmin.c b/tools/crmadmin.c index 7f0929fbb27..99fe9be6308 100644 --- a/tools/crmadmin.c +++ b/tools/crmadmin.c @@ -28,12 +28,14 @@ #include #include +#include static int message_timer_id = -1; static int message_timeout_ms = 30 * 1000; static GMainLoop *mainloop = NULL; static crm_ipc_t *ipc_channel = NULL; +static pacemakerd_t *pacemakerd_api = NULL; static char *admin_uuid = NULL; static char *ipc_name = NULL; @@ -243,13 +245,16 @@ do_work(void) } else if (DO_PACEMAKERD_HEALTH) { crm_trace("Querying pacemakerd state"); - sys_to = CRM_SYSTEM_MCP; - crmd_operation = CRM_OP_PING; + ret = pacemakerd_api->cmds->ping(pacemakerd_api, + ipc_name?ipc_name:crm_system_name, + admin_uuid, 0); if (BE_VERBOSE) { expected_responses = 1; } + return ret; + } else if (DO_ELECT_DC) { /* tell the local node to initiate an election */ @@ -344,10 +349,62 @@ struct ipc_client_callbacks crm_callbacks = { .destroy = crmadmin_ipc_connection_destroy }; +static void +ping_callback (pacemakerd_t *pacemakerd, + time_t last_good, + enum pacemakerd_state state, + int rc, gpointer userdata) +{ + crm_time_t *crm_when = crm_time_new(NULL); + char *pinged_buf = NULL; + static int received_responses = 0; + + crm_time_set_timet(crm_when, &last_good); + pinged_buf = crm_time_as_string(crm_when, + crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone); + printf("Status of pacemakerd: %s (%s%s%s)\n", + pacemakerd_state_enum2text(state), + (rc == pcmk_ok)?"ok":"unclean", + (last_good != (time_t) 0)?" @ ":"", + (last_good != (time_t) 0)?pinged_buf:""); + + free(pinged_buf); + crm_time_free(crm_when); + + if (BE_SILENT && pacemakerd_state_enum2text(state) != NULL) { + fprintf(stderr, "%s\n", pacemakerd_state_enum2text(state)); + } + + received_responses++; + + if (received_responses >= expected_responses) { + crm_trace("Received expected number (%d) of replies, exiting normally", + expected_responses); + crm_exit(CRM_EX_OK); + } +} + gboolean do_init(void) { - mainloop_io_t *ipc_source = + mainloop_io_t *ipc_source; + + if (DO_PACEMAKERD_HEALTH) { + gboolean rv = FALSE; + + pacemakerd_api = pacemakerd_api_new(); + if (pacemakerd_api) { + pacemakerd_api->cmds->set_disconnect_callback(pacemakerd_api, + crmadmin_ipc_connection_destroy, NULL); + pacemakerd_api->cmds->set_ping_callback(pacemakerd_api, + ping_callback, NULL); + rv = (pacemakerd_api->cmds->connect(pacemakerd_api, + ipc_name?ipc_name:crm_system_name) == pcmk_ok); + } + return rv; + } + + ipc_source = mainloop_add_ipc_client( DO_PACEMAKERD_HEALTH?CRM_SYSTEM_MCP:CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks); @@ -356,7 +413,7 @@ do_init(void) ipc_channel = mainloop_get_ipc_client(ipc_source); - if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST || DO_PACEMAKERD_HEALTH) { + if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST) { return TRUE; } else if (ipc_channel != NULL) { @@ -431,33 +488,6 @@ admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata) fprintf(stderr, "%s\n", state); } - } else if (DO_PACEMAKERD_HEALTH) { - xmlNode *data = get_message_xml(xml, F_CRM_DATA); - const char *state = - crm_element_value(data, XML_PING_ATTR_PACEMAKERDSTATE); - time_t pinged = (time_t) 0; - long long value_ll = 0; - crm_time_t *crm_when = crm_time_new(NULL); - char *pinged_buf = NULL; - - crm_element_value_ll(data, XML_ATTR_TSTAMP, &value_ll); - pinged = (time_t) value_ll; - crm_time_set_timet(crm_when, &pinged); - pinged_buf = crm_time_as_string(crm_when, - crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone); - printf("Status of %s: %s (%s%s%s)\n", - crm_element_value(data, XML_PING_ATTR_SYSFROM), - state, crm_element_value(data, XML_PING_ATTR_STATUS), - (pinged != (time_t) 0)?" @ ":"", - (pinged != (time_t) 0)?pinged_buf:""); - - free(pinged_buf); - crm_time_free(crm_when); - - if (BE_SILENT && state != NULL) { - fprintf(stderr, "%s\n", state); - } - } else if (DO_WHOIS_DC) { const char *dc = crm_element_value(xml, F_CRM_HOST_FROM);