corosync  2.3.2
exec/votequorum.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2009-2012 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Authors: Christine Caulfield (ccaulfie@redhat.com)
7  * Fabio M. Di Nitto (fdinitto@redhat.com)
8  *
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 #include <config.h>
37 
38 #include <sys/types.h>
39 #include <stdint.h>
40 
41 #include <qb/qbipc_common.h>
42 
43 #include "quorum.h"
44 #include <corosync/corodefs.h>
45 #include <corosync/list.h>
46 #include <corosync/logsys.h>
47 #include <corosync/coroapi.h>
48 #include <corosync/icmap.h>
50 
51 #include "service.h"
52 
53 LOGSYS_DECLARE_SUBSYS ("VOTEQ");
54 
55 /*
56  * interface with corosync
57  */
58 
59 static struct corosync_api_v1 *corosync_api;
60 
61 /*
62  * votequorum global config vars
63  */
64 
65 
66 static char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN];
67 static struct cluster_node *qdevice = NULL;
68 static unsigned int qdevice_timeout = VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT;
69 static uint8_t qdevice_can_operate = 1;
70 static void *qdevice_reg_conn = NULL;
71 static uint8_t qdevice_master_wins = 0;
72 
73 static uint8_t two_node = 0;
74 
75 static uint8_t wait_for_all = 0;
76 static uint8_t wait_for_all_status = 0;
77 
78 static uint8_t auto_tie_breaker = 0;
79 static int lowest_node_id = -1;
80 
81 #define DEFAULT_LMS_WIN 10000
82 static uint8_t last_man_standing = 0;
83 static uint32_t last_man_standing_window = DEFAULT_LMS_WIN;
84 
85 static uint8_t allow_downscale = 0;
86 static uint32_t ev_barrier = 0;
87 
88 /*
89  * votequorum_exec defines/structs/forward definitions
90  */
91 
93  struct qb_ipc_request_header header __attribute__((aligned(8)));
94  uint32_t nodeid;
95  uint32_t votes;
96  uint32_t expected_votes;
97  uint32_t flags;
98 } __attribute__((packed));
99 
101  struct qb_ipc_request_header header __attribute__((aligned(8)));
102  uint32_t nodeid;
103  uint32_t value;
104  uint8_t param;
105  uint8_t _pad0;
106  uint8_t _pad1;
107  uint8_t _pad2;
108 } __attribute__((packed));
109 
111  struct qb_ipc_request_header header __attribute__((aligned(8)));
112  uint32_t operation;
114 } __attribute__((packed));
115 
117  struct qb_ipc_request_header header __attribute__((aligned(8)));
120 } __attribute__((packed));
121 
122 /*
123  * votequorum_exec onwire version (via totem)
124  */
125 
126 #include "votequorum.h"
127 
128 /*
129  * votequorum_exec onwire messages (via totem)
130  */
131 
132 #define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO 0
133 #define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE 1
134 #define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG 2
135 #define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE 3
136 
137 static void votequorum_exec_send_expectedvotes_notification(void);
138 static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context);
139 
140 #define VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES 1
141 #define VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES 2
142 
143 static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid, uint32_t value);
144 
145 /*
146  * used by req_exec_quorum_qdevice_reg
147  */
148 #define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER 0
149 #define VOTEQUORUM_QDEVICE_OPERATION_REGISTER 1
150 
151 /*
152  * votequorum internal node status/view
153  */
154 
155 #define NODE_FLAGS_QUORATE 1
156 #define NODE_FLAGS_LEAVING 2
157 #define NODE_FLAGS_WFASTATUS 4
158 #define NODE_FLAGS_FIRST 8
159 #define NODE_FLAGS_QDEVICE_REGISTERED 16
160 #define NODE_FLAGS_QDEVICE_ALIVE 32
161 #define NODE_FLAGS_QDEVICE_CAST_VOTE 64
162 #define NODE_FLAGS_QDEVICE_MASTER_WINS 128
163 
164 typedef enum {
168 } nodestate_t;
169 
170 struct cluster_node {
171  int node_id;
173  uint32_t votes;
174  uint32_t expected_votes;
175  uint32_t flags;
176  struct list_head list;
177 };
178 
179 /*
180  * votequorum internal quorum status
181  */
182 
183 static uint8_t quorum;
184 static uint8_t cluster_is_quorate;
185 
186 /*
187  * votequorum membership data
188  */
189 
190 static struct cluster_node *us;
191 static struct list_head cluster_members_list;
192 static unsigned int quorum_members[PROCESSOR_COUNT_MAX];
193 static int quorum_members_entries = 0;
194 static struct memb_ring_id quorum_ringid;
195 
196 /*
197  * pre allocate all cluster_nodes + one for qdevice
198  */
199 static struct cluster_node cluster_nodes[PROCESSOR_COUNT_MAX+2];
200 static int cluster_nodes_entries = 0;
201 
202 /*
203  * votequorum tracking
204  */
205 struct quorum_pd {
206  unsigned char track_flags;
209  struct list_head list;
210  void *conn;
211 };
212 
213 static struct list_head trackers_list;
214 
215 /*
216  * votequorum timers
217  */
218 
219 static corosync_timer_handle_t qdevice_timer;
220 static int qdevice_timer_set = 0;
221 static corosync_timer_handle_t last_man_standing_timer;
222 static int last_man_standing_timer_set = 0;
223 
224 /*
225  * Service Interfaces required by service_message_handler struct
226  */
227 
228 static int sync_in_progress = 0;
229 
230 static void votequorum_sync_init (
231  const unsigned int *trans_list,
232  size_t trans_list_entries,
233  const unsigned int *member_list,
234  size_t member_list_entries,
235  const struct memb_ring_id *ring_id);
236 
237 static int votequorum_sync_process (void);
238 static void votequorum_sync_activate (void);
239 static void votequorum_sync_abort (void);
240 
241 static quorum_set_quorate_fn_t quorum_callback;
242 
243 /*
244  * votequorum_exec handler and definitions
245  */
246 
247 static char *votequorum_exec_init_fn (struct corosync_api_v1 *api);
248 static int votequorum_exec_exit_fn (void);
249 static int votequorum_exec_send_nodeinfo(uint32_t nodeid);
250 
251 static void message_handler_req_exec_votequorum_nodeinfo (
252  const void *message,
253  unsigned int nodeid);
254 static void exec_votequorum_nodeinfo_endian_convert (void *message);
255 
256 static void message_handler_req_exec_votequorum_reconfigure (
257  const void *message,
258  unsigned int nodeid);
259 static void exec_votequorum_reconfigure_endian_convert (void *message);
260 
261 static void message_handler_req_exec_votequorum_qdevice_reg (
262  const void *message,
263  unsigned int nodeid);
264 static void exec_votequorum_qdevice_reg_endian_convert (void *message);
265 
266 static void message_handler_req_exec_votequorum_qdevice_reconfigure (
267  const void *message,
268  unsigned int nodeid);
269 static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message);
270 
271 static struct corosync_exec_handler votequorum_exec_engine[] =
272 {
273  { /* 0 */
274  .exec_handler_fn = message_handler_req_exec_votequorum_nodeinfo,
275  .exec_endian_convert_fn = exec_votequorum_nodeinfo_endian_convert
276  },
277  { /* 1 */
278  .exec_handler_fn = message_handler_req_exec_votequorum_reconfigure,
279  .exec_endian_convert_fn = exec_votequorum_reconfigure_endian_convert
280  },
281  { /* 2 */
282  .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reg,
283  .exec_endian_convert_fn = exec_votequorum_qdevice_reg_endian_convert
284  },
285  { /* 3 */
286  .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reconfigure,
287  .exec_endian_convert_fn = exec_votequorum_qdevice_reconfigure_endian_convert
288  },
289 };
290 
291 /*
292  * Library Handler and Functions Definitions
293  */
294 
295 static int quorum_lib_init_fn (void *conn);
296 
297 static int quorum_lib_exit_fn (void *conn);
298 
299 static void message_handler_req_lib_votequorum_getinfo (void *conn,
300  const void *message);
301 
302 static void message_handler_req_lib_votequorum_setexpected (void *conn,
303  const void *message);
304 
305 static void message_handler_req_lib_votequorum_setvotes (void *conn,
306  const void *message);
307 
308 static void message_handler_req_lib_votequorum_trackstart (void *conn,
309  const void *message);
310 
311 static void message_handler_req_lib_votequorum_trackstop (void *conn,
312  const void *message);
313 
314 static void message_handler_req_lib_votequorum_qdevice_register (void *conn,
315  const void *message);
316 
317 static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
318  const void *message);
319 
320 static void message_handler_req_lib_votequorum_qdevice_update (void *conn,
321  const void *message);
322 
323 static void message_handler_req_lib_votequorum_qdevice_poll (void *conn,
324  const void *message);
325 
326 static void message_handler_req_lib_votequorum_qdevice_master_wins (void *conn,
327  const void *message);
328 
329 static struct corosync_lib_handler quorum_lib_service[] =
330 {
331  { /* 0 */
332  .lib_handler_fn = message_handler_req_lib_votequorum_getinfo,
334  },
335  { /* 1 */
336  .lib_handler_fn = message_handler_req_lib_votequorum_setexpected,
338  },
339  { /* 2 */
340  .lib_handler_fn = message_handler_req_lib_votequorum_setvotes,
342  },
343  { /* 3 */
344  .lib_handler_fn = message_handler_req_lib_votequorum_trackstart,
346  },
347  { /* 4 */
348  .lib_handler_fn = message_handler_req_lib_votequorum_trackstop,
350  },
351  { /* 5 */
352  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_register,
354  },
355  { /* 6 */
356  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_unregister,
358  },
359  { /* 7 */
360  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_update,
362  },
363  { /* 8 */
364  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_poll,
366  },
367  { /* 9 */
368  .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_master_wins,
370  }
371 };
372 
373 static struct corosync_service_engine votequorum_service_engine = {
374  .name = "corosync vote quorum service v1.0",
375  .id = VOTEQUORUM_SERVICE,
376  .priority = 2,
377  .private_data_size = sizeof (struct quorum_pd),
378  .allow_inquorate = CS_LIB_ALLOW_INQUORATE,
379  .flow_control = COROSYNC_LIB_FLOW_CONTROL_REQUIRED,
380  .lib_init_fn = quorum_lib_init_fn,
381  .lib_exit_fn = quorum_lib_exit_fn,
382  .lib_engine = quorum_lib_service,
383  .lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler),
384  .exec_init_fn = votequorum_exec_init_fn,
385  .exec_exit_fn = votequorum_exec_exit_fn,
386  .exec_engine = votequorum_exec_engine,
387  .exec_engine_count = sizeof (votequorum_exec_engine) / sizeof (struct corosync_exec_handler),
388  .sync_init = votequorum_sync_init,
389  .sync_process = votequorum_sync_process,
390  .sync_activate = votequorum_sync_activate,
391  .sync_abort = votequorum_sync_abort
392 };
393 
395 {
396  return (&votequorum_service_engine);
397 }
398 
399 static struct default_service votequorum_service[] = {
400  {
401  .name = "corosync_votequorum",
402  .ver = 0,
404  },
405 };
406 
407 /*
408  * common/utility macros/functions
409  */
410 
411 #define max(a,b) (((a) > (b)) ? (a) : (b))
412 
413 #define list_iterate(v, head) \
414  for (v = (head)->next; v != head; v = v->next)
415 
416 static void node_add_ordered(struct cluster_node *newnode)
417 {
418  struct cluster_node *node = NULL;
419  struct list_head *tmp;
420  struct list_head *newlist = &newnode->list;
421 
422  ENTER();
423 
424  list_iterate(tmp, &cluster_members_list) {
425  node = list_entry(tmp, struct cluster_node, list);
426  if (newnode->node_id < node->node_id) {
427  break;
428  }
429  }
430 
431  if (!node) {
432  list_add(&newnode->list, &cluster_members_list);
433  } else {
434  newlist->prev = tmp->prev;
435  newlist->next = tmp;
436  tmp->prev->next = newlist;
437  tmp->prev = newlist;
438  }
439 
440  LEAVE();
441 }
442 
443 static struct cluster_node *allocate_node(unsigned int nodeid)
444 {
445  struct cluster_node *cl = NULL;
446  struct list_head *tmp;
447 
448  ENTER();
449 
450  if (cluster_nodes_entries <= PROCESSOR_COUNT_MAX + 1) {
451  cl = (struct cluster_node *)&cluster_nodes[cluster_nodes_entries];
452  cluster_nodes_entries++;
453  } else {
454  list_iterate(tmp, &cluster_members_list) {
455  cl = list_entry(tmp, struct cluster_node, list);
456  if (cl->state == NODESTATE_DEAD) {
457  break;
458  }
459  }
460  /*
461  * this should never happen
462  */
463  if (!cl) {
464  log_printf(LOGSYS_LEVEL_CRIT, "Unable to find memory for node %u data!!", nodeid);
465  goto out;
466  }
467  list_del(tmp);
468  }
469 
470  memset(cl, 0, sizeof(struct cluster_node));
471  cl->node_id = nodeid;
472  if (nodeid != VOTEQUORUM_QDEVICE_NODEID) {
473  node_add_ordered(cl);
474  }
475 
476 out:
477  LEAVE();
478 
479  return cl;
480 }
481 
482 static struct cluster_node *find_node_by_nodeid(unsigned int nodeid)
483 {
484  struct cluster_node *node;
485  struct list_head *tmp;
486 
487  ENTER();
488 
489  if (nodeid == us->node_id) {
490  LEAVE();
491  return us;
492  }
493 
494  if (nodeid == VOTEQUORUM_QDEVICE_NODEID) {
495  LEAVE();
496  return qdevice;
497  }
498 
499  list_iterate(tmp, &cluster_members_list) {
500  node = list_entry(tmp, struct cluster_node, list);
501  if (node->node_id == nodeid) {
502  LEAVE();
503  return node;
504  }
505  }
506 
507  LEAVE();
508  return NULL;
509 }
510 
511 static void get_lowest_node_id(void)
512 {
513  struct cluster_node *node = NULL;
514  struct list_head *tmp;
515 
516  ENTER();
517 
518  lowest_node_id = us->node_id;
519 
520  list_iterate(tmp, &cluster_members_list) {
521  node = list_entry(tmp, struct cluster_node, list);
522  if ((node->state == NODESTATE_MEMBER) &&
523  (node->node_id < lowest_node_id)) {
524  lowest_node_id = node->node_id;
525  }
526  }
527  log_printf(LOGSYS_LEVEL_DEBUG, "lowest node id: %d us: %d", lowest_node_id, us->node_id);
528  icmap_set_uint32("runtime.votequorum.lowest_node_id", lowest_node_id);
529 
530  LEAVE();
531 }
532 
533 static int check_low_node_id_partition(void)
534 {
535  struct cluster_node *node = NULL;
536  struct list_head *tmp;
537  int found = 0;
538 
539  ENTER();
540 
541  list_iterate(tmp, &cluster_members_list) {
542  node = list_entry(tmp, struct cluster_node, list);
543  if ((node->state == NODESTATE_MEMBER) &&
544  (node->node_id == lowest_node_id)) {
545  found = 1;
546  }
547  }
548 
549  LEAVE();
550  return found;
551 }
552 
553 static int check_qdevice_master(void)
554 {
555  struct cluster_node *node = NULL;
556  struct list_head *tmp;
557  int found = 0;
558 
559  ENTER();
560 
561  list_iterate(tmp, &cluster_members_list) {
562  node = list_entry(tmp, struct cluster_node, list);
563  if ((node->state == NODESTATE_MEMBER) &&
566  found = 1;
567  }
568  }
569 
570  LEAVE();
571  return found;
572 }
573 
574 static void decode_flags(uint32_t flags)
575 {
576  ENTER();
577 
579  "flags: quorate: %s Leaving: %s WFA Status: %s First: %s Qdevice: %s QdeviceAlive: %s QdeviceCastVote: %s QdeviceMasterWins: %s",
580  (flags & NODE_FLAGS_QUORATE)?"Yes":"No",
581  (flags & NODE_FLAGS_LEAVING)?"Yes":"No",
582  (flags & NODE_FLAGS_WFASTATUS)?"Yes":"No",
583  (flags & NODE_FLAGS_FIRST)?"Yes":"No",
584  (flags & NODE_FLAGS_QDEVICE_REGISTERED)?"Yes":"No",
585  (flags & NODE_FLAGS_QDEVICE_ALIVE)?"Yes":"No",
586  (flags & NODE_FLAGS_QDEVICE_CAST_VOTE)?"Yes":"No",
587  (flags & NODE_FLAGS_QDEVICE_MASTER_WINS)?"Yes":"No");
588 
589  LEAVE();
590 }
591 
592 static void update_wait_for_all_status(uint8_t wfa_status)
593 {
594  ENTER();
595 
596  wait_for_all_status = wfa_status;
597  if (wait_for_all_status) {
599  } else {
600  us->flags &= ~NODE_FLAGS_WFASTATUS;
601  }
602  icmap_set_uint8("runtime.votequorum.wait_for_all_status",
603  wait_for_all_status);
604 
605  LEAVE();
606 }
607 
608 static void update_two_node(void)
609 {
610  ENTER();
611 
612  icmap_set_uint8("runtime.votequorum.two_node", two_node);
613 
614  LEAVE();
615 }
616 
617 static void update_ev_barrier(uint32_t expected_votes)
618 {
619  ENTER();
620 
621  ev_barrier = expected_votes;
622  icmap_set_uint32("runtime.votequorum.ev_barrier", ev_barrier);
623 
624  LEAVE();
625 }
626 
627 static void update_qdevice_can_operate(uint8_t status)
628 {
629  ENTER();
630 
631  qdevice_can_operate = status;
632  icmap_set_uint8("runtime.votequorum.qdevice_can_operate", qdevice_can_operate);
633 
634  LEAVE();
635 }
636 
637 static void update_qdevice_master_wins(uint8_t allow)
638 {
639  ENTER();
640 
641  qdevice_master_wins = allow;
642  icmap_set_uint8("runtime.votequorum.qdevice_master_wins", qdevice_master_wins);
643 
644  LEAVE();
645 }
646 
647 /*
648  * quorum calculation core bits
649  */
650 
651 static int calculate_quorum(int allow_decrease, unsigned int max_expected, unsigned int *ret_total_votes)
652 {
653  struct list_head *nodelist;
654  struct cluster_node *node;
655  unsigned int total_votes = 0;
656  unsigned int highest_expected = 0;
657  unsigned int newquorum, q1, q2;
658  unsigned int total_nodes = 0;
659 
660  ENTER();
661 
662  if ((allow_downscale) && (allow_decrease) && (max_expected)) {
663  max_expected = max(ev_barrier, max_expected);
664  }
665 
666  list_iterate(nodelist, &cluster_members_list) {
667  node = list_entry(nodelist, struct cluster_node, list);
668 
669  log_printf(LOGSYS_LEVEL_DEBUG, "node %u state=%d, votes=%u, expected=%u",
670  node->node_id, node->state, node->votes, node->expected_votes);
671 
672  if (node->state == NODESTATE_MEMBER) {
673  if (max_expected) {
674  node->expected_votes = max_expected;
675  } else {
676  highest_expected = max(highest_expected, node->expected_votes);
677  }
678  total_votes += node->votes;
679  total_nodes++;
680  }
681  }
682 
684  log_printf(LOGSYS_LEVEL_DEBUG, "node 0 state=1, votes=%u", qdevice->votes);
685  total_votes += qdevice->votes;
686  total_nodes++;
687  }
688 
689  if (max_expected > 0) {
690  highest_expected = max_expected;
691  }
692 
693  /*
694  * This quorum calculation is taken from the OpenVMS Cluster Systems
695  * manual, but, then, you guessed that didn't you
696  */
697  q1 = (highest_expected + 2) / 2;
698  q2 = (total_votes + 2) / 2;
699  newquorum = max(q1, q2);
700 
701  /*
702  * Normally quorum never decreases but the system administrator can
703  * force it down by setting expected votes to a maximum value
704  */
705  if (!allow_decrease) {
706  newquorum = max(quorum, newquorum);
707  }
708 
709  /*
710  * The special two_node mode allows each of the two nodes to retain
711  * quorum if the other fails. Only one of the two should live past
712  * fencing (as both nodes try to fence each other in split-brain.)
713  * Also: if there are more than two nodes, force us inquorate to avoid
714  * any damage or confusion.
715  */
716  if (two_node && total_nodes <= 2) {
717  newquorum = 1;
718  }
719 
720  if (ret_total_votes) {
721  *ret_total_votes = total_votes;
722  }
723 
724  LEAVE();
725  return newquorum;
726 }
727 
728 static void are_we_quorate(unsigned int total_votes)
729 {
730  int quorate;
731  int quorum_change = 0;
732 
733  ENTER();
734 
735  /*
736  * wait for all nodes to show up before granting quorum
737  */
738 
739  if ((wait_for_all) && (wait_for_all_status)) {
740  if (total_votes != us->expected_votes) {
742  "Waiting for all cluster members. "
743  "Current votes: %d expected_votes: %d",
744  total_votes, us->expected_votes);
745  cluster_is_quorate = 0;
746  return;
747  }
748  update_wait_for_all_status(0);
749  }
750 
751  if (quorum > total_votes) {
752  quorate = 0;
753  } else {
754  quorate = 1;
755  get_lowest_node_id();
756  }
757 
758  if ((auto_tie_breaker) &&
759  (total_votes == (us->expected_votes / 2)) &&
760  (check_low_node_id_partition() == 1)) {
761  quorate = 1;
762  }
763 
764  if ((qdevice_master_wins) &&
765  (!quorate) &&
766  (check_qdevice_master() == 1)) {
767  log_printf(LOGSYS_LEVEL_DEBUG, "node is quorate as part of master_wins partition");
768  quorate = 1;
769  }
770 
771  if (cluster_is_quorate && !quorate) {
772  quorum_change = 1;
773  log_printf(LOGSYS_LEVEL_DEBUG, "quorum lost, blocking activity");
774  }
775  if (!cluster_is_quorate && quorate) {
776  quorum_change = 1;
777  log_printf(LOGSYS_LEVEL_DEBUG, "quorum regained, resuming activity");
778  }
779 
780  cluster_is_quorate = quorate;
781  if (cluster_is_quorate) {
782  us->flags |= NODE_FLAGS_QUORATE;
783  } else {
784  us->flags &= ~NODE_FLAGS_QUORATE;
785  }
786 
787  if (wait_for_all) {
788  if (quorate) {
789  update_wait_for_all_status(0);
790  } else {
791  update_wait_for_all_status(1);
792  }
793  }
794 
795  if ((quorum_change) &&
796  (sync_in_progress == 0)) {
797  quorum_callback(quorum_members, quorum_members_entries,
798  cluster_is_quorate, &quorum_ringid);
799  }
800 
801  LEAVE();
802 }
803 
804 static void get_total_votes(unsigned int *totalvotes, unsigned int *current_members)
805 {
806  unsigned int total_votes = 0;
807  unsigned int cluster_members = 0;
808  struct list_head *nodelist;
809  struct cluster_node *node;
810 
811  ENTER();
812 
813  list_iterate(nodelist, &cluster_members_list) {
814  node = list_entry(nodelist, struct cluster_node, list);
815  if (node->state == NODESTATE_MEMBER) {
816  cluster_members++;
817  total_votes += node->votes;
818  }
819  }
820 
821  if (qdevice->votes) {
822  total_votes += qdevice->votes;
823  cluster_members++;
824  }
825 
826  *totalvotes = total_votes;
827  *current_members = cluster_members;
828 
829  LEAVE();
830 }
831 
832 /*
833  * Recalculate cluster quorum, set quorate and notify changes
834  */
835 static void recalculate_quorum(int allow_decrease, int by_current_nodes)
836 {
837  unsigned int total_votes = 0;
838  unsigned int cluster_members = 0;
839 
840  ENTER();
841 
842  get_total_votes(&total_votes, &cluster_members);
843 
844  if (!by_current_nodes) {
845  cluster_members = 0;
846  }
847 
848  /*
849  * Keep expected_votes at the highest number of votes in the cluster
850  */
851  log_printf(LOGSYS_LEVEL_DEBUG, "total_votes=%d, expected_votes=%d", total_votes, us->expected_votes);
852  if (total_votes > us->expected_votes) {
853  us->expected_votes = total_votes;
854  votequorum_exec_send_expectedvotes_notification();
855  }
856 
857  quorum = calculate_quorum(allow_decrease, cluster_members, &total_votes);
858  are_we_quorate(total_votes);
859 
860  votequorum_exec_send_quorum_notification(NULL, 0L);
861 
862  LEAVE();
863 }
864 
865 /*
866  * configuration bits and pieces
867  */
868 
869 static int votequorum_read_nodelist_configuration(uint32_t *votes,
870  uint32_t *nodes,
871  uint32_t *expected_votes)
872 {
873  icmap_iter_t iter;
874  const char *iter_key;
875  char tmp_key[ICMAP_KEYNAME_MAXLEN];
876  uint32_t our_pos, node_pos;
877  uint32_t nodecount = 0;
878  uint32_t nodelist_expected_votes = 0;
879  uint32_t node_votes = 0;
880  int res = 0;
881 
882  ENTER();
883 
884  if (icmap_get_uint32("nodelist.local_node_pos", &our_pos) != CS_OK) {
886  "No nodelist defined or our node is not in the nodelist");
887  return 0;
888  }
889 
890  iter = icmap_iter_init("nodelist.node.");
891 
892  while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) {
893 
894  res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key);
895  if (res != 2) {
896  continue;
897  }
898 
899  if (strcmp(tmp_key, "ring0_addr") != 0) {
900  continue;
901  }
902 
903  nodecount++;
904 
905  snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.quorum_votes", node_pos);
906  if (icmap_get_uint32(tmp_key, &node_votes) != CS_OK) {
907  node_votes = 1;
908  }
909 
910  nodelist_expected_votes = nodelist_expected_votes + node_votes;
911 
912  if (node_pos == our_pos) {
913  *votes = node_votes;
914  }
915  }
916 
917  *expected_votes = nodelist_expected_votes;
918  *nodes = nodecount;
919 
920  icmap_iter_finalize(iter);
921 
922  LEAVE();
923 
924  return 1;
925 }
926 
927 static int votequorum_qdevice_is_configured(uint32_t *qdevice_votes)
928 {
929  char *qdevice_model = NULL;
930  int ret = 0;
931 
932  ENTER();
933 
934  if (icmap_get_string("quorum.device.model", &qdevice_model) == CS_OK) {
935  if (strlen(qdevice_model)) {
936  if (icmap_get_uint32("quorum.device.votes", qdevice_votes) != CS_OK) {
937  *qdevice_votes = -1;
938  }
939  if (icmap_get_uint32("quorum.device.timeout", &qdevice_timeout) != CS_OK) {
940  qdevice_timeout = VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT;
941  }
942  update_qdevice_can_operate(1);
943  ret = 1;
944  }
945 
946  free(qdevice_model);
947  }
948 
949  LEAVE();
950 
951  return ret;
952 }
953 
954 #define VOTEQUORUM_READCONFIG_STARTUP 0
955 #define VOTEQUORUM_READCONFIG_RUNTIME 1
956 
957 static char *votequorum_readconfig(int runtime)
958 {
959  uint32_t node_votes = 0, qdevice_votes = 0;
960  uint32_t node_expected_votes = 0, expected_votes = 0;
961  uint32_t node_count = 0;
962  int have_nodelist, have_qdevice;
963  char *error = NULL;
964 
965  ENTER();
966 
967  log_printf(LOGSYS_LEVEL_DEBUG, "Reading configuration (runtime: %d)", runtime);
968 
969  /*
970  * gather basic data here
971  */
972  icmap_get_uint32("quorum.expected_votes", &expected_votes);
973  have_nodelist = votequorum_read_nodelist_configuration(&node_votes, &node_count, &node_expected_votes);
974  have_qdevice = votequorum_qdevice_is_configured(&qdevice_votes);
975  icmap_get_uint8("quorum.two_node", &two_node);
976 
977  /*
978  * do config verification and enablement
979  */
980 
981  if ((!have_nodelist) && (!expected_votes)) {
982  if (!runtime) {
983  error = (char *)"configuration error: nodelist or quorum.expected_votes must be configured!";
984  } else {
985  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: nodelist or quorum.expected_votes must be configured!");
986  log_printf(LOGSYS_LEVEL_CRIT, "will continue with current runtime data");
987  }
988  goto out;
989  }
990 
991  /*
992  * two_node and qdevice are not compatible in the same config.
993  * try to make an educated guess of what to do
994  */
995 
996  if ((two_node) && (have_qdevice)) {
997  if (!runtime) {
998  error = (char *)"configuration error: two_node and quorum device cannot be configured at the same time!";
999  goto out;
1000  } else {
1001  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: two_node and quorum device cannot be configured at the same time!");
1002  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1003  log_printf(LOGSYS_LEVEL_CRIT, "quorum device is registered, disabling two_node");
1004  two_node = 0;
1005  } else {
1006  log_printf(LOGSYS_LEVEL_CRIT, "quorum device is not registered, allowing two_node");
1007  update_qdevice_can_operate(0);
1008  }
1009  }
1010  }
1011 
1012  /*
1013  * Enable special features
1014  */
1015  if (!runtime) {
1016  if (two_node) {
1017  wait_for_all = 1;
1018  }
1019 
1020  icmap_get_uint8("quorum.allow_downscale", &allow_downscale);
1021  icmap_get_uint8("quorum.wait_for_all", &wait_for_all);
1022  icmap_get_uint8("quorum.auto_tie_breaker", &auto_tie_breaker);
1023  icmap_get_uint8("quorum.last_man_standing", &last_man_standing);
1024  icmap_get_uint32("quorum.last_man_standing_window", &last_man_standing_window);
1025  }
1026 
1027  /*
1028  * quorum device is not compatible with last_man_standing and auto_tie_breaker
1029  * neither lms or atb can be set at runtime, so there is no need to check for
1030  * runtime incompatibilities, but qdevice can be configured _after_ LMS and ATB have
1031  * been enabled at startup.
1032  */
1033 
1034  if ((have_qdevice) && (last_man_standing)) {
1035  if (!runtime) {
1036  error = (char *)"configuration error: quorum.device is not compatible with last_man_standing";
1037  goto out;
1038  } else {
1039  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with last_man_standing");
1040  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1041  update_qdevice_can_operate(0);
1042  }
1043  }
1044 
1045  if ((have_qdevice) && (auto_tie_breaker)) {
1046  if (!runtime) {
1047  error = (char *)"configuration error: quorum.device is not compatible with auto_tie_breaker";
1048  goto out;
1049  } else {
1050  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with auto_tie_breaker");
1051  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1052  update_qdevice_can_operate(0);
1053  }
1054  }
1055 
1056  if ((have_qdevice) && (wait_for_all)) {
1057  if (!runtime) {
1058  error = (char *)"configuration error: quorum.device is not compatible with wait_for_all";
1059  goto out;
1060  } else {
1061  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with wait_for_all");
1062  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1063  update_qdevice_can_operate(0);
1064  }
1065  }
1066 
1067  if ((have_qdevice) && (allow_downscale)) {
1068  if (!runtime) {
1069  error = (char *)"configuration error: quorum.device is not compatible with allow_downscale";
1070  goto out;
1071  } else {
1072  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with allow_downscale");
1073  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1074  update_qdevice_can_operate(0);
1075  }
1076  }
1077 
1078  /*
1079  * if user specifies quorum.expected_votes + quorum.device but NOT the device.votes
1080  * we don't know what the quorum device should vote.
1081  */
1082 
1083  if ((expected_votes) && (have_qdevice) && (qdevice_votes == -1)) {
1084  if (!runtime) {
1085  error = (char *)"configuration error: quorum.device.votes must be specified when quorum.expected_votes is set";
1086  goto out;
1087  } else {
1088  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes must be specified when quorum.expected_votes is set");
1089  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1090  update_qdevice_can_operate(0);
1091  }
1092  }
1093 
1094  /*
1095  * if user specifies a node list with uneven votes and no device.votes
1096  * we cannot autocalculate the votes
1097  */
1098 
1099  if ((have_qdevice) &&
1100  (qdevice_votes == -1) &&
1101  (have_nodelist) &&
1102  (node_count != node_expected_votes)) {
1103  if (!runtime) {
1104  error = (char *)"configuration error: quorum.device.votes must be specified when not all nodes votes 1";
1105  goto out;
1106  } else {
1107  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes must be specified when not all nodes votes 1");
1108  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1109  update_qdevice_can_operate(0);
1110  }
1111  }
1112 
1113  /*
1114  * validate quorum device votes vs expected_votes
1115  */
1116 
1117  if ((qdevice_votes > 0) && (expected_votes)) {
1118  int delta = expected_votes - qdevice_votes;
1119  if (delta < 2) {
1120  if (!runtime) {
1121  error = (char *)"configuration error: quorum.device.votes is too high or expected_votes is too low";
1122  goto out;
1123  } else {
1124  log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes is too high or expected_votes is too low");
1125  log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations");
1126  update_qdevice_can_operate(0);
1127  }
1128  }
1129  }
1130 
1131  /*
1132  * automatically calculate device votes and adjust expected_votes from nodelist
1133  */
1134 
1135  if ((have_qdevice) &&
1136  (qdevice_votes == -1) &&
1137  (!expected_votes) &&
1138  (have_nodelist) &&
1139  (node_count == node_expected_votes)) {
1140  qdevice_votes = node_expected_votes - 1;
1141  node_expected_votes = node_expected_votes + qdevice_votes;
1142  }
1143 
1144  /*
1145  * set this node votes and expected_votes
1146  */
1147 
1148  if (have_nodelist) {
1149  us->votes = node_votes;
1150  us->expected_votes = node_expected_votes;
1151  } else {
1152  us->votes = 1;
1153  icmap_get_uint32("quorum.votes", &us->votes);
1154  }
1155 
1156  if (expected_votes) {
1158  }
1159 
1160  /*
1161  * set qdevice votes
1162  */
1163 
1164  if (!have_qdevice) {
1165  qdevice->votes = 0;
1166  }
1167 
1168  if (qdevice_votes != -1) {
1169  qdevice->votes = qdevice_votes;
1170  }
1171 
1172  update_ev_barrier(us->expected_votes);
1173  update_two_node();
1174  if (wait_for_all) {
1175  update_wait_for_all_status(1);
1176  }
1177 
1178 out:
1179  LEAVE();
1180  return error;
1181 }
1182 
1183 static void votequorum_refresh_config(
1184  int32_t event,
1185  const char *key_name,
1186  struct icmap_notify_value new_val,
1187  struct icmap_notify_value old_val,
1188  void *user_data)
1189 {
1190  int old_votes, old_expected_votes;
1191 
1192  ENTER();
1193 
1194  old_votes = us->votes;
1195  old_expected_votes = us->expected_votes;
1196 
1197  /*
1198  * Reload the configuration
1199  */
1200  votequorum_readconfig(VOTEQUORUM_READCONFIG_RUNTIME);
1201 
1202  /*
1203  * activate new config
1204  */
1205  votequorum_exec_send_nodeinfo(us->node_id);
1206  votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID);
1207  if (us->votes != old_votes) {
1208  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES,
1209  us->node_id, us->votes);
1210  }
1211  if (us->expected_votes != old_expected_votes) {
1212  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES,
1213  us->node_id, us->expected_votes);
1214  }
1215 
1216  LEAVE();
1217 }
1218 
1219 static void votequorum_exec_add_config_notification(void)
1220 {
1221  icmap_track_t icmap_track_nodelist = NULL;
1222  icmap_track_t icmap_track_quorum = NULL;
1223 
1224  ENTER();
1225 
1226  icmap_track_add("nodelist.",
1228  votequorum_refresh_config,
1229  NULL,
1230  &icmap_track_nodelist);
1231 
1232  icmap_track_add("quorum.",
1234  votequorum_refresh_config,
1235  NULL,
1236  &icmap_track_quorum);
1237 
1238  LEAVE();
1239 }
1240 
1241 /*
1242  * votequorum_exec core
1243  */
1244 
1245 static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid, uint32_t value)
1246 {
1248  struct iovec iov[1];
1249  int ret;
1250 
1251  ENTER();
1252 
1259 
1262 
1263  iov[0].iov_base = (void *)&req_exec_quorum_reconfigure;
1264  iov[0].iov_len = sizeof(req_exec_quorum_reconfigure);
1265 
1266  ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
1267 
1268  LEAVE();
1269  return ret;
1270 }
1271 
1272 static int votequorum_exec_send_nodeinfo(uint32_t nodeid)
1273 {
1275  struct iovec iov[1];
1276  struct cluster_node *node;
1277  int ret;
1278 
1279  ENTER();
1280 
1281  node = find_node_by_nodeid(nodeid);
1282  if (!node) {
1283  return -1;
1284  }
1285 
1290  if (nodeid != VOTEQUORUM_QDEVICE_NODEID) {
1291  decode_flags(node->flags);
1292  }
1293 
1295  req_exec_quorum_nodeinfo.header.size = sizeof(req_exec_quorum_nodeinfo);
1296 
1297  iov[0].iov_base = (void *)&req_exec_quorum_nodeinfo;
1298  iov[0].iov_len = sizeof(req_exec_quorum_nodeinfo);
1299 
1300  ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
1301 
1302  LEAVE();
1303  return ret;
1304 }
1305 
1306 static int votequorum_exec_send_qdevice_reconfigure(const char *oldname, const char *newname)
1307 {
1309  struct iovec iov[1];
1310  int ret;
1311 
1312  ENTER();
1313 
1318 
1319  iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reconfigure;
1320  iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reconfigure);
1321 
1322  ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
1323 
1324  LEAVE();
1325  return ret;
1326 }
1327 
1328 static int votequorum_exec_send_qdevice_reg(uint32_t operation, const char *qdevice_name_req)
1329 {
1331  struct iovec iov[1];
1332  int ret;
1333 
1334  ENTER();
1335 
1339  strcpy(req_exec_quorum_qdevice_reg.qdevice_name, qdevice_name_req);
1340 
1341  iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reg;
1342  iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reg);
1343 
1344  ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
1345 
1346  LEAVE();
1347  return ret;
1348 }
1349 
1350 static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context)
1351 {
1353  struct list_head *tmp;
1354  struct cluster_node *node;
1355  int cluster_members = 0;
1356  int i = 0;
1357  int size;
1358  char buf[sizeof(struct res_lib_votequorum_notification) + sizeof(struct votequorum_node) * (PROCESSOR_COUNT_MAX + 2)];
1359 
1360  ENTER();
1361 
1362  list_iterate(tmp, &cluster_members_list) {
1363  node = list_entry(tmp, struct cluster_node, list);
1364  cluster_members++;
1365  }
1366  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1367  cluster_members++;
1368  }
1369 
1370  size = sizeof(struct res_lib_votequorum_notification) + sizeof(struct votequorum_node) * cluster_members;
1371 
1372  res_lib_votequorum_notification = (struct res_lib_votequorum_notification *)&buf;
1373  res_lib_votequorum_notification->quorate = cluster_is_quorate;
1374  res_lib_votequorum_notification->node_list_entries = cluster_members;
1375  res_lib_votequorum_notification->context = context;
1376  list_iterate(tmp, &cluster_members_list) {
1377  node = list_entry(tmp, struct cluster_node, list);
1378  res_lib_votequorum_notification->node_list[i].nodeid = node->node_id;
1379  res_lib_votequorum_notification->node_list[i++].state = node->state;
1380  }
1381  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1382  res_lib_votequorum_notification->node_list[i].nodeid = VOTEQUORUM_QDEVICE_NODEID;
1383  res_lib_votequorum_notification->node_list[i++].state = qdevice->state;
1384  }
1385  res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_NOTIFICATION;
1386  res_lib_votequorum_notification->header.size = size;
1387  res_lib_votequorum_notification->header.error = CS_OK;
1388 
1389  /* Send it to all interested parties */
1390  if (conn) {
1391  int ret = corosync_api->ipc_dispatch_send(conn, &buf, size);
1392  LEAVE();
1393  return ret;
1394  } else {
1395  struct quorum_pd *qpd;
1396 
1397  list_iterate(tmp, &trackers_list) {
1398  qpd = list_entry(tmp, struct quorum_pd, list);
1399  res_lib_votequorum_notification->context = qpd->tracking_context;
1400  corosync_api->ipc_dispatch_send(qpd->conn, &buf, size);
1401  }
1402  }
1403 
1404  LEAVE();
1405 
1406  return 0;
1407 }
1408 
1409 static void votequorum_exec_send_expectedvotes_notification(void)
1410 {
1412  struct quorum_pd *qpd;
1413  struct list_head *tmp;
1414 
1415  ENTER();
1416 
1417  log_printf(LOGSYS_LEVEL_DEBUG, "Sending expected votes callback");
1418 
1423 
1424  list_iterate(tmp, &trackers_list) {
1425  qpd = list_entry(tmp, struct quorum_pd, list);
1429  }
1430 
1431  LEAVE();
1432 }
1433 
1434 static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message)
1435 {
1436  ENTER();
1437 
1438  LEAVE();
1439 }
1440 
1441 static void message_handler_req_exec_votequorum_qdevice_reconfigure (
1442  const void *message,
1443  unsigned int nodeid)
1444 {
1446 
1447  ENTER();
1448 
1449  log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice name change req from node %u [from: %s to: %s]",
1450  nodeid,
1451  req_exec_quorum_qdevice_reconfigure->oldname,
1452  req_exec_quorum_qdevice_reconfigure->newname);
1453 
1454  if (!strcmp(req_exec_quorum_qdevice_reconfigure->oldname, qdevice_name)) {
1455  log_printf(LOGSYS_LEVEL_DEBUG, "Allowing qdevice rename");
1456  memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN);
1457  strcpy(qdevice_name, req_exec_quorum_qdevice_reconfigure->newname);
1458  /*
1459  * TODO: notify qdevices about name change?
1460  * this is not relevant for now and can wait later on since
1461  * qdevices are local only and libvotequorum is not final
1462  */
1463  }
1464 
1465  LEAVE();
1466 }
1467 
1468 static void exec_votequorum_qdevice_reg_endian_convert (void *message)
1469 {
1471 
1472  ENTER();
1473 
1474  req_exec_quorum_qdevice_reg->operation = swab32(req_exec_quorum_qdevice_reg->operation);
1475 
1476  LEAVE();
1477 }
1478 
1479 static void message_handler_req_exec_votequorum_qdevice_reg (
1480  const void *message,
1481  unsigned int nodeid)
1482 {
1485  int wipe_qdevice_name = 1;
1486  struct cluster_node *node = NULL;
1487  struct list_head *tmp;
1488  cs_error_t error = CS_OK;
1489 
1490  ENTER();
1491 
1492  log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice op %u req from node %u [%s]",
1493  req_exec_quorum_qdevice_reg->operation,
1494  nodeid, req_exec_quorum_qdevice_reg->qdevice_name);
1495 
1496  switch(req_exec_quorum_qdevice_reg->operation)
1497  {
1499  if (nodeid != us->node_id) {
1500  if (!strlen(qdevice_name)) {
1501  log_printf(LOGSYS_LEVEL_DEBUG, "Remote qdevice name recorded");
1502  strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name);
1503  }
1504  LEAVE();
1505  return;
1506  }
1507 
1508  /*
1509  * protect against the case where we broadcast qdevice registration
1510  * to new memebers, we receive the message back, but there is no registration
1511  * connection in progress
1512  */
1513  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
1514  LEAVE();
1515  return;
1516  }
1517 
1518  /*
1519  * this should NEVER happen
1520  */
1521  if (!qdevice_reg_conn) {
1522  log_printf(LOGSYS_LEVEL_WARNING, "Unable to determine origin of the qdevice register call!");
1523  LEAVE();
1524  return;
1525  }
1526 
1527  /*
1528  * registering our own device in this case
1529  */
1530  if (!strlen(qdevice_name)) {
1531  strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name);
1532  }
1533 
1534  /*
1535  * check if it is our device or something else
1536  */
1537  if ((!strncmp(req_exec_quorum_qdevice_reg->qdevice_name,
1538  qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN))) {
1540  votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID);
1541  votequorum_exec_send_nodeinfo(us->node_id);
1542  } else {
1544  "A new qdevice with different name (new: %s old: %s) is trying to register!",
1545  req_exec_quorum_qdevice_reg->qdevice_name, qdevice_name);
1546  error = CS_ERR_EXIST;
1547  }
1548 
1551  res_lib_votequorum_status.header.error = error;
1552  corosync_api->ipc_response_send(qdevice_reg_conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status));
1553  qdevice_reg_conn = NULL;
1554  break;
1556  list_iterate(tmp, &cluster_members_list) {
1557  node = list_entry(tmp, struct cluster_node, list);
1558  if ((node->state == NODESTATE_MEMBER) &&
1559  (node->flags & NODE_FLAGS_QDEVICE_REGISTERED)) {
1560  wipe_qdevice_name = 0;
1561  }
1562  }
1563 
1564  if (wipe_qdevice_name) {
1565  memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN);
1566  }
1567 
1568  break;
1569  }
1570  LEAVE();
1571 }
1572 
1573 static void exec_votequorum_nodeinfo_endian_convert (void *message)
1574 {
1575  struct req_exec_quorum_nodeinfo *nodeinfo = message;
1576 
1577  ENTER();
1578 
1579  nodeinfo->nodeid = swab32(nodeinfo->nodeid);
1580  nodeinfo->votes = swab32(nodeinfo->votes);
1581  nodeinfo->expected_votes = swab32(nodeinfo->expected_votes);
1582  nodeinfo->flags = swab32(nodeinfo->flags);
1583 
1584  LEAVE();
1585 }
1586 
1587 static void message_handler_req_exec_votequorum_nodeinfo (
1588  const void *message,
1589  unsigned int sender_nodeid)
1590 {
1591  const struct req_exec_quorum_nodeinfo *req_exec_quorum_nodeinfo = message;
1592  struct cluster_node *node = NULL;
1593  int old_votes;
1594  int old_expected;
1595  uint32_t old_flags;
1596  nodestate_t old_state;
1597  int new_node = 0;
1598  int allow_downgrade = 0;
1599  int by_node = 0;
1600  unsigned int nodeid = req_exec_quorum_nodeinfo->nodeid;
1601 
1602  ENTER();
1603 
1604  log_printf(LOGSYS_LEVEL_DEBUG, "got nodeinfo message from cluster node %u", sender_nodeid);
1605  log_printf(LOGSYS_LEVEL_DEBUG, "nodeinfo message[%u]: votes: %d, expected: %d flags: %d",
1606  nodeid,
1607  req_exec_quorum_nodeinfo->votes,
1608  req_exec_quorum_nodeinfo->expected_votes,
1609  req_exec_quorum_nodeinfo->flags);
1610 
1611  if (nodeid != VOTEQUORUM_QDEVICE_NODEID) {
1612  decode_flags(req_exec_quorum_nodeinfo->flags);
1613  }
1614 
1615  node = find_node_by_nodeid(nodeid);
1616  if (!node) {
1617  node = allocate_node(nodeid);
1618  new_node = 1;
1619  }
1620  if (!node) {
1621  corosync_api->error_memory_failure();
1622  LEAVE();
1623  return;
1624  }
1625 
1626  if (new_node) {
1627  old_votes = 0;
1628  old_expected = 0;
1629  old_state = NODESTATE_DEAD;
1630  old_flags = 0;
1631  } else {
1632  old_votes = node->votes;
1633  old_expected = node->expected_votes;
1634  old_state = node->state;
1635  old_flags = node->flags;
1636  }
1637 
1638  if (nodeid == VOTEQUORUM_QDEVICE_NODEID) {
1639  struct cluster_node *sender_node = find_node_by_nodeid(sender_nodeid);
1640 
1641  assert(sender_node != NULL);
1642 
1643  if ((!cluster_is_quorate) &&
1644  (sender_node->flags & NODE_FLAGS_QUORATE)) {
1645  node->votes = req_exec_quorum_nodeinfo->votes;
1646  } else {
1647  node->votes = max(node->votes, req_exec_quorum_nodeinfo->votes);
1648  }
1649  goto recalculate;
1650  }
1651 
1652  /* Update node state */
1653  node->flags = req_exec_quorum_nodeinfo->flags;
1654  node->votes = req_exec_quorum_nodeinfo->votes;
1655  node->state = NODESTATE_MEMBER;
1656 
1657  if (node->flags & NODE_FLAGS_LEAVING) {
1658  node->state = NODESTATE_LEAVING;
1659  allow_downgrade = 1;
1660  by_node = 1;
1661  }
1662 
1663  if ((!cluster_is_quorate) &&
1664  (node->flags & NODE_FLAGS_QUORATE)) {
1665  allow_downgrade = 1;
1666  us->expected_votes = req_exec_quorum_nodeinfo->expected_votes;
1667  }
1668 
1669  if (node->flags & NODE_FLAGS_QUORATE) {
1670  node->expected_votes = req_exec_quorum_nodeinfo->expected_votes;
1671  } else {
1672  node->expected_votes = us->expected_votes;
1673  }
1674 
1675  if ((last_man_standing) && (node->votes > 1)) {
1676  log_printf(LOGSYS_LEVEL_WARNING, "Last Man Standing feature is supported only when all"
1677  "cluster nodes votes are set to 1. Disabling LMS.");
1678  last_man_standing = 0;
1679  if (last_man_standing_timer_set) {
1680  corosync_api->timer_delete(last_man_standing_timer);
1681  last_man_standing_timer_set = 0;
1682  }
1683  }
1684 
1685 recalculate:
1686 
1687  if ((new_node) ||
1688  (nodeid == us->node_id) ||
1689  (node->flags & NODE_FLAGS_FIRST) ||
1690  (old_votes != node->votes) ||
1691  (old_expected != node->expected_votes) ||
1692  (old_flags != node->flags) ||
1693  (old_state != node->state)) {
1694  recalculate_quorum(allow_downgrade, by_node);
1695  }
1696 
1697  if ((wait_for_all) &&
1698  (!(node->flags & NODE_FLAGS_WFASTATUS)) &&
1699  (node->flags & NODE_FLAGS_QUORATE)) {
1700  update_wait_for_all_status(0);
1701  }
1702 
1703  LEAVE();
1704 }
1705 
1706 static void exec_votequorum_reconfigure_endian_convert (void *message)
1707 {
1708  struct req_exec_quorum_reconfigure *reconfigure = message;
1709 
1710  ENTER();
1711 
1712  reconfigure->nodeid = swab32(reconfigure->nodeid);
1713  reconfigure->value = swab32(reconfigure->value);
1714 
1715  LEAVE();
1716 }
1717 
1718 static void message_handler_req_exec_votequorum_reconfigure (
1719  const void *message,
1720  unsigned int nodeid)
1721 {
1723  struct cluster_node *node;
1724  struct list_head *nodelist;
1725 
1726  ENTER();
1727 
1728  log_printf(LOGSYS_LEVEL_DEBUG, "got reconfigure message from cluster node %u for %u",
1729  nodeid, req_exec_quorum_reconfigure->nodeid);
1730 
1731  switch(req_exec_quorum_reconfigure->param)
1732  {
1734  list_iterate(nodelist, &cluster_members_list) {
1735  node = list_entry(nodelist, struct cluster_node, list);
1736  if (node->state == NODESTATE_MEMBER) {
1737  node->expected_votes = req_exec_quorum_reconfigure->value;
1738  }
1739  }
1740  votequorum_exec_send_expectedvotes_notification();
1741  update_ev_barrier(req_exec_quorum_reconfigure->value);
1742  recalculate_quorum(1, 0); /* Allow decrease */
1743  break;
1744 
1746  node = find_node_by_nodeid(req_exec_quorum_reconfigure->nodeid);
1747  if (!node) {
1748  LEAVE();
1749  return;
1750  }
1751  node->votes = req_exec_quorum_reconfigure->value;
1752  recalculate_quorum(1, 0); /* Allow decrease */
1753  break;
1754 
1755  }
1756 
1757  LEAVE();
1758 }
1759 
1760 static int votequorum_exec_exit_fn (void)
1761 {
1762  int ret = 0;
1763 
1764  ENTER();
1765 
1766  /*
1767  * tell the other nodes we are leaving
1768  */
1769 
1770  if (allow_downscale) {
1771  us->flags |= NODE_FLAGS_LEAVING;
1772  ret = votequorum_exec_send_nodeinfo(us->node_id);
1773  }
1774 
1775  LEAVE();
1776  return ret;
1777 }
1778 
1779 static char *votequorum_exec_init_fn (struct corosync_api_v1 *api)
1780 {
1781  char *error = NULL;
1782 
1783  ENTER();
1784 
1785  /*
1786  * make sure we start clean
1787  */
1788  list_init(&cluster_members_list);
1789  list_init(&trackers_list);
1790  qdevice = NULL;
1791  us = NULL;
1792  memset(cluster_nodes, 0, sizeof(cluster_nodes));
1793 
1794  /*
1795  * Allocate a cluster_node for qdevice
1796  */
1797  qdevice = allocate_node(VOTEQUORUM_QDEVICE_NODEID);
1798  if (!qdevice) {
1799  LEAVE();
1800  return ((char *)"Could not allocate node.");
1801  }
1802  qdevice->votes = 0;
1803  memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN);
1804 
1805  /*
1806  * Allocate a cluster_node for us
1807  */
1808  us = allocate_node(corosync_api->totem_nodeid_get());
1809  if (!us) {
1810  LEAVE();
1811  return ((char *)"Could not allocate node.");
1812  }
1813 
1814  icmap_set_uint32("runtime.votequorum.this_node_id", us->node_id);
1815 
1816  us->state = NODESTATE_MEMBER;
1817  us->votes = 1;
1818  us->flags |= NODE_FLAGS_FIRST;
1819 
1820  error = votequorum_readconfig(VOTEQUORUM_READCONFIG_STARTUP);
1821  if (error) {
1822  return error;
1823  }
1824  recalculate_quorum(0, 0);
1825 
1826  /*
1827  * Listen for changes
1828  */
1829  votequorum_exec_add_config_notification();
1830 
1831  /*
1832  * Start us off with one node
1833  */
1834  votequorum_exec_send_nodeinfo(us->node_id);
1835 
1836  LEAVE();
1837 
1838  return (NULL);
1839 }
1840 
1841 /*
1842  * votequorum service core
1843  */
1844 
1845 static void votequorum_last_man_standing_timer_fn(void *arg)
1846 {
1847  ENTER();
1848 
1849  last_man_standing_timer_set = 0;
1850  if (cluster_is_quorate) {
1851  recalculate_quorum(1,1);
1852  }
1853 
1854  LEAVE();
1855 }
1856 
1857 static void votequorum_sync_init (
1858  const unsigned int *trans_list, size_t trans_list_entries,
1859  const unsigned int *member_list, size_t member_list_entries,
1860  const struct memb_ring_id *ring_id)
1861 {
1862  int i, j;
1863  int found;
1864  int left_nodes;
1865  struct cluster_node *node;
1866 
1867  ENTER();
1868 
1869  sync_in_progress = 1;
1870 
1871  if (member_list_entries > 1) {
1872  us->flags &= ~NODE_FLAGS_FIRST;
1873  }
1874 
1875  /*
1876  * we don't need to track which nodes have left directly,
1877  * since that info is in the node db, but we need to know
1878  * if somebody has left for last_man_standing
1879  */
1880  left_nodes = 0;
1881  for (i = 0; i < quorum_members_entries; i++) {
1882  found = 0;
1883  for (j = 0; j < member_list_entries; j++) {
1884  if (quorum_members[i] == member_list[j]) {
1885  found = 1;
1886  break;
1887  }
1888  }
1889  if (found == 0) {
1890  left_nodes = 1;
1891  node = find_node_by_nodeid(quorum_members[i]);
1892  if (node) {
1893  node->state = NODESTATE_DEAD;
1894  }
1895  }
1896  }
1897 
1898  if (last_man_standing) {
1899  if (((member_list_entries >= quorum) && (left_nodes)) ||
1900  ((member_list_entries <= quorum) && (auto_tie_breaker) && (check_low_node_id_partition() == 1))) {
1901  if (last_man_standing_timer_set) {
1902  corosync_api->timer_delete(last_man_standing_timer);
1903  last_man_standing_timer_set = 0;
1904  }
1905  corosync_api->timer_add_duration((unsigned long long)last_man_standing_window*1000000,
1906  NULL, votequorum_last_man_standing_timer_fn,
1907  &last_man_standing_timer);
1908  last_man_standing_timer_set = 1;
1909  }
1910  }
1911 
1912  memcpy(quorum_members, member_list, sizeof(unsigned int) * member_list_entries);
1913  quorum_members_entries = member_list_entries;
1914  memcpy(&quorum_ringid, ring_id, sizeof(*ring_id));
1915 
1916  LEAVE();
1917 }
1918 
1919 static int votequorum_sync_process (void)
1920 {
1921  votequorum_exec_send_nodeinfo(us->node_id);
1922  votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID);
1923  if (strlen(qdevice_name)) {
1924  votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
1925  qdevice_name);
1926  }
1927  return 0;
1928 }
1929 
1930 static void votequorum_sync_activate (void)
1931 {
1932  recalculate_quorum(0, 0);
1933  quorum_callback(quorum_members, quorum_members_entries,
1934  cluster_is_quorate, &quorum_ringid);
1935  sync_in_progress = 0;
1936 }
1937 
1938 static void votequorum_sync_abort (void)
1939 {
1940 
1941 }
1942 
1944  quorum_set_quorate_fn_t q_set_quorate_fn)
1945 {
1946  char *error;
1947 
1948  ENTER();
1949 
1950  if (q_set_quorate_fn == NULL) {
1951  return ((char *)"Quorate function not set");
1952  }
1953 
1954  corosync_api = api;
1955  quorum_callback = q_set_quorate_fn;
1956 
1957  error = corosync_service_link_and_init(corosync_api,
1958  &votequorum_service[0]);
1959  if (error) {
1960  return (error);
1961  }
1962 
1963  LEAVE();
1964 
1965  return (NULL);
1966 }
1967 
1968 /*
1969  * Library Handler init/fini
1970  */
1971 
1972 static int quorum_lib_init_fn (void *conn)
1973 {
1974  struct quorum_pd *pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
1975 
1976  ENTER();
1977 
1978  list_init (&pd->list);
1979  pd->conn = conn;
1980 
1981  LEAVE();
1982  return (0);
1983 }
1984 
1985 static int quorum_lib_exit_fn (void *conn)
1986 {
1987  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
1988 
1989  ENTER();
1990 
1991  if (quorum_pd->tracking_enabled) {
1992  list_del (&quorum_pd->list);
1993  list_init (&quorum_pd->list);
1994  }
1995 
1996  LEAVE();
1997 
1998  return (0);
1999 }
2000 
2001 /*
2002  * library internal functions
2003  */
2004 
2005 static void qdevice_timer_fn(void *arg)
2006 {
2007  ENTER();
2008 
2009  if ((!(us->flags & NODE_FLAGS_QDEVICE_ALIVE)) ||
2010  (!qdevice_timer_set)) {
2011  LEAVE();
2012  return;
2013  }
2014 
2015  us->flags &= ~NODE_FLAGS_QDEVICE_ALIVE;
2017  log_printf(LOGSYS_LEVEL_INFO, "lost contact with quorum device %s", qdevice_name);
2018  votequorum_exec_send_nodeinfo(us->node_id);
2019 
2020  qdevice_timer_set = 0;
2021 
2022  LEAVE();
2023 }
2024 
2025 /*
2026  * Library Handler Functions
2027  */
2028 
2029 static void message_handler_req_lib_votequorum_getinfo (void *conn, const void *message)
2030 {
2033  struct cluster_node *node;
2034  unsigned int highest_expected = 0;
2035  unsigned int total_votes = 0;
2036  cs_error_t error = CS_OK;
2037  uint32_t nodeid = req_lib_votequorum_getinfo->nodeid;
2038 
2039  ENTER();
2040 
2041  log_printf(LOGSYS_LEVEL_DEBUG, "got getinfo request on %p for node %u", conn, req_lib_votequorum_getinfo->nodeid);
2042 
2043  if (nodeid == VOTEQUORUM_QDEVICE_NODEID) {
2044  nodeid = us->node_id;
2045  }
2046 
2047  node = find_node_by_nodeid(nodeid);
2048  if (node) {
2049  struct cluster_node *iternode;
2050  struct list_head *nodelist;
2051 
2052  list_iterate(nodelist, &cluster_members_list) {
2053  iternode = list_entry(nodelist, struct cluster_node, list);
2054 
2055  if (iternode->state == NODESTATE_MEMBER) {
2056  highest_expected =
2057  max(highest_expected, iternode->expected_votes);
2058  total_votes += iternode->votes;
2059  }
2060  }
2061 
2062  if (node->flags & NODE_FLAGS_QDEVICE_CAST_VOTE) {
2063  total_votes += qdevice->votes;
2064  }
2065 
2066  switch(node->state) {
2067  case NODESTATE_MEMBER:
2069  break;
2070  case NODESTATE_DEAD:
2072  break;
2073  case NODESTATE_LEAVING:
2075  break;
2076  default:
2078  break;
2079  }
2083  res_lib_votequorum_getinfo.highest_expected = highest_expected;
2084 
2089 
2090  if (two_node) {
2092  }
2093  if (cluster_is_quorate) {
2095  }
2096  if (wait_for_all) {
2098  }
2099  if (last_man_standing) {
2101  }
2102  if (auto_tie_breaker) {
2104  }
2105  if (allow_downscale) {
2107  }
2108 
2110  strcpy(res_lib_votequorum_getinfo.qdevice_name, qdevice_name);
2112 
2113  if (node->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2115  }
2116  if (node->flags & NODE_FLAGS_QDEVICE_ALIVE) {
2118  }
2119  if (node->flags & NODE_FLAGS_QDEVICE_CAST_VOTE) {
2121  }
2122  if (node->flags & NODE_FLAGS_QDEVICE_MASTER_WINS) {
2124  }
2125  } else {
2126  error = CS_ERR_NOT_EXIST;
2127  }
2128 
2131  res_lib_votequorum_getinfo.header.error = error;
2133  log_printf(LOGSYS_LEVEL_DEBUG, "getinfo response error: %d", error);
2134 
2135  LEAVE();
2136 }
2137 
2138 static void message_handler_req_lib_votequorum_setexpected (void *conn, const void *message)
2139 {
2142  cs_error_t error = CS_OK;
2143  unsigned int newquorum;
2144  unsigned int total_votes;
2145  uint8_t allow_downscale_status = 0;
2146 
2147  ENTER();
2148 
2149  allow_downscale_status = allow_downscale;
2150  allow_downscale = 0;
2151 
2152  /*
2153  * Validate new expected votes
2154  */
2155  newquorum = calculate_quorum(1, req_lib_votequorum_setexpected->expected_votes, &total_votes);
2156  allow_downscale = allow_downscale_status;
2157  if (newquorum < total_votes / 2 ||
2158  newquorum > total_votes) {
2159  error = CS_ERR_INVALID_PARAM;
2160  goto error_exit;
2161  }
2162 
2163  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES, us->node_id,
2164  req_lib_votequorum_setexpected->expected_votes);
2165 
2166 error_exit:
2169  res_lib_votequorum_status.header.error = error;
2171 
2172  LEAVE();
2173 }
2174 
2175 static void message_handler_req_lib_votequorum_setvotes (void *conn, const void *message)
2176 {
2179  struct cluster_node *node;
2180  unsigned int newquorum;
2181  unsigned int total_votes;
2182  unsigned int saved_votes;
2183  cs_error_t error = CS_OK;
2184  unsigned int nodeid;
2185 
2186  ENTER();
2187 
2188  nodeid = req_lib_votequorum_setvotes->nodeid;
2189  node = find_node_by_nodeid(nodeid);
2190  if (!node) {
2191  error = CS_ERR_NAME_NOT_FOUND;
2192  goto error_exit;
2193  }
2194 
2195  /*
2196  * Check votes is valid
2197  */
2198  saved_votes = node->votes;
2199  node->votes = req_lib_votequorum_setvotes->votes;
2200 
2201  newquorum = calculate_quorum(1, 0, &total_votes);
2202 
2203  if (newquorum < total_votes / 2 ||
2204  newquorum > total_votes) {
2205  node->votes = saved_votes;
2206  error = CS_ERR_INVALID_PARAM;
2207  goto error_exit;
2208  }
2209 
2210  votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES, nodeid,
2211  req_lib_votequorum_setvotes->votes);
2212 
2213 error_exit:
2216  res_lib_votequorum_status.header.error = error;
2218 
2219  LEAVE();
2220 }
2221 
2222 static void message_handler_req_lib_votequorum_trackstart (void *conn,
2223  const void *message)
2224 {
2227  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
2228 
2229  ENTER();
2230  /*
2231  * If an immediate listing of the current cluster membership
2232  * is requested, generate membership list
2233  */
2234  if (req_lib_votequorum_trackstart->track_flags & CS_TRACK_CURRENT ||
2235  req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES) {
2236  log_printf(LOGSYS_LEVEL_DEBUG, "sending initial status to %p", conn);
2237  votequorum_exec_send_quorum_notification(conn, req_lib_votequorum_trackstart->context);
2238  }
2239 
2240  /*
2241  * Record requests for tracking
2242  */
2243  if (req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES ||
2244  req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES_ONLY) {
2245 
2246  quorum_pd->track_flags = req_lib_votequorum_trackstart->track_flags;
2247  quorum_pd->tracking_enabled = 1;
2248  quorum_pd->tracking_context = req_lib_votequorum_trackstart->context;
2249 
2250  list_add (&quorum_pd->list, &trackers_list);
2251  }
2252 
2255  res_lib_votequorum_status.header.error = CS_OK;
2257 
2258  LEAVE();
2259 }
2260 
2261 static void message_handler_req_lib_votequorum_trackstop (void *conn,
2262  const void *message)
2263 {
2265  struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn);
2266  int error = CS_OK;
2267 
2268  ENTER();
2269 
2270  if (quorum_pd->tracking_enabled) {
2271  error = CS_OK;
2272  quorum_pd->tracking_enabled = 0;
2273  list_del (&quorum_pd->list);
2274  list_init (&quorum_pd->list);
2275  } else {
2276  error = CS_ERR_NOT_EXIST;
2277  }
2278 
2281  res_lib_votequorum_status.header.error = error;
2283 
2284  LEAVE();
2285 }
2286 
2287 static void message_handler_req_lib_votequorum_qdevice_register (void *conn,
2288  const void *message)
2289 {
2292  cs_error_t error = CS_OK;
2293 
2294  ENTER();
2295 
2296  if (!qdevice_can_operate) {
2297  log_printf(LOGSYS_LEVEL_INFO, "Registration of quorum device is disabled by incorrect corosync.conf. See logs for more information");
2298  error = CS_ERR_ACCESS;
2299  goto out;
2300  }
2301 
2302  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2303  if ((!strncmp(req_lib_votequorum_qdevice_register->name,
2304  qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN))) {
2305  goto out;
2306  } else {
2308  "A new qdevice with different name (new: %s old: %s) is trying to re-register!",
2309  req_lib_votequorum_qdevice_register->name, qdevice_name);
2310  error = CS_ERR_EXIST;
2311  goto out;
2312  }
2313  } else {
2314  if (qdevice_reg_conn != NULL) {
2316  "Registration request already in progress");
2317  error = CS_ERR_TRY_AGAIN;
2318  goto out;
2319  }
2320  qdevice_reg_conn = conn;
2321  if (votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
2322  req_lib_votequorum_qdevice_register->name) != 0) {
2324  "Unable to send qdevice registration request to cluster");
2325  error = CS_ERR_TRY_AGAIN;
2326  qdevice_reg_conn = NULL;
2327  } else {
2328  LEAVE();
2329  return;
2330  }
2331  }
2332 
2333 out:
2334 
2337  res_lib_votequorum_status.header.error = error;
2339 
2340  LEAVE();
2341 }
2342 
2343 static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
2344  const void *message)
2345 {
2348  cs_error_t error = CS_OK;
2349 
2350  ENTER();
2351 
2352  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2353  if (strncmp(req_lib_votequorum_qdevice_unregister->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) {
2354  error = CS_ERR_INVALID_PARAM;
2355  goto out;
2356  }
2357  if (qdevice_timer_set) {
2358  corosync_api->timer_delete(qdevice_timer);
2359  qdevice_timer_set = 0;
2360  }
2361  us->flags &= ~NODE_FLAGS_QDEVICE_REGISTERED;
2362  us->flags &= ~NODE_FLAGS_QDEVICE_ALIVE;
2365  votequorum_exec_send_nodeinfo(us->node_id);
2366  votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER,
2367  req_lib_votequorum_qdevice_unregister->name);
2368  } else {
2369  error = CS_ERR_NOT_EXIST;
2370  }
2371 
2372 out:
2375  res_lib_votequorum_status.header.error = error;
2377 
2378  LEAVE();
2379 }
2380 
2381 static void message_handler_req_lib_votequorum_qdevice_update (void *conn,
2382  const void *message)
2383 {
2386  cs_error_t error = CS_OK;
2387 
2388  ENTER();
2389 
2390  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2391  if (strncmp(req_lib_votequorum_qdevice_update->oldname, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) {
2392  error = CS_ERR_INVALID_PARAM;
2393  goto out;
2394  }
2395  votequorum_exec_send_qdevice_reconfigure(req_lib_votequorum_qdevice_update->oldname,
2396  req_lib_votequorum_qdevice_update->newname);
2397  } else {
2398  error = CS_ERR_NOT_EXIST;
2399  }
2400 
2401 out:
2404  res_lib_votequorum_status.header.error = error;
2406 
2407  LEAVE();
2408 }
2409 
2410 static void message_handler_req_lib_votequorum_qdevice_poll (void *conn,
2411  const void *message)
2412 {
2415  cs_error_t error = CS_OK;
2416  uint32_t oldflags;
2417 
2418  ENTER();
2419 
2420  if (!qdevice_can_operate) {
2421  error = CS_ERR_ACCESS;
2422  goto out;
2423  }
2424 
2425  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2426  if (strncmp(req_lib_votequorum_qdevice_poll->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) {
2427  error = CS_ERR_INVALID_PARAM;
2428  goto out;
2429  }
2430 
2431  if (qdevice_timer_set) {
2432  corosync_api->timer_delete(qdevice_timer);
2433  qdevice_timer_set = 0;
2434  }
2435 
2436  oldflags = us->flags;
2437 
2439 
2440  if (req_lib_votequorum_qdevice_poll->cast_vote) {
2442  } else {
2444  }
2445 
2446  if (us->flags != oldflags) {
2447  votequorum_exec_send_nodeinfo(us->node_id);
2448  }
2449 
2450  corosync_api->timer_add_duration((unsigned long long)qdevice_timeout*1000000, qdevice,
2451  qdevice_timer_fn, &qdevice_timer);
2452  qdevice_timer_set = 1;
2453  } else {
2454  error = CS_ERR_NOT_EXIST;
2455  }
2456 
2457 out:
2460  res_lib_votequorum_status.header.error = error;
2462 
2463  LEAVE();
2464 }
2465 
2466 static void message_handler_req_lib_votequorum_qdevice_master_wins (void *conn,
2467  const void *message)
2468 {
2471  cs_error_t error = CS_OK;
2472  uint32_t oldflags = us->flags;
2473 
2474  ENTER();
2475 
2476  if (!qdevice_can_operate) {
2477  error = CS_ERR_ACCESS;
2478  goto out;
2479  }
2480 
2481  if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) {
2482  if (strncmp(req_lib_votequorum_qdevice_master_wins->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) {
2483  error = CS_ERR_INVALID_PARAM;
2484  goto out;
2485  }
2486 
2487  if (req_lib_votequorum_qdevice_master_wins->allow) {
2489  } else {
2491  }
2492 
2493  if (us->flags != oldflags) {
2494  votequorum_exec_send_nodeinfo(us->node_id);
2495  }
2496 
2497  update_qdevice_master_wins(req_lib_votequorum_qdevice_master_wins->allow);
2498  } else {
2499  error = CS_ERR_NOT_EXIST;
2500  }
2501 
2502 out:
2505  res_lib_votequorum_status.header.error = error;
2507 
2508  LEAVE();
2509 }
uint32_t expected_votes
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_INFO_QUORATE
#define TOTEM_AGREED
Definition: coroapi.h:89
const char * name
Definition: coroapi.h:432
uint32_t votes
char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_READCONFIG_STARTUP
const char * icmap_iter_next(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type)
Definition: icmap.c:1103
#define NODE_FLAGS_WFASTATUS
void(* lib_handler_fn)(void *conn, const void *msg)
Definition: coroapi.h:418
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:73
uint32_t value
struct list_head * next
Definition: list.h:47
#define NODE_FLAGS_QUORATE
void(* timer_delete)(corosync_timer_handle_t timer_handle)
Definition: coroapi.h:193
struct list_head list
void icmap_iter_finalize(icmap_iter_t iter)
Definition: icmap.c:1124
#define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER
#define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE
#define max(a, b)
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
Definition: coroapi.h:181
#define list_iterate(v, head)
char * votequorum_init(struct corosync_api_v1 *api, quorum_set_quorate_fn_t q_set_quorate_fn)
nodestate_t
int tracking_enabled
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define CS_TRACK_CURRENT
Definition: corotypes.h:74
#define NODE_FLAGS_QDEVICE_MASTER_WINS
nodestate_t state
#define VOTEQUORUM_INFO_LAST_MAN_STANDING
struct message_header header
Definition: totemsrp.c:149
#define VOTEQUORUM_INFO_WAIT_FOR_ALL
#define NODE_FLAGS_QDEVICE_CAST_VOTE
uint32_t operation
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE
#define VOTEQUORUM_INFO_TWONODE
char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
Definition: list.h:46
#define VOTEQUORUM_INFO_QDEVICE_REGISTERED
#define log_printf(level, format, args...)
Definition: logsys.h:217
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
Definition: coroapi.h:423
char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_QDEVICE_NODEID
#define VOTEQUORUM_INFO_QDEVICE_MASTER_WINS
#define VOTEQUORUM_NODESTATE_MEMBER
#define CS_TRACK_CHANGES
Definition: corotypes.h:75
#define SERVICE_ID_MAKE(a, b)
Definition: coroapi.h:411
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
uint32_t expected_votes
#define ICMAP_KEYNAME_MAXLEN
Definition: icmap.h:48
#define VOTEQUORUM_QDEVICE_OPERATION_REGISTER
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:842
#define VOTEQUORUM_INFO_ALLOW_DOWNSCALE
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:71
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
#define VOTEQUORUM_INFO_QDEVICE_ALIVE
void *(* ipc_private_data_get)(void *conn)
Definition: coroapi.h:208
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
Definition: icmap.c:611
void * user_data
Definition: sam.c:126
struct list_head list
#define LOGSYS_DECLARE_SUBSYS(subsys)
Definition: logsys.h:197
#define CS_TRACK_CHANGES_ONLY
Definition: corotypes.h:76
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
uint32_t flags
char name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
Linked list API.
#define COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED
Definition: coroapi.h:129
cs_error_t
Definition: corotypes.h:78
unsigned char track_flags
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
int(* ipc_dispatch_send)(void *conn, const void *msg, size_t mlen)
Definition: coroapi.h:215
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:866
uint8_t param
uint32_t quorate
Definition: sam.c:133
#define swab32(x)
Definition: swab.h:43
int(* totem_mcast)(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
Definition: coroapi.h:233
#define VOTEQUORUM_INFO_AUTO_TIE_BREAKER
#define ENTER
Definition: logsys.h:218
int(* ipc_response_send)(void *conn, const void *msg, size_t mlen)
Definition: coroapi.h:210
char * corosync_service_link_and_init(struct corosync_api_v1 *corosync_api, struct default_service *service)
Link and initialize a service.
Definition: service.c:117
struct corosync_service_engine * votequorum_get_service_engine_ver0(void)
char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define VOTEQUORUM_NODESTATE_LEAVING
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:83
void(* error_memory_failure)(void) __attribute__((noreturn))
Definition: coroapi.h:375
#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG
#define VOTEQUORUM_READCONFIG_RUNTIME
struct list_head * prev
Definition: list.h:48
#define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO
#define VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES
#define VOTEQUORUM_QDEVICE_MAX_NAME_LEN
qb_loop_timer_handle corosync_timer_handle_t
Definition: coroapi.h:64
cs_error_t icmap_get_string(const char *key_name, char **str)
Definition: icmap.c:896
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:69
char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define NODE_FLAGS_LEAVING
#define list_entry(ptr, type, member)
Definition: list.h:84
char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]
#define COROSYNC_LIB_FLOW_CONTROL_REQUIRED
Definition: coroapi.h:128
typedef __attribute__
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:72
cs_error_t icmap_set_uint8(const char *key_name, uint8_t value)
Definition: icmap.c:587
#define VOTEQUORUM_NODESTATE_DEAD
#define VOTEQUORUM_INFO_QDEVICE_CAST_VOTE
#define VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT
const char * name
Definition: service.h:43
unsigned int nodeid
Definition: coroapi.h:65
icmap_iter_t icmap_iter_init(const char *prefix)
Definition: icmap.c:1097
struct memb_ring_id ring_id
Definition: totemsrp.c:153
uint64_t tracking_context
void(* quorum_set_quorate_fn_t)(const unsigned int *view_list, size_t view_list_entries, int quorate, struct memb_ring_id *)
Definition: exec/quorum.h:42
unsigned int(* totem_nodeid_get)(void)
Definition: coroapi.h:227
#define VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES
#define DEFAULT_LMS_WIN
#define LEAVE
Definition: logsys.h:219
#define NODE_FLAGS_QDEVICE_ALIVE
qb_map_iter_t * icmap_iter_t
Definition: icmap.h:121
#define NODE_FLAGS_QDEVICE_REGISTERED
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Definition: icmap.c:1167
#define NODE_FLAGS_FIRST
struct qb_ipc_request_header header __attribute__((aligned(8)))
#define ICMAP_TRACK_PREFIX
Definition: icmap.h:84