corosync  2.4.2
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2006 MontaVista Software, Inc.
3  * Copyright (c) 2006-2012 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8  *
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
72 #include <config.h>
73 
74 #include <pthread.h>
75 #include <assert.h>
76 #include <sys/types.h>
77 #include <sys/file.h>
78 #include <sys/poll.h>
79 #include <sys/uio.h>
80 #include <sys/mman.h>
81 #include <sys/socket.h>
82 #include <sys/un.h>
83 #include <sys/time.h>
84 #include <sys/resource.h>
85 #include <sys/stat.h>
86 #include <netinet/in.h>
87 #include <arpa/inet.h>
88 #include <unistd.h>
89 #include <fcntl.h>
90 #include <stdlib.h>
91 #include <stdio.h>
92 #include <errno.h>
93 #include <signal.h>
94 #include <sched.h>
95 #include <time.h>
96 #include <semaphore.h>
97 #include <string.h>
98 
99 #include <qb/qbdefs.h>
100 #include <qb/qblog.h>
101 #include <qb/qbloop.h>
102 #include <qb/qbutil.h>
103 #include <qb/qbipcs.h>
104 
105 #include <corosync/swab.h>
106 #include <corosync/corotypes.h>
107 #include <corosync/corodefs.h>
108 #include <corosync/list.h>
109 #include <corosync/totem/totempg.h>
110 #include <corosync/logsys.h>
111 #include <corosync/icmap.h>
112 
113 #include "quorum.h"
114 #include "totemsrp.h"
115 #include "logconfig.h"
116 #include "totemconfig.h"
117 #include "main.h"
118 #include "sync.h"
119 #include "timer.h"
120 #include "util.h"
121 #include "apidef.h"
122 #include "service.h"
123 #include "schedwrk.h"
124 
125 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
126 #define IPC_LOGSYS_SIZE 1024*64
127 #else
128 #define IPC_LOGSYS_SIZE 8192*128
129 #endif
130 
131 LOGSYS_DECLARE_SYSTEM ("corosync",
133  LOG_DAEMON,
134  LOG_INFO);
135 
136 LOGSYS_DECLARE_SUBSYS ("MAIN");
137 
138 #define SERVER_BACKLOG 5
139 
140 static int sched_priority = 0;
141 
142 static unsigned int service_count = 32;
143 
145 
146 static struct corosync_api_v1 *api = NULL;
147 
148 static int sync_in_process = 1;
149 
150 static qb_loop_t *corosync_poll_handle;
151 
152 struct sched_param global_sched_param;
153 
154 static corosync_timer_handle_t corosync_stats_timer_handle;
155 
156 static const char *corosync_lock_file = LOCALSTATEDIR"/run/corosync.pid";
157 
158 static int ip_version = AF_INET;
159 
160 qb_loop_t *cs_poll_handle_get (void)
161 {
162  return (corosync_poll_handle);
163 }
164 
165 int cs_poll_dispatch_add (qb_loop_t * handle,
166  int fd,
167  int events,
168  void *data,
169 
170  int (*dispatch_fn) (int fd,
171  int revents,
172  void *data))
173 {
174  return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data,
175  dispatch_fn);
176 }
177 
178 int cs_poll_dispatch_delete(qb_loop_t * handle, int fd)
179 {
180  return qb_loop_poll_del(handle, fd);
181 }
182 
184 {
185  int i;
186 
187  for (i = 0; i < SERVICES_COUNT_MAX; i++) {
188  if (corosync_service[i] && corosync_service[i]->exec_dump_fn) {
190  }
191  }
192 }
193 
194 static void corosync_blackbox_write_to_file (void)
195 {
196  char fname[PATH_MAX];
197  char fdata_fname[PATH_MAX];
198  char time_str[PATH_MAX];
199  struct tm cur_time_tm;
200  time_t cur_time_t;
201  ssize_t res;
202 
203  cur_time_t = time(NULL);
204  localtime_r(&cur_time_t, &cur_time_tm);
205 
206  strftime(time_str, PATH_MAX, "%Y-%m-%dT%H:%M:%S", &cur_time_tm);
207  snprintf(fname, PATH_MAX, "%s/fdata-%s-%lld",
208  get_run_dir(),
209  time_str,
210  (long long int)getpid());
211 
212  if ((res = qb_log_blackbox_write_to_file(fname)) < 0) {
213  LOGSYS_PERROR(-res, LOGSYS_LEVEL_ERROR, "Can't store blackbox file");
214  }
215  snprintf(fdata_fname, sizeof(fdata_fname), "%s/fdata", get_run_dir());
216  unlink(fdata_fname);
217  if (symlink(fname, fdata_fname) == -1) {
218  log_printf(LOGSYS_LEVEL_ERROR, "Can't create symlink to '%s' for corosync blackbox file '%s'",
219  fname, fdata_fname);
220  }
221 }
222 
223 static void unlink_all_completed (void)
224 {
225  api->timer_delete (corosync_stats_timer_handle);
226  qb_loop_stop (corosync_poll_handle);
227  icmap_fini();
228 }
229 
231 {
232  corosync_service_unlink_all (api, unlink_all_completed);
233 }
234 
235 static int32_t sig_diag_handler (int num, void *data)
236 {
238  return 0;
239 }
240 
241 static int32_t sig_exit_handler (int num, void *data)
242 {
243  log_printf(LOGSYS_LEVEL_NOTICE, "Node was shut down by a signal");
244  corosync_service_unlink_all (api, unlink_all_completed);
245  return 0;
246 }
247 
248 static void sigsegv_handler (int num)
249 {
250  (void)signal (SIGSEGV, SIG_DFL);
251  corosync_blackbox_write_to_file ();
252  qb_log_fini();
253  raise (SIGSEGV);
254 }
255 
256 /*
257  * QB wrapper for real signal handler
258  */
259 static int32_t sig_segv_handler (int num, void *data)
260 {
261 
262  sigsegv_handler(num);
263 
264  return 0;
265 }
266 
267 static void sigabrt_handler (int num)
268 {
269  (void)signal (SIGABRT, SIG_DFL);
270  corosync_blackbox_write_to_file ();
271  qb_log_fini();
272  raise (SIGABRT);
273 }
274 
275 /*
276  * QB wrapper for real signal handler
277  */
278 static int32_t sig_abrt_handler (int num, void *data)
279 {
280 
281  sigabrt_handler(num);
282 
283  return 0;
284 }
285 
286 #define LOCALHOST_IP inet_addr("127.0.0.1")
287 
288 static void *corosync_group_handle;
289 
290 static struct totempg_group corosync_group = {
291  .group = "a",
292  .group_len = 1
293 };
294 
295 static void serialize_lock (void)
296 {
297 }
298 
299 static void serialize_unlock (void)
300 {
301 }
302 
303 static void corosync_sync_completed (void)
304 {
306  "Completed service synchronization, ready to provide service.");
307  sync_in_process = 0;
308 
309  cs_ipcs_sync_state_changed(sync_in_process);
311  /*
312  * Inform totem to start using new message queue again
313  */
315 }
316 
317 static int corosync_sync_callbacks_retrieve (
318  int service_id,
319  struct sync_callbacks *callbacks)
320 {
321  if (corosync_service[service_id] == NULL) {
322  return (-1);
323  }
324 
325  if (callbacks == NULL) {
326  return (0);
327  }
328 
329  callbacks->name = corosync_service[service_id]->name;
330 
331  callbacks->sync_init = corosync_service[service_id]->sync_init;
332  callbacks->sync_process = corosync_service[service_id]->sync_process;
333  callbacks->sync_activate = corosync_service[service_id]->sync_activate;
334  callbacks->sync_abort = corosync_service[service_id]->sync_abort;
335  return (0);
336 }
337 
338 static struct memb_ring_id corosync_ring_id;
339 
340 static void member_object_joined (unsigned int nodeid)
341 {
342  char member_ip[ICMAP_KEYNAME_MAXLEN];
343  char member_join_count[ICMAP_KEYNAME_MAXLEN];
344  char member_status[ICMAP_KEYNAME_MAXLEN];
345 
346  snprintf(member_ip, ICMAP_KEYNAME_MAXLEN,
347  "runtime.totem.pg.mrp.srp.members.%u.ip", nodeid);
348  snprintf(member_join_count, ICMAP_KEYNAME_MAXLEN,
349  "runtime.totem.pg.mrp.srp.members.%u.join_count", nodeid);
350  snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
351  "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
352 
353  if (icmap_get(member_ip, NULL, NULL, NULL) == CS_OK) {
354  icmap_inc(member_join_count);
355  icmap_set_string(member_status, "joined");
356  } else {
357  icmap_set_string(member_ip, (char*)api->totem_ifaces_print (nodeid));
358  icmap_set_uint32(member_join_count, 1);
359  icmap_set_string(member_status, "joined");
360  }
361 
363  "Member joined: %s", api->totem_ifaces_print (nodeid));
364 }
365 
366 static void member_object_left (unsigned int nodeid)
367 {
368  char member_status[ICMAP_KEYNAME_MAXLEN];
369 
370  snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
371  "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
372  icmap_set_string(member_status, "left");
373 
375  "Member left: %s", api->totem_ifaces_print (nodeid));
376 }
377 
378 static void confchg_fn (
379  enum totem_configuration_type configuration_type,
380  const unsigned int *member_list, size_t member_list_entries,
381  const unsigned int *left_list, size_t left_list_entries,
382  const unsigned int *joined_list, size_t joined_list_entries,
383  const struct memb_ring_id *ring_id)
384 {
385  int i;
386  int abort_activate = 0;
387 
388  if (sync_in_process == 1) {
389  abort_activate = 1;
390  }
391  sync_in_process = 1;
392  cs_ipcs_sync_state_changed(sync_in_process);
393  memcpy (&corosync_ring_id, ring_id, sizeof (struct memb_ring_id));
394 
395  for (i = 0; i < left_list_entries; i++) {
396  member_object_left (left_list[i]);
397  }
398  for (i = 0; i < joined_list_entries; i++) {
399  member_object_joined (joined_list[i]);
400  }
401  /*
402  * Call configuration change for all services
403  */
404  for (i = 0; i < service_count; i++) {
405  if (corosync_service[i] && corosync_service[i]->confchg_fn) {
406  corosync_service[i]->confchg_fn (configuration_type,
407  member_list, member_list_entries,
408  left_list, left_list_entries,
409  joined_list, joined_list_entries, ring_id);
410  }
411  }
412 
413  if (abort_activate) {
414  sync_abort ();
415  }
416  if (configuration_type == TOTEM_CONFIGURATION_TRANSITIONAL) {
417  sync_save_transitional (member_list, member_list_entries, ring_id);
418  }
419  if (configuration_type == TOTEM_CONFIGURATION_REGULAR) {
420  sync_start (member_list, member_list_entries, ring_id);
421  }
422 }
423 
424 static void priv_drop (void)
425 {
426  return; /* TODO: we are still not dropping privs */
427 }
428 
429 static void corosync_tty_detach (void)
430 {
431  int devnull;
432 
433  /*
434  * Disconnect from TTY if this is not a debug run
435  */
436 
437  switch (fork ()) {
438  case -1:
440  break;
441  case 0:
442  /*
443  * child which is disconnected, run this process
444  */
445  break;
446  default:
447  exit (0);
448  break;
449  }
450 
451  /* Create new session */
452  (void)setsid();
453 
454  /*
455  * Map stdin/out/err to /dev/null.
456  */
457  devnull = open("/dev/null", O_RDWR);
458  if (devnull == -1) {
460  }
461 
462  if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0
463  || dup2(devnull, 2) < 0) {
464  close(devnull);
466  }
467  close(devnull);
468 }
469 
470 static void corosync_mlockall (void)
471 {
472  int res;
473  struct rlimit rlimit;
474 
475  rlimit.rlim_cur = RLIM_INFINITY;
476  rlimit.rlim_max = RLIM_INFINITY;
477 
478 #ifndef RLIMIT_MEMLOCK
479 #define RLIMIT_MEMLOCK RLIMIT_VMEM
480 #endif
481 
482  setrlimit (RLIMIT_MEMLOCK, &rlimit);
483 
484  res = mlockall (MCL_CURRENT | MCL_FUTURE);
485  if (res == -1) {
487  "Could not lock memory of service to avoid page faults");
488  };
489 }
490 
491 
492 static void corosync_totem_stats_updater (void *data)
493 {
494  totempg_stats_t * stats;
495  uint32_t total_mtt_rx_token;
496  uint32_t total_backlog_calc;
497  uint32_t total_token_holdtime;
498  int t, prev, i;
499  int32_t token_count;
500  char key_name[ICMAP_KEYNAME_MAXLEN];
501 
502  stats = api->totem_get_stats();
503 
504  icmap_set_uint32("runtime.totem.pg.msg_reserved", stats->msg_reserved);
505  icmap_set_uint32("runtime.totem.pg.msg_queue_avail", stats->msg_queue_avail);
506  icmap_set_uint64("runtime.totem.pg.mrp.srp.orf_token_tx", stats->mrp->srp->orf_token_tx);
507  icmap_set_uint64("runtime.totem.pg.mrp.srp.orf_token_rx", stats->mrp->srp->orf_token_rx);
508  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_merge_detect_tx", stats->mrp->srp->memb_merge_detect_tx);
509  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_merge_detect_rx", stats->mrp->srp->memb_merge_detect_rx);
510  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_join_tx", stats->mrp->srp->memb_join_tx);
511  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_join_rx", stats->mrp->srp->memb_join_rx);
512  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_tx", stats->mrp->srp->mcast_tx);
513  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_retx", stats->mrp->srp->mcast_retx);
514  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_rx", stats->mrp->srp->mcast_rx);
515  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_commit_token_tx", stats->mrp->srp->memb_commit_token_tx);
516  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_commit_token_rx", stats->mrp->srp->memb_commit_token_rx);
517  icmap_set_uint64("runtime.totem.pg.mrp.srp.token_hold_cancel_tx", stats->mrp->srp->token_hold_cancel_tx);
518  icmap_set_uint64("runtime.totem.pg.mrp.srp.token_hold_cancel_rx", stats->mrp->srp->token_hold_cancel_rx);
519  icmap_set_uint64("runtime.totem.pg.mrp.srp.operational_entered", stats->mrp->srp->operational_entered);
520  icmap_set_uint64("runtime.totem.pg.mrp.srp.operational_token_lost", stats->mrp->srp->operational_token_lost);
521  icmap_set_uint64("runtime.totem.pg.mrp.srp.gather_entered", stats->mrp->srp->gather_entered);
522  icmap_set_uint64("runtime.totem.pg.mrp.srp.gather_token_lost", stats->mrp->srp->gather_token_lost);
523  icmap_set_uint64("runtime.totem.pg.mrp.srp.commit_entered", stats->mrp->srp->commit_entered);
524  icmap_set_uint64("runtime.totem.pg.mrp.srp.commit_token_lost", stats->mrp->srp->commit_token_lost);
525  icmap_set_uint64("runtime.totem.pg.mrp.srp.recovery_entered", stats->mrp->srp->recovery_entered);
526  icmap_set_uint64("runtime.totem.pg.mrp.srp.recovery_token_lost", stats->mrp->srp->recovery_token_lost);
527  icmap_set_uint64("runtime.totem.pg.mrp.srp.consensus_timeouts", stats->mrp->srp->consensus_timeouts);
528  icmap_set_uint64("runtime.totem.pg.mrp.srp.rx_msg_dropped", stats->mrp->srp->rx_msg_dropped);
529  icmap_set_uint32("runtime.totem.pg.mrp.srp.continuous_gather", stats->mrp->srp->continuous_gather);
530  icmap_set_uint32("runtime.totem.pg.mrp.srp.continuous_sendmsg_failures",
532 
533  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure",
534  stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ? 1 : 0);
535 
536  if (stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ||
539  "Totem is unable to form a cluster because of an "
540  "operating system or network fault. The most common "
541  "cause of this message is that the local firewall is "
542  "configured improperly.");
543  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 1);
544  } else {
545  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 0);
546  }
547 
548  for (i = 0; i < stats->mrp->srp->rrp->interface_count; i++) {
549  snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "runtime.totem.pg.mrp.rrp.%u.faulty", i);
550  icmap_set_uint8(key_name, stats->mrp->srp->rrp->faulty[i]);
551  }
552  total_mtt_rx_token = 0;
553  total_token_holdtime = 0;
554  total_backlog_calc = 0;
555  token_count = 0;
556  t = stats->mrp->srp->latest_token;
557  while (1) {
558  if (t == 0)
559  prev = TOTEM_TOKEN_STATS_MAX - 1;
560  else
561  prev = t - 1;
562  if (prev == stats->mrp->srp->earliest_token)
563  break;
564  /* if tx == 0, then dropped token (not ours) */
565  if (stats->mrp->srp->token[t].tx != 0 ||
566  (stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx) > 0 ) {
567  total_mtt_rx_token += (stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx);
568  total_token_holdtime += (stats->mrp->srp->token[t].tx - stats->mrp->srp->token[t].rx);
569  total_backlog_calc += stats->mrp->srp->token[t].backlog_calc;
570  token_count++;
571  }
572  t = prev;
573  }
574  if (token_count) {
575  icmap_set_uint32("runtime.totem.pg.mrp.srp.mtt_rx_token", (total_mtt_rx_token / token_count));
576  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_token_workload", (total_token_holdtime / token_count));
577  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_backlog_calc", (total_backlog_calc / token_count));
578  }
579 
581 
582  api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
583  corosync_totem_stats_updater,
584  &corosync_stats_timer_handle);
585 }
586 
587 static void corosync_totem_stats_init (void)
588 {
589  icmap_set_uint32("runtime.totem.pg.mrp.srp.mtt_rx_token", 0);
590  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_token_workload", 0);
591  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_backlog_calc", 0);
592 
593  /* start stats timer */
594  api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
595  corosync_totem_stats_updater,
596  &corosync_stats_timer_handle);
597 }
598 
599 static void deliver_fn (
600  unsigned int nodeid,
601  const void *msg,
602  unsigned int msg_len,
603  int endian_conversion_required)
604 {
605  const struct qb_ipc_request_header *header;
606  int32_t service;
607  int32_t fn_id;
608  uint32_t id;
609 
610  header = msg;
611  if (endian_conversion_required) {
612  id = swab32 (header->id);
613  } else {
614  id = header->id;
615  }
616 
617  /*
618  * Call the proper executive handler
619  */
620  service = id >> 16;
621  fn_id = id & 0xffff;
622 
623  if (!corosync_service[service]) {
624  return;
625  }
626  if (fn_id >= corosync_service[service]->exec_engine_count) {
627  log_printf(LOGSYS_LEVEL_WARNING, "discarded unknown message %d for service %d (max id %d)",
628  fn_id, service, corosync_service[service]->exec_engine_count);
629  return;
630  }
631 
632  icmap_fast_inc(service_stats_rx[service][fn_id]);
633 
634  if (endian_conversion_required) {
635  assert(corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL);
637  ((void *)msg);
638  }
639 
641  (msg, nodeid);
642 }
643 
645  const struct iovec *iovec,
646  unsigned int iov_len,
647  unsigned int guarantee)
648 {
649  const struct qb_ipc_request_header *req = iovec->iov_base;
650  int32_t service;
651  int32_t fn_id;
652 
653  service = req->id >> 16;
654  fn_id = req->id & 0xffff;
655 
656  if (corosync_service[service]) {
657  icmap_fast_inc(service_stats_tx[service][fn_id]);
658  }
659 
660  return (totempg_groups_mcast_joined (corosync_group_handle, iovec, iov_len, guarantee));
661 }
662 
663 static void corosync_ring_id_create_or_load (
664  struct memb_ring_id *memb_ring_id,
665  const struct totem_ip_address *addr)
666 {
667  int fd;
668  int res = 0;
669  char filename[PATH_MAX];
670 
671  snprintf (filename, sizeof(filename), "%s/ringid_%s",
672  get_run_dir(), totemip_print (addr));
673  fd = open (filename, O_RDONLY, 0700);
674  /*
675  * If file can be opened and read, read the ring id
676  */
677  if (fd != -1) {
678  res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
679  close (fd);
680  }
681  /*
682  * If file could not be opened or read, create a new ring id
683  */
684  if ((fd == -1) || (res != sizeof (uint64_t))) {
685  memb_ring_id->seq = 0;
686  umask(0);
687  fd = open (filename, O_CREAT|O_RDWR, 0700);
688  if (fd != -1) {
689  res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
690  close (fd);
691  if (res == -1) {
693  "Couldn't write ringid file '%s'", filename);
694 
696  }
697  } else {
699  "Couldn't create ringid file '%s'", filename);
700 
702  }
703  }
704 
705  totemip_copy(&memb_ring_id->rep, addr);
706  assert (!totemip_zero_check(&memb_ring_id->rep));
707 }
708 
709 static void corosync_ring_id_store (
710  const struct memb_ring_id *memb_ring_id,
711  const struct totem_ip_address *addr)
712 {
713  char filename[PATH_MAX];
714  int fd;
715  int res;
716 
717  snprintf (filename, sizeof(filename), "%s/ringid_%s",
718  get_run_dir(), totemip_print (addr));
719 
720  fd = open (filename, O_WRONLY, 0700);
721  if (fd == -1) {
722  fd = open (filename, O_CREAT|O_RDWR, 0700);
723  }
724  if (fd == -1) {
726  "Couldn't store new ring id %llx to stable storage",
727  memb_ring_id->seq);
728 
730  }
732  "Storing new sequence id for ring %llx", memb_ring_id->seq);
733  res = write (fd, &memb_ring_id->seq, sizeof(memb_ring_id->seq));
734  close (fd);
735  if (res != sizeof(memb_ring_id->seq)) {
737  "Couldn't store new ring id %llx to stable storage",
738  memb_ring_id->seq);
739 
741  }
742 }
743 
744 static qb_loop_timer_handle recheck_the_q_level_timer;
746 {
747  totempg_check_q_level(corosync_group_handle);
749  qb_loop_timer_add(cs_poll_handle_get(), QB_LOOP_MED, 1*QB_TIME_NS_IN_MSEC,
750  NULL, corosync_recheck_the_q_level, &recheck_the_q_level_timer);
751  }
752 }
753 
756 };
757 
758 
760  unsigned int service,
761  unsigned int id,
762  const void *msg,
763  void *sending_allowed_private_data)
764 {
766  (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
767  struct iovec reserve_iovec;
768  struct qb_ipc_request_header *header = (struct qb_ipc_request_header *)msg;
769  int sending_allowed;
770 
771  reserve_iovec.iov_base = (char *)header;
772  reserve_iovec.iov_len = header->size;
773 
775  corosync_group_handle,
776  &reserve_iovec, 1);
777  if (pd->reserved_msgs == -1) {
778  return -EINVAL;
779  }
780 
781  sending_allowed = QB_FALSE;
782  if (corosync_quorum_is_quorate() == 1 ||
783  corosync_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) {
784  // we are quorate
785  // now check flow control
786  if (corosync_service[service]->lib_engine[id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) {
787  sending_allowed = QB_TRUE;
788  } else if (pd->reserved_msgs && sync_in_process == 0) {
789  sending_allowed = QB_TRUE;
790  } else if (pd->reserved_msgs == 0) {
791  return -ENOBUFS;
792  } else /* (sync_in_process) */ {
793  return -EINPROGRESS;
794  }
795  } else {
796  return -EHOSTUNREACH;
797  }
798 
799  return (sending_allowed);
800 }
801 
802 void corosync_sending_allowed_release (void *sending_allowed_private_data)
803 {
805  (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
806 
807  if (pd->reserved_msgs == -1) {
808  return;
809  }
811 }
812 
814 {
815  int ret = 0;
816 
817  assert (source != NULL);
818  if (source->nodeid == totempg_my_nodeid_get ()) {
819  ret = 1;
820  }
821  return ret;
822 }
823 
825  mar_message_source_t *source,
826  void *conn)
827 {
828  assert ((source != NULL) && (conn != NULL));
829  memset (source, 0, sizeof (mar_message_source_t));
830  source->nodeid = totempg_my_nodeid_get ();
831  source->conn = conn;
832 }
833 
836  qb_loop_timer_handle handle;
837  unsigned long long tv_prev;
838  unsigned long long max_tv_diff;
839 };
840 
841 static void timer_function_scheduler_timeout (void *data)
842 {
843  struct scheduler_pause_timeout_data *timeout_data = (struct scheduler_pause_timeout_data *)data;
844  unsigned long long tv_current;
845  unsigned long long tv_diff;
846 
847  tv_current = qb_util_nano_current_get ();
848 
849  if (timeout_data->tv_prev == 0) {
850  /*
851  * Initial call -> just pretent everything is ok
852  */
853  timeout_data->tv_prev = tv_current;
854  timeout_data->max_tv_diff = 0;
855  }
856 
857  tv_diff = tv_current - timeout_data->tv_prev;
858  timeout_data->tv_prev = tv_current;
859 
860  if (tv_diff > timeout_data->max_tv_diff) {
861  log_printf (LOGSYS_LEVEL_WARNING, "Corosync main process was not scheduled for %0.4f ms "
862  "(threshold is %0.4f ms). Consider token timeout increase.",
863  (float)tv_diff / QB_TIME_NS_IN_MSEC, (float)timeout_data->max_tv_diff / QB_TIME_NS_IN_MSEC);
864  }
865 
866  /*
867  * Set next threshold, because token_timeout can change
868  */
869  timeout_data->max_tv_diff = timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC * 0.8;
870  qb_loop_timer_add (corosync_poll_handle,
871  QB_LOOP_MED,
872  timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC / 3,
873  timeout_data,
874  timer_function_scheduler_timeout,
875  &timeout_data->handle);
876 }
877 
878 
879 static void corosync_setscheduler (void)
880 {
881 #if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER)
882  int res;
883 
884  sched_priority = sched_get_priority_max (SCHED_RR);
885  if (sched_priority != -1) {
886  global_sched_param.sched_priority = sched_priority;
887  res = sched_setscheduler (0, SCHED_RR, &global_sched_param);
888  if (res == -1) {
890  "Could not set SCHED_RR at priority %d",
891  global_sched_param.sched_priority);
892 
893  global_sched_param.sched_priority = 0;
894 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
895  qb_log_thread_priority_set (SCHED_OTHER, 0);
896 #endif
897  } else {
898 
899  /*
900  * Turn on SCHED_RR in logsys system
901  */
902 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
903  res = qb_log_thread_priority_set (SCHED_RR, sched_priority);
904 #else
905  res = -1;
906 #endif
907  if (res == -1) {
909  "Could not set logsys thread priority."
910  " Can't continue because of priority inversions.");
912  }
913  }
914  } else {
916  "Could not get maximum scheduler priority");
917  sched_priority = 0;
918  }
919 #else
921  "The Platform is missing process priority setting features. Leaving at default.");
922 #endif
923 }
924 
925 
926 /* The basename man page contains scary warnings about
927  thread-safety and portability, hence this */
928 static const char *corosync_basename(const char *file_name)
929 {
930  char *base;
931  base = strrchr (file_name, '/');
932  if (base) {
933  return base + 1;
934  }
935 
936  return file_name;
937 }
938 
939 static void
940 _logsys_log_printf(int level, int subsys,
941  const char *function_name,
942  const char *file_name,
943  int file_line,
944  const char *format,
945  ...) __attribute__((format(printf, 6, 7)));
946 
947 static void
948 _logsys_log_printf(int level, int subsys,
949  const char *function_name,
950  const char *file_name,
951  int file_line,
952  const char *format, ...)
953 {
954  va_list ap;
955 
956  va_start(ap, format);
957  qb_log_from_external_source_va(function_name, corosync_basename(file_name),
958  format, level, file_line,
959  subsys, ap);
960  va_end(ap);
961 }
962 
963 static void fplay_key_change_notify_fn (
964  int32_t event,
965  const char *key_name,
966  struct icmap_notify_value new_val,
967  struct icmap_notify_value old_val,
968  void *user_data)
969 {
970  if (strcmp(key_name, "runtime.blackbox.dump_flight_data") == 0) {
971  fprintf(stderr,"Writetofile\n");
972  corosync_blackbox_write_to_file ();
973  }
974  if (strcmp(key_name, "runtime.blackbox.dump_state") == 0) {
975  fprintf(stderr,"statefump\n");
977  }
978 }
979 
980 static void corosync_fplay_control_init (void)
981 {
982  icmap_track_t track = NULL;
983 
984  icmap_set_string("runtime.blackbox.dump_flight_data", "no");
985  icmap_set_string("runtime.blackbox.dump_state", "no");
986 
987  icmap_track_add("runtime.blackbox.dump_flight_data",
989  fplay_key_change_notify_fn,
990  NULL, &track);
991  icmap_track_add("runtime.blackbox.dump_state",
993  fplay_key_change_notify_fn,
994  NULL, &track);
995 }
996 
997 /*
998  * Set RO flag for keys, which ether doesn't make sense to change by user (statistic)
999  * or which when changed are not reflected by runtime (totem.crypto_cipher, ...).
1000  *
1001  * Also some RO keys cannot be determined in this stage, so they are set later in
1002  * other functions (like nodelist.local_node_pos, ...)
1003  */
1004 static void set_icmap_ro_keys_flag (void)
1005 {
1006  /*
1007  * Set RO flag for all keys of internal configuration and runtime statistics
1008  */
1009  icmap_set_ro_access("internal_configuration.", CS_TRUE, CS_TRUE);
1010  icmap_set_ro_access("runtime.connections.", CS_TRUE, CS_TRUE);
1011  icmap_set_ro_access("runtime.totem.", CS_TRUE, CS_TRUE);
1012  icmap_set_ro_access("runtime.services.", CS_TRUE, CS_TRUE);
1013  icmap_set_ro_access("runtime.config.", CS_TRUE, CS_TRUE);
1014  icmap_set_ro_access("uidgid.config.", CS_TRUE, CS_TRUE);
1015 
1016  /*
1017  * Set RO flag for constrete keys of configuration which can't be changed
1018  * during runtime
1019  */
1020  icmap_set_ro_access("totem.crypto_cipher", CS_FALSE, CS_TRUE);
1021  icmap_set_ro_access("totem.crypto_hash", CS_FALSE, CS_TRUE);
1022  icmap_set_ro_access("totem.secauth", CS_FALSE, CS_TRUE);
1023  icmap_set_ro_access("totem.ip_version", CS_FALSE, CS_TRUE);
1024  icmap_set_ro_access("totem.rrp_mode", CS_FALSE, CS_TRUE);
1025  icmap_set_ro_access("totem.transport", CS_FALSE, CS_TRUE);
1026  icmap_set_ro_access("totem.cluster_name", CS_FALSE, CS_TRUE);
1027  icmap_set_ro_access("totem.netmtu", CS_FALSE, CS_TRUE);
1028  icmap_set_ro_access("totem.threads", CS_FALSE, CS_TRUE);
1029  icmap_set_ro_access("totem.version", CS_FALSE, CS_TRUE);
1030  icmap_set_ro_access("totem.nodeid", CS_FALSE, CS_TRUE);
1031  icmap_set_ro_access("totem.clear_node_high_bit", CS_FALSE, CS_TRUE);
1032  icmap_set_ro_access("qb.ipc_type", CS_FALSE, CS_TRUE);
1033  icmap_set_ro_access("config.reload_in_progress", CS_FALSE, CS_TRUE);
1034  icmap_set_ro_access("config.totemconfig_reload_in_progress", CS_FALSE, CS_TRUE);
1035 }
1036 
1037 static void main_service_ready (void)
1038 {
1039  int res;
1040 
1041  /*
1042  * This must occur after totempg is initialized because "this_ip" must be set
1043  */
1045  if (res == -1) {
1046  log_printf (LOGSYS_LEVEL_ERROR, "Could not initialize default services");
1048  }
1049  cs_ipcs_init();
1050  corosync_totem_stats_init ();
1051  corosync_fplay_control_init ();
1052  sync_init (
1053  corosync_sync_callbacks_retrieve,
1054  corosync_sync_completed);
1055 }
1056 
1057 static enum e_corosync_done corosync_flock (const char *lockfile, pid_t pid)
1058 {
1059  struct flock lock;
1060  enum e_corosync_done err;
1061  char pid_s[17];
1062  int fd_flag;
1063  int lf;
1064 
1065  err = COROSYNC_DONE_EXIT;
1066 
1067  lf = open (lockfile, O_WRONLY | O_CREAT, 0640);
1068  if (lf == -1) {
1069  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't create lock file.");
1070  return (COROSYNC_DONE_AQUIRE_LOCK);
1071  }
1072 
1073 retry_fcntl:
1074  lock.l_type = F_WRLCK;
1075  lock.l_start = 0;
1076  lock.l_whence = SEEK_SET;
1077  lock.l_len = 0;
1078  if (fcntl (lf, F_SETLK, &lock) == -1) {
1079  switch (errno) {
1080  case EINTR:
1081  goto retry_fcntl;
1082  break;
1083  case EAGAIN:
1084  case EACCES:
1085  log_printf (LOGSYS_LEVEL_ERROR, "Another Corosync instance is already running.");
1087  goto error_close;
1088  break;
1089  default:
1090  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't acquire lock. Error was %s",
1091  strerror(errno));
1093  goto error_close;
1094  break;
1095  }
1096  }
1097 
1098  if (ftruncate (lf, 0) == -1) {
1099  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't truncate lock file. Error was %s",
1100  strerror (errno));
1102  goto error_close_unlink;
1103  }
1104 
1105  memset (pid_s, 0, sizeof (pid_s));
1106  snprintf (pid_s, sizeof (pid_s) - 1, "%u\n", pid);
1107 
1108 retry_write:
1109  if (write (lf, pid_s, strlen (pid_s)) != strlen (pid_s)) {
1110  if (errno == EINTR) {
1111  goto retry_write;
1112  } else {
1113  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't write pid to lock file. "
1114  "Error was %s", strerror (errno));
1116  goto error_close_unlink;
1117  }
1118  }
1119 
1120  if ((fd_flag = fcntl (lf, F_GETFD, 0)) == -1) {
1121  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't get close-on-exec flag from lock file. "
1122  "Error was %s", strerror (errno));
1124  goto error_close_unlink;
1125  }
1126  fd_flag |= FD_CLOEXEC;
1127  if (fcntl (lf, F_SETFD, fd_flag) == -1) {
1128  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't set close-on-exec flag to lock file. "
1129  "Error was %s", strerror (errno));
1131  goto error_close_unlink;
1132  }
1133 
1134  return (err);
1135 
1136 error_close_unlink:
1137  unlink (lockfile);
1138 error_close:
1139  close (lf);
1140 
1141  return (err);
1142 }
1143 
1144 int main (int argc, char **argv, char **envp)
1145 {
1146  const char *error_string;
1147  struct totem_config totem_config;
1148  int res, ch;
1149  int background, setprio, testonly;
1150  struct stat stat_out;
1151  enum e_corosync_done flock_err;
1152  uint64_t totem_config_warnings;
1153  struct scheduler_pause_timeout_data scheduler_pause_timeout_data;
1154 
1155  /* default configuration
1156  */
1157  background = 1;
1158  setprio = 1;
1159  testonly = 0;
1160 
1161  while ((ch = getopt (argc, argv, "fprtv")) != EOF) {
1162 
1163  switch (ch) {
1164  case 'f':
1165  background = 0;
1166  break;
1167  case 'p':
1168  setprio = 0;
1169  break;
1170  case 'r':
1171  setprio = 1;
1172  break;
1173  case 't':
1174  testonly = 1;
1175  break;
1176  case 'v':
1177  printf ("Corosync Cluster Engine, version '%s'\n", VERSION);
1178  printf ("Copyright (c) 2006-2009 Red Hat, Inc.\n");
1180  return EXIT_SUCCESS;
1181 
1182  break;
1183  default:
1184  fprintf(stderr, \
1185  "usage:\n"\
1186  " -f : Start application in foreground.\n"\
1187  " -p : Do not set process priority.\n"\
1188  " -t : Test configuration and exit.\n"\
1189  " -r : Set round robin realtime scheduling (default).\n"\
1190  " -v : Display version and SVN revision of Corosync and exit.\n");
1192  return EXIT_FAILURE;
1193  }
1194  }
1195 
1196  /*
1197  * Set round robin realtime scheduling with priority 99
1198  * Lock all memory to avoid page faults which may interrupt
1199  * application healthchecking
1200  */
1201  if (setprio) {
1202  corosync_setscheduler ();
1203  }
1204 
1205  corosync_mlockall ();
1206 
1207  /*
1208  * Other signals are registered later via qb_loop_signal_add
1209  */
1210  (void)signal (SIGSEGV, sigsegv_handler);
1211  (void)signal (SIGABRT, sigabrt_handler);
1212 #if MSG_NOSIGNAL != 0
1213  (void)signal (SIGPIPE, SIG_IGN);
1214 #endif
1215 
1216  if (icmap_init() != CS_OK) {
1217  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize configuration component.");
1219  }
1220  set_icmap_ro_keys_flag();
1221 
1222  /*
1223  * Initialize the corosync_api_v1 definition
1224  */
1225  api = apidef_get ();
1226 
1227  res = coroparse_configparse(icmap_get_global_map(), &error_string);
1228  if (res == -1) {
1229  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1231  }
1232 
1233  res = corosync_log_config_read (&error_string);
1234  if (res == -1) {
1235  /*
1236  * if we are here, we _must_ flush the logsys queue
1237  * and try to inform that we couldn't read the config.
1238  * this is a desperate attempt before certain death
1239  * and there is no guarantee that we can print to stderr
1240  * nor that logsys is sending the messages where we expect.
1241  */
1242  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1243  fprintf(stderr, "%s", error_string);
1244  syslog (LOGSYS_LEVEL_ERROR, "%s", error_string);
1246  }
1247 
1248  if (!testonly) {
1249  log_printf (LOGSYS_LEVEL_NOTICE, "Corosync Cluster Engine ('%s'): started and ready to provide service.", VERSION);
1250  log_printf (LOGSYS_LEVEL_INFO, "Corosync built-in features:" PACKAGE_FEATURES "");
1251  }
1252 
1253  /*
1254  * Make sure required directory is present
1255  */
1256  res = stat (get_run_dir(), &stat_out);
1257  if ((res == -1) || (res == 0 && !S_ISDIR(stat_out.st_mode))) {
1258  log_printf (LOGSYS_LEVEL_ERROR, "Required directory not present %s. Please create it.", get_run_dir());
1260  }
1261 
1262  res = chdir(get_run_dir());
1263  if (res == -1) {
1264  log_printf (LOGSYS_LEVEL_ERROR, "Cannot chdir to run directory %s. "
1265  "Please make sure it has correct context and rights.", get_run_dir());
1267  }
1268 
1269  res = totem_config_read (&totem_config, &error_string, &totem_config_warnings);
1270  if (res == -1) {
1271  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1273  }
1274 
1275  if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_IGNORED) {
1276  log_printf (LOGSYS_LEVEL_WARNING, "member section is used together with nodelist. Members ignored.");
1277  }
1278 
1279  if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED) {
1280  log_printf (LOGSYS_LEVEL_WARNING, "member section is deprecated.");
1281  }
1282 
1283  if (totem_config_warnings & TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED) {
1284  log_printf (LOGSYS_LEVEL_WARNING, "nodeid appears both in totem section and nodelist. Nodelist one is used.");
1285  }
1286 
1287  if (totem_config_warnings != 0) {
1288  log_printf (LOGSYS_LEVEL_WARNING, "Please migrate config file to nodelist.");
1289  }
1290 
1291  res = totem_config_keyread (&totem_config, &error_string);
1292  if (res == -1) {
1293  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1295  }
1296 
1297  res = totem_config_validate (&totem_config, &error_string);
1298  if (res == -1) {
1299  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1301  }
1302 
1303  if (testonly) {
1305  }
1306 
1307  ip_version = totem_config.ip_version;
1308 
1309  totem_config.totem_memb_ring_id_create_or_load = corosync_ring_id_create_or_load;
1310  totem_config.totem_memb_ring_id_store = corosync_ring_id_store;
1311 
1313  totem_config.totem_logging_configuration.log_subsys_id = _logsys_subsys_create("TOTEM", "totem,"
1314  "totemmrp.c,totemrrp.c,totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
1315  "totempg.c,totemiba.c,totemudp.c,totemudpu.c,totemnet.c");
1316 
1323  totem_config.totem_logging_configuration.log_printf = _logsys_log_printf;
1325 
1326  /*
1327  * Now we are fully initialized.
1328  */
1329  if (background) {
1330  corosync_tty_detach ();
1331  }
1332 
1333  corosync_poll_handle = qb_loop_create ();
1334 
1335  memset(&scheduler_pause_timeout_data, 0, sizeof(scheduler_pause_timeout_data));
1336  scheduler_pause_timeout_data.totem_config = &totem_config;
1337  timer_function_scheduler_timeout (&scheduler_pause_timeout_data);
1338 
1339  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
1340  SIGUSR2, NULL, sig_diag_handler, NULL);
1341  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1342  SIGINT, NULL, sig_exit_handler, NULL);
1343  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1344  SIGSEGV, NULL, sig_segv_handler, NULL);
1345  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1346  SIGABRT, NULL, sig_abrt_handler, NULL);
1347  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1348  SIGQUIT, NULL, sig_exit_handler, NULL);
1349  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1350  SIGTERM, NULL, sig_exit_handler, NULL);
1351 
1352  if (logsys_thread_start() != 0) {
1353  log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize log thread");
1355  }
1356 
1357  if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) != COROSYNC_DONE_EXIT) {
1358  corosync_exit_error (flock_err);
1359  }
1360 
1361  /*
1362  * if totempg_initialize doesn't have root priveleges, it cannot
1363  * bind to a specific interface. This only matters if
1364  * there is more then one interface in a system, so
1365  * in this case, only a warning is printed
1366  */
1367  /*
1368  * Join multicast group and setup delivery
1369  * and configuration change functions
1370  */
1372  corosync_poll_handle,
1373  &totem_config);
1374 
1376  main_service_ready);
1377 
1379  &corosync_group_handle,
1380  deliver_fn,
1381  confchg_fn);
1382 
1384  corosync_group_handle,
1385  &corosync_group,
1386  1);
1387 
1388  /*
1389  * Drop root privleges to user 'corosync'
1390  * TODO: Don't really need full root capabilities;
1391  * needed capabilities are:
1392  * CAP_NET_RAW (bindtodevice)
1393  * CAP_SYS_NICE (setscheduler)
1394  * CAP_IPC_LOCK (mlockall)
1395  */
1396  priv_drop ();
1397 
1398  schedwrk_init (
1399  serialize_lock,
1400  serialize_unlock);
1401 
1402  /*
1403  * Start main processing loop
1404  */
1405  qb_loop_run (corosync_poll_handle);
1406 
1407  /*
1408  * Exit was requested
1409  */
1410  totempg_finalize ();
1411 
1412  /*
1413  * free the loop resources
1414  */
1415  qb_loop_destroy (corosync_poll_handle);
1416 
1417  /*
1418  * free up the icmap
1419  */
1420 
1421  /*
1422  * Remove pid lock file
1423  */
1424  unlink (corosync_lock_file);
1425 
1427 
1428  return EXIT_SUCCESS;
1429 }
#define CS_TRUE
Definition: corotypes.h:54
const char * name
Definition: coroapi.h:492
int32_t cs_ipcs_q_level_get(void)
Definition: ipc_glue.c:690
unsigned int corosync_service_defaults_link_and_init(struct corosync_api_v1 *corosync_api)
Load all of the default services.
Definition: service.c:339
void(*) in log_level_security)
Definition: totem.h:82
uint64_t gather_entered
Definition: totem.h:260
const char * get_run_dir(void)
Definition: util.c:174
int corosync_quorum_is_quorate(void)
Definition: exec/quorum.c:67
void totempg_finalize(void)
Definition: totempg.c:833
Totem Single Ring Protocol.
uint64_t memb_commit_token_rx
Definition: totem.h:255
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:73
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.h:39
#define LOGSYS_LEVEL_TRACE
Definition: logsys.h:75
void totempg_trans_ack(void)
Definition: totempg.c:1530
struct totem_config * totem_config
Definition: main.c:835
#define PACKAGE_FEATURES
Definition: config.h:364
#define CS_FALSE
Definition: corotypes.h:53
void sync_start(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:571
uint64_t memb_join_tx
Definition: totem.h:249
void(* timer_delete)(corosync_timer_handle_t timer_handle)
Definition: coroapi.h:241
qb_loop_timer_handle handle
Definition: main.c:836
void sync_abort(void)
Definition: sync.c:600
The totem_ip_address struct.
Definition: coroapi.h:111
totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX]
Definition: totem.h:274
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:214
Totem Single Ring Protocol.
int(* sync_process)(void)
Definition: coroapi.h:522
void(* exec_dump_fn)(void)
Definition: coroapi.h:502
int sync_init(int(*sync_callbacks_retrieve)(int service_id, struct sync_callbacks *callbacks), void(*synchronization_completed)(void))
Definition: sync.c:173
uint64_t mcast_rx
Definition: totem.h:253
int coroparse_configparse(icmap_map_t config_map, const char **error_string)
Definition: coroparse.c:251
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
Definition: coroapi.h:229
qb_loop_t * cs_poll_handle_get(void)
Definition: main.c:160
void corosync_recheck_the_q_level(void *data)
Definition: main.c:745
#define corosync_exit_error(err)
Definition: exec/util.h:70
void(* exec_endian_convert_fn)(void *msg)
Definition: coroapi.h:478
int totempg_groups_joined_release(int msg_count)
Definition: totempg.c:1324
#define VERSION
Definition: config.h:424
totem_configuration_type
The totem_configuration_type enum.
Definition: coroapi.h:132
#define LOCALSTATEDIR
Definition: config.h:349
struct message_header header
Definition: totemsrp.c:60
uint64_t memb_merge_detect_rx
Definition: totem.h:248
LOGSYS_DECLARE_SUBSYS("MAIN")
int guarantee
Definition: totemsrp.c:66
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:77
void schedwrk_init(void(*serialize_lock_fn)(void), void(*serialize_unlock_fn)(void))
Definition: schedwrk.c:83
cs_error_t icmap_set_string(const char *key_name, const char *value)
Definition: icmap.c:641
void(* sync_abort)(void)
Definition: coroapi.h:524
int earliest_token
Definition: totem.h:271
uint64_t orf_token_tx
Definition: totem.h:245
int cs_poll_dispatch_add(qb_loop_t *handle, int fd, int events, void *data, int(*dispatch_fn)(int fd, int revents, void *data))
Definition: main.c:165
int totempg_groups_initialize(void **instance, void(*deliver_fn)(unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required), void(*confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id))
Initialize a groups instance.
Definition: totempg.c:1112
uint64_t gather_token_lost
Definition: totem.h:261
#define RLIMIT_MEMLOCK
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:95
icmap_map_t icmap_get_global_map(void)
Return global icmap.
Definition: icmap.c:278
unsigned long long max_tv_diff
Definition: main.c:838
struct corosync_service_engine * corosync_service[SERVICES_COUNT_MAX]
Definition: service.c:110
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:436
uint64_t memb_commit_token_tx
Definition: totem.h:254
#define LOGSYS_PERROR(err_num, level, fmt, args...)
The LOGSYS_PERROR macro.
Definition: logsys.h:313
cs_error_t icmap_inc(const char *key_name)
Increase stored value by one.
Definition: icmap.c:1057
#define log_printf(level, format, args...)
Definition: logsys.h:319
#define MAX_NO_CONT_SENDMSG_FAILURES
Definition: totem.h:62
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
Definition: coroapi.h:477
#define TOTEM_TOKEN_STATS_MAX
Definition: totem.h:273
void cs_ipcs_sync_state_changed(int32_t sync_in_process)
Definition: ipc_glue.c:753
int corosync_sending_allowed(unsigned int service, unsigned int id, const void *msg, void *sending_allowed_private_data)
Definition: main.c:759
void icmap_fini(void)
Finalize global icmap.
Definition: icmap.c:261
int main_mcast(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
Definition: main.c:644
void cs_ipc_allow_connections(int32_t allow)
Definition: ipc_glue.c:155
void corosync_service_unlink_all(struct corosync_api_v1 *api, void(*unlink_all_complete)(void))
Unlink and exit all corosync services.
Definition: service.c:394
uint64_t operational_entered
Definition: totem.h:258
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
void totempg_service_ready_register(void(*totem_service_ready)(void))
Definition: totempg.c:1499
void(* sync_activate)(void)
Definition: sync.h:46
#define ICMAP_KEYNAME_MAXLEN
Maximum length of key in icmap.
Definition: icmap.h:48
int latest_token
Definition: totem.h:272
uint64_t operational_token_lost
Definition: totem.h:259
#define MAX_NO_CONT_GATHER
Maximum number of continuous gather states.
Definition: totem.h:58
uint64_t consensus_timeouts
Definition: totem.h:266
int totem_config_keyread(struct totem_config *totem_config, const char **error_string)
Definition: totemconfig.c:1450
const char * name
Definition: sync.h:48
uint64_t recovery_token_lost
Definition: totem.h:265
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:71
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
struct corosync_exec_handler * exec_engine
Definition: coroapi.h:507
unsigned int totempg_my_nodeid_get(void)
Definition: totempg.c:1490
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
Definition: icmap.c:611
pthread_mutex_t lock
Definition: sam.c:131
uint64_t commit_token_lost
Definition: totem.h:263
void * user_data
Definition: sam.c:127
void(* sync_abort)(void)
Definition: sync.h:47
uint64_t token_hold_cancel_rx
Definition: totem.h:257
int logsys_thread_start(void)
logsys_thread_start
Definition: logsys.c:820
int totem_config_validate(struct totem_config *totem_config, const char **error_string)
Definition: totemconfig.c:1264
#define LOGSYS_MODE_OUTPUT_SYSLOG
Definition: logsys.h:60
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
int totempg_groups_joined_reserve(void *instance, const struct iovec *iovec, unsigned int iov_len)
Definition: totempg.c:1280
const void * group
Definition: totempg.h:56
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:70
Linked list API.
struct totem_ip_address rep
Definition: coroapi.h:123
void sync_save_transitional(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:589
int totempg_groups_mcast_joined(void *instance, const struct iovec *iovec, unsigned int iov_len, int guarantee)
Definition: totempg.c:1210
const char * service_stats_tx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition: service.c:113
uint64_t mcast_retx
Definition: totem.h:252
cs_error_t icmap_get(const char *key_name, void *value, size_t *value_len, icmap_value_types_t *type)
Retrieve value of key key_name and store it in user preallocated value pointer.
Definition: icmap.c:739
int corosync_log_config_read(cmap_handle_t cmap_h, const char *default_logfile, const char **error_string)
Definition: logconfig.c:652
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:75
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
void cs_ipcs_init(void)
Definition: ipc_glue.c:896
totemmrp_stats_t * mrp
Definition: totem.h:288
void(* sync_activate)(void)
Definition: coroapi.h:523
The corosync_api_v1 struct.
Definition: coroapi.h:225
typedef __attribute__
void(* confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id)
Definition: coroapi.h:510
#define TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED
Definition: totemconfig.h:47
uint32_t msg_queue_avail
Definition: totem.h:290
uint64_t rx_msg_dropped
Definition: totem.h:267
#define TOTEM_CONFIG_WARNING_MEMBERS_IGNORED
Definition: totemconfig.h:46
#define swab32(x)
The swab32 macro.
Definition: swab.h:51
uint32_t continuous_gather
Definition: totem.h:268
cs_error_t icmap_set_uint64(const char *key_name, uint64_t value)
Definition: icmap.c:623
totemrrp_stats_t * rrp
Definition: totem.h:244
cs_error_t icmap_fast_inc(const char *key_name)
Increase stored value by one.
Definition: icmap.c:1077
#define MILLI_2_NANO_SECONDS
Definition: coroapi.h:105
void message_source_set(mar_message_source_t *source, void *conn)
Definition: main.c:824
int(* sync_process)(void)
Definition: sync.h:45
void logsys_config_apply(void)
logsys_config_apply
Definition: logsys.c:770
uint64_t token_hold_cancel_tx
Definition: totem.h:256
int ip_version
Definition: totem.h:189
unsigned int token_timeout
Definition: totem.h:129
void totempg_check_q_level(void *instance)
Definition: totempg.c:1272
uint64_t mcast_tx
Definition: totem.h:251
const char * service_stats_rx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition: service.c:112
The memb_ring_id struct.
Definition: coroapi.h:122
struct corosync_api_v1 * apidef_get(void)
Definition: apidef.c:147
void corosync_sending_allowed_release(void *sending_allowed_private_data)
Definition: main.c:802
void cs_ipcs_stats_update(void)
Definition: ipc_glue.c:759
uint64_t memb_merge_detect_tx
Definition: totem.h:247
uint32_t msg_reserved
Definition: totem.h:289
totemsrp_stats_t * srp
Definition: totem.h:283
#define SERVICES_COUNT_MAX
Definition: coroapi.h:463
unsigned long long tv_prev
Definition: main.c:837
uint64_t memb_join_rx
Definition: totem.h:250
void corosync_state_dump(void)
Definition: main.c:183
qb_loop_timer_handle corosync_timer_handle_t
corosync_timer_handle_t
Definition: coroapi.h:74
int cs_poll_dispatch_delete(qb_loop_t *handle, int fd)
Definition: main.c:178
e_corosync_done
Definition: exec/util.h:47
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: coroapi.h:516
int message_source_is_local(const mar_message_source_t *source)
Definition: main.c:813
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:163
int totempg_groups_join(void *instance, const struct totempg_group *groups, size_t group_cnt)
Definition: totempg.c:1160
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:72
unsigned long long seq
Definition: coroapi.h:124
cs_error_t icmap_set_uint8(const char *key_name, uint8_t value)
Definition: icmap.c:587
uint64_t recovery_entered
Definition: totem.h:264
cs_error_t icmap_set_ro_access(const char *key_name, int prefix, int ro_access)
Set read-only access for given key (key_name) or prefix, If prefix is set.
Definition: icmap.c:1233
int totempg_initialize(qb_loop_t *poll_handle, struct totem_config *totem_config)
Initialize the totem process groups abstraction.
Definition: totempg.c:787
void corosync_shutdown_request(void)
Definition: main.c:230
uint32_t interface_count
Definition: totem.h:232
struct sched_param global_sched_param
Definition: main.c:152
unsigned int nodeid
Definition: coroapi.h:75
uint32_t continuous_sendmsg_failures
Definition: totem.h:269
uint8_t * faulty
Definition: totem.h:231
struct memb_ring_id ring_id
Definition: totemsrp.c:64
const char *(* totem_ifaces_print)(unsigned int nodeid)
Definition: coroapi.h:291
int main(int argc, char **argv, char **envp)
Definition: main.c:1144
cs_error_t icmap_init(void)
Initialize global icmap.
Definition: icmap.c:221
uint64_t commit_entered
Definition: totem.h:262
#define TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED
Definition: totemconfig.h:48
LOGSYS_DECLARE_SYSTEM("corosync", LOGSYS_MODE_OUTPUT_STDERR|LOGSYS_MODE_OUTPUT_SYSLOG, LOG_DAEMON, LOG_INFO)
#define LOGSYS_MODE_OUTPUT_STDERR
Definition: logsys.h:59
void *(* totem_get_stats)(void)
Definition: coroapi.h:436
uint64_t orf_token_rx
Definition: totem.h:246
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1167
int totem_config_read(struct totem_config *totem_config, const char **error_string, uint64_t *warnings)
Definition: totemconfig.c:978
void(* totem_memb_ring_id_create_or_load)(struct memb_ring_id *memb_ring_id, const struct totem_ip_address *addr)
Definition: totem.h:191
The mar_message_source_t struct.
Definition: coroapi.h:50
void logsys_system_fini(void)
logsys_system_fini
Definition: logsys.c:285
void(* totem_memb_ring_id_store)(const struct memb_ring_id *memb_ring_id, const struct totem_ip_address *addr)
Definition: totem.h:195