corosync  3.0.4
totemknet.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2020 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7 
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of the MontaVista Software, Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <assert.h>
38 #include <sys/mman.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/socket.h>
42 #include <netdb.h>
43 #include <sys/un.h>
44 #include <sys/ioctl.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <net/ethernet.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <pthread.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <sys/uio.h>
60 #include <limits.h>
61 
62 #include <qb/qbdefs.h>
63 #include <qb/qbloop.h>
64 #ifdef HAVE_LIBNOZZLE
65 #include <libgen.h>
66 #include <libnozzle.h>
67 #endif
68 
69 #include <corosync/sq.h>
70 #include <corosync/swab.h>
71 #include <corosync/logsys.h>
72 #include <corosync/icmap.h>
73 #include <corosync/totem/totemip.h>
74 #include "totemknet.h"
75 
76 #include "main.h"
77 #include "util.h"
78 
79 #include <libknet.h>
81 
82 #ifndef MSG_NOSIGNAL
83 #define MSG_NOSIGNAL 0
84 #endif
85 
86 #ifdef HAVE_LIBNOZZLE
87 static int setup_nozzle(void *knet_context);
88 #endif
89 
90 /* Should match that used by cfg */
91 #define CFG_INTERFACE_STATUS_MAX_LEN 512
92 
94  struct crypto_instance *crypto_inst;
95 
96  qb_loop_t *poll_handle;
97 
98  knet_handle_t knet_handle;
99 
101 
102  void *context;
103 
105  void *context,
106  const void *msg,
107  unsigned int msg_len,
108  const struct sockaddr_storage *system_from);
109 
111  void *context,
112  const struct totem_ip_address *iface_address,
113  unsigned int link_no);
114 
116  void *context,
117  int net_mtu);
118 
120 
121  /*
122  * Function and data used to log messages
123  */
125 
127 
129 
131 
133 
135 
137 
139  int level,
140  int subsys,
141  const char *function,
142  const char *file,
143  int line,
144  const char *format,
145  ...)__attribute__((format(printf, 6, 7)));
146 
148 
149  char iov_buffer[KNET_MAX_PACKET_SIZE];
150 
152 
154 
156 
158 
160 
162 
164 
165  qb_loop_timer_handle timer_netif_check_timeout;
166 
167  qb_loop_timer_handle timer_merge_detect_timeout;
168 
170 
172 
173  int logpipes[2];
174  int knet_fd;
175 
176  pthread_mutex_t log_mutex;
177 #ifdef HAVE_LIBNOZZLE
178  char *nozzle_name;
179  char *nozzle_ipaddr;
180  char *nozzle_prefix;
181  char *nozzle_macaddr;
182  nozzle_t nozzle_handle;
183 #endif
184 };
185 
186 /* Awkward. But needed to get stats from knet */
188 
189 struct work_item {
190  const void *msg;
191  unsigned int msg_len;
193 };
194 
196  void *knet_context);
197 
198 static void totemknet_start_merge_detect_timeout(
199  void *knet_context);
200 
201 static void totemknet_stop_merge_detect_timeout(
202  void *knet_context);
203 
204 static void log_flush_messages (
205  void *knet_context);
206 
207 static void totemknet_instance_initialize (struct totemknet_instance *instance)
208 {
209  int res;
210 
211  memset (instance, 0, sizeof (struct totemknet_instance));
212  res = pthread_mutex_init(&instance->log_mutex, NULL);
213  /*
214  * There is not too much else what can be done.
215  */
216  assert(res == 0);
217 }
218 
219 #define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
220 do { \
221  (void)pthread_mutex_lock(&instance->log_mutex); \
222  instance->totemknet_log_printf ( \
223  level, subsys, function, file, line, \
224  (const char *)format, ##args); \
225  (void)pthread_mutex_unlock(&instance->log_mutex); \
226 } while (0);
227 
228 #define knet_log_printf(level, format, args...) \
229 do { \
230  knet_log_printf_lock ( \
231  level, instance->totemknet_subsys_id, \
232  __FUNCTION__, __FILE__, __LINE__, \
233  (const char *)format, ##args); \
234 } while (0);
235 
236 #define libknet_log_printf(level, format, args...) \
237 do { \
238  knet_log_printf_lock ( \
239  level, instance->knet_subsys_id, \
240  __FUNCTION__, "libknet.h", __LINE__, \
241  (const char *)format, ##args); \
242 } while (0);
243 
244 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
245 do { \
246  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
247  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
248  instance->totemknet_log_printf ( \
249  level, instance->totemknet_subsys_id, \
250  __FUNCTION__, __FILE__, __LINE__, \
251  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
252  } while(0)
253 
254 
255 #ifdef HAVE_LIBNOZZLE
256 static inline int is_ether_addr_multicast(const uint8_t *addr)
257 {
258  return (addr[0] & 0x01);
259 }
260 static inline int is_ether_addr_zero(const uint8_t *addr)
261 {
262  return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
263 }
264 
265 static int ether_host_filter_fn(void *private_data,
266  const unsigned char *outdata,
267  ssize_t outdata_len,
268  uint8_t tx_rx,
269  knet_node_id_t this_host_id,
270  knet_node_id_t src_host_id,
271  int8_t *channel,
272  knet_node_id_t *dst_host_ids,
273  size_t *dst_host_ids_entries)
274 {
275  struct ether_header *eth_h = (struct ether_header *)outdata;
276  uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
277  uint16_t dst_host_id;
278 
279  if (is_ether_addr_zero(dst_mac))
280  return -1;
281 
282  if (is_ether_addr_multicast(dst_mac)) {
283  return 1;
284  }
285 
286  memmove(&dst_host_id, &dst_mac[4], 2);
287 
288  dst_host_ids[0] = ntohs(dst_host_id);
289  *dst_host_ids_entries = 1;
290 
291  return 0;
292 }
293 #endif
294 
295 static int dst_host_filter_callback_fn(void *private_data,
296  const unsigned char *outdata,
297  ssize_t outdata_len,
298  uint8_t tx_rx,
299  knet_node_id_t this_host_id,
300  knet_node_id_t src_host_id,
301  int8_t *channel,
302  knet_node_id_t *dst_host_ids,
303  size_t *dst_host_ids_entries)
304 {
305  struct totem_message_header *header = (struct totem_message_header *)outdata;
306  int res;
307 
308 #ifdef HAVE_LIBNOZZLE
309  if (*channel != 0) {
310  return ether_host_filter_fn(private_data,
311  outdata, outdata_len,
312  tx_rx,
313  this_host_id, src_host_id,
314  channel,
315  dst_host_ids,
316  dst_host_ids_entries);
317  }
318 #endif
319  if (header->target_nodeid) {
320  dst_host_ids[0] = header->target_nodeid;
321  *dst_host_ids_entries = 1;
322  res = 0; /* unicast message */
323  }
324  else {
325  *dst_host_ids_entries = 0;
326  res = 1; /* multicast message */
327  }
328  return res;
329 }
330 
331 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
332 {
333  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
334 
335  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
336  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
337  knet_handle_remove_datafd(instance->knet_handle, datafd);
338  }
339 }
340 
341 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
342 {
343  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
344 
345  // TODO: what? if anything.
346  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
347 }
348 
349 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
350 {
351  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
352  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
353 
354  /* We don't need to tell corosync the actual knet MTU */
355 // instance->totemknet_mtu_changed(instance->context, data_mtu);
356 }
357 
359  void *knet_context,
360  const char *cipher_type,
361  const char *hash_type)
362 {
363  return (0);
364 }
365 
366 
367 static inline void ucast_sendmsg (
368  struct totemknet_instance *instance,
369  struct totem_ip_address *system_to,
370  const void *msg,
371  unsigned int msg_len)
372 {
373  int res = 0;
374  struct totem_message_header *header = (struct totem_message_header *)msg;
375  struct msghdr msg_ucast;
376  struct iovec iovec;
377 
378  header->target_nodeid = system_to->nodeid;
379 
380  iovec.iov_base = (void *)msg;
381  iovec.iov_len = msg_len;
382 
383  /*
384  * Build unicast message
385  */
386  memset(&msg_ucast, 0, sizeof(msg_ucast));
387  msg_ucast.msg_iov = (void *)&iovec;
388  msg_ucast.msg_iovlen = 1;
389 #ifdef HAVE_MSGHDR_CONTROL
390  msg_ucast.msg_control = 0;
391 #endif
392 #ifdef HAVE_MSGHDR_CONTROLLEN
393  msg_ucast.msg_controllen = 0;
394 #endif
395 #ifdef HAVE_MSGHDR_FLAGS
396  msg_ucast.msg_flags = 0;
397 #endif
398 #ifdef HAVE_MSGHDR_ACCRIGHTS
399  msg_ucast.msg_accrights = NULL;
400 #endif
401 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
402  msg_ucast.msg_accrightslen = 0;
403 #endif
404 
405  /*
406  * Transmit unicast message
407  * An error here is recovered by totemsrp
408  */
409 
410  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
411  if (res < 0) {
413  "sendmsg(ucast) failed (non-critical)");
414  }
415 }
416 
417 static inline void mcast_sendmsg (
418  struct totemknet_instance *instance,
419  const void *msg,
420  unsigned int msg_len,
421  int only_active)
422 {
423  int res;
424  struct totem_message_header *header = (struct totem_message_header *)msg;
425  struct msghdr msg_mcast;
426  struct iovec iovec;
427 
428  iovec.iov_base = (void *)msg;
429  iovec.iov_len = msg_len;
430 
431  header->target_nodeid = 0;
432 
433  /*
434  * Build multicast message
435  */
436  memset(&msg_mcast, 0, sizeof(msg_mcast));
437  msg_mcast.msg_iov = (void *)&iovec;
438  msg_mcast.msg_iovlen = 1;
439 #ifdef HAVE_MSGHDR_CONTROL
440  msg_mcast.msg_control = 0;
441 #endif
442 #ifdef HAVE_MSGHDR_CONTROLLEN
443  msg_mcast.msg_controllen = 0;
444 #endif
445 #ifdef HAVE_MSGHDR_FLAGS
446  msg_mcast.msg_flags = 0;
447 #endif
448 #ifdef HAVE_MSGHDR_ACCRIGHTS
449  msg_mcast.msg_accrights = NULL;
450 #endif
451 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
452  msg_mcast.msg_accrightslen = 0;
453 #endif
454 
455 
456 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
457 
458  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
459  if (res < msg_len) {
460  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
461  }
462 
463  if (!only_active || instance->send_merge_detect_message) {
464  /*
465  * Current message was sent to all nodes
466  */
468  instance->send_merge_detect_message = 0;
469  }
470 }
471 
472 static int node_compare(const void *aptr, const void *bptr)
473 {
474  uint16_t a,b;
475 
476  a = *(uint16_t *)aptr;
477  b = *(uint16_t *)bptr;
478 
479  return a > b;
480 }
481 
482 #ifndef OWN_INDEX_NONE
483 #define OWN_INDEX_NONE -1
484 #endif
485 
486 int totemknet_ifaces_get (void *knet_context,
487  char ***status,
488  unsigned int *iface_count)
489 {
490  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
491  struct knet_link_status link_status;
492  knet_node_id_t host_list[KNET_MAX_HOST];
493  uint8_t link_list[KNET_MAX_LINK];
494  size_t num_hosts;
495  size_t num_links;
496  size_t link_idx;
497  int i,j;
498  char *ptr;
499  int res = 0;
500 
501  /*
502  * Don't do the whole 'link_info' bit if the caller just wants
503  * a count of interfaces.
504  */
505  if (status) {
506  int own_idx = OWN_INDEX_NONE;
507 
508  res = knet_host_get_host_list(instance->knet_handle,
509  host_list, &num_hosts);
510  if (res) {
511  return (-1);
512  }
513  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
514 
515  for (j=0; j<num_hosts; j++) {
516  if (host_list[j] == instance->our_nodeid) {
517  own_idx = j;
518  break;
519  }
520  }
521 
522  for (i=0; i<INTERFACE_MAX; i++) {
523  memset(instance->link_status[i], 'd', CFG_INTERFACE_STATUS_MAX_LEN-1);
524  if (own_idx != OWN_INDEX_NONE) {
525  instance->link_status[i][own_idx] = 'n';
526  }
527  instance->link_status[i][num_hosts] = '\0';
528  }
529 
530  /* This is all a bit "inside-out" because "status" is a set of strings per link
531  * and knet orders things by host
532  */
533  for (j=0; j<num_hosts; j++) {
534  if (own_idx != OWN_INDEX_NONE && j == own_idx) {
535  continue ;
536  }
537 
538  res = knet_link_get_link_list(instance->knet_handle,
539  host_list[j], link_list, &num_links);
540  if (res) {
541  return (-1);
542  }
543 
544  link_idx = 0;
545  for (i=0; i < num_links; i++) {
546  /*
547  * Skip over links that are unconfigured to corosync. This is basically
548  * link0 if corosync isn't using it for comms, as we will still
549  * have it set up for loopback.
550  */
551  if (!instance->totem_config->interfaces[link_list[i]].configured) {
552  continue;
553  }
554  ptr = instance->link_status[link_idx++];
555 
556  res = knet_link_get_status(instance->knet_handle,
557  host_list[j],
558  link_list[i],
559  &link_status,
560  sizeof(link_status));
561  if (res == 0) {
562  ptr[j] = '0' + (link_status.enabled |
563  link_status.connected<<1 |
564  link_status.dynconnected<<2);
565  }
566  else {
568  "totemknet_ifaces_get: Cannot get link status: %s", strerror(errno));
569  ptr[j] = '?';
570  }
571  }
572  }
573  *status = instance->link_status;
574  }
575 
576  *iface_count = INTERFACE_MAX;
577 
578  return (res);
579 }
580 
582  void *knet_context)
583 {
584  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
585  int res = 0;
586  int i,j;
587  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
588  uint8_t links[KNET_MAX_LINK];
589  size_t num_nodes;
590  size_t num_links;
591 
592  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
593 
594  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
595  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
596 
597  /*
598  * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
599  */
600  res = knet_handle_setfwd(instance->knet_handle, 0);
601  if (res) {
602  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
603  }
604 
605  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
606  if (res) {
607  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
608  /* Crash out anyway */
609  goto finalise_error;
610  }
611 
612  /* Tidily shut down all nodes & links. */
613  for (i=0; i<num_nodes; i++) {
614 
615  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
616  if (res) {
617  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
618  goto finalise_error;
619  }
620  for (j=0; j<num_links; j++) {
621  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
622  if (res) {
623  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
624  }
625  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
626  if (res) {
627  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
628  }
629  }
630  res = knet_host_remove(instance->knet_handle, nodes[i]);
631  if (res) {
632  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
633  }
634  }
635 
636 finalise_error:
637  res = knet_handle_free(instance->knet_handle);
638  if (res) {
639  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
640  }
641 
642  totemknet_stop_merge_detect_timeout(instance);
643 
644  log_flush_messages(instance);
645 
646  /*
647  * Error is deliberately ignored
648  */
649  (void)pthread_mutex_destroy(&instance->log_mutex);
650 
651  return (res);
652 }
653 
654 static int log_deliver_fn (
655  int fd,
656  int revents,
657  void *data)
658 {
659  struct totemknet_instance *instance = (struct totemknet_instance *)data;
660  char buffer[sizeof(struct knet_log_msg)*4];
661  char *bufptr = buffer;
662  int done = 0;
663  int len;
664 
665  len = read(fd, buffer, sizeof(buffer));
666  while (done < len) {
667  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
668  switch (msg->msglevel) {
669  case KNET_LOG_ERR:
671  knet_log_get_subsystem_name(msg->subsystem),
672  msg->msg);
673  break;
674  case KNET_LOG_WARN:
676  knet_log_get_subsystem_name(msg->subsystem),
677  msg->msg);
678  break;
679  case KNET_LOG_INFO:
681  knet_log_get_subsystem_name(msg->subsystem),
682  msg->msg);
683  break;
684  case KNET_LOG_DEBUG:
686  knet_log_get_subsystem_name(msg->subsystem),
687  msg->msg);
688  break;
689  }
690  bufptr += sizeof(struct knet_log_msg);
691  done += sizeof(struct knet_log_msg);
692  }
693  return 0;
694 }
695 
696 static int data_deliver_fn (
697  int fd,
698  int revents,
699  void *data)
700 {
701  struct totemknet_instance *instance = (struct totemknet_instance *)data;
702  struct msghdr msg_hdr;
703  struct iovec iov_recv;
704  struct sockaddr_storage system_from;
705  ssize_t msg_len;
706  int truncated_packet;
707 
708  iov_recv.iov_base = instance->iov_buffer;
709  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
710 
711  msg_hdr.msg_name = &system_from;
712  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
713  msg_hdr.msg_iov = &iov_recv;
714  msg_hdr.msg_iovlen = 1;
715 #ifdef HAVE_MSGHDR_CONTROL
716  msg_hdr.msg_control = 0;
717 #endif
718 #ifdef HAVE_MSGHDR_CONTROLLEN
719  msg_hdr.msg_controllen = 0;
720 #endif
721 #ifdef HAVE_MSGHDR_FLAGS
722  msg_hdr.msg_flags = 0;
723 #endif
724 #ifdef HAVE_MSGHDR_ACCRIGHTS
725  msg_hdr.msg_accrights = NULL;
726 #endif
727 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
728  msg_hdr.msg_accrightslen = 0;
729 #endif
730 
731  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
732  if (msg_len <= 0) {
733  return (0);
734  }
735 
736  truncated_packet = 0;
737 
738 #ifdef HAVE_MSGHDR_FLAGS
739  if (msg_hdr.msg_flags & MSG_TRUNC) {
740  truncated_packet = 1;
741  }
742 #else
743  /*
744  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
745  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
746  */
747  if (bytes_received == KNET_MAX_PACKET_SIZE) {
748  truncated_packet = 1;
749  }
750 #endif
751 
752  if (truncated_packet) {
754  "Received too big message. This may be because something bad is happening"
755  "on the network (attack?), or you tried join more nodes than corosync is"
756  "compiled with (%u) or bug in the code (bad estimation of "
757  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
758  return (0);
759  }
760 
761  /*
762  * Handle incoming message
763  */
764  instance->totemknet_deliver_fn (
765  instance->context,
766  instance->iov_buffer,
767  msg_len,
768  &system_from);
769 
770  return (0);
771 }
772 
773 static void timer_function_netif_check_timeout (
774  void *data)
775 {
776  struct totemknet_instance *instance = (struct totemknet_instance *)data;
777  int i;
778 
779  for (i=0; i < INTERFACE_MAX; i++) {
780  if (!instance->totem_config->interfaces[i].configured) {
781  continue;
782  }
783  instance->totemknet_iface_change_fn (instance->context,
784  &instance->my_ids[i],
785  i);
786  }
787 }
788 
789 static void knet_set_access_list_config(struct totemknet_instance *instance)
790 {
791 #ifdef HAVE_KNET_ACCESS_LIST
792  uint32_t value;
793  cs_error_t err;
794 
796  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
797 
798  err = knet_handle_enable_access_lists(instance->knet_handle, value);
799  if (err) {
800  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
801  }
802 #endif
803 }
804 
805 
806 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
807 static void totemknet_refresh_config(
808  int32_t event,
809  const char *key_name,
810  struct icmap_notify_value new_val,
811  struct icmap_notify_value old_val,
812  void *user_data)
813 {
814  uint8_t reloading;
815  uint32_t value;
816  uint32_t link_no;
817  size_t num_nodes;
818  knet_node_id_t host_ids[KNET_MAX_HOST];
819  int i;
820  int err;
821  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
822 
823  ENTER();
824 
825  /*
826  * If a full reload is in progress then don't do anything until it's done and
827  * can reconfigure it all atomically
828  */
829  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
830  return;
831  }
832 
833  knet_set_access_list_config(instance);
834 
835  if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
836 
838  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
839  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
840  if (err) {
841  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
842  }
843  }
844 
845  /* Configure link parameters for each node */
846  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
847  if (err != 0) {
848  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
849  }
850 
851  for (i=0; i<num_nodes; i++) {
852  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
853  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
854  continue;
855  }
856 
857  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
858  instance->totem_config->interfaces[link_no].knet_ping_interval,
859  instance->totem_config->interfaces[link_no].knet_ping_timeout,
860  instance->totem_config->interfaces[link_no].knet_ping_precision);
861  if (err) {
862  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
863  }
864  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
865  instance->totem_config->interfaces[link_no].knet_pong_count);
866  if (err) {
867  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
868  }
869  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
870  instance->totem_config->interfaces[link_no].knet_link_priority);
871  if (err) {
872  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
873  }
874 
875  }
876  }
877 
878  LEAVE();
879 }
880 
881 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
882 {
883  icmap_track_t icmap_track_totem = NULL;
884  icmap_track_t icmap_track_reload = NULL;
885 
886  ENTER();
887 
888  icmap_track_add("totem.",
890  totemknet_refresh_config,
891  instance,
892  &icmap_track_totem);
893 
894  icmap_track_add("config.totemconfig_reload_in_progress",
896  totemknet_refresh_config,
897  instance,
898  &icmap_track_reload);
899 
900  LEAVE();
901 }
902 
903 /*
904  * Create an instance
905  */
907  qb_loop_t *poll_handle,
908  void **knet_context,
909  struct totem_config *totem_config,
910  totemsrp_stats_t *stats,
911  void *context,
912 
913  void (*deliver_fn) (
914  void *context,
915  const void *msg,
916  unsigned int msg_len,
917  const struct sockaddr_storage *system_from),
918 
919  void (*iface_change_fn) (
920  void *context,
921  const struct totem_ip_address *iface_address,
922  unsigned int link_no),
923 
924  void (*mtu_changed) (
925  void *context,
926  int net_mtu),
927 
928  void (*target_set_completed) (
929  void *context))
930 {
931  struct totemknet_instance *instance;
932  int8_t channel=0;
933  int res;
934  int i;
935 
936  instance = malloc (sizeof (struct totemknet_instance));
937  if (instance == NULL) {
938  return (-1);
939  }
940 
941  totemknet_instance_initialize (instance);
942 
943  instance->totem_config = totem_config;
944 
945  /*
946  * Configure logging
947  */
948  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
955 
956  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
957 
958  /*
959  * Initialize local variables for totemknet
960  */
961 
962  instance->our_nodeid = instance->totem_config->node_id;
963 
964  for (i=0; i< INTERFACE_MAX; i++) {
965  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
966  instance->my_ids[i].nodeid = instance->our_nodeid;
967  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
968 
969  /* Needed for totemsrp */
971  }
972 
973  instance->poll_handle = poll_handle;
974 
975  instance->context = context;
976  instance->totemknet_deliver_fn = deliver_fn;
977 
978  instance->totemknet_iface_change_fn = iface_change_fn;
979 
980  instance->totemknet_mtu_changed = mtu_changed;
981 
982  instance->totemknet_target_set_completed = target_set_completed;
983 
984  instance->loopback_link = 0;
985 
986  res = pipe(instance->logpipes);
987  if (res == -1) {
988  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
989  goto exit_error;
990  }
991  if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
992  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
993  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
994  goto exit_error;
995  }
996 
997 
998 #if !defined(KNET_API_VER) || (KNET_API_VER == 1)
999  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
1000 #endif
1001 #if KNET_API_VER == 2
1002  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
1003 #endif
1004 
1005  if (!instance->knet_handle) {
1006  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
1007  goto exit_error;
1008  }
1009 
1010  knet_set_access_list_config(instance);
1011 
1012  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
1013  if (res) {
1014  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
1015  }
1016  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
1017  if (res) {
1018  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
1019  }
1020  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
1021  if (res) {
1022  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
1023  }
1024  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1025  if (res) {
1026  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1027  }
1028  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1029  if (res) {
1030  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1031  }
1032  global_instance = instance;
1033 
1034  /* Get an fd into knet */
1035  instance->knet_fd = 0;
1036  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1037  if (res) {
1038  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1039  goto exit_error;
1040  }
1041 
1042  /* Enable crypto if requested */
1043  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
1044  struct knet_handle_crypto_cfg crypto_cfg;
1045 
1046  assert(strlen(instance->totem_config->crypto_model) < sizeof(crypto_cfg.crypto_model));
1047  strcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model);
1048 
1049  assert(strlen(instance->totem_config->crypto_cipher_type) < sizeof(crypto_cfg.crypto_cipher_type));
1050  strcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type);
1051 
1052  assert(strlen(instance->totem_config->crypto_hash_type) < sizeof(crypto_cfg.crypto_hash_type));
1053  strcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type);
1054 
1055  assert(instance->totem_config->private_key_len <= sizeof(crypto_cfg.private_key));
1056  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1057  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1058 
1059  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1060  if (res == -1) {
1061  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1062  goto exit_error;
1063  }
1064  if (res == -2) {
1065  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1066  goto exit_error;
1067  }
1068  knet_log_printf(LOG_INFO, "kronosnet crypto initialized: %s/%s", crypto_cfg.crypto_cipher_type, crypto_cfg.crypto_hash_type);
1069  }
1070 
1071  /* Set up compression */
1072  totemknet_reconfigure(instance, instance->totem_config);
1073 
1074  knet_handle_setfwd(instance->knet_handle, 1);
1075 
1076  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1077  if (strcmp(instance->totem_config->link_mode, "active")==0) {
1078  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1079  }
1080  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1081  instance->link_mode = KNET_LINK_POLICY_RR;
1082  }
1083 
1084  for (i=0; i<INTERFACE_MAX; i++) {
1085  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1086  if (!instance->link_status[i]) {
1087  goto exit_error;
1088  }
1089  }
1090 
1091  qb_loop_poll_add (instance->poll_handle,
1092  QB_LOOP_MED,
1093  instance->logpipes[0],
1094  POLLIN, instance, log_deliver_fn);
1095 
1096  qb_loop_poll_add (instance->poll_handle,
1097  QB_LOOP_HIGH,
1098  instance->knet_fd,
1099  POLLIN, instance, data_deliver_fn);
1100 
1101  /*
1102  * Upper layer isn't ready to receive message because it hasn't
1103  * initialized yet. Add short timer to check the interfaces.
1104  */
1105  qb_loop_timer_add (instance->poll_handle,
1106  QB_LOOP_MED,
1107  100*QB_TIME_NS_IN_MSEC,
1108  (void *)instance,
1109  timer_function_netif_check_timeout,
1110  &instance->timer_netif_check_timeout);
1111 
1112  totemknet_start_merge_detect_timeout(instance);
1113 
1114  /* Start listening for config changes */
1115  totemknet_add_config_notifications(instance);
1116 
1117  /* Add stats keys to icmap */
1119 
1120  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1121  *knet_context = instance;
1122 
1123  return (0);
1124 
1125 exit_error:
1126  log_flush_messages(instance);
1127  free(instance);
1128  return (-1);
1129 }
1130 
1132 {
1133  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1134  return malloc(KNET_MAX_PACKET_SIZE + 512);
1135 }
1136 
1138 {
1139  return free (ptr);
1140 }
1141 
1143  void *knet_context,
1144  int processor_count)
1145 {
1146  return (0);
1147 }
1148 
1149 int totemknet_recv_flush (void *knet_context)
1150 {
1151  return (0);
1152 }
1153 
1154 int totemknet_send_flush (void *knet_context)
1155 {
1156  return (0);
1157 }
1158 
1160  void *knet_context,
1161  const void *msg,
1162  unsigned int msg_len)
1163 {
1164  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1165  int res = 0;
1166 
1167  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1168 
1169  return (res);
1170 }
1172  void *knet_context,
1173  const void *msg,
1174  unsigned int msg_len)
1175 {
1176  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1177  int res = 0;
1178 
1179  mcast_sendmsg (instance, msg, msg_len, 0);
1180 
1181  return (res);
1182 }
1183 
1185  void *knet_context,
1186  const void *msg,
1187  unsigned int msg_len)
1188 {
1189  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1190  int res = 0;
1191 
1192  mcast_sendmsg (instance, msg, msg_len, 1);
1193 
1194  return (res);
1195 }
1196 
1197 
1199 {
1200  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1201  int res = 0;
1202 
1203  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1204 
1205  return (res);
1206 }
1207 
1209 {
1210  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1211 
1212  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1213 }
1214 
1216  void *knet_context,
1217  unsigned int nodeid)
1218 {
1219  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1220  int res = 0;
1221 
1222  instance->token_target.nodeid = nodeid;
1223 
1224  instance->totemknet_target_set_completed (instance->context);
1225 
1226  return (res);
1227 }
1228 
1230  void *knet_context)
1231 {
1232  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1233  unsigned int res;
1234  struct sockaddr_storage system_from;
1235  struct msghdr msg_hdr;
1236  struct iovec iov_recv;
1237  struct pollfd ufd;
1238  int nfds;
1239  int msg_processed = 0;
1240 
1241  iov_recv.iov_base = instance->iov_buffer;
1242  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1243 
1244  msg_hdr.msg_name = &system_from;
1245  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1246  msg_hdr.msg_iov = &iov_recv;
1247  msg_hdr.msg_iovlen = 1;
1248 #ifdef HAVE_MSGHDR_CONTROL
1249  msg_hdr.msg_control = 0;
1250 #endif
1251 #ifdef HAVE_MSGHDR_CONTROLLEN
1252  msg_hdr.msg_controllen = 0;
1253 #endif
1254 #ifdef HAVE_MSGHDR_FLAGS
1255  msg_hdr.msg_flags = 0;
1256 #endif
1257 #ifdef HAVE_MSGHDR_ACCRIGHTS
1258  msg_msg_hdr.msg_accrights = NULL;
1259 #endif
1260 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1261  msg_msg_hdr.msg_accrightslen = 0;
1262 #endif
1263 
1264  do {
1265  ufd.fd = instance->knet_fd;
1266  ufd.events = POLLIN;
1267  nfds = poll (&ufd, 1, 0);
1268  if (nfds == 1 && ufd.revents & POLLIN) {
1269  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1270  if (res != -1) {
1271  msg_processed = 1;
1272  } else {
1273  msg_processed = -1;
1274  }
1275  }
1276  } while (nfds == 1);
1277 
1278  return (msg_processed);
1279 }
1280 
1281 int totemknet_iface_set (void *knet_context,
1282  const struct totem_ip_address *local_addr,
1283  unsigned short ip_port,
1284  unsigned int iface_no)
1285 {
1286  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1287 
1288  totemip_copy(&instance->my_ids[iface_no], local_addr);
1289 
1290  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1291 
1292  instance->ip_port[iface_no] = ip_port;
1293 
1294  return 0;
1295 }
1296 
1297 
1299  void *knet_context,
1300  const struct totem_ip_address *local,
1301  const struct totem_ip_address *member,
1302  int link_no)
1303 {
1304  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1305  int err;
1306  int port = instance->ip_port[link_no];
1307  struct sockaddr_storage remote_ss;
1308  struct sockaddr_storage local_ss;
1309  int addrlen;
1310  int i;
1311  int host_found = 0;
1312  knet_node_id_t host_ids[KNET_MAX_HOST];
1313  size_t num_host_ids;
1314 
1315  /* Only create 1 loopback link and use link 0 */
1316  if (member->nodeid == instance->our_nodeid) {
1317  if (!instance->loopback_link) {
1318  link_no = 0;
1319  instance->loopback_link = 1;
1320  } else {
1321  /* Already done */
1322  return 0;
1323  }
1324  }
1325 
1326  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1327  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1328 
1329 
1330  /* Only add the host if it doesn't already exist in knet */
1331  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1332  if (err) {
1333  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1334  return -1;
1335  }
1336  for (i=0; i<num_host_ids; i++) {
1337  if (host_ids[i] == member->nodeid) {
1338  host_found = 1;
1339  }
1340  }
1341 
1342  if (!host_found) {
1343  err = knet_host_add(instance->knet_handle, member->nodeid);
1344  if (err != 0 && errno != EEXIST) {
1345  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1346  return -1;
1347  }
1348  } else {
1349  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1350  }
1351 
1352 
1353  if (err == 0) {
1354  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1355  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1356  return -1;
1357  }
1358  }
1359 
1360  memset(&local_ss, 0, sizeof(local_ss));
1361  memset(&remote_ss, 0, sizeof(remote_ss));
1362  /* Casts to remove const */
1363  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1364  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1365 
1366  if (member->nodeid == instance->our_nodeid) {
1367  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1368 
1369  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1370  KNET_TRANSPORT_LOOPBACK,
1371  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1372  }
1373  else {
1374  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1375  instance->totem_config->interfaces[link_no].knet_transport,
1376  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1377  }
1378  if (err) {
1379  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1380  return -1;
1381  }
1382 
1383  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1384  instance->totem_config->interfaces[link_no].knet_link_priority);
1385 
1386  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1387  instance->totem_config->interfaces[link_no].knet_link_priority);
1388  if (err) {
1389  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1390  }
1391 
1392  /* ping timeouts maybe 0 here for a newly added interface so we leave this till later, it will
1393  get done in totemknet_refresh_config */
1394  if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1395  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1396  instance->totem_config->interfaces[link_no].knet_ping_interval,
1397  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1398  instance->totem_config->interfaces[link_no].knet_ping_precision);
1399  if (err) {
1400  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1401  }
1402  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1403  instance->totem_config->interfaces[link_no].knet_pong_count);
1404  if (err) {
1405  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1406  }
1407  }
1408 
1409  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1410  if (err) {
1411  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1412  return -1;
1413  }
1414 
1415  /* register stats */
1416  stats_knet_add_member(member->nodeid, link_no);
1417  return (0);
1418 }
1419 
1421  void *knet_context,
1422  const struct totem_ip_address *token_target,
1423  int link_no)
1424 {
1425  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1426  int res;
1427  uint8_t link_list[KNET_MAX_LINK];
1428  size_t num_links;
1429 
1430  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1431 
1432  /* Don't remove the link with the loopback on it until we shut down */
1433  if (token_target->nodeid == instance->our_nodeid) {
1434  return 0;
1435  }
1436 
1437  /* Tidy stats */
1439 
1440  /* Remove the link first */
1441  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1442  if (res != 0) {
1443  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1444  return res;
1445  }
1446 
1447  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1448  if (res != 0) {
1449  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1450  return res;
1451  }
1452 
1453  /* If this is the last link, then remove the node */
1454  res = knet_link_get_link_list(instance->knet_handle,
1455  token_target->nodeid, link_list, &num_links);
1456  if (res) {
1457  return (0); /* not really failure */
1458  }
1459 
1460  if (num_links == 0) {
1461  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1462  }
1463  return res;
1464 }
1465 
1467  void *knet_context)
1468 {
1469  return (0);
1470 }
1471 
1473  void *knet_context,
1474  struct totem_config *totem_config)
1475 {
1476  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1477  struct knet_handle_compress_cfg compress_cfg;
1478  int res = 0;
1479 
1481  assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
1482  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1483 
1484  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1485  compress_cfg.compress_level = totem_config->knet_compression_level;
1486 
1487  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1488  if (res) {
1489  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1490  }
1491  }
1492 
1493 #ifdef HAVE_LIBNOZZLE
1494  /* Set up nozzle device(s). Return code is ignored, because unability
1495  * configure nozzle is not fatal problem, errors are logged and
1496  * there is not much else we can do */
1497  (void)setup_nozzle(instance);
1498 #endif
1499  return (res);
1500 }
1501 
1502 
1504  void *knet_context)
1505 {
1506  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1507 
1508  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1509 }
1510 
1511 /* For the stats module */
1513  knet_node_id_t node, uint8_t link_no,
1514  struct knet_link_status *status)
1515 {
1516  int res;
1517  int ret = CS_OK;
1518 
1519  /* We are probably not using knet */
1520  if (!global_instance) {
1521  return CS_ERR_NOT_EXIST;
1522  }
1523 
1524  if (link_no >= INTERFACE_MAX) {
1525  return CS_ERR_NOT_EXIST; /* Invalid link number */
1526  }
1527 
1528  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1529  if (res) {
1530  switch (errno) {
1531  case EINVAL:
1532  ret = CS_ERR_INVALID_PARAM;
1533  break;
1534  case EBUSY:
1535  ret = CS_ERR_BUSY;
1536  break;
1537  case EDEADLK:
1538  ret = CS_ERR_TRY_AGAIN;
1539  break;
1540  default:
1541  ret = CS_ERR_LIBRARY;
1542  break;
1543  }
1544  }
1545 
1546  return (ret);
1547 }
1548 
1550  struct knet_handle_stats *stats)
1551 {
1552  int res;
1553 
1554  /* We are probably not using knet */
1555  if (!global_instance) {
1556  return CS_ERR_NOT_EXIST;
1557  }
1558 
1559  res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1560  if (res != 0) {
1561  return (qb_to_cs_error(-errno));
1562  }
1563 
1564  return CS_OK;
1565 }
1566 
1567 static void timer_function_merge_detect_timeout (
1568  void *data)
1569 {
1570  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1571 
1572  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1573  instance->send_merge_detect_message = 1;
1574  }
1575 
1577 
1578  totemknet_start_merge_detect_timeout(instance);
1579 }
1580 
1581 static void totemknet_start_merge_detect_timeout(
1582  void *knet_context)
1583 {
1584  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1585 
1586  qb_loop_timer_add(instance->poll_handle,
1587  QB_LOOP_MED,
1588  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1589  (void *)instance,
1590  timer_function_merge_detect_timeout,
1591  &instance->timer_merge_detect_timeout);
1592 
1593 }
1594 
1595 static void totemknet_stop_merge_detect_timeout(
1596  void *knet_context)
1597 {
1598  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1599 
1600  qb_loop_timer_del(instance->poll_handle,
1601  instance->timer_merge_detect_timeout);
1602 }
1603 
1604 static void log_flush_messages (void *knet_context)
1605 {
1606  struct pollfd pfd;
1607  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1608  int cont;
1609 
1610  cont = 1;
1611 
1612  while (cont) {
1613  pfd.fd = instance->logpipes[0];
1614  pfd.events = POLLIN;
1615  pfd.revents = 0;
1616 
1617  if ((poll(&pfd, 1, 0) > 0) &&
1618  (pfd.revents & POLLIN) &&
1619  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1620  cont = 1;
1621  } else {
1622  cont = 0;
1623  }
1624  }
1625 }
1626 
1627 
1628 #ifdef HAVE_LIBNOZZLE
1629 #define NOZZLE_NAME "nozzle.name"
1630 #define NOZZLE_IPADDR "nozzle.ipaddr"
1631 #define NOZZLE_PREFIX "nozzle.ipprefix"
1632 #define NOZZLE_MACADDR "nozzle.macaddr"
1633 
1634 #define NOZZLE_CHANNEL 1
1635 
1636 
1637 static char *get_nozzle_script_dir(void *knet_context)
1638 {
1639  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1640  char filename[PATH_MAX + FILENAME_MAX + 1];
1641  static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
1642  int res;
1643  const char *dirname_res;
1644 
1645  /*
1646  * Build script directory based on corosync.conf file location
1647  */
1648  res = snprintf(filename, sizeof(filename), "%s",
1650  if (res >= sizeof(filename)) {
1651  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1652  return NULL;
1653  }
1654 
1655  dirname_res = dirname(filename);
1656 
1657  res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
1658  dirname_res, "updown.d");
1659  if (res >= sizeof(updown_dirname)) {
1660  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1661  return NULL;
1662  }
1663  return updown_dirname;
1664 }
1665 
1666 /*
1667  * Deliberately doesn't return the status as caller doesn't care.
1668  * The result will be logged though
1669  */
1670 static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
1671 {
1672  int res;
1673  char *exec_string;
1674 
1675  res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
1676  if (res == -1 && errno != ENOENT) {
1677  knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
1678  } else if (res == -2) {
1679  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
1680  knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
1681  }
1682 }
1683 
1684 /*
1685  * Reparse IP address to add in our node ID
1686  * IPv6 addresses must end in '::'
1687  * IPv4 addresses must just be valid
1688  * '/xx' lengths are optional for IPv6, mandatory for IPv4
1689  *
1690  * Returns the modified IP address as a string to pass into libnozzle
1691  */
1692 static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
1693  const char *input_addr,
1694  const char *prefix, int nodeid,
1695  char *output_addr, size_t output_len)
1696 {
1697  char *coloncolon;
1698  int bits;
1699  int max_prefix = 64;
1700  uint32_t nodeid_mask;
1701  uint32_t addr_mask;
1702  uint32_t masked_nodeid;
1703  struct in_addr *addr;
1704  struct totem_ip_address totemip;
1705 
1706  coloncolon = strstr(input_addr, "::");
1707  if (!coloncolon) {
1708  max_prefix = 30;
1709  }
1710 
1711  bits = atoi(prefix);
1712  if (bits < 8 || bits > max_prefix) {
1713  knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
1714  return -1;
1715  }
1716 
1717  /* IPv6 is easy */
1718  if (coloncolon) {
1719  memcpy(output_addr, input_addr, coloncolon-input_addr);
1720  sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
1721  return 0;
1722  }
1723 
1724  /* For IPv4 we need to parse the address into binary, mask off the required bits,
1725  * add in the masked_nodeid and 'print' it out again
1726  */
1727  nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
1728  addr_mask = UINT32_MAX ^ nodeid_mask;
1729  masked_nodeid = nodeid & nodeid_mask;
1730 
1731  if (totemip_parse(&totemip, input_addr, AF_INET)) {
1732  knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
1733  return -1;
1734  }
1735  addr = (struct in_addr *)&totemip.addr;
1736  addr->s_addr &= htonl(addr_mask);
1737  addr->s_addr |= htonl(masked_nodeid);
1738 
1739  inet_ntop(AF_INET, addr, output_addr, output_len);
1740  return 0;
1741 }
1742 
1743 static int create_nozzle_device(void *knet_context, const char *name,
1744  const char *ipaddr, const char *prefix,
1745  const char *macaddr)
1746 {
1747  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1748  char device_name[IFNAMSIZ+1];
1749  size_t size = IFNAMSIZ;
1750  int8_t channel = NOZZLE_CHANNEL;
1751  nozzle_t nozzle_dev;
1752  int nozzle_fd;
1753  int res;
1754  char *updown_dir;
1755  char parsed_ipaddr[INET6_ADDRSTRLEN];
1756  char mac[19];
1757 
1758  memset(device_name, 0, size);
1759  memset(&mac, 0, sizeof(mac));
1760  strncpy(device_name, name, size);
1761 
1762  updown_dir = get_nozzle_script_dir(knet_context);
1763  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
1764 
1765  nozzle_dev = nozzle_open(device_name, size, updown_dir);
1766  if (!nozzle_dev) {
1767  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
1768  return -1;
1769  }
1770  instance->nozzle_handle = nozzle_dev;
1771 
1772  if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
1773  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
1774  goto out_clean;
1775  }
1776 
1777  if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
1778  /* Prints its own errors */
1779  goto out_clean;
1780  }
1781  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
1782  if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
1783  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
1784  goto out_clean;
1785  }
1786 
1787  nozzle_fd = nozzle_get_fd(nozzle_dev);
1788  knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
1789 
1790  res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
1791  if (res != 0) {
1792  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
1793  goto out_clean;
1794  }
1795 
1796  run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
1797 
1798  res = nozzle_set_up(nozzle_dev);
1799  if (res != 0) {
1800  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
1801  goto out_clean;
1802  }
1803  run_nozzle_script(instance, NOZZLE_UP, "up");
1804 
1805  return 0;
1806 
1807 out_clean:
1808  nozzle_close(nozzle_dev);
1809  return -1;
1810 }
1811 
1812 static int remove_nozzle_device(void *knet_context)
1813 {
1814  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1815  int res;
1816  int datafd;
1817 
1818  res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
1819  if (res != 0) {
1820  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1821  return -1;
1822  }
1823 
1824  res = knet_handle_remove_datafd(instance->knet_handle, datafd);
1825  if (res != 0) {
1826  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1827  return -1;
1828  }
1829 
1830  run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
1831  res = nozzle_set_down(instance->nozzle_handle);
1832  if (res != 0) {
1833  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
1834  return -1;
1835  }
1836  run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
1837 
1838  res = nozzle_close(instance->nozzle_handle);
1839  if (res != 0) {
1840  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
1841  return -1;
1842  }
1843  knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
1844  return 0;
1845 }
1846 
1847 static void free_nozzle(struct totemknet_instance *instance)
1848 {
1849  free(instance->nozzle_name);
1850  free(instance->nozzle_ipaddr);
1851  free(instance->nozzle_prefix);
1852  free(instance->nozzle_macaddr);
1853 
1854  instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
1855  instance->nozzle_macaddr = NULL;
1856 }
1857 
1858 static int setup_nozzle(void *knet_context)
1859 {
1860  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1861  char *ipaddr_str = NULL;
1862  char *name_str = NULL;
1863  char *prefix_str = NULL;
1864  char *macaddr_str = NULL;
1865  char mac[32];
1866  int name_res;
1867  int macaddr_res;
1868  int res = -1;
1869 
1870  /*
1871  * Return value ignored on purpose. icmap_get_string changes
1872  * ipaddr_str/prefix_str only on success.
1873  */
1874  (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
1875  (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
1876  macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
1877  name_res = icmap_get_string(NOZZLE_NAME, &name_str);
1878 
1879  /* Is is being removed? */
1880  if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
1881  remove_nozzle_device(instance);
1882  free_nozzle(instance);
1883  goto out_free;
1884  }
1885 
1886  if (!name_str) {
1887  /* no nozzle */
1888  goto out_free;
1889  }
1890 
1891  if (!ipaddr_str) {
1892  knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
1893  goto out_free;
1894  }
1895 
1896  if (!prefix_str) {
1897  knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
1898  goto out_free;
1899  }
1900 
1901  if (macaddr_str && strlen(macaddr_str) != 17) {
1902  knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
1903  goto out_free;
1904  }
1905  if (!macaddr_str) {
1906  macaddr_str = (char*)"54:54:01:00:00:00";
1907  }
1908 
1909  if (instance->nozzle_name &&
1910  (strcmp(name_str, instance->nozzle_name) == 0) &&
1911  (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
1912  (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
1913  (instance->nozzle_macaddr == NULL ||
1914  strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
1915  /* Nothing has changed */
1916  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
1917  goto out_free;
1918  }
1919 
1920  /* Add nodeid into MAC address */
1921  memcpy(mac, macaddr_str, 12);
1922  snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
1923  instance->our_nodeid >> 8,
1924  instance->our_nodeid & 0xFF);
1925  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
1926 
1927  if (name_res == CS_OK && name_str) {
1928  /* Reconfigure */
1929  if (instance->nozzle_name) {
1930  remove_nozzle_device(instance);
1931  free_nozzle(instance);
1932  }
1933 
1934  res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
1935  mac);
1936 
1937  instance->nozzle_name = strdup(name_str);
1938  instance->nozzle_ipaddr = strdup(ipaddr_str);
1939  instance->nozzle_prefix = strdup(prefix_str);
1940  instance->nozzle_macaddr = strdup(macaddr_str);
1941  if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
1942  !instance->nozzle_prefix) {
1943  knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
1944  /*
1945  * This 'free' will cause a complete reconfigure of the device next time we reload
1946  * but will also let the the current device keep working until then.
1947  * remove_nozzle() only needs the, statically-allocated, nozzle_handle
1948  */
1949  free_nozzle(instance);
1950  }
1951  }
1952 
1953 out_free:
1954  free(name_str);
1955  free(ipaddr_str);
1956  free(prefix_str);
1957  if (macaddr_res == CS_OK) {
1958  free(macaddr_str);
1959  }
1960 
1961  return res;
1962 }
1963 #endif // HAVE_LIBNOZZLE
totemknet_ifaces_get
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:486
knet_log_printf
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:228
totem_logging_configuration::log_level_error
int log_level_error
Definition: totem.h:109
totem_config::node_id
unsigned int node_id
Definition: totem.h:160
value
uint32_t value
Definition: exec/votequorum.c:2
LOGSYS_LEVEL_CRIT
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
totemknet_instance::totemknet_log_level_error
int totemknet_log_level_error
Definition: totemknet.c:126
totemknet_mcast_flush_send
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1171
totemknet_instance::totemknet_iface_change_fn
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:110
totem_interface::knet_ping_precision
int knet_ping_precision
Definition: totem.h:92
totemknet_instance::totemknet_mtu_changed
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:115
MSG_NOSIGNAL
#define MSG_NOSIGNAL
Definition: totemknet.c:83
totemknet_member_add
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1298
work_item::msg_len
unsigned int msg_len
Definition: totemknet.c:191
stats_knet_del_member
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:723
totemknet_buffer_release
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:1137
totemknet_token_send
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1159
CS_ERR_LIBRARY
@ CS_ERR_LIBRARY
Definition: corotypes.h:99
CS_ERR_TRY_AGAIN
@ CS_ERR_TRY_AGAIN
Definition: corotypes.h:103
LOGSYS_LEVEL_ERROR
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
totemknet_crypto_set
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:358
totemknet_reconfigure
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1472
totemknet_iface_set
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1281
totem_config::knet_pmtud_interval
unsigned int knet_pmtud_interval
Definition: totem.h:162
work_item
Definition: totemknet.c:189
LOGSYS_LEVEL_DEBUG
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
totem_logging_configuration::log_level_debug
int log_level_debug
Definition: totem.h:112
totemip_print
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:256
totemknet_net_mtu_adjust
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1208
totemsrp_stats_t
Definition: totemstats.h:53
type
char type
Definition: totem.h:2
totemknet_instance::poll_handle
qb_loop_t * poll_handle
Definition: totemknet.c:96
totem_logging_configuration::log_level_notice
int log_level_notice
Definition: totem.h:111
OWN_INDEX_NONE
#define OWN_INDEX_NONE
Definition: totemknet.c:483
CS_ERR_NOT_EXIST
@ CS_ERR_NOT_EXIST
Definition: corotypes.h:109
totem_interface::knet_link_priority
int knet_link_priority
Definition: totem.h:89
KNET_LOGSYS_PERROR
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:244
totemknet_instance::totemknet_log_level_notice
int totemknet_log_level_notice
Definition: totemknet.c:130
totemknet_finalize
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:581
libknet_log_printf
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:236
stats_knet_add_member
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:713
totemknet_instance::totem_config
struct totem_config * totem_config
Definition: totemknet.c:161
totemknet_instance::context
void * context
Definition: totemknet.c:102
CFG_INTERFACE_STATUS_MAX_LEN
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:91
ICMAP_TRACK_PREFIX
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem....
Definition: icmap.h:85
totem_ip_address::nodeid
unsigned int nodeid
Definition: coroapi.h:112
totemknet_processor_count_set
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:1142
ENTER
#define ENTER
Definition: logsys.h:324
totemknet_instance::iov_buffer
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:149
corosync_get_config_file
const char * corosync_get_config_file(void)
Definition: main.c:204
totemknet_link_get_status
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1512
totem_config::merge_timeout
unsigned int merge_timeout
Definition: totem.h:190
CS_PRI_NODE_ID
#define CS_PRI_NODE_ID
Definition: corotypes.h:59
totemknet_stats_clear
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1503
swab.h
totem_config::interfaces
struct totem_interface * interfaces
Definition: totem.h:158
totem_interface::knet_transport
int knet_transport
Definition: totem.h:94
totemknet_instance::totemknet_log_level_debug
int totemknet_log_level_debug
Definition: totemknet.c:132
addr
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:2
totem_config::knet_compression_level
int knet_compression_level
Definition: totem.h:228
totemip_copy
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:123
ICMAP_TRACK_ADD
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
icmap.h
header
struct totem_message_header header
Definition: totemsrp.c:0
totemknet_instance::totemknet_log_level_security
int totemknet_log_level_security
Definition: totemknet.c:124
CS_OK
@ CS_OK
Definition: corotypes.h:98
LOGSYS_LEVEL_INFO
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
totem_config::net_mtu
unsigned int net_mtu
Definition: totem.h:202
totemip.h
INTERFACE_MAX
#define INTERFACE_MAX
Definition: coroapi.h:88
totemknet_instance::my_ids
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:153
totem_config::knet_compression_threshold
uint32_t knet_compression_threshold
Definition: totem.h:226
CS_ERR_BUSY
@ CS_ERR_BUSY
Definition: corotypes.h:107
totem_interface::configured
uint8_t configured
Definition: totem.h:87
totemknet_instance::send_merge_detect_message
int send_merge_detect_message
Definition: totemknet.c:169
cs_error_t
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:97
__attribute__
typedef __attribute__
totem_interface::boundto
struct totem_ip_address boundto
Definition: totem.h:83
LEAVE
#define LEAVE
Definition: logsys.h:325
totemknet_instance::crypto_inst
struct crypto_instance * crypto_inst
Definition: totemknet.c:94
icmap_track_add
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1159
totemknet_instance::knet_fd
int knet_fd
Definition: totemknet.c:174
totemknet_iface_check
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1198
totemknet_instance::timer_netif_check_timeout
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:165
totem_config::crypto_model
char * crypto_model
Definition: totem.h:218
totem_logging_configuration::log_subsys_id
int log_subsys_id
Definition: totem.h:114
totemknet_instance
Definition: totemknet.c:93
totem_config::link_mode
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:198
totemknet_instance::knet_context
void(*) void knet_context)
Definition: totemknet.c:145
totemknet_recv_flush
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1149
totemknet_instance::ip_port
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:155
totemknet_instance::our_nodeid
int our_nodeid
Definition: totemknet.c:157
totem_config
Definition: totem.h:152
qb_to_cs_error
cs_error_t qb_to_cs_error(int result)
qb_to_cs_error
totemknet_token_target_set
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1215
totemknet.h
totem_ip_address
The totem_ip_address struct.
Definition: coroapi.h:111
totemknet_instance::token_target
struct totem_ip_address token_target
Definition: totemknet.c:163
sq.h
icmap_get_uint32
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:892
totemknet_instance::log_mutex
pthread_mutex_t log_mutex
Definition: totemknet.c:176
totemknet_initialize
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:906
global_instance
struct totemknet_instance * global_instance
Definition: totemknet.c:187
totem_message_header
Definition: totem.h:124
totem_interface::knet_pong_count
int knet_pong_count
Definition: totem.h:93
totemknet_member_list_rebind_ip
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1466
totem_interface::ip_port
uint16_t ip_port
Definition: totem.h:85
totem_interface::knet_ping_interval
int knet_ping_interval
Definition: totem.h:90
_logsys_subsys_create
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
totemknet_instance::loopback_link
int loopback_link
Definition: totemknet.c:159
totemknet_instance::knet_handle
knet_handle_t knet_handle
Definition: totemknet.c:98
totemip_parse
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition: totemip.c:306
ICMAP_TRACK_DELETE
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
work_item::msg
const void * msg
Definition: totemknet.c:190
totemknet_instance::timer_merge_detect_timeout
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:167
user_data
void * user_data
Definition: sam.c:127
system_from
struct srp_addr system_from
Definition: totemsrp.c:1
ICMAP_TRACK_MODIFY
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
totem_logging_configuration::log_level_warning
int log_level_warning
Definition: totem.h:110
totemknet_recv_mcast_empty
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1229
totemknet_instance::merge_detect_messages_sent_before_timeout
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:171
icmap_get_string
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:856
totemknet_member_remove
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1420
main.h
util.h
totemknet_send_flush
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1154
totemknet_mcast_noflush_send
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1184
totemknet_instance::knet_subsys_id
int knet_subsys_id
Definition: totemknet.c:136
totemknet_instance::totemknet_deliver_fn
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:104
totemknet_buffer_alloc
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:1131
nodeid
unsigned int nodeid
Definition: coroapi.h:0
totem_interface::knet_ping_timeout
int knet_ping_timeout
Definition: totem.h:91
totem_config::private_key
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:167
PROCESSOR_COUNT_MAX
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
config.h
totemknet_handle_get_stats
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1549
CS_ERR_INVALID_PARAM
@ CS_ERR_INVALID_PARAM
Definition: corotypes.h:104
totemknet_instance::totemknet_subsys_id
int totemknet_subsys_id
Definition: totemknet.c:134
logsys.h
totem_config::crypto_hash_type
char * crypto_hash_type
Definition: totem.h:222
totemknet_instance::totemknet_target_set_completed
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:119
totem_logging_configuration::log_printf
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:99
LOGSYS_LEVEL_WARNING
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
totem_interface::bindnet
struct totem_ip_address bindnet
Definition: totem.h:82
icmap_track
Definition: icmap.c:61
totem_config::private_key_len
unsigned int private_key_len
Definition: totem.h:169
totemknet_instance::link_mode
int link_mode
Definition: totemknet.c:100
totem_config::crypto_cipher_type
char * crypto_cipher_type
Definition: totem.h:220
totemknet_instance::totemknet_log_printf
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:138
icmap_notify_value
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
stats_knet_add_handle
void stats_knet_add_handle(void)
Definition: stats.c:736
totemip_totemip_to_sockaddr_convert
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:264
totem_config::block_unlisted_ips
unsigned int block_unlisted_ips
Definition: totem.h:236
icmap_get_uint8
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:868
totemknet_instance::totemknet_log_level_warning
int totemknet_log_level_warning
Definition: totemknet.c:128
totemknet_instance::logpipes
int logpipes[2]
Definition: totemknet.c:173
totemstats.h
totem_message_header::target_nodeid
unsigned int target_nodeid
Definition: totem.h:130
work_item::instance
struct totemknet_instance * instance
Definition: totemknet.c:192
totemknet_instance::link_status
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:151
totem_config::knet_compression_model
char * knet_compression_model
Definition: totem.h:224
totem_config::totem_logging_configuration
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:200