corosync  3.0.3
totemknet.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7 
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of the MontaVista Software, Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <assert.h>
38 #include <sys/mman.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/socket.h>
42 #include <netdb.h>
43 #include <sys/un.h>
44 #include <sys/ioctl.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <net/ethernet.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <pthread.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <sys/uio.h>
60 #include <limits.h>
61 
62 #include <qb/qbdefs.h>
63 #include <qb/qbloop.h>
64 #ifdef HAVE_LIBNOZZLE
65 #include <libgen.h>
66 #include <libnozzle.h>
67 #endif
68 
69 #include <corosync/sq.h>
70 #include <corosync/swab.h>
71 #include <corosync/logsys.h>
72 #include <corosync/icmap.h>
73 #include <corosync/totem/totemip.h>
74 #include "totemknet.h"
75 
76 #include "main.h"
77 #include "util.h"
78 
79 #include <libknet.h>
81 
82 #ifndef MSG_NOSIGNAL
83 #define MSG_NOSIGNAL 0
84 #endif
85 
86 #ifdef HAVE_LIBNOZZLE
87 static int setup_nozzle(void *knet_context);
88 #endif
89 
90 /* Should match that used by cfg */
91 #define CFG_INTERFACE_STATUS_MAX_LEN 512
92 
94  struct crypto_instance *crypto_inst;
95 
96  qb_loop_t *poll_handle;
97 
98  knet_handle_t knet_handle;
99 
101 
102  void *context;
103 
105  void *context,
106  const void *msg,
107  unsigned int msg_len,
108  const struct sockaddr_storage *system_from);
109 
111  void *context,
112  const struct totem_ip_address *iface_address,
113  unsigned int link_no);
114 
116  void *context,
117  int net_mtu);
118 
120 
121  /*
122  * Function and data used to log messages
123  */
125 
127 
129 
131 
133 
135 
137 
139  int level,
140  int subsys,
141  const char *function,
142  const char *file,
143  int line,
144  const char *format,
145  ...)__attribute__((format(printf, 6, 7)));
146 
147  void *knet_context;
148 
149  char iov_buffer[KNET_MAX_PACKET_SIZE];
150 
152 
154 
156 
158 
160 
162 
164 
165  qb_loop_timer_handle timer_netif_check_timeout;
166 
167  qb_loop_timer_handle timer_merge_detect_timeout;
168 
170 
172 
173  int logpipes[2];
174  int knet_fd;
175 
176  pthread_mutex_t log_mutex;
177 #ifdef HAVE_LIBNOZZLE
178  char *nozzle_name;
182  nozzle_t nozzle_handle;
183 #endif
184 };
185 
186 /* Awkward. But needed to get stats from knet */
188 
189 struct work_item {
190  const void *msg;
191  unsigned int msg_len;
193 };
194 
196  void *knet_context);
197 
198 static void totemknet_start_merge_detect_timeout(
199  void *knet_context);
200 
201 static void totemknet_stop_merge_detect_timeout(
202  void *knet_context);
203 
204 static void log_flush_messages (
205  void *knet_context);
206 
207 static void totemknet_instance_initialize (struct totemknet_instance *instance)
208 {
209  int res;
210 
211  memset (instance, 0, sizeof (struct totemknet_instance));
212  res = pthread_mutex_init(&instance->log_mutex, NULL);
213  /*
214  * There is not too much else what can be done.
215  */
216  assert(res == 0);
217 }
218 
219 #define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
220 do { \
221  (void)pthread_mutex_lock(&instance->log_mutex); \
222  instance->totemknet_log_printf ( \
223  level, subsys, function, file, line, \
224  (const char *)format, ##args); \
225  (void)pthread_mutex_unlock(&instance->log_mutex); \
226 } while (0);
227 
228 #define knet_log_printf(level, format, args...) \
229 do { \
230  knet_log_printf_lock ( \
231  level, instance->totemknet_subsys_id, \
232  __FUNCTION__, __FILE__, __LINE__, \
233  (const char *)format, ##args); \
234 } while (0);
235 
236 #define libknet_log_printf(level, format, args...) \
237 do { \
238  knet_log_printf_lock ( \
239  level, instance->knet_subsys_id, \
240  __FUNCTION__, "libknet.h", __LINE__, \
241  (const char *)format, ##args); \
242 } while (0);
243 
244 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
245 do { \
246  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
247  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
248  instance->totemknet_log_printf ( \
249  level, instance->totemknet_subsys_id, \
250  __FUNCTION__, __FILE__, __LINE__, \
251  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
252  } while(0)
253 
254 
255 #ifdef HAVE_LIBNOZZLE
256 static inline int is_ether_addr_multicast(const uint8_t *addr)
257 {
258  return (addr[0] & 0x01);
259 }
260 static inline int is_ether_addr_zero(const uint8_t *addr)
261 {
262  return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
263 }
264 
265 static int ether_host_filter_fn(void *private_data,
266  const unsigned char *outdata,
267  ssize_t outdata_len,
268  uint8_t tx_rx,
269  knet_node_id_t this_host_id,
270  knet_node_id_t src_host_id,
271  int8_t *channel,
272  knet_node_id_t *dst_host_ids,
273  size_t *dst_host_ids_entries)
274 {
275  struct ether_header *eth_h = (struct ether_header *)outdata;
276  uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
277  uint16_t dst_host_id;
278 
279  if (is_ether_addr_zero(dst_mac))
280  return -1;
281 
282  if (is_ether_addr_multicast(dst_mac)) {
283  return 1;
284  }
285 
286  memmove(&dst_host_id, &dst_mac[4], 2);
287 
288  dst_host_ids[0] = ntohs(dst_host_id);
289  *dst_host_ids_entries = 1;
290 
291  return 0;
292 }
293 #endif
294 
295 static int dst_host_filter_callback_fn(void *private_data,
296  const unsigned char *outdata,
297  ssize_t outdata_len,
298  uint8_t tx_rx,
299  knet_node_id_t this_host_id,
300  knet_node_id_t src_host_id,
301  int8_t *channel,
302  knet_node_id_t *dst_host_ids,
303  size_t *dst_host_ids_entries)
304 {
305  struct totem_message_header *header = (struct totem_message_header *)outdata;
306  int res;
307 
308 #ifdef HAVE_LIBNOZZLE
309  if (*channel != 0) {
310  return ether_host_filter_fn(private_data,
311  outdata, outdata_len,
312  tx_rx,
313  this_host_id, src_host_id,
314  channel,
315  dst_host_ids,
316  dst_host_ids_entries);
317  }
318 #endif
319  if (header->target_nodeid) {
320  dst_host_ids[0] = header->target_nodeid;
321  *dst_host_ids_entries = 1;
322  res = 0; /* unicast message */
323  }
324  else {
325  *dst_host_ids_entries = 0;
326  res = 1; /* multicast message */
327  }
328  return res;
329 }
330 
331 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
332 {
333  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
334 
335  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
336  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
337  knet_handle_remove_datafd(instance->knet_handle, datafd);
338  }
339 }
340 
341 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
342 {
343  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
344 
345  // TODO: what? if anything.
346  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
347 }
348 
349 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
350 {
351  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
352  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
353 
354  /* We don't need to tell corosync the actual knet MTU */
355 // instance->totemknet_mtu_changed(instance->context, data_mtu);
356 }
357 
359  void *knet_context,
360  const char *cipher_type,
361  const char *hash_type)
362 {
363  return (0);
364 }
365 
366 
367 static inline void ucast_sendmsg (
368  struct totemknet_instance *instance,
369  struct totem_ip_address *system_to,
370  const void *msg,
371  unsigned int msg_len)
372 {
373  int res = 0;
374  struct totem_message_header *header = (struct totem_message_header *)msg;
375  struct msghdr msg_ucast;
376  struct iovec iovec;
377 
378  header->target_nodeid = system_to->nodeid;
379 
380  iovec.iov_base = (void *)msg;
381  iovec.iov_len = msg_len;
382 
383  /*
384  * Build unicast message
385  */
386  memset(&msg_ucast, 0, sizeof(msg_ucast));
387  msg_ucast.msg_iov = (void *)&iovec;
388  msg_ucast.msg_iovlen = 1;
389 #ifdef HAVE_MSGHDR_CONTROL
390  msg_ucast.msg_control = 0;
391 #endif
392 #ifdef HAVE_MSGHDR_CONTROLLEN
393  msg_ucast.msg_controllen = 0;
394 #endif
395 #ifdef HAVE_MSGHDR_FLAGS
396  msg_ucast.msg_flags = 0;
397 #endif
398 #ifdef HAVE_MSGHDR_ACCRIGHTS
399  msg_ucast.msg_accrights = NULL;
400 #endif
401 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
402  msg_ucast.msg_accrightslen = 0;
403 #endif
404 
405  /*
406  * Transmit unicast message
407  * An error here is recovered by totemsrp
408  */
409 
410  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
411  if (res < 0) {
413  "sendmsg(ucast) failed (non-critical)");
414  }
415 }
416 
417 static inline void mcast_sendmsg (
418  struct totemknet_instance *instance,
419  const void *msg,
420  unsigned int msg_len,
421  int only_active)
422 {
423  int res;
424  struct totem_message_header *header = (struct totem_message_header *)msg;
425  struct msghdr msg_mcast;
426  struct iovec iovec;
427 
428  iovec.iov_base = (void *)msg;
429  iovec.iov_len = msg_len;
430 
431  header->target_nodeid = 0;
432 
433  /*
434  * Build multicast message
435  */
436  memset(&msg_mcast, 0, sizeof(msg_mcast));
437  msg_mcast.msg_iov = (void *)&iovec;
438  msg_mcast.msg_iovlen = 1;
439 #ifdef HAVE_MSGHDR_CONTROL
440  msg_mcast.msg_control = 0;
441 #endif
442 #ifdef HAVE_MSGHDR_CONTROLLEN
443  msg_mcast.msg_controllen = 0;
444 #endif
445 #ifdef HAVE_MSGHDR_FLAGS
446  msg_mcast.msg_flags = 0;
447 #endif
448 #ifdef HAVE_MSGHDR_ACCRIGHTS
449  msg_mcast.msg_accrights = NULL;
450 #endif
451 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
452  msg_mcast.msg_accrightslen = 0;
453 #endif
454 
455 
456 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
457 
458  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
459  if (res < msg_len) {
460  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
461  }
462 
463  if (!only_active || instance->send_merge_detect_message) {
464  /*
465  * Current message was sent to all nodes
466  */
468  instance->send_merge_detect_message = 0;
469  }
470 }
471 
472 static int node_compare(const void *aptr, const void *bptr)
473 {
474  uint16_t a,b;
475 
476  a = *(uint16_t *)aptr;
477  b = *(uint16_t *)bptr;
478 
479  return a > b;
480 }
481 
482 int totemknet_ifaces_get (void *knet_context,
483  char ***status,
484  unsigned int *iface_count)
485 {
486  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
487  struct knet_link_status link_status;
488  knet_node_id_t host_list[KNET_MAX_HOST];
489  uint8_t link_list[KNET_MAX_LINK];
490  size_t num_hosts;
491  size_t num_links;
492  size_t link_idx;
493  int i,j;
494  char *ptr;
495  int res = 0;
496 
497  /*
498  * Don't do the whole 'link_info' bit if the caller just wants
499  * a count of interfaces.
500  */
501  if (status) {
502 
503  res = knet_host_get_host_list(instance->knet_handle,
504  host_list, &num_hosts);
505  if (res) {
506  return (-1);
507  }
508  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
509 
510  for (i=0; i<INTERFACE_MAX; i++) {
511  memset(instance->link_status[i], 'n', CFG_INTERFACE_STATUS_MAX_LEN-1);
512  instance->link_status[i][num_hosts] = '\0';
513  }
514 
515  /* This is all a bit "inside-out" because "status" is a set of strings per link
516  * and knet orders things by host
517  */
518  for (j=0; j<num_hosts; j++) {
519  res = knet_link_get_link_list(instance->knet_handle,
520  host_list[j], link_list, &num_links);
521  if (res) {
522  return (-1);
523  }
524 
525  link_idx = 0;
526  for (i=0; i < num_links; i++) {
527  /*
528  * Skip over links that are unconfigured to corosync. This is basically
529  * link0 if corosync isn't using it for comms, as we will still
530  * have it set up for loopback.
531  */
532  if (!instance->totem_config->interfaces[link_list[i]].configured) {
533  continue;
534  }
535  ptr = instance->link_status[link_idx++];
536 
537  res = knet_link_get_status(instance->knet_handle,
538  host_list[j],
539  link_list[i],
540  &link_status,
541  sizeof(link_status));
542  if (res == 0) {
543  ptr[j] = '0' + (link_status.enabled |
544  link_status.connected<<1 |
545  link_status.dynconnected<<2);
546  }
547  else {
548  ptr[j] = '?';
549  }
550  }
551  }
552  *status = instance->link_status;
553  }
554 
555  *iface_count = INTERFACE_MAX;
556 
557  return (res);
558 }
559 
561  void *knet_context)
562 {
563  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
564  int res = 0;
565  int i,j;
566  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
567  uint8_t links[KNET_MAX_LINK];
568  size_t num_nodes;
569  size_t num_links;
570 
571  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
572 
573  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
574  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
575 
576  /*
577  * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
578  */
579  res = knet_handle_setfwd(instance->knet_handle, 0);
580  if (res) {
581  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
582  }
583 
584  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
585  if (res) {
586  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
587  /* Crash out anyway */
588  goto finalise_error;
589  }
590 
591  /* Tidily shut down all nodes & links. */
592  for (i=0; i<num_nodes; i++) {
593 
594  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
595  if (res) {
596  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
597  goto finalise_error;
598  }
599  for (j=0; j<num_links; j++) {
600  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
601  if (res) {
602  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
603  }
604  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
605  if (res) {
606  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
607  }
608  }
609  res = knet_host_remove(instance->knet_handle, nodes[i]);
610  if (res) {
611  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
612  }
613  }
614 
615 finalise_error:
616  res = knet_handle_free(instance->knet_handle);
617  if (res) {
618  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
619  }
620 
621  totemknet_stop_merge_detect_timeout(instance);
622 
623  log_flush_messages(instance);
624 
625  /*
626  * Error is deliberately ignored
627  */
628  (void)pthread_mutex_destroy(&instance->log_mutex);
629 
630  return (res);
631 }
632 
633 static int log_deliver_fn (
634  int fd,
635  int revents,
636  void *data)
637 {
638  struct totemknet_instance *instance = (struct totemknet_instance *)data;
639  char buffer[sizeof(struct knet_log_msg)*4];
640  char *bufptr = buffer;
641  int done = 0;
642  int len;
643 
644  len = read(fd, buffer, sizeof(buffer));
645  while (done < len) {
646  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
647  switch (msg->msglevel) {
648  case KNET_LOG_ERR:
650  knet_log_get_subsystem_name(msg->subsystem),
651  msg->msg);
652  break;
653  case KNET_LOG_WARN:
655  knet_log_get_subsystem_name(msg->subsystem),
656  msg->msg);
657  break;
658  case KNET_LOG_INFO:
660  knet_log_get_subsystem_name(msg->subsystem),
661  msg->msg);
662  break;
663  case KNET_LOG_DEBUG:
665  knet_log_get_subsystem_name(msg->subsystem),
666  msg->msg);
667  break;
668  }
669  bufptr += sizeof(struct knet_log_msg);
670  done += sizeof(struct knet_log_msg);
671  }
672  return 0;
673 }
674 
675 static int data_deliver_fn (
676  int fd,
677  int revents,
678  void *data)
679 {
680  struct totemknet_instance *instance = (struct totemknet_instance *)data;
681  struct msghdr msg_hdr;
682  struct iovec iov_recv;
683  struct sockaddr_storage system_from;
684  ssize_t msg_len;
685  int truncated_packet;
686 
687  iov_recv.iov_base = instance->iov_buffer;
688  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
689 
690  msg_hdr.msg_name = &system_from;
691  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
692  msg_hdr.msg_iov = &iov_recv;
693  msg_hdr.msg_iovlen = 1;
694 #ifdef HAVE_MSGHDR_CONTROL
695  msg_hdr.msg_control = 0;
696 #endif
697 #ifdef HAVE_MSGHDR_CONTROLLEN
698  msg_hdr.msg_controllen = 0;
699 #endif
700 #ifdef HAVE_MSGHDR_FLAGS
701  msg_hdr.msg_flags = 0;
702 #endif
703 #ifdef HAVE_MSGHDR_ACCRIGHTS
704  msg_hdr.msg_accrights = NULL;
705 #endif
706 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
707  msg_hdr.msg_accrightslen = 0;
708 #endif
709 
710  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
711  if (msg_len <= 0) {
712  return (0);
713  }
714 
715  truncated_packet = 0;
716 
717 #ifdef HAVE_MSGHDR_FLAGS
718  if (msg_hdr.msg_flags & MSG_TRUNC) {
719  truncated_packet = 1;
720  }
721 #else
722  /*
723  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
724  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
725  */
726  if (bytes_received == KNET_MAX_PACKET_SIZE) {
727  truncated_packet = 1;
728  }
729 #endif
730 
731  if (truncated_packet) {
733  "Received too big message. This may be because something bad is happening"
734  "on the network (attack?), or you tried join more nodes than corosync is"
735  "compiled with (%u) or bug in the code (bad estimation of "
736  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
737  return (0);
738  }
739 
740  /*
741  * Handle incoming message
742  */
743  instance->totemknet_deliver_fn (
744  instance->context,
745  instance->iov_buffer,
746  msg_len,
747  &system_from);
748 
749  return (0);
750 }
751 
752 static void timer_function_netif_check_timeout (
753  void *data)
754 {
755  struct totemknet_instance *instance = (struct totemknet_instance *)data;
756  int i;
757 
758  for (i=0; i < INTERFACE_MAX; i++) {
759  if (!instance->totem_config->interfaces[i].configured) {
760  continue;
761  }
762  instance->totemknet_iface_change_fn (instance->context,
763  &instance->my_ids[i],
764  i);
765  }
766 }
767 
768 static void knet_set_access_list_config(struct totemknet_instance *instance)
769 {
770 #ifdef HAVE_KNET_ACCESS_LIST
771  uint32_t value;
772  cs_error_t err;
773 
775  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
776 
777  err = knet_handle_enable_access_lists(instance->knet_handle, value);
778  if (err) {
779  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
780  }
781 #endif
782 }
783 
784 
785 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
786 static void totemknet_refresh_config(
787  int32_t event,
788  const char *key_name,
789  struct icmap_notify_value new_val,
790  struct icmap_notify_value old_val,
791  void *user_data)
792 {
793  uint8_t reloading;
794  uint32_t value;
795  uint32_t link_no;
796  size_t num_nodes;
797  knet_node_id_t host_ids[KNET_MAX_HOST];
798  int i;
799  int err;
800  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
801 
802  ENTER();
803 
804  /*
805  * If a full reload is in progress then don't do anything until it's done and
806  * can reconfigure it all atomically
807  */
808  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
809  return;
810  }
811 
812  knet_set_access_list_config(instance);
813 
814  if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
815 
817  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
818  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
819  if (err) {
820  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
821  }
822  }
823 
824  /* Configure link parameters for each node */
825  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
826  if (err != 0) {
827  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
828  }
829 
830  for (i=0; i<num_nodes; i++) {
831  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
832  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
833  continue;
834  }
835 
836  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
837  instance->totem_config->interfaces[link_no].knet_ping_interval,
838  instance->totem_config->interfaces[link_no].knet_ping_timeout,
839  instance->totem_config->interfaces[link_no].knet_ping_precision);
840  if (err) {
841  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
842  }
843  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
844  instance->totem_config->interfaces[link_no].knet_pong_count);
845  if (err) {
846  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
847  }
848  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
849  instance->totem_config->interfaces[link_no].knet_link_priority);
850  if (err) {
851  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
852  }
853 
854  }
855  }
856 
857  LEAVE();
858 }
859 
860 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
861 {
862  icmap_track_t icmap_track_totem = NULL;
863  icmap_track_t icmap_track_reload = NULL;
864 
865  ENTER();
866 
867  icmap_track_add("totem.",
869  totemknet_refresh_config,
870  instance,
871  &icmap_track_totem);
872 
873  icmap_track_add("config.totemconfig_reload_in_progress",
875  totemknet_refresh_config,
876  instance,
877  &icmap_track_reload);
878 
879  LEAVE();
880 }
881 
882 /*
883  * Create an instance
884  */
886  qb_loop_t *poll_handle,
887  void **knet_context,
888  struct totem_config *totem_config,
889  totemsrp_stats_t *stats,
890  void *context,
891 
892  void (*deliver_fn) (
893  void *context,
894  const void *msg,
895  unsigned int msg_len,
896  const struct sockaddr_storage *system_from),
897 
898  void (*iface_change_fn) (
899  void *context,
900  const struct totem_ip_address *iface_address,
901  unsigned int link_no),
902 
903  void (*mtu_changed) (
904  void *context,
905  int net_mtu),
906 
907  void (*target_set_completed) (
908  void *context))
909 {
910  struct totemknet_instance *instance;
911  int8_t channel=0;
912  int res;
913  int i;
914 
915  instance = malloc (sizeof (struct totemknet_instance));
916  if (instance == NULL) {
917  return (-1);
918  }
919 
920  totemknet_instance_initialize (instance);
921 
922  instance->totem_config = totem_config;
923 
924  /*
925  * Configure logging
926  */
927  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
934 
935  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
936 
937  /*
938  * Initialize local variables for totemknet
939  */
940 
941  instance->our_nodeid = instance->totem_config->node_id;
942 
943  for (i=0; i< INTERFACE_MAX; i++) {
944  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
945  instance->my_ids[i].nodeid = instance->our_nodeid;
946  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
947 
948  /* Needed for totemsrp */
950  }
951 
952  instance->poll_handle = poll_handle;
953 
954  instance->context = context;
955  instance->totemknet_deliver_fn = deliver_fn;
956 
957  instance->totemknet_iface_change_fn = iface_change_fn;
958 
959  instance->totemknet_mtu_changed = mtu_changed;
960 
961  instance->totemknet_target_set_completed = target_set_completed;
962 
963  instance->loopback_link = 0;
964 
965  res = pipe(instance->logpipes);
966  if (res == -1) {
967  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
968  goto exit_error;
969  }
970  fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK);
971  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK);
972 
973 #if !defined(KNET_API_VER) || (KNET_API_VER == 1)
974  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
975 #endif
976 #if KNET_API_VER == 2
977  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
978 #endif
979 
980  if (!instance->knet_handle) {
981  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
982  goto exit_error;
983  }
984 
985  knet_set_access_list_config(instance);
986 
987  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
988  if (res) {
989  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
990  }
991  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
992  if (res) {
993  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
994  }
995  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
996  if (res) {
997  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
998  }
999  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1000  if (res) {
1001  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1002  }
1003  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1004  if (res) {
1005  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1006  }
1007  global_instance = instance;
1008 
1009  /* Get an fd into knet */
1010  instance->knet_fd = 0;
1011  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1012  if (res) {
1013  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1014  goto exit_error;
1015  }
1016 
1017  /* Enable crypto if requested */
1018  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
1019  struct knet_handle_crypto_cfg crypto_cfg;
1020 
1021  strcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model);
1022  strcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type);
1023  strcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type);
1024  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1025  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1026 
1027  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1028  if (res == -1) {
1029  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1030  goto exit_error;
1031  }
1032  if (res == -2) {
1033  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1034  goto exit_error;
1035  }
1036  knet_log_printf(LOG_INFO, "kronosnet crypto initialized: %s/%s", crypto_cfg.crypto_cipher_type, crypto_cfg.crypto_hash_type);
1037  }
1038 
1039  /* Set up compression */
1040  totemknet_reconfigure(instance, instance->totem_config);
1041 
1042  knet_handle_setfwd(instance->knet_handle, 1);
1043 
1044  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1045  if (strcmp(instance->totem_config->link_mode, "active")==0) {
1046  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1047  }
1048  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1049  instance->link_mode = KNET_LINK_POLICY_RR;
1050  }
1051 
1052  for (i=0; i<INTERFACE_MAX; i++) {
1053  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1054  if (!instance->link_status[i]) {
1055  goto exit_error;
1056  }
1057  }
1058 
1059  qb_loop_poll_add (instance->poll_handle,
1060  QB_LOOP_MED,
1061  instance->logpipes[0],
1062  POLLIN, instance, log_deliver_fn);
1063 
1064  qb_loop_poll_add (instance->poll_handle,
1065  QB_LOOP_HIGH,
1066  instance->knet_fd,
1067  POLLIN, instance, data_deliver_fn);
1068 
1069  /*
1070  * Upper layer isn't ready to receive message because it hasn't
1071  * initialized yet. Add short timer to check the interfaces.
1072  */
1073  qb_loop_timer_add (instance->poll_handle,
1074  QB_LOOP_MED,
1075  100*QB_TIME_NS_IN_MSEC,
1076  (void *)instance,
1077  timer_function_netif_check_timeout,
1078  &instance->timer_netif_check_timeout);
1079 
1080  totemknet_start_merge_detect_timeout(instance);
1081 
1082  /* Start listening for config changes */
1083  totemknet_add_config_notifications(instance);
1084 
1085  /* Add stats keys to icmap */
1087 
1088  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1089  *knet_context = instance;
1090 
1091  return (0);
1092 
1093 exit_error:
1094  log_flush_messages(instance);
1095  free(instance);
1096  return (-1);
1097 }
1098 
1100 {
1101  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1102  return malloc(KNET_MAX_PACKET_SIZE + 512);
1103 }
1104 
1106 {
1107  return free (ptr);
1108 }
1109 
1111  void *knet_context,
1112  int processor_count)
1113 {
1114  return (0);
1115 }
1116 
1117 int totemknet_recv_flush (void *knet_context)
1118 {
1119  return (0);
1120 }
1121 
1122 int totemknet_send_flush (void *knet_context)
1123 {
1124  return (0);
1125 }
1126 
1128  void *knet_context,
1129  const void *msg,
1130  unsigned int msg_len)
1131 {
1132  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1133  int res = 0;
1134 
1135  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1136 
1137  return (res);
1138 }
1140  void *knet_context,
1141  const void *msg,
1142  unsigned int msg_len)
1143 {
1144  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1145  int res = 0;
1146 
1147  mcast_sendmsg (instance, msg, msg_len, 0);
1148 
1149  return (res);
1150 }
1151 
1153  void *knet_context,
1154  const void *msg,
1155  unsigned int msg_len)
1156 {
1157  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1158  int res = 0;
1159 
1160  mcast_sendmsg (instance, msg, msg_len, 1);
1161 
1162  return (res);
1163 }
1164 
1165 
1167 {
1168  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1169  int res = 0;
1170 
1171  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1172 
1173  return (res);
1174 }
1175 
1177 {
1178  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1179 
1180  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1181 }
1182 
1184  void *knet_context,
1185  unsigned int nodeid)
1186 {
1187  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1188  int res = 0;
1189 
1190  instance->token_target.nodeid = nodeid;
1191 
1192  instance->totemknet_target_set_completed (instance->context);
1193 
1194  return (res);
1195 }
1196 
1198  void *knet_context)
1199 {
1200  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1201  unsigned int res;
1202  struct sockaddr_storage system_from;
1203  struct msghdr msg_hdr;
1204  struct iovec iov_recv;
1205  struct pollfd ufd;
1206  int nfds;
1207  int msg_processed = 0;
1208 
1209  iov_recv.iov_base = instance->iov_buffer;
1210  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1211 
1212  msg_hdr.msg_name = &system_from;
1213  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1214  msg_hdr.msg_iov = &iov_recv;
1215  msg_hdr.msg_iovlen = 1;
1216 #ifdef HAVE_MSGHDR_CONTROL
1217  msg_hdr.msg_control = 0;
1218 #endif
1219 #ifdef HAVE_MSGHDR_CONTROLLEN
1220  msg_hdr.msg_controllen = 0;
1221 #endif
1222 #ifdef HAVE_MSGHDR_FLAGS
1223  msg_hdr.msg_flags = 0;
1224 #endif
1225 #ifdef HAVE_MSGHDR_ACCRIGHTS
1226  msg_msg_hdr.msg_accrights = NULL;
1227 #endif
1228 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1229  msg_msg_hdr.msg_accrightslen = 0;
1230 #endif
1231 
1232  do {
1233  ufd.fd = instance->knet_fd;
1234  ufd.events = POLLIN;
1235  nfds = poll (&ufd, 1, 0);
1236  if (nfds == 1 && ufd.revents & POLLIN) {
1237  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1238  if (res != -1) {
1239  msg_processed = 1;
1240  } else {
1241  msg_processed = -1;
1242  }
1243  }
1244  } while (nfds == 1);
1245 
1246  return (msg_processed);
1247 }
1248 
1249 int totemknet_iface_set (void *knet_context,
1250  const struct totem_ip_address *local_addr,
1251  unsigned short ip_port,
1252  unsigned int iface_no)
1253 {
1254  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1255 
1256  totemip_copy(&instance->my_ids[iface_no], local_addr);
1257 
1258  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1259 
1260  instance->ip_port[iface_no] = ip_port;
1261 
1262  return 0;
1263 }
1264 
1265 
1267  void *knet_context,
1268  const struct totem_ip_address *local,
1269  const struct totem_ip_address *member,
1270  int link_no)
1271 {
1272  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1273  int err;
1274  int port = instance->ip_port[link_no];
1275  struct sockaddr_storage remote_ss;
1276  struct sockaddr_storage local_ss;
1277  int addrlen;
1278  int i;
1279  int host_found = 0;
1280  knet_node_id_t host_ids[KNET_MAX_HOST];
1281  size_t num_host_ids;
1282 
1283  /* Only create 1 loopback link and use link 0 */
1284  if (member->nodeid == instance->our_nodeid) {
1285  if (!instance->loopback_link) {
1286  link_no = 0;
1287  instance->loopback_link = 1;
1288  } else {
1289  /* Already done */
1290  return 0;
1291  }
1292  }
1293 
1294  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1295  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1296 
1297 
1298  /* Only add the host if it doesn't already exist in knet */
1299  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1300  if (err) {
1301  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1302  return -1;
1303  }
1304  for (i=0; i<num_host_ids; i++) {
1305  if (host_ids[i] == member->nodeid) {
1306  host_found = 1;
1307  }
1308  }
1309 
1310  if (!host_found) {
1311  err = knet_host_add(instance->knet_handle, member->nodeid);
1312  if (err != 0 && errno != EEXIST) {
1313  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1314  return -1;
1315  }
1316  } else {
1317  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1318  }
1319 
1320 
1321  if (err == 0) {
1322  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1323  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1324  return -1;
1325  }
1326  }
1327 
1328  memset(&local_ss, 0, sizeof(local_ss));
1329  memset(&remote_ss, 0, sizeof(remote_ss));
1330  /* Casts to remove const */
1331  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1332  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1333 
1334  if (member->nodeid == instance->our_nodeid) {
1335  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1336 
1337  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1338  KNET_TRANSPORT_LOOPBACK,
1339  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1340  }
1341  else {
1342  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1343  instance->totem_config->interfaces[link_no].knet_transport,
1344  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1345  }
1346  if (err) {
1347  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1348  return -1;
1349  }
1350 
1351  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1352  instance->totem_config->interfaces[link_no].knet_link_priority);
1353 
1354  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1355  instance->totem_config->interfaces[link_no].knet_link_priority);
1356  if (err) {
1357  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1358  }
1359 
1360  /* ping timeouts maybe 0 here for a newly added interface so we leave this till later, it will
1361  get done in totemknet_refresh_config */
1362  if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1363  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1364  instance->totem_config->interfaces[link_no].knet_ping_interval,
1365  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1366  instance->totem_config->interfaces[link_no].knet_ping_precision);
1367  if (err) {
1368  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1369  }
1370  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1371  instance->totem_config->interfaces[link_no].knet_pong_count);
1372  if (err) {
1373  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1374  }
1375  }
1376 
1377  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1378  if (err) {
1379  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1380  return -1;
1381  }
1382 
1383  /* register stats */
1384  stats_knet_add_member(member->nodeid, link_no);
1385  return (0);
1386 }
1387 
1389  void *knet_context,
1390  const struct totem_ip_address *token_target,
1391  int link_no)
1392 {
1393  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1394  int res;
1395  uint8_t link_list[KNET_MAX_LINK];
1396  size_t num_links;
1397 
1398  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1399 
1400  /* Don't remove the link with the loopback on it until we shut down */
1401  if (token_target->nodeid == instance->our_nodeid) {
1402  return 0;
1403  }
1404 
1405  /* Tidy stats */
1407 
1408  /* Remove the link first */
1409  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1410  if (res != 0) {
1411  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1412  return res;
1413  }
1414 
1415  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1416  if (res != 0) {
1417  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1418  return res;
1419  }
1420 
1421  /* If this is the last link, then remove the node */
1422  res = knet_link_get_link_list(instance->knet_handle,
1423  token_target->nodeid, link_list, &num_links);
1424  if (res) {
1425  return (0); /* not really failure */
1426  }
1427 
1428  if (num_links == 0) {
1429  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1430  }
1431  return res;
1432 }
1433 
1435  void *knet_context)
1436 {
1437  return (0);
1438 }
1439 
1441  void *knet_context,
1442  struct totem_config *totem_config)
1443 {
1444  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1445  struct knet_handle_compress_cfg compress_cfg;
1446  int res = 0;
1447 
1449  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1450  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1451  compress_cfg.compress_level = totem_config->knet_compression_level;
1452 
1453  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1454  if (res) {
1455  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1456  }
1457  }
1458 
1459 #ifdef HAVE_LIBNOZZLE
1460  /* Set up nozzle device(s). Return code is ignored, because unability
1461  * configure nozzle is not fatal problem, errors are logged and
1462  * there is not much else we can do */
1463  (void)setup_nozzle(instance);
1464 #endif
1465  return (res);
1466 }
1467 
1468 
1470  void *knet_context)
1471 {
1472  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1473 
1474  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1475 }
1476 
1477 /* For the stats module */
1479  knet_node_id_t node, uint8_t link_no,
1480  struct knet_link_status *status)
1481 {
1482  int res;
1483  int ret = CS_OK;
1484 
1485  /* We are probably not using knet */
1486  if (!global_instance) {
1487  return CS_ERR_NOT_EXIST;
1488  }
1489 
1490  if (link_no >= INTERFACE_MAX) {
1491  return CS_ERR_NOT_EXIST; /* Invalid link number */
1492  }
1493 
1494  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1495  if (res) {
1496  switch (errno) {
1497  case EINVAL:
1498  ret = CS_ERR_INVALID_PARAM;
1499  break;
1500  case EBUSY:
1501  ret = CS_ERR_BUSY;
1502  break;
1503  case EDEADLK:
1504  ret = CS_ERR_TRY_AGAIN;
1505  break;
1506  default:
1507  ret = CS_ERR_LIBRARY;
1508  break;
1509  }
1510  }
1511 
1512  return (ret);
1513 }
1514 
1516  struct knet_handle_stats *stats)
1517 {
1518  /* We are probably not using knet */
1519  if (!global_instance) {
1520  return CS_ERR_NOT_EXIST;
1521  }
1522 
1523  return knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1524 }
1525 
1526 static void timer_function_merge_detect_timeout (
1527  void *data)
1528 {
1529  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1530 
1531  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1532  instance->send_merge_detect_message = 1;
1533  }
1534 
1536 
1537  totemknet_start_merge_detect_timeout(instance);
1538 }
1539 
1540 static void totemknet_start_merge_detect_timeout(
1541  void *knet_context)
1542 {
1543  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1544 
1545  qb_loop_timer_add(instance->poll_handle,
1546  QB_LOOP_MED,
1547  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1548  (void *)instance,
1549  timer_function_merge_detect_timeout,
1550  &instance->timer_merge_detect_timeout);
1551 
1552 }
1553 
1554 static void totemknet_stop_merge_detect_timeout(
1555  void *knet_context)
1556 {
1557  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1558 
1559  qb_loop_timer_del(instance->poll_handle,
1560  instance->timer_merge_detect_timeout);
1561 }
1562 
1563 static void log_flush_messages (void *knet_context)
1564 {
1565  struct pollfd pfd;
1566  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1567  int cont;
1568 
1569  cont = 1;
1570 
1571  while (cont) {
1572  pfd.fd = instance->logpipes[0];
1573  pfd.events = POLLIN;
1574  pfd.revents = 0;
1575 
1576  if ((poll(&pfd, 1, 0) > 0) &&
1577  (pfd.revents & POLLIN) &&
1578  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1579  cont = 1;
1580  } else {
1581  cont = 0;
1582  }
1583  }
1584 }
1585 
1586 
1587 #ifdef HAVE_LIBNOZZLE
1588 #define NOZZLE_NAME "nozzle.name"
1589 #define NOZZLE_IPADDR "nozzle.ipaddr"
1590 #define NOZZLE_PREFIX "nozzle.ipprefix"
1591 #define NOZZLE_MACADDR "nozzle.macaddr"
1592 
1593 #define NOZZLE_CHANNEL 1
1594 
1595 
1596 static char *get_nozzle_script_dir(void *knet_context)
1597 {
1598  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1599  char filename[PATH_MAX + FILENAME_MAX + 1];
1600  static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
1601  int res;
1602  const char *dirname_res;
1603 
1604  /*
1605  * Build script directory based on corosync.conf file location
1606  */
1607  res = snprintf(filename, sizeof(filename), "%s",
1609  if (res >= sizeof(filename)) {
1610  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1611  return NULL;
1612  }
1613 
1614  dirname_res = dirname(filename);
1615 
1616  res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
1617  dirname_res, "updown.d");
1618  if (res >= sizeof(updown_dirname)) {
1619  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1620  return NULL;
1621  }
1622  return updown_dirname;
1623 }
1624 
1625 /*
1626  * Deliberately doesn't return the status as caller doesn't care.
1627  * The result will be logged though
1628  */
1629 static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
1630 {
1631  int res;
1632  char *exec_string;
1633 
1634  res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
1635  if (res == -1 && errno != ENOENT) {
1636  knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
1637  } else if (res == -2) {
1638  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
1639  knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
1640  }
1641 }
1642 
1643 /*
1644  * Reparse IP address to add in our node ID
1645  * IPv6 addresses must end in '::'
1646  * IPv4 addresses must just be valid
1647  * '/xx' lengths are optional for IPv6, mandatory for IPv4
1648  *
1649  * Returns the modified IP address as a string to pass into libnozzle
1650  */
1651 static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
1652  const char *input_addr,
1653  const char *prefix, int nodeid,
1654  char *output_addr, size_t output_len)
1655 {
1656  char *coloncolon;
1657  int bits;
1658  int max_prefix = 64;
1659  uint32_t nodeid_mask;
1660  uint32_t addr_mask;
1661  uint32_t masked_nodeid;
1662  struct in_addr *addr;
1663  struct totem_ip_address totemip;
1664 
1665  coloncolon = strstr(input_addr, "::");
1666  if (!coloncolon) {
1667  max_prefix = 30;
1668  }
1669 
1670  bits = atoi(prefix);
1671  if (bits < 8 || bits > max_prefix) {
1672  knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
1673  return -1;
1674  }
1675 
1676  /* IPv6 is easy */
1677  if (coloncolon) {
1678  memcpy(output_addr, input_addr, coloncolon-input_addr);
1679  sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
1680  return 0;
1681  }
1682 
1683  /* For IPv4 we need to parse the address into binary, mask off the required bits,
1684  * add in the masked_nodeid and 'print' it out again
1685  */
1686  nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
1687  addr_mask = UINT32_MAX ^ nodeid_mask;
1688  masked_nodeid = nodeid & nodeid_mask;
1689 
1690  if (totemip_parse(&totemip, input_addr, AF_INET)) {
1691  knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
1692  return -1;
1693  }
1694  addr = (struct in_addr *)&totemip.addr;
1695  addr->s_addr &= htonl(addr_mask);
1696  addr->s_addr |= htonl(masked_nodeid);
1697 
1698  inet_ntop(AF_INET, addr, output_addr, output_len);
1699  return 0;
1700 }
1701 
1702 static int create_nozzle_device(void *knet_context, const char *name,
1703  const char *ipaddr, const char *prefix,
1704  const char *macaddr)
1705 {
1706  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1707  char device_name[IFNAMSIZ+1];
1708  size_t size = IFNAMSIZ;
1709  int8_t channel = NOZZLE_CHANNEL;
1710  nozzle_t nozzle_dev;
1711  int nozzle_fd;
1712  int res;
1713  char *updown_dir;
1714  char parsed_ipaddr[INET6_ADDRSTRLEN];
1715  char mac[19];
1716 
1717  memset(device_name, 0, size);
1718  memset(&mac, 0, sizeof(mac));
1719  strncpy(device_name, name, size);
1720 
1721  updown_dir = get_nozzle_script_dir(knet_context);
1722  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
1723 
1724  nozzle_dev = nozzle_open(device_name, size, updown_dir);
1725  if (!nozzle_dev) {
1726  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
1727  return -1;
1728  }
1729  instance->nozzle_handle = nozzle_dev;
1730 
1731  if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
1732  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
1733  goto out_clean;
1734  }
1735 
1736  if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
1737  /* Prints its own errors */
1738  goto out_clean;
1739  }
1740  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
1741  if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
1742  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
1743  goto out_clean;
1744  }
1745 
1746  nozzle_fd = nozzle_get_fd(nozzle_dev);
1747  knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
1748 
1749  res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
1750  if (res != 0) {
1751  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
1752  goto out_clean;
1753  }
1754 
1755  run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
1756 
1757  res = nozzle_set_up(nozzle_dev);
1758  if (res != 0) {
1759  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
1760  goto out_clean;
1761  }
1762  run_nozzle_script(instance, NOZZLE_UP, "up");
1763 
1764  return 0;
1765 
1766 out_clean:
1767  nozzle_close(nozzle_dev);
1768  return -1;
1769 }
1770 
1771 static int remove_nozzle_device(void *knet_context)
1772 {
1773  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1774  int res;
1775  int datafd;
1776 
1777  res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
1778  if (res != 0) {
1779  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1780  return -1;
1781  }
1782 
1783  res = knet_handle_remove_datafd(instance->knet_handle, datafd);
1784  if (res != 0) {
1785  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1786  return -1;
1787  }
1788 
1789  run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
1790  res = nozzle_set_down(instance->nozzle_handle);
1791  if (res != 0) {
1792  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
1793  return -1;
1794  }
1795  run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
1796 
1797  res = nozzle_close(instance->nozzle_handle);
1798  if (res != 0) {
1799  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
1800  return -1;
1801  }
1802  knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
1803  return 0;
1804 }
1805 
1806 static void free_nozzle(struct totemknet_instance *instance)
1807 {
1808  free(instance->nozzle_name);
1809  free(instance->nozzle_ipaddr);
1810  free(instance->nozzle_prefix);
1811  free(instance->nozzle_macaddr);
1812 
1813  instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
1814  instance->nozzle_macaddr = NULL;
1815 }
1816 
1817 static int setup_nozzle(void *knet_context)
1818 {
1819  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1820  char *ipaddr_str = NULL;
1821  char *name_str = NULL;
1822  char *prefix_str = NULL;
1823  char *macaddr_str = NULL;
1824  char mac[32];
1825  int name_res;
1826  int macaddr_res;
1827  int res = -1;
1828 
1829  /*
1830  * Return value ignored on purpose. icmap_get_string changes
1831  * ipaddr_str/prefix_str only on success.
1832  */
1833  (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
1834  (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
1835  macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
1836  name_res = icmap_get_string(NOZZLE_NAME, &name_str);
1837 
1838  /* Is is being removed? */
1839  if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
1840  remove_nozzle_device(instance);
1841  free_nozzle(instance);
1842  goto out_free;
1843  }
1844 
1845  if (!name_str) {
1846  /* no nozzle */
1847  goto out_free;
1848  }
1849 
1850  if (!ipaddr_str) {
1851  knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
1852  goto out_free;
1853  }
1854 
1855  if (!prefix_str) {
1856  knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
1857  goto out_free;
1858  }
1859 
1860  if (macaddr_str && strlen(macaddr_str) != 17) {
1861  knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
1862  goto out_free;
1863  }
1864  if (!macaddr_str) {
1865  macaddr_str = (char*)"54:54:01:00:00:00";
1866  }
1867 
1868  if (instance->nozzle_name &&
1869  (strcmp(name_str, instance->nozzle_name) == 0) &&
1870  (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
1871  (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
1872  (instance->nozzle_macaddr == NULL ||
1873  strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
1874  /* Nothing has changed */
1875  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
1876  goto out_free;
1877  }
1878 
1879  /* Add nodeid into MAC address */
1880  memcpy(mac, macaddr_str, 12);
1881  snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
1882  instance->our_nodeid >> 8,
1883  instance->our_nodeid & 0xFF);
1884  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
1885 
1886  if (name_res == CS_OK && name_str) {
1887  /* Reconfigure */
1888  if (instance->nozzle_name) {
1889  remove_nozzle_device(instance);
1890  free_nozzle(instance);
1891  }
1892 
1893  res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
1894  mac);
1895 
1896  instance->nozzle_name = strdup(name_str);
1897  instance->nozzle_ipaddr = strdup(ipaddr_str);
1898  instance->nozzle_prefix = strdup(prefix_str);
1899  instance->nozzle_macaddr = strdup(macaddr_str);
1900  if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
1901  !instance->nozzle_prefix) {
1902  knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
1903  /*
1904  * This 'free' will cause a complete reconfigure of the device next time we reload
1905  * but will also let the the current device keep working until then.
1906  * remove_nozzle() only needs the, statically-allocated, nozzle_handle
1907  */
1908  free_nozzle(instance);
1909  }
1910  }
1911 
1912 out_free:
1913  free(name_str);
1914  free(ipaddr_str);
1915  free(prefix_str);
1916  if (macaddr_res == CS_OK) {
1917  free(macaddr_str);
1918  }
1919 
1920  return res;
1921 }
1922 #endif // HAVE_LIBNOZZLE
totemknet_ifaces_get
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:482
knet_log_printf
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:228
totem_logging_configuration::log_level_error
int log_level_error
Definition: totem.h:109
totemknet_instance::nozzle_prefix
char * nozzle_prefix
Definition: totemknet.c:180
totem_config::node_id
unsigned int node_id
Definition: totem.h:160
NOZZLE_CHANNEL
#define NOZZLE_CHANNEL
Definition: totemknet.c:1593
value
uint32_t value
Definition: exec/votequorum.c:100
LOGSYS_LEVEL_CRIT
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
totemknet_instance::nozzle_name
char * nozzle_name
Definition: totemknet.c:178
totemknet_instance::totemknet_log_level_error
int totemknet_log_level_error
Definition: totemknet.c:126
totemknet_mcast_flush_send
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1139
totemknet_instance::totemknet_iface_change_fn
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:110
totem_interface::knet_ping_precision
int knet_ping_precision
Definition: totem.h:92
NOZZLE_PREFIX
#define NOZZLE_PREFIX
Definition: totemknet.c:1590
totemknet_instance::totemknet_mtu_changed
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:115
MSG_NOSIGNAL
#define MSG_NOSIGNAL
Definition: totemknet.c:83
totemknet_member_add
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1266
work_item::msg_len
unsigned int msg_len
Definition: totemknet.c:191
stats_knet_del_member
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:624
totemknet_instance::nozzle_handle
nozzle_t nozzle_handle
Definition: totemknet.c:182
totemknet_buffer_release
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:1105
totemknet_token_send
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1127
NOZZLE_MACADDR
#define NOZZLE_MACADDR
Definition: totemknet.c:1591
CS_ERR_LIBRARY
Definition: corotypes.h:99
CS_ERR_TRY_AGAIN
Definition: corotypes.h:103
LOGSYS_LEVEL_ERROR
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
totemknet_crypto_set
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:358
totemknet_reconfigure
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1440
totemknet_iface_set
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1249
totem_config::knet_pmtud_interval
unsigned int knet_pmtud_interval
Definition: totem.h:162
work_item
Definition: totemknet.c:189
LOGSYS_LEVEL_DEBUG
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
totem_logging_configuration::log_level_debug
int log_level_debug
Definition: totem.h:112
totemip_print
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:264
totemknet_net_mtu_adjust
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1176
totemsrp_stats_t
Definition: totemstats.h:53
type
char type
Definition: totem.h:55
totemknet_instance::poll_handle
qb_loop_t * poll_handle
Definition: totemknet.c:96
totem_logging_configuration::log_level_notice
int log_level_notice
Definition: totem.h:111
CS_ERR_NOT_EXIST
Definition: corotypes.h:109
totem_interface::knet_link_priority
int knet_link_priority
Definition: totem.h:89
KNET_LOGSYS_PERROR
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:244
totemknet_instance::totemknet_log_level_notice
int totemknet_log_level_notice
Definition: totemknet.c:130
totemknet_finalize
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:560
libknet_log_printf
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:236
stats_knet_add_member
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:614
totemknet_instance::totem_config
struct totem_config * totem_config
Definition: totemknet.c:161
totemknet_instance::context
void * context
Definition: totemknet.c:102
CFG_INTERFACE_STATUS_MAX_LEN
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:91
ICMAP_TRACK_PREFIX
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem....
Definition: icmap.h:85
NOZZLE_NAME
#define NOZZLE_NAME
Definition: totemknet.c:1588
totem_ip_address::nodeid
unsigned int nodeid
Definition: coroapi.h:112
totemknet_processor_count_set
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:1110
ENTER
#define ENTER
Definition: logsys.h:324
totemknet_instance::iov_buffer
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:149
corosync_get_config_file
const char * corosync_get_config_file(void)
Definition: main.c:204
totemknet_link_get_status
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1478
totem_config::merge_timeout
unsigned int merge_timeout
Definition: totem.h:190
CS_PRI_NODE_ID
#define CS_PRI_NODE_ID
Definition: corotypes.h:59
totemknet_stats_clear
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1469
swab.h
totem_config::interfaces
struct totem_interface * interfaces
Definition: totem.h:158
totem_interface::knet_transport
int knet_transport
Definition: totem.h:94
totemknet_instance::totemknet_log_level_debug
int totemknet_log_level_debug
Definition: totemknet.c:132
addr
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:77
totem_config::knet_compression_level
int knet_compression_level
Definition: totem.h:228
totemip_copy
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:123
ICMAP_TRACK_ADD
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
icmap.h
header
struct totem_message_header header
Definition: totemsrp.c:260
totemknet_instance::nozzle_ipaddr
char * nozzle_ipaddr
Definition: totemknet.c:179
totemknet_instance::nozzle_macaddr
char * nozzle_macaddr
Definition: totemknet.c:181
totemknet_instance::totemknet_log_level_security
int totemknet_log_level_security
Definition: totemknet.c:124
CS_OK
Definition: corotypes.h:98
LOGSYS_LEVEL_INFO
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
totem_config::net_mtu
unsigned int net_mtu
Definition: totem.h:202
totemip.h
INTERFACE_MAX
#define INTERFACE_MAX
Definition: coroapi.h:88
totemknet_instance::my_ids
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:153
totem_config::knet_compression_threshold
uint32_t knet_compression_threshold
Definition: totem.h:226
CS_ERR_BUSY
Definition: corotypes.h:107
totem_interface::configured
uint8_t configured
Definition: totem.h:87
totemknet_instance::send_merge_detect_message
int send_merge_detect_message
Definition: totemknet.c:169
cs_error_t
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:97
__attribute__
typedef __attribute__
NOZZLE_IPADDR
#define NOZZLE_IPADDR
Definition: totemknet.c:1589
totem_interface::boundto
struct totem_ip_address boundto
Definition: totem.h:83
LEAVE
#define LEAVE
Definition: logsys.h:325
totemknet_instance::crypto_inst
struct crypto_instance * crypto_inst
Definition: totemknet.c:94
icmap_track_add
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1157
totemknet_instance::knet_fd
int knet_fd
Definition: totemknet.c:174
totemknet_iface_check
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1166
totemknet_instance::timer_netif_check_timeout
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:165
totem_config::crypto_model
char * crypto_model
Definition: totem.h:218
totem_logging_configuration::log_subsys_id
int log_subsys_id
Definition: totem.h:114
totemknet_instance
Definition: totemknet.c:93
totem_config::link_mode
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:198
totemknet_instance::knet_context
void(*) void knet_context)
Definition: totemknet.c:145
totemknet_recv_flush
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1117
totemknet_instance::ip_port
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:155
totemknet_instance::our_nodeid
int our_nodeid
Definition: totemknet.c:157
totem_config
Definition: totem.h:152
totemknet_token_target_set
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1183
totemknet.h
totem_ip_address
The totem_ip_address struct.
Definition: coroapi.h:111
totemknet_instance::token_target
struct totem_ip_address token_target
Definition: totemknet.c:163
sq.h
icmap_get_uint32
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:890
totemknet_instance::log_mutex
pthread_mutex_t log_mutex
Definition: totemknet.c:176
totemknet_initialize
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:885
global_instance
struct totemknet_instance * global_instance
Definition: totemknet.c:187
totem_message_header
Definition: totem.h:124
totem_interface::knet_pong_count
int knet_pong_count
Definition: totem.h:93
totemknet_member_list_rebind_ip
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1434
totem_interface::ip_port
uint16_t ip_port
Definition: totem.h:85
totem_interface::knet_ping_interval
int knet_ping_interval
Definition: totem.h:90
_logsys_subsys_create
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
totemknet_instance::loopback_link
int loopback_link
Definition: totemknet.c:159
totemknet_instance::knet_handle
knet_handle_t knet_handle
Definition: totemknet.c:98
totemip_parse
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition: totemip.c:314
ICMAP_TRACK_DELETE
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
work_item::msg
const void * msg
Definition: totemknet.c:190
totemknet_instance::timer_merge_detect_timeout
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:167
user_data
void * user_data
Definition: sam.c:127
system_from
struct srp_addr system_from
Definition: totemsrp.c:261
ICMAP_TRACK_MODIFY
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
totem_logging_configuration::log_level_warning
int log_level_warning
Definition: totem.h:110
totemknet_recv_mcast_empty
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1197
totemknet_instance::merge_detect_messages_sent_before_timeout
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:171
icmap_get_string
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:854
totemknet_member_remove
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1388
main.h
util.h
totemknet_send_flush
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1122
totemknet_mcast_noflush_send
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1152
totemknet_instance::knet_subsys_id
int knet_subsys_id
Definition: totemknet.c:136
totemknet_instance::totemknet_deliver_fn
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:104
totemknet_buffer_alloc
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:1099
nodeid
unsigned int nodeid
Definition: coroapi.h:75
totem_interface::knet_ping_timeout
int knet_ping_timeout
Definition: totem.h:91
totem_config::private_key
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:167
PROCESSOR_COUNT_MAX
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
config.h
totemknet_handle_get_stats
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1515
CS_ERR_INVALID_PARAM
Definition: corotypes.h:104
totemknet_instance::totemknet_subsys_id
int totemknet_subsys_id
Definition: totemknet.c:134
logsys.h
totem_config::crypto_hash_type
char * crypto_hash_type
Definition: totem.h:222
totemknet_instance::totemknet_target_set_completed
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:119
totem_logging_configuration::log_printf
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:99
LOGSYS_LEVEL_WARNING
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
totem_interface::bindnet
struct totem_ip_address bindnet
Definition: totem.h:82
icmap_track
Definition: icmap.c:61
totem_config::private_key_len
unsigned int private_key_len
Definition: totem.h:169
totemknet_instance::link_mode
int link_mode
Definition: totemknet.c:100
totem_config::crypto_cipher_type
char * crypto_cipher_type
Definition: totem.h:220
totemknet_instance::totemknet_log_printf
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:138
icmap_notify_value
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
stats_knet_add_handle
void stats_knet_add_handle(void)
Definition: stats.c:637
totemip_totemip_to_sockaddr_convert
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:272
totem_config::block_unlisted_ips
unsigned int block_unlisted_ips
Definition: totem.h:236
icmap_get_uint8
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:866
totemknet_instance::totemknet_log_level_warning
int totemknet_log_level_warning
Definition: totemknet.c:128
totemknet_instance::logpipes
int logpipes[2]
Definition: totemknet.c:173
totemstats.h
totem_message_header::target_nodeid
unsigned int target_nodeid
Definition: totem.h:130
work_item::instance
struct totemknet_instance * instance
Definition: totemknet.c:192
totemknet_instance::link_status
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:151
totem_config::knet_compression_model
char * knet_compression_model
Definition: totem.h:224
totem_config::totem_logging_configuration
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:200