corosync  3.1.0
totemknet.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2020 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Christine Caulfield (ccaulfie@redhat.com)
7 
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of the MontaVista Software, Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <config.h>
36 
37 #include <assert.h>
38 #include <sys/mman.h>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/socket.h>
42 #include <netdb.h>
43 #include <sys/un.h>
44 #include <sys/ioctl.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <net/ethernet.h>
48 #include <arpa/inet.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <stdlib.h>
52 #include <stdio.h>
53 #include <errno.h>
54 #include <pthread.h>
55 #include <sched.h>
56 #include <time.h>
57 #include <sys/time.h>
58 #include <sys/poll.h>
59 #include <sys/uio.h>
60 #include <limits.h>
61 
62 #include <qb/qbdefs.h>
63 #include <qb/qbloop.h>
64 #ifdef HAVE_LIBNOZZLE
65 #include <libgen.h>
66 #include <libnozzle.h>
67 #endif
68 
69 #include <corosync/sq.h>
70 #include <corosync/swab.h>
71 #include <corosync/logsys.h>
72 #include <corosync/icmap.h>
73 #include <corosync/totem/totemip.h>
74 #include "totemknet.h"
75 
76 #include "main.h"
77 #include "util.h"
78 
79 #include <libknet.h>
81 
82 #ifndef MSG_NOSIGNAL
83 #define MSG_NOSIGNAL 0
84 #endif
85 
86 #ifdef HAVE_LIBNOZZLE
87 static int setup_nozzle(void *knet_context);
88 #endif
89 
90 /* Should match that used by cfg */
91 #define CFG_INTERFACE_STATUS_MAX_LEN 512
92 
94  struct crypto_instance *crypto_inst;
95 
96  qb_loop_t *poll_handle;
97 
98  knet_handle_t knet_handle;
99 
101 
102  void *context;
103 
105  void *context,
106  const void *msg,
107  unsigned int msg_len,
108  const struct sockaddr_storage *system_from);
109 
111  void *context,
112  const struct totem_ip_address *iface_address,
113  unsigned int link_no);
114 
116  void *context,
117  int net_mtu);
118 
120 
121  /*
122  * Function and data used to log messages
123  */
125 
127 
129 
131 
133 
135 
137 
139  int level,
140  int subsys,
141  const char *function,
142  const char *file,
143  int line,
144  const char *format,
145  ...)__attribute__((format(printf, 6, 7)));
146 
148 
149  char iov_buffer[KNET_MAX_PACKET_SIZE];
150 
152 
154 
156 
158 
160 
162 
164 
165  qb_loop_timer_handle timer_netif_check_timeout;
166 
167  qb_loop_timer_handle timer_merge_detect_timeout;
168 
170 
172 
173  int logpipes[2];
174  int knet_fd;
175 
176  pthread_mutex_t log_mutex;
177 #ifdef HAVE_LIBNOZZLE
178  char *nozzle_name;
179  char *nozzle_ipaddr;
180  char *nozzle_prefix;
181  char *nozzle_macaddr;
182  nozzle_t nozzle_handle;
183 #endif
184 };
185 
186 /* Awkward. But needed to get stats from knet */
188 
189 struct work_item {
190  const void *msg;
191  unsigned int msg_len;
193 };
194 
196  void *knet_context);
197 
198 
199 static int totemknet_configure_compression (
200  void *knet_context,
201  struct totem_config *totem_config);
202 
203 static void totemknet_start_merge_detect_timeout(
204  void *knet_context);
205 
206 static void totemknet_stop_merge_detect_timeout(
207  void *knet_context);
208 
209 static void log_flush_messages (
210  void *knet_context);
211 
212 static void totemknet_instance_initialize (struct totemknet_instance *instance)
213 {
214  int res;
215 
216  memset (instance, 0, sizeof (struct totemknet_instance));
217  res = pthread_mutex_init(&instance->log_mutex, NULL);
218  /*
219  * There is not too much else what can be done.
220  */
221  assert(res == 0);
222 }
223 
224 #define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
225 do { \
226  (void)pthread_mutex_lock(&instance->log_mutex); \
227  instance->totemknet_log_printf ( \
228  level, subsys, function, file, line, \
229  (const char *)format, ##args); \
230  (void)pthread_mutex_unlock(&instance->log_mutex); \
231 } while (0);
232 
233 #define knet_log_printf(level, format, args...) \
234 do { \
235  knet_log_printf_lock ( \
236  level, instance->totemknet_subsys_id, \
237  __FUNCTION__, __FILE__, __LINE__, \
238  (const char *)format, ##args); \
239 } while (0);
240 
241 #define libknet_log_printf(level, format, args...) \
242 do { \
243  knet_log_printf_lock ( \
244  level, instance->knet_subsys_id, \
245  __FUNCTION__, "libknet.h", __LINE__, \
246  (const char *)format, ##args); \
247 } while (0);
248 
249 #define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
250 do { \
251  char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
252  const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
253  instance->totemknet_log_printf ( \
254  level, instance->totemknet_subsys_id, \
255  __FUNCTION__, __FILE__, __LINE__, \
256  fmt ": %s (%d)", ##args, _error_ptr, err_num); \
257  } while(0)
258 
259 
260 #ifdef HAVE_LIBNOZZLE
261 static inline int is_ether_addr_multicast(const uint8_t *addr)
262 {
263  return (addr[0] & 0x01);
264 }
265 static inline int is_ether_addr_zero(const uint8_t *addr)
266 {
267  return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
268 }
269 
270 static int ether_host_filter_fn(void *private_data,
271  const unsigned char *outdata,
272  ssize_t outdata_len,
273  uint8_t tx_rx,
274  knet_node_id_t this_host_id,
275  knet_node_id_t src_host_id,
276  int8_t *channel,
277  knet_node_id_t *dst_host_ids,
278  size_t *dst_host_ids_entries)
279 {
280  struct ether_header *eth_h = (struct ether_header *)outdata;
281  uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
282  uint16_t dst_host_id;
283 
284  if (is_ether_addr_zero(dst_mac))
285  return -1;
286 
287  if (is_ether_addr_multicast(dst_mac)) {
288  return 1;
289  }
290 
291  memmove(&dst_host_id, &dst_mac[4], 2);
292 
293  dst_host_ids[0] = ntohs(dst_host_id);
294  *dst_host_ids_entries = 1;
295 
296  return 0;
297 }
298 #endif
299 
300 static int dst_host_filter_callback_fn(void *private_data,
301  const unsigned char *outdata,
302  ssize_t outdata_len,
303  uint8_t tx_rx,
304  knet_node_id_t this_host_id,
305  knet_node_id_t src_host_id,
306  int8_t *channel,
307  knet_node_id_t *dst_host_ids,
308  size_t *dst_host_ids_entries)
309 {
310  struct totem_message_header *header = (struct totem_message_header *)outdata;
311  int res;
312 
313 #ifdef HAVE_LIBNOZZLE
314  if (*channel != 0) {
315  return ether_host_filter_fn(private_data,
316  outdata, outdata_len,
317  tx_rx,
318  this_host_id, src_host_id,
319  channel,
320  dst_host_ids,
321  dst_host_ids_entries);
322  }
323 #endif
324  if (header->target_nodeid) {
325  dst_host_ids[0] = header->target_nodeid;
326  *dst_host_ids_entries = 1;
327  res = 0; /* unicast message */
328  }
329  else {
330  *dst_host_ids_entries = 0;
331  res = 1; /* multicast message */
332  }
333  return res;
334 }
335 
336 static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
337 {
338  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
339 
340  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
341  if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
342  knet_handle_remove_datafd(instance->knet_handle, datafd);
343  }
344 }
345 
346 static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
347 {
348  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
349 
350  // TODO: what? if anything.
351  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
352 }
353 
354 static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
355 {
356  struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
357  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
358 
359  /* We don't need to tell corosync the actual knet MTU */
360 // instance->totemknet_mtu_changed(instance->context, data_mtu);
361 }
362 
364  void *knet_context,
365  const char *cipher_type,
366  const char *hash_type)
367 {
368  return (0);
369 }
370 
371 
372 static inline void ucast_sendmsg (
373  struct totemknet_instance *instance,
374  struct totem_ip_address *system_to,
375  const void *msg,
376  unsigned int msg_len)
377 {
378  int res = 0;
379  struct totem_message_header *header = (struct totem_message_header *)msg;
380  struct msghdr msg_ucast;
381  struct iovec iovec;
382 
383  header->target_nodeid = system_to->nodeid;
384 
385  iovec.iov_base = (void *)msg;
386  iovec.iov_len = msg_len;
387 
388  /*
389  * Build unicast message
390  */
391  memset(&msg_ucast, 0, sizeof(msg_ucast));
392  msg_ucast.msg_iov = (void *)&iovec;
393  msg_ucast.msg_iovlen = 1;
394 #ifdef HAVE_MSGHDR_CONTROL
395  msg_ucast.msg_control = 0;
396 #endif
397 #ifdef HAVE_MSGHDR_CONTROLLEN
398  msg_ucast.msg_controllen = 0;
399 #endif
400 #ifdef HAVE_MSGHDR_FLAGS
401  msg_ucast.msg_flags = 0;
402 #endif
403 #ifdef HAVE_MSGHDR_ACCRIGHTS
404  msg_ucast.msg_accrights = NULL;
405 #endif
406 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
407  msg_ucast.msg_accrightslen = 0;
408 #endif
409 
410  /*
411  * Transmit unicast message
412  * An error here is recovered by totemsrp
413  */
414 
415  res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
416  if (res < 0) {
418  "sendmsg(ucast) failed (non-critical)");
419  }
420 }
421 
422 static inline void mcast_sendmsg (
423  struct totemknet_instance *instance,
424  const void *msg,
425  unsigned int msg_len,
426  int only_active)
427 {
428  int res;
429  struct totem_message_header *header = (struct totem_message_header *)msg;
430  struct msghdr msg_mcast;
431  struct iovec iovec;
432 
433  iovec.iov_base = (void *)msg;
434  iovec.iov_len = msg_len;
435 
436  header->target_nodeid = 0;
437 
438  /*
439  * Build multicast message
440  */
441  memset(&msg_mcast, 0, sizeof(msg_mcast));
442  msg_mcast.msg_iov = (void *)&iovec;
443  msg_mcast.msg_iovlen = 1;
444 #ifdef HAVE_MSGHDR_CONTROL
445  msg_mcast.msg_control = 0;
446 #endif
447 #ifdef HAVE_MSGHDR_CONTROLLEN
448  msg_mcast.msg_controllen = 0;
449 #endif
450 #ifdef HAVE_MSGHDR_FLAGS
451  msg_mcast.msg_flags = 0;
452 #endif
453 #ifdef HAVE_MSGHDR_ACCRIGHTS
454  msg_mcast.msg_accrights = NULL;
455 #endif
456 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
457  msg_mcast.msg_accrightslen = 0;
458 #endif
459 
460 
461 // log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
462 
463  res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
464  if (res < msg_len) {
465  knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
466  }
467 
468  if (!only_active || instance->send_merge_detect_message) {
469  /*
470  * Current message was sent to all nodes
471  */
473  instance->send_merge_detect_message = 0;
474  }
475 }
476 
477 static int node_compare(const void *aptr, const void *bptr)
478 {
479  uint16_t a,b;
480 
481  a = *(uint16_t *)aptr;
482  b = *(uint16_t *)bptr;
483 
484  return a > b;
485 }
486 
487 #ifndef OWN_INDEX_NONE
488 #define OWN_INDEX_NONE -1
489 #endif
490 
491 int totemknet_ifaces_get (void *knet_context,
492  char ***status,
493  unsigned int *iface_count)
494 {
495  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
496  struct knet_link_status link_status;
497  knet_node_id_t host_list[KNET_MAX_HOST];
498  uint8_t link_list[KNET_MAX_LINK];
499  size_t num_hosts;
500  size_t num_links;
501  size_t link_idx;
502  int i,j;
503  char *ptr;
504  int res = 0;
505 
506  /*
507  * Don't do the whole 'link_info' bit if the caller just wants
508  * a count of interfaces.
509  */
510  if (status) {
511  int own_idx = OWN_INDEX_NONE;
512 
513  res = knet_host_get_host_list(instance->knet_handle,
514  host_list, &num_hosts);
515  if (res) {
516  return (-1);
517  }
518  qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
519 
520  for (j=0; j<num_hosts; j++) {
521  if (host_list[j] == instance->our_nodeid) {
522  own_idx = j;
523  break;
524  }
525  }
526 
527  for (i=0; i<INTERFACE_MAX; i++) {
528  memset(instance->link_status[i], 'd', CFG_INTERFACE_STATUS_MAX_LEN-1);
529  if (own_idx != OWN_INDEX_NONE) {
530  instance->link_status[i][own_idx] = 'n';
531  }
532  instance->link_status[i][num_hosts] = '\0';
533  }
534 
535  /* This is all a bit "inside-out" because "status" is a set of strings per link
536  * and knet orders things by host
537  */
538  for (j=0; j<num_hosts; j++) {
539  if (own_idx != OWN_INDEX_NONE && j == own_idx) {
540  continue ;
541  }
542 
543  res = knet_link_get_link_list(instance->knet_handle,
544  host_list[j], link_list, &num_links);
545  if (res) {
546  return (-1);
547  }
548 
549  link_idx = 0;
550  for (i=0; i < num_links; i++) {
551  /*
552  * Skip over links that are unconfigured to corosync. This is basically
553  * link0 if corosync isn't using it for comms, as we will still
554  * have it set up for loopback.
555  */
556  if (!instance->totem_config->interfaces[link_list[i]].configured) {
557  continue;
558  }
559  ptr = instance->link_status[link_idx++];
560 
561  res = knet_link_get_status(instance->knet_handle,
562  host_list[j],
563  link_list[i],
564  &link_status,
565  sizeof(link_status));
566  if (res == 0) {
567  ptr[j] = '0' + (link_status.enabled |
568  link_status.connected<<1 |
569  link_status.dynconnected<<2);
570  }
571  else {
573  "totemknet_ifaces_get: Cannot get link status: %s", strerror(errno));
574  ptr[j] = '?';
575  }
576  }
577  }
578  *status = instance->link_status;
579  }
580 
581  *iface_count = INTERFACE_MAX;
582 
583  return (res);
584 }
585 
587  void *knet_context)
588 {
589  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
590  int res = 0;
591  int i,j;
592  static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
593  uint8_t links[KNET_MAX_LINK];
594  size_t num_nodes;
595  size_t num_links;
596 
597  knet_log_printf(LOG_DEBUG, "totemknet: finalize");
598 
599  qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
600  qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
601 
602  /*
603  * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
604  */
605  res = knet_handle_setfwd(instance->knet_handle, 0);
606  if (res) {
607  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
608  }
609 
610  res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
611  if (res) {
612  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
613  /* Crash out anyway */
614  goto finalise_error;
615  }
616 
617  /* Tidily shut down all nodes & links. */
618  for (i=0; i<num_nodes; i++) {
619 
620  res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
621  if (res) {
622  knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
623  goto finalise_error;
624  }
625  for (j=0; j<num_links; j++) {
626  res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
627  if (res) {
628  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
629  }
630  res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
631  if (res) {
632  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
633  }
634  }
635  res = knet_host_remove(instance->knet_handle, nodes[i]);
636  if (res) {
637  knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
638  }
639  }
640 
641 finalise_error:
642  res = knet_handle_free(instance->knet_handle);
643  if (res) {
644  knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
645  }
646 
647  totemknet_stop_merge_detect_timeout(instance);
648 
649  log_flush_messages(instance);
650 
651  /*
652  * Error is deliberately ignored
653  */
654  (void)pthread_mutex_destroy(&instance->log_mutex);
655 
656  return (res);
657 }
658 
659 static int log_deliver_fn (
660  int fd,
661  int revents,
662  void *data)
663 {
664  struct totemknet_instance *instance = (struct totemknet_instance *)data;
665  char buffer[sizeof(struct knet_log_msg)*4];
666  char *bufptr = buffer;
667  int done = 0;
668  int len;
669 
670  len = read(fd, buffer, sizeof(buffer));
671  while (done < len) {
672  struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
673  switch (msg->msglevel) {
674  case KNET_LOG_ERR:
676  knet_log_get_subsystem_name(msg->subsystem),
677  msg->msg);
678  break;
679  case KNET_LOG_WARN:
681  knet_log_get_subsystem_name(msg->subsystem),
682  msg->msg);
683  break;
684  case KNET_LOG_INFO:
686  knet_log_get_subsystem_name(msg->subsystem),
687  msg->msg);
688  break;
689  case KNET_LOG_DEBUG:
691  knet_log_get_subsystem_name(msg->subsystem),
692  msg->msg);
693  break;
694  }
695  bufptr += sizeof(struct knet_log_msg);
696  done += sizeof(struct knet_log_msg);
697  }
698  return 0;
699 }
700 
701 static int data_deliver_fn (
702  int fd,
703  int revents,
704  void *data)
705 {
706  struct totemknet_instance *instance = (struct totemknet_instance *)data;
707  struct msghdr msg_hdr;
708  struct iovec iov_recv;
709  struct sockaddr_storage system_from;
710  ssize_t msg_len;
711  int truncated_packet;
712 
713  iov_recv.iov_base = instance->iov_buffer;
714  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
715 
716  msg_hdr.msg_name = &system_from;
717  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
718  msg_hdr.msg_iov = &iov_recv;
719  msg_hdr.msg_iovlen = 1;
720 #ifdef HAVE_MSGHDR_CONTROL
721  msg_hdr.msg_control = 0;
722 #endif
723 #ifdef HAVE_MSGHDR_CONTROLLEN
724  msg_hdr.msg_controllen = 0;
725 #endif
726 #ifdef HAVE_MSGHDR_FLAGS
727  msg_hdr.msg_flags = 0;
728 #endif
729 #ifdef HAVE_MSGHDR_ACCRIGHTS
730  msg_hdr.msg_accrights = NULL;
731 #endif
732 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
733  msg_hdr.msg_accrightslen = 0;
734 #endif
735 
736  msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
737  if (msg_len <= 0) {
738  return (0);
739  }
740 
741  truncated_packet = 0;
742 
743 #ifdef HAVE_MSGHDR_FLAGS
744  if (msg_hdr.msg_flags & MSG_TRUNC) {
745  truncated_packet = 1;
746  }
747 #else
748  /*
749  * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
750  * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
751  */
752  if (bytes_received == KNET_MAX_PACKET_SIZE) {
753  truncated_packet = 1;
754  }
755 #endif
756 
757  if (truncated_packet) {
759  "Received too big message. This may be because something bad is happening"
760  "on the network (attack?), or you tried join more nodes than corosync is"
761  "compiled with (%u) or bug in the code (bad estimation of "
762  "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
763  return (0);
764  }
765 
766  /*
767  * Handle incoming message
768  */
769  instance->totemknet_deliver_fn (
770  instance->context,
771  instance->iov_buffer,
772  msg_len,
773  &system_from);
774 
775  return (0);
776 }
777 
778 static void timer_function_netif_check_timeout (
779  void *data)
780 {
781  struct totemknet_instance *instance = (struct totemknet_instance *)data;
782  int i;
783 
784  for (i=0; i < INTERFACE_MAX; i++) {
785  if (!instance->totem_config->interfaces[i].configured) {
786  continue;
787  }
788  instance->totemknet_iface_change_fn (instance->context,
789  &instance->my_ids[i],
790  i);
791  }
792 }
793 
794 static void knet_set_access_list_config(struct totemknet_instance *instance)
795 {
796 #ifdef HAVE_KNET_ACCESS_LIST
797  uint32_t value;
798  cs_error_t err;
799 
801  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
802 
803  err = knet_handle_enable_access_lists(instance->knet_handle, value);
804  if (err) {
805  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
806  }
807 #endif
808 }
809 
810 
811 /* NOTE: this relies on the fact that totem_reload_notify() is called first */
812 static void totemknet_refresh_config(
813  int32_t event,
814  const char *key_name,
815  struct icmap_notify_value new_val,
816  struct icmap_notify_value old_val,
817  void *user_data)
818 {
819  uint8_t reloading;
820  uint32_t value;
821  uint32_t link_no;
822  size_t num_nodes;
823  knet_node_id_t host_ids[KNET_MAX_HOST];
824  int i;
825  int err;
826  struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
827 
828  ENTER();
829 
830  /*
831  * If a full reload is in progress then don't do anything until it's done and
832  * can reconfigure it all atomically
833  */
834  if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
835  return;
836  }
837 
838  knet_set_access_list_config(instance);
839 
840  if (icmap_get_uint32("totem.knet_pmtud_interval", &value) == CS_OK) {
841 
843  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %d", value);
844  err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
845  if (err) {
846  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
847  }
848  }
849 
850  /* Configure link parameters for each node */
851  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
852  if (err != 0) {
853  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
854  }
855 
856  for (i=0; i<num_nodes; i++) {
857  for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
858  if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
859  continue;
860  }
861 
862  err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
863  instance->totem_config->interfaces[link_no].knet_ping_interval,
864  instance->totem_config->interfaces[link_no].knet_ping_timeout,
865  instance->totem_config->interfaces[link_no].knet_ping_precision);
866  if (err) {
867  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
868  }
869  err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
870  instance->totem_config->interfaces[link_no].knet_pong_count);
871  if (err) {
872  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
873  }
874  err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
875  instance->totem_config->interfaces[link_no].knet_link_priority);
876  if (err) {
877  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
878  }
879 
880  }
881  }
882 
883  LEAVE();
884 }
885 
886 static void totemknet_add_config_notifications(struct totemknet_instance *instance)
887 {
888  icmap_track_t icmap_track_totem = NULL;
889  icmap_track_t icmap_track_reload = NULL;
890 
891  ENTER();
892 
893  icmap_track_add("totem.",
895  totemknet_refresh_config,
896  instance,
897  &icmap_track_totem);
898 
899  icmap_track_add("config.totemconfig_reload_in_progress",
901  totemknet_refresh_config,
902  instance,
903  &icmap_track_reload);
904 
905  LEAVE();
906 }
907 
908 static int totemknet_set_knet_crypto(struct totemknet_instance *instance)
909 {
910  struct knet_handle_crypto_cfg crypto_cfg;
911  int res;
912 
913  /* These have already been validated */
914  memcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model, sizeof(crypto_cfg.crypto_model));
915  memcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type, sizeof(crypto_cfg.crypto_model));
916  memcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type, sizeof(crypto_cfg.crypto_model));
917  memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
918  crypto_cfg.private_key_len = instance->totem_config->private_key_len;
919 
920 #ifdef HAVE_KNET_CRYPTO_RECONF
921 
922  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s on index %d",
923  crypto_cfg.crypto_model,
924  crypto_cfg.crypto_cipher_type,
925  crypto_cfg.crypto_hash_type,
926  instance->totem_config->crypto_index
927  );
928 
929  /* If crypto is being disabled we need to explicitly allow cleartext traffic in knet */
930  if (strcmp(instance->totem_config->crypto_cipher_type, "none") == 0) {
931  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
932  if (res) {
933  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(ALLOW) failed %s", strerror(errno));
934  }
935  }
936 
937  /* use_config will be called later when all nodes are synced */
938  res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, instance->totem_config->crypto_index);
939  if (res == -1) {
940  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: %s", instance->totem_config->crypto_index, strerror(errno));
941  goto exit_error;
942  }
943  if (res == -2) {
944  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: -2", instance->totem_config->crypto_index);
945  goto exit_error;
946  }
947 #else
948  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s",
949  crypto_cfg.crypto_model,
950  crypto_cfg.crypto_cipher_type,
951  crypto_cfg.crypto_hash_type
952  );
953 
954  res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
955  if (res == -1) {
956  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
957  goto exit_error;
958  }
959  if (res == -2) {
960  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
961  goto exit_error;
962  }
963 #endif
964 
965 
966 exit_error:
967  return res;
968 }
969 
970 /*
971  * Create an instance
972  */
974  qb_loop_t *poll_handle,
975  void **knet_context,
976  struct totem_config *totem_config,
977  totemsrp_stats_t *stats,
978  void *context,
979 
980  void (*deliver_fn) (
981  void *context,
982  const void *msg,
983  unsigned int msg_len,
984  const struct sockaddr_storage *system_from),
985 
986  void (*iface_change_fn) (
987  void *context,
988  const struct totem_ip_address *iface_address,
989  unsigned int link_no),
990 
991  void (*mtu_changed) (
992  void *context,
993  int net_mtu),
994 
995  void (*target_set_completed) (
996  void *context))
997 {
998  struct totemknet_instance *instance;
999  int8_t channel=0;
1000  int res;
1001  int i;
1002 
1003  instance = malloc (sizeof (struct totemknet_instance));
1004  if (instance == NULL) {
1005  return (-1);
1006  }
1007 
1008  totemknet_instance_initialize (instance);
1009 
1010  instance->totem_config = totem_config;
1011 
1012  /*
1013  * Configure logging
1014  */
1015  instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
1022 
1023  instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
1024 
1025  /*
1026  * Initialize local variables for totemknet
1027  */
1028 
1029  instance->our_nodeid = instance->totem_config->node_id;
1030 
1031  for (i=0; i< INTERFACE_MAX; i++) {
1032  totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
1033  instance->my_ids[i].nodeid = instance->our_nodeid;
1034  instance->ip_port[i] = totem_config->interfaces[i].ip_port;
1035 
1036  /* Needed for totemsrp */
1038  }
1039 
1040  instance->poll_handle = poll_handle;
1041 
1042  instance->context = context;
1043  instance->totemknet_deliver_fn = deliver_fn;
1044 
1045  instance->totemknet_iface_change_fn = iface_change_fn;
1046 
1047  instance->totemknet_mtu_changed = mtu_changed;
1048 
1049  instance->totemknet_target_set_completed = target_set_completed;
1050 
1051  instance->loopback_link = 0;
1052 
1053  res = pipe(instance->logpipes);
1054  if (res == -1) {
1055  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
1056  goto exit_error;
1057  }
1058  if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
1059  fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
1060  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
1061  goto exit_error;
1062  }
1063 
1064 
1065 #if !defined(KNET_API_VER) || (KNET_API_VER == 1)
1066  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
1067 #endif
1068 #if KNET_API_VER == 2
1069  instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
1070 #endif
1071 
1072  if (!instance->knet_handle) {
1073  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
1074  goto exit_error;
1075  }
1076 
1077  knet_set_access_list_config(instance);
1078 
1079  res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
1080  if (res) {
1081  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
1082  }
1083  res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
1084  if (res) {
1085  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
1086  }
1087  res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
1088  if (res) {
1089  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
1090  }
1091  res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1092  if (res) {
1093  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1094  }
1095  res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1096  if (res) {
1097  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1098  }
1099  global_instance = instance;
1100 
1101  /* Get an fd into knet */
1102  instance->knet_fd = 0;
1103  res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1104  if (res) {
1105  knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1106  goto exit_error;
1107  }
1108 
1109  /* Enable crypto if requested */
1110 #ifdef HAVE_KNET_CRYPTO_RECONF
1111  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
1112  res = totemknet_set_knet_crypto(instance);
1113  if (res == 0) {
1114  res = knet_handle_crypto_use_config(instance->knet_handle, totem_config->crypto_index);
1115  if (res) {
1116  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_use_config failed: %s", strerror(errno));
1117  goto exit_error;
1118  }
1119  } else {
1120  knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1121  goto exit_error;
1122  }
1123  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1124  if (res) {
1125  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (DISALLOW) failed: %s", strerror(errno));
1126  goto exit_error;
1127  }
1128 
1129  } else {
1130  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1131  if (res) {
1132  knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (ALLOW) failed: %s", strerror(errno));
1133  goto exit_error;
1134  }
1135  }
1136 #else
1137  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
1138  res = totemknet_set_knet_crypto(instance);
1139  if (res) {
1140  knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1141  goto exit_error;
1142  }
1143  }
1144 #endif
1145 
1146  /* Set up compression */
1147  if (strcmp(totem_config->knet_compression_model, "none") != 0) {
1148  /* Not fatal, but will log */
1149  (void)totemknet_configure_compression(knet_context, totem_config);
1150  }
1151 
1152  knet_handle_setfwd(instance->knet_handle, 1);
1153 
1154  instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1155  if (strcmp(instance->totem_config->link_mode, "active")==0) {
1156  instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1157  }
1158  if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1159  instance->link_mode = KNET_LINK_POLICY_RR;
1160  }
1161 
1162  for (i=0; i<INTERFACE_MAX; i++) {
1163  instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1164  if (!instance->link_status[i]) {
1165  goto exit_error;
1166  }
1167  }
1168 
1169  qb_loop_poll_add (instance->poll_handle,
1170  QB_LOOP_MED,
1171  instance->logpipes[0],
1172  POLLIN, instance, log_deliver_fn);
1173 
1174  qb_loop_poll_add (instance->poll_handle,
1175  QB_LOOP_HIGH,
1176  instance->knet_fd,
1177  POLLIN, instance, data_deliver_fn);
1178 
1179  /*
1180  * Upper layer isn't ready to receive message because it hasn't
1181  * initialized yet. Add short timer to check the interfaces.
1182  */
1183  qb_loop_timer_add (instance->poll_handle,
1184  QB_LOOP_MED,
1185  100*QB_TIME_NS_IN_MSEC,
1186  (void *)instance,
1187  timer_function_netif_check_timeout,
1188  &instance->timer_netif_check_timeout);
1189 
1190  totemknet_start_merge_detect_timeout(instance);
1191 
1192  /* Start listening for config changes */
1193  totemknet_add_config_notifications(instance);
1194 
1195  /* Add stats keys to icmap */
1197 
1198  knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1199  *knet_context = instance;
1200 
1201  return (0);
1202 
1203 exit_error:
1204  log_flush_messages(instance);
1205  free(instance);
1206  return (-1);
1207 }
1208 
1210 {
1211  /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1212  return malloc(KNET_MAX_PACKET_SIZE + 512);
1213 }
1214 
1216 {
1217  return free (ptr);
1218 }
1219 
1221  void *knet_context,
1222  int processor_count)
1223 {
1224  return (0);
1225 }
1226 
1228 {
1229  return (0);
1230 }
1231 
1233 {
1234  return (0);
1235 }
1236 
1238  void *knet_context,
1239  const void *msg,
1240  unsigned int msg_len)
1241 {
1242  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1243  int res = 0;
1244 
1245  ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1246 
1247  return (res);
1248 }
1250  void *knet_context,
1251  const void *msg,
1252  unsigned int msg_len)
1253 {
1254  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1255  int res = 0;
1256 
1257  mcast_sendmsg (instance, msg, msg_len, 0);
1258 
1259  return (res);
1260 }
1261 
1263  void *knet_context,
1264  const void *msg,
1265  unsigned int msg_len)
1266 {
1267  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1268  int res = 0;
1269 
1270  mcast_sendmsg (instance, msg, msg_len, 1);
1271 
1272  return (res);
1273 }
1274 
1275 
1277 {
1278  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1279  int res = 0;
1280 
1281  knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1282 
1283  return (res);
1284 }
1285 
1287 {
1288  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1289 
1290  knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1291 }
1292 
1294  void *knet_context,
1295  unsigned int nodeid)
1296 {
1297  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1298  int res = 0;
1299 
1300  instance->token_target.nodeid = nodeid;
1301 
1302  instance->totemknet_target_set_completed (instance->context);
1303 
1304  return (res);
1305 }
1306 
1308  void *knet_context)
1309 {
1310  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1311  unsigned int res;
1312  struct sockaddr_storage system_from;
1313  struct msghdr msg_hdr;
1314  struct iovec iov_recv;
1315  struct pollfd ufd;
1316  int nfds;
1317  int msg_processed = 0;
1318 
1319  iov_recv.iov_base = instance->iov_buffer;
1320  iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1321 
1322  msg_hdr.msg_name = &system_from;
1323  msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1324  msg_hdr.msg_iov = &iov_recv;
1325  msg_hdr.msg_iovlen = 1;
1326 #ifdef HAVE_MSGHDR_CONTROL
1327  msg_hdr.msg_control = 0;
1328 #endif
1329 #ifdef HAVE_MSGHDR_CONTROLLEN
1330  msg_hdr.msg_controllen = 0;
1331 #endif
1332 #ifdef HAVE_MSGHDR_FLAGS
1333  msg_hdr.msg_flags = 0;
1334 #endif
1335 #ifdef HAVE_MSGHDR_ACCRIGHTS
1336  msg_msg_hdr.msg_accrights = NULL;
1337 #endif
1338 #ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1339  msg_msg_hdr.msg_accrightslen = 0;
1340 #endif
1341 
1342  do {
1343  ufd.fd = instance->knet_fd;
1344  ufd.events = POLLIN;
1345  nfds = poll (&ufd, 1, 0);
1346  if (nfds == 1 && ufd.revents & POLLIN) {
1347  res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1348  if (res != -1) {
1349  msg_processed = 1;
1350  } else {
1351  msg_processed = -1;
1352  }
1353  }
1354  } while (nfds == 1);
1355 
1356  return (msg_processed);
1357 }
1358 
1359 int totemknet_iface_set (void *knet_context,
1360  const struct totem_ip_address *local_addr,
1361  unsigned short ip_port,
1362  unsigned int iface_no)
1363 {
1364  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1365 
1366  totemip_copy(&instance->my_ids[iface_no], local_addr);
1367 
1368  knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1369 
1370  instance->ip_port[iface_no] = ip_port;
1371 
1372  return 0;
1373 }
1374 
1375 
1377  void *knet_context,
1378  const struct totem_ip_address *local,
1379  const struct totem_ip_address *member,
1380  int link_no)
1381 {
1382  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1383  int err;
1384  int port = instance->ip_port[link_no];
1385  struct sockaddr_storage remote_ss;
1386  struct sockaddr_storage local_ss;
1387  int addrlen;
1388  int i;
1389  int host_found = 0;
1390  knet_node_id_t host_ids[KNET_MAX_HOST];
1391  size_t num_host_ids;
1392 
1393  /* Only create 1 loopback link and use link 0 */
1394  if (member->nodeid == instance->our_nodeid) {
1395  if (!instance->loopback_link) {
1396  link_no = 0;
1397  instance->loopback_link = 1;
1398  } else {
1399  /* Already done */
1400  return 0;
1401  }
1402  }
1403 
1404  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1405  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1406 
1407 
1408  /* Only add the host if it doesn't already exist in knet */
1409  err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1410  if (err) {
1411  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1412  return -1;
1413  }
1414  for (i=0; i<num_host_ids; i++) {
1415  if (host_ids[i] == member->nodeid) {
1416  host_found = 1;
1417  }
1418  }
1419 
1420  if (!host_found) {
1421  err = knet_host_add(instance->knet_handle, member->nodeid);
1422  if (err != 0 && errno != EEXIST) {
1423  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1424  return -1;
1425  }
1426  } else {
1427  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1428  }
1429 
1430 
1431  if (err == 0) {
1432  if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1433  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1434  return -1;
1435  }
1436  }
1437 
1438  memset(&local_ss, 0, sizeof(local_ss));
1439  memset(&remote_ss, 0, sizeof(remote_ss));
1440  /* Casts to remove const */
1441  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1442  totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1443 
1444  if (member->nodeid == instance->our_nodeid) {
1445  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1446 
1447  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1448  KNET_TRANSPORT_LOOPBACK,
1449  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1450  }
1451  else {
1452  err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1453  instance->totem_config->interfaces[link_no].knet_transport,
1454  &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1455  }
1456  if (err) {
1457  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1458  return -1;
1459  }
1460 
1461  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1462  instance->totem_config->interfaces[link_no].knet_link_priority);
1463 
1464  err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1465  instance->totem_config->interfaces[link_no].knet_link_priority);
1466  if (err) {
1467  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1468  }
1469 
1470  /* ping timeouts maybe 0 here for a newly added interface so we leave this till later, it will
1471  get done in totemknet_refresh_config */
1472  if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1473  err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1474  instance->totem_config->interfaces[link_no].knet_ping_interval,
1475  instance->totem_config->interfaces[link_no].knet_ping_timeout,
1476  instance->totem_config->interfaces[link_no].knet_ping_precision);
1477  if (err) {
1478  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1479  }
1480  err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1481  instance->totem_config->interfaces[link_no].knet_pong_count);
1482  if (err) {
1483  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1484  }
1485  }
1486 
1487  err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1488  if (err) {
1489  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1490  return -1;
1491  }
1492 
1493  /* register stats */
1494  stats_knet_add_member(member->nodeid, link_no);
1495  return (0);
1496 }
1497 
1499  void *knet_context,
1500  const struct totem_ip_address *token_target,
1501  int link_no)
1502 {
1503  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1504  int res;
1505  uint8_t link_list[KNET_MAX_LINK];
1506  size_t num_links;
1507 
1508  knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1509 
1510  /* Don't remove the link with the loopback on it until we shut down */
1511  if (token_target->nodeid == instance->our_nodeid) {
1512  return 0;
1513  }
1514 
1515  /* Tidy stats */
1517 
1518  /* Remove the link first */
1519  res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1520  if (res != 0) {
1521  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1522  return res;
1523  }
1524 
1525  res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1526  if (res != 0) {
1527  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1528  return res;
1529  }
1530 
1531  /* If this is the last link, then remove the node */
1532  res = knet_link_get_link_list(instance->knet_handle,
1533  token_target->nodeid, link_list, &num_links);
1534  if (res) {
1535  return (0); /* not really failure */
1536  }
1537 
1538  if (num_links == 0) {
1539  res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1540  }
1541  return res;
1542 }
1543 
1545  void *knet_context)
1546 {
1547  return (0);
1548 }
1549 
1550 
1551 static int totemknet_configure_compression (
1552  void *knet_context,
1553  struct totem_config *totem_config)
1554 {
1555  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1556  struct knet_handle_compress_cfg compress_cfg;
1557  int res = 0;
1558 
1559  assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
1560  strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1561 
1562  compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1563  compress_cfg.compress_level = totem_config->knet_compression_level;
1564 
1565  res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1566  if (res) {
1567  KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1568  }
1569  return res;
1570 }
1571 
1573  void *knet_context,
1574  struct totem_config *totem_config)
1575 {
1576  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1577  int res = 0;
1578 
1579  (void)totemknet_configure_compression(knet_context, totem_config);
1580 
1581 #ifdef HAVE_LIBNOZZLE
1582  /* Set up nozzle device(s). Return code is ignored, because inability
1583  * configure nozzle is not fatal problem, errors are logged and
1584  * there is not much else we can do */
1585  (void)setup_nozzle(instance);
1586 #endif
1587 
1589  /* Flip crypto_index */
1591  res = totemknet_set_knet_crypto(instance);
1592 
1593  knet_log_printf(LOG_INFO, "kronosnet crypto reconfigured on index %d: %s/%s/%s", totem_config->crypto_index,
1597  }
1598  return (res);
1599 }
1600 
1601 
1603  void *knet_context,
1604  struct totem_config *totem_config,
1606 {
1607 #ifdef HAVE_KNET_CRYPTO_RECONF
1608  int res;
1609  int config_to_use;
1610  int config_to_clear;
1611  struct knet_handle_crypto_cfg crypto_cfg;
1612  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1613 
1614  knet_log_printf(LOGSYS_LEVEL_DEBUG, "totemknet_crypto_reconfigure_phase %d, index=%d\n", phase, totem_config->crypto_index);
1615 
1616  switch (phase) {
1618  config_to_use = totem_config->crypto_index;
1619  if (strcmp(instance->totem_config->crypto_cipher_type, "none") == 0) {
1620  config_to_use = 0; /* we are clearing it */
1621  }
1622 
1623  /* Enable the new config on this node */
1624  res = knet_handle_crypto_use_config(instance->knet_handle, config_to_use);
1625  if (res == -1) {
1626  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_use_config %d failed: %s", config_to_use, strerror(errno));
1627  }
1628  break;
1629 
1631  /*
1632  * All nodes should now have the new config. clear the old one out
1633  * OR disable crypto entirely if that's what the new config insists on.
1634  */
1635  config_to_clear = 3-totem_config->crypto_index;
1636  knet_log_printf(LOGSYS_LEVEL_DEBUG, "Clearing old knet crypto config %d\n", config_to_clear);
1637 
1638  strcpy(crypto_cfg.crypto_model, "none");
1639  strcpy(crypto_cfg.crypto_cipher_type, "none");
1640  strcpy(crypto_cfg.crypto_hash_type, "none");
1641  res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, config_to_clear);
1642  if (res == -1) {
1643  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: %s", config_to_clear, strerror(errno));
1644  }
1645  if (res == -2) {
1646  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: -2", config_to_clear);
1647  }
1648 
1649  /* If crypto is enabled then disable all cleartext reception */
1650  if (strcmp(instance->totem_config->crypto_cipher_type, "none") != 0) {
1651  res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1652  if (res) {
1653  knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(DISALLOW) failed %s", strerror(errno));
1654  }
1655  }
1656  }
1657 #endif
1658  return 0;
1659 }
1660 
1662  void *knet_context)
1663 {
1664  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1665 
1666  (void) knet_handle_clear_stats(instance->knet_handle, KNET_CLEARSTATS_HANDLE_AND_LINK);
1667 }
1668 
1669 /* For the stats module */
1671  knet_node_id_t node, uint8_t link_no,
1672  struct knet_link_status *status)
1673 {
1674  int res;
1675  int ret = CS_OK;
1676 
1677  /* We are probably not using knet */
1678  if (!global_instance) {
1679  return CS_ERR_NOT_EXIST;
1680  }
1681 
1682  if (link_no >= INTERFACE_MAX) {
1683  return CS_ERR_NOT_EXIST; /* Invalid link number */
1684  }
1685 
1686  res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1687  if (res) {
1688  switch (errno) {
1689  case EINVAL:
1690  ret = CS_ERR_INVALID_PARAM;
1691  break;
1692  case EBUSY:
1693  ret = CS_ERR_BUSY;
1694  break;
1695  case EDEADLK:
1696  ret = CS_ERR_TRY_AGAIN;
1697  break;
1698  default:
1699  ret = CS_ERR_LIBRARY;
1700  break;
1701  }
1702  }
1703 
1704  return (ret);
1705 }
1706 
1708  struct knet_handle_stats *stats)
1709 {
1710  int res;
1711 
1712  /* We are probably not using knet */
1713  if (!global_instance) {
1714  return CS_ERR_NOT_EXIST;
1715  }
1716 
1717  res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1718  if (res != 0) {
1719  return (qb_to_cs_error(-errno));
1720  }
1721 
1722  return CS_OK;
1723 }
1724 
1725 static void timer_function_merge_detect_timeout (
1726  void *data)
1727 {
1728  struct totemknet_instance *instance = (struct totemknet_instance *)data;
1729 
1730  if (instance->merge_detect_messages_sent_before_timeout == 0) {
1731  instance->send_merge_detect_message = 1;
1732  }
1733 
1735 
1736  totemknet_start_merge_detect_timeout(instance);
1737 }
1738 
1739 static void totemknet_start_merge_detect_timeout(
1740  void *knet_context)
1741 {
1742  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1743 
1744  qb_loop_timer_add(instance->poll_handle,
1745  QB_LOOP_MED,
1746  instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1747  (void *)instance,
1748  timer_function_merge_detect_timeout,
1749  &instance->timer_merge_detect_timeout);
1750 
1751 }
1752 
1753 static void totemknet_stop_merge_detect_timeout(
1754  void *knet_context)
1755 {
1756  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1757 
1758  qb_loop_timer_del(instance->poll_handle,
1759  instance->timer_merge_detect_timeout);
1760 }
1761 
1762 static void log_flush_messages (void *knet_context)
1763 {
1764  struct pollfd pfd;
1765  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1766  int cont;
1767 
1768  cont = 1;
1769 
1770  while (cont) {
1771  pfd.fd = instance->logpipes[0];
1772  pfd.events = POLLIN;
1773  pfd.revents = 0;
1774 
1775  if ((poll(&pfd, 1, 0) > 0) &&
1776  (pfd.revents & POLLIN) &&
1777  (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1778  cont = 1;
1779  } else {
1780  cont = 0;
1781  }
1782  }
1783 }
1784 
1785 
1786 #ifdef HAVE_LIBNOZZLE
1787 #define NOZZLE_NAME "nozzle.name"
1788 #define NOZZLE_IPADDR "nozzle.ipaddr"
1789 #define NOZZLE_PREFIX "nozzle.ipprefix"
1790 #define NOZZLE_MACADDR "nozzle.macaddr"
1791 
1792 #define NOZZLE_CHANNEL 1
1793 
1794 
1795 static char *get_nozzle_script_dir(void *knet_context)
1796 {
1797  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1798  char filename[PATH_MAX + FILENAME_MAX + 1];
1799  static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
1800  int res;
1801  const char *dirname_res;
1802 
1803  /*
1804  * Build script directory based on corosync.conf file location
1805  */
1806  res = snprintf(filename, sizeof(filename), "%s",
1808  if (res >= sizeof(filename)) {
1809  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1810  return NULL;
1811  }
1812 
1813  dirname_res = dirname(filename);
1814 
1815  res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
1816  dirname_res, "updown.d");
1817  if (res >= sizeof(updown_dirname)) {
1818  knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
1819  return NULL;
1820  }
1821  return updown_dirname;
1822 }
1823 
1824 /*
1825  * Deliberately doesn't return the status as caller doesn't care.
1826  * The result will be logged though
1827  */
1828 static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
1829 {
1830  int res;
1831  char *exec_string;
1832 
1833  res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
1834  if (res == -1 && errno != ENOENT) {
1835  knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
1836  } else if (res == -2) {
1837  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
1838  knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
1839  }
1840 }
1841 
1842 /*
1843  * Reparse IP address to add in our node ID
1844  * IPv6 addresses must end in '::'
1845  * IPv4 addresses must just be valid
1846  * '/xx' lengths are optional for IPv6, mandatory for IPv4
1847  *
1848  * Returns the modified IP address as a string to pass into libnozzle
1849  */
1850 static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
1851  const char *input_addr,
1852  const char *prefix, int nodeid,
1853  char *output_addr, size_t output_len)
1854 {
1855  char *coloncolon;
1856  int bits;
1857  int max_prefix = 64;
1858  uint32_t nodeid_mask;
1859  uint32_t addr_mask;
1860  uint32_t masked_nodeid;
1861  struct in_addr *addr;
1862  struct totem_ip_address totemip;
1863 
1864  coloncolon = strstr(input_addr, "::");
1865  if (!coloncolon) {
1866  max_prefix = 30;
1867  }
1868 
1869  bits = atoi(prefix);
1870  if (bits < 8 || bits > max_prefix) {
1871  knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
1872  return -1;
1873  }
1874 
1875  /* IPv6 is easy */
1876  if (coloncolon) {
1877  memcpy(output_addr, input_addr, coloncolon-input_addr);
1878  sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
1879  return 0;
1880  }
1881 
1882  /* For IPv4 we need to parse the address into binary, mask off the required bits,
1883  * add in the masked_nodeid and 'print' it out again
1884  */
1885  nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
1886  addr_mask = UINT32_MAX ^ nodeid_mask;
1887  masked_nodeid = nodeid & nodeid_mask;
1888 
1889  if (totemip_parse(&totemip, input_addr, AF_INET)) {
1890  knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
1891  return -1;
1892  }
1893  addr = (struct in_addr *)&totemip.addr;
1894  addr->s_addr &= htonl(addr_mask);
1895  addr->s_addr |= htonl(masked_nodeid);
1896 
1897  inet_ntop(AF_INET, addr, output_addr, output_len);
1898  return 0;
1899 }
1900 
1901 static int create_nozzle_device(void *knet_context, const char *name,
1902  const char *ipaddr, const char *prefix,
1903  const char *macaddr)
1904 {
1905  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1906  char device_name[IFNAMSIZ+1];
1907  size_t size = IFNAMSIZ;
1908  int8_t channel = NOZZLE_CHANNEL;
1909  nozzle_t nozzle_dev;
1910  int nozzle_fd;
1911  int res;
1912  char *updown_dir;
1913  char parsed_ipaddr[INET6_ADDRSTRLEN];
1914  char mac[19];
1915 
1916  memset(device_name, 0, size);
1917  memset(&mac, 0, sizeof(mac));
1918  strncpy(device_name, name, size);
1919 
1920  updown_dir = get_nozzle_script_dir(knet_context);
1921  knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
1922 
1923  nozzle_dev = nozzle_open(device_name, size, updown_dir);
1924  if (!nozzle_dev) {
1925  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
1926  return -1;
1927  }
1928  instance->nozzle_handle = nozzle_dev;
1929 
1930  if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
1931  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
1932  goto out_clean;
1933  }
1934 
1935  if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
1936  /* Prints its own errors */
1937  goto out_clean;
1938  }
1939  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
1940  if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
1941  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
1942  goto out_clean;
1943  }
1944 
1945  nozzle_fd = nozzle_get_fd(nozzle_dev);
1946  knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
1947 
1948  res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
1949  if (res != 0) {
1950  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
1951  goto out_clean;
1952  }
1953 
1954  run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
1955 
1956  res = nozzle_set_up(nozzle_dev);
1957  if (res != 0) {
1958  knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
1959  goto out_clean;
1960  }
1961  run_nozzle_script(instance, NOZZLE_UP, "up");
1962 
1963  return 0;
1964 
1965 out_clean:
1966  nozzle_close(nozzle_dev);
1967  return -1;
1968 }
1969 
1970 static int remove_nozzle_device(void *knet_context)
1971 {
1972  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1973  int res;
1974  int datafd;
1975 
1976  res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
1977  if (res != 0) {
1978  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1979  return -1;
1980  }
1981 
1982  res = knet_handle_remove_datafd(instance->knet_handle, datafd);
1983  if (res != 0) {
1984  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
1985  return -1;
1986  }
1987 
1988  run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
1989  res = nozzle_set_down(instance->nozzle_handle);
1990  if (res != 0) {
1991  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
1992  return -1;
1993  }
1994  run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
1995 
1996  res = nozzle_close(instance->nozzle_handle);
1997  if (res != 0) {
1998  knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
1999  return -1;
2000  }
2001  knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
2002  return 0;
2003 }
2004 
2005 static void free_nozzle(struct totemknet_instance *instance)
2006 {
2007  free(instance->nozzle_name);
2008  free(instance->nozzle_ipaddr);
2009  free(instance->nozzle_prefix);
2010  free(instance->nozzle_macaddr);
2011 
2012  instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
2013  instance->nozzle_macaddr = NULL;
2014 }
2015 
2016 static int setup_nozzle(void *knet_context)
2017 {
2018  struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2019  char *ipaddr_str = NULL;
2020  char *name_str = NULL;
2021  char *prefix_str = NULL;
2022  char *macaddr_str = NULL;
2023  char mac[32];
2024  int name_res;
2025  int macaddr_res;
2026  int res = -1;
2027 
2028  /*
2029  * Return value ignored on purpose. icmap_get_string changes
2030  * ipaddr_str/prefix_str only on success.
2031  */
2032  (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
2033  (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
2034  macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
2035  name_res = icmap_get_string(NOZZLE_NAME, &name_str);
2036 
2037  /* Is is being removed? */
2038  if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
2039  remove_nozzle_device(instance);
2040  free_nozzle(instance);
2041  goto out_free;
2042  }
2043 
2044  if (!name_str) {
2045  /* no nozzle */
2046  goto out_free;
2047  }
2048 
2049  if (!ipaddr_str) {
2050  knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
2051  goto out_free;
2052  }
2053 
2054  if (!prefix_str) {
2055  knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
2056  goto out_free;
2057  }
2058 
2059  if (macaddr_str && strlen(macaddr_str) != 17) {
2060  knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
2061  goto out_free;
2062  }
2063  if (!macaddr_str) {
2064  macaddr_str = (char*)"54:54:01:00:00:00";
2065  }
2066 
2067  if (instance->nozzle_name &&
2068  (strcmp(name_str, instance->nozzle_name) == 0) &&
2069  (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
2070  (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
2071  (instance->nozzle_macaddr == NULL ||
2072  strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
2073  /* Nothing has changed */
2074  knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
2075  goto out_free;
2076  }
2077 
2078  /* Add nodeid into MAC address */
2079  memcpy(mac, macaddr_str, 12);
2080  snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
2081  instance->our_nodeid >> 8,
2082  instance->our_nodeid & 0xFF);
2083  knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
2084 
2085  if (name_res == CS_OK && name_str) {
2086  /* Reconfigure */
2087  if (instance->nozzle_name) {
2088  remove_nozzle_device(instance);
2089  free_nozzle(instance);
2090  }
2091 
2092  res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
2093  mac);
2094 
2095  instance->nozzle_name = strdup(name_str);
2096  instance->nozzle_ipaddr = strdup(ipaddr_str);
2097  instance->nozzle_prefix = strdup(prefix_str);
2098  instance->nozzle_macaddr = strdup(macaddr_str);
2099  if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
2100  !instance->nozzle_prefix) {
2101  knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
2102  /*
2103  * This 'free' will cause a complete reconfigure of the device next time we reload
2104  * but will also let the the current device keep working until then.
2105  * remove_nozzle() only needs the, statically-allocated, nozzle_handle
2106  */
2107  free_nozzle(instance);
2108  }
2109  }
2110 
2111 out_free:
2112  free(name_str);
2113  free(ipaddr_str);
2114  free(prefix_str);
2115  if (macaddr_res == CS_OK) {
2116  free(macaddr_str);
2117  }
2118 
2119  return res;
2120 }
2121 #endif // HAVE_LIBNOZZLE
#define INTERFACE_MAX
Definition: coroapi.h:88
unsigned int nodeid
Definition: coroapi.h:0
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:2
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
#define CS_PRI_NODE_ID
Definition: corotypes.h:59
cs_error_t qb_to_cs_error(int result)
qb_to_cs_error
cs_error_t
The cs_error_t enum.
Definition: corotypes.h:97
@ CS_ERR_BUSY
Definition: corotypes.h:107
@ CS_ERR_TRY_AGAIN
Definition: corotypes.h:103
@ CS_OK
Definition: corotypes.h:98
@ CS_ERR_INVALID_PARAM
Definition: corotypes.h:104
@ CS_ERR_LIBRARY
Definition: corotypes.h:99
@ CS_ERR_NOT_EXIST
Definition: corotypes.h:109
uint32_t value
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition: icmap.c:868
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
Definition: icmap.c:892
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition: icmap.c:1159
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem....
Definition: icmap.h:85
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition: icmap.c:856
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:72
#define LEAVE
Definition: logsys.h:325
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:75
#define LOGSYS_LEVEL_CRIT
Definition: logsys.h:71
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:73
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition: logsys.c:433
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:76
#define ENTER
Definition: logsys.h:324
const char * corosync_get_config_file(void)
Definition: main.c:204
void * user_data
Definition: sam.c:127
Structure passed as new_value and old_value in change callback.
Definition: icmap.h:91
char crypto_model[CONFIG_STRING_LEN_MAX]
Definition: totem.h:223
unsigned int private_key_len
Definition: totem.h:176
unsigned int node_id
Definition: totem.h:167
uint32_t knet_compression_threshold
Definition: totem.h:235
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:207
struct totem_interface * interfaces
Definition: totem.h:165
int crypto_changed
Definition: totem.h:231
unsigned int merge_timeout
Definition: totem.h:197
int knet_compression_level
Definition: totem.h:237
unsigned int net_mtu
Definition: totem.h:209
char knet_compression_model[CONFIG_STRING_LEN_MAX]
Definition: totem.h:233
unsigned int block_unlisted_ips
Definition: totem.h:245
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition: totem.h:174
int crypto_index
Definition: totem.h:229
unsigned int knet_pmtud_interval
Definition: totem.h:169
char crypto_cipher_type[CONFIG_STRING_LEN_MAX]
Definition: totem.h:225
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition: totem.h:205
char crypto_hash_type[CONFIG_STRING_LEN_MAX]
Definition: totem.h:227
int knet_ping_timeout
Definition: totem.h:93
int knet_link_priority
Definition: totem.h:91
struct totem_ip_address boundto
Definition: totem.h:84
uint16_t ip_port
Definition: totem.h:87
int knet_ping_interval
Definition: totem.h:92
uint8_t configured
Definition: totem.h:89
int knet_ping_precision
Definition: totem.h:94
int knet_pong_count
Definition: totem.h:95
int knet_transport
Definition: totem.h:96
struct totem_ip_address bindnet
Definition: totem.h:83
The totem_ip_address struct.
Definition: coroapi.h:111
unsigned int nodeid
Definition: coroapi.h:112
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:101
unsigned int target_nodeid
Definition: totem.h:132
void(* totemknet_target_set_completed)(void *context)
Definition: totemknet.c:119
pthread_mutex_t log_mutex
Definition: totemknet.c:176
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition: totemknet.c:115
struct crypto_instance * crypto_inst
Definition: totemknet.c:94
struct totem_config * totem_config
Definition: totemknet.c:161
qb_loop_timer_handle timer_netif_check_timeout
Definition: totemknet.c:165
char * link_status[INTERFACE_MAX]
Definition: totemknet.c:151
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition: totemknet.c:138
knet_handle_t knet_handle
Definition: totemknet.c:98
uint16_t ip_port[INTERFACE_MAX]
Definition: totemknet.c:155
int totemknet_log_level_debug
Definition: totemknet.c:132
void(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition: totemknet.c:104
struct totem_ip_address token_target
Definition: totemknet.c:163
qb_loop_timer_handle timer_merge_detect_timeout
Definition: totemknet.c:167
void(*) void knet_context)
Definition: totemknet.c:145
int totemknet_log_level_warning
Definition: totemknet.c:128
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition: totemknet.c:153
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition: totemknet.c:149
unsigned int merge_detect_messages_sent_before_timeout
Definition: totemknet.c:171
int send_merge_detect_message
Definition: totemknet.c:169
void(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition: totemknet.c:110
int totemknet_log_level_error
Definition: totemknet.c:126
int totemknet_log_level_security
Definition: totemknet.c:124
qb_loop_t * poll_handle
Definition: totemknet.c:96
int totemknet_log_level_notice
Definition: totemknet.c:130
const void * msg
Definition: totemknet.c:190
unsigned int msg_len
Definition: totemknet.c:191
struct totemknet_instance * instance
Definition: totemknet.c:192
typedef __attribute__
cfg_message_crypto_reconfig_phase_t
Definition: totem.h:154
@ CRYPTO_RECONFIG_PHASE_CLEANUP
Definition: totem.h:156
@ CRYPTO_RECONFIG_PHASE_ACTIVATE
Definition: totem.h:155
char type
Definition: totem.h:2
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition: totemip.c:306
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:123
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition: totemip.c:264
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:256
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1249
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition: totemknet.c:1359
int totemknet_finalize(void *knet_context)
Definition: totemknet.c:586
int totemknet_recv_flush(void *knet_context)
Definition: totemknet.c:1227
int totemknet_member_list_rebind_ip(void *knet_context)
Definition: totemknet.c:1544
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition: totemknet.c:1220
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1262
struct totemknet_instance * global_instance
Definition: totemknet.c:187
void totemknet_buffer_release(void *ptr)
Definition: totemknet.c:1215
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition: totemknet.c:491
void * totemknet_buffer_alloc(void)
Definition: totemknet.c:1209
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition: totemknet.c:1376
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition: totemknet.c:363
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition: totemknet.c:1498
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, void(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), void(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition: totemknet.c:973
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition: totemknet.c:1237
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition: totemknet.c:91
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition: totemknet.c:1670
#define OWN_INDEX_NONE
Definition: totemknet.c:488
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition: totemknet.c:1707
#define MSG_NOSIGNAL
Definition: totemknet.c:83
void totemknet_stats_clear(void *knet_context)
Definition: totemknet.c:1661
int totemknet_send_flush(void *knet_context)
Definition: totemknet.c:1232
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1286
#define knet_log_printf(level, format, args...)
Definition: totemknet.c:233
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition: totemknet.c:1293
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: totemknet.c:249
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition: totemknet.c:1572
int totemknet_crypto_reconfigure_phase(void *knet_context, struct totem_config *totem_config, cfg_message_crypto_reconfig_phase_t phase)
Definition: totemknet.c:1602
#define libknet_log_printf(level, format, args...)
Definition: totemknet.c:241
int totemknet_recv_mcast_empty(void *knet_context)
Definition: totemknet.c:1307
int totemknet_iface_check(void *knet_context)
Definition: totemknet.c:1276
struct totem_message_header header
Definition: totemsrp.c:0
struct srp_addr system_from
Definition: totemsrp.c:1
void stats_knet_add_handle(void)
Definition: stats.c:736
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:723
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition: stats.c:713