[PATCH] add optional heartbeat to remote protocol.

Fourth in a series. Ok, this is taking more patches than I originally thought ;-) (http://www.redhat.com/archives/linux-audit/2008-August/msg00198.html) This patch adds a heartbeat feature to both the server and the client. In the client, a configurable timeout triggers a heartbeat packet being sent to the server to ensure that the connection is still alive and that the server is OK. It has an admin-configurable action on failure. The server side is a bit more nebulous. It has an admin-defined maximum idle time, but at the moment all it does if the client is idle too long is print a message. Suggestions for what to do in that case? (search for fprintf in the patch below) DJ diff -x .svn -U 3 -r pristine/audisp/plugins/remote/audisp-remote.c trunk/audisp/plugins/remote/audisp-remote.c --- pristine/audisp/plugins/remote/audisp-remote.c 2008-08-29 12:13:30.000000000 -0400 +++ trunk/audisp/plugins/remote/audisp-remote.c 2008-09-02 22:43:20.000000000 -0400 @@ -31,6 +31,7 @@ #include <stdlib.h> #include <errno.h> #include <time.h> +#include <sys/select.h> #include <sys/socket.h> #include <netinet/in.h> #include <netinet/tcp.h> @@ -236,6 +237,10 @@ config.generic_warning_action, config.generic_warning_exe); } +static void send_heartbeat () +{ + relay_event (0, 0); +} int main(int argc, char *argv[]) { @@ -270,9 +275,30 @@ reload_config(); } + if (config.heartbeat_timeout > 0) { + fd_set rfd; + struct timeval tv; + int n; + + FD_ZERO (&rfd); + FD_SET (fileno (stdin), &rfd); + tv.tv_sec = config.heartbeat_timeout; + tv.tv_usec = 0; + + n = select (fileno (stdin) + 1, &rfd, NULL, &rfd, &tv); + + if (n <= 0) { + /* We attempt a hearbeat if select + fails, which may give us more + heartbeats than we need. This is + safer than too few heartbeats. */ + send_heartbeat (); + continue; + } + } - /* Now the event loop */ - while (fgets_unlocked(tmp, MAX_AUDIT_MESSAGE_LENGTH, stdin) && + /* Now read the [next] message. */ + if (fgets_unlocked(tmp, MAX_AUDIT_MESSAGE_LENGTH, stdin) && hup==0 && stop==0) { if (!suspend) { rc = relay_event(tmp, strnlen(tmp, @@ -436,6 +462,9 @@ { int rc; + if (len == 0) + return 0; + if (!transport_ok) if (init_transport ()) return -1; @@ -492,7 +521,9 @@ goto try_again; } - AUDIT_RMW_PACK_HEADER (header, 0, 0, len, sequence_id); + type = (s != NULL) ? AUDIT_RMW_TYPE_MESSAGE : AUDIT_RMW_TYPE_HEARTBEAT; + + AUDIT_RMW_PACK_HEADER (header, 0, type, len, sequence_id); rc = ar_write(sock, header, AUDIT_RMW_HEADER_SIZE); if (rc <= 0) { if (config.network_failure_action == FA_SYSLOG) @@ -502,13 +533,16 @@ goto try_again; } - rc = ar_write(sock, s, len); - if (rc <= 0) { - if (config.network_failure_action == FA_SYSLOG) - syslog(LOG_ERR, "connection to %s closed unexpectedly", - config.remote_server); - stop_transport(); - goto try_again; + if (s != NULL && len > 0) + { + rc = ar_write(sock, s, len); + if (rc <= 0) { + if (config.network_failure_action == FA_SYSLOG) + syslog(LOG_ERR, "connection to %s closed unexpectedly", + config.remote_server); + stop_transport(); + goto try_again; + } } rc = ar_read (sock, header, AUDIT_RMW_HEADER_SIZE); diff -x .svn -U 3 -r pristine/audisp/plugins/remote/remote-config.c trunk/audisp/plugins/remote/remote-config.c --- pristine/audisp/plugins/remote/remote-config.c 2008-08-29 12:13:30.000000000 -0400 +++ trunk/audisp/plugins/remote/remote-config.c 2008-08-29 15:53:37.000000000 -0400 @@ -72,6 +72,8 @@ remote_conf_t *config); static int format_parser(struct nv_pair *nv, int line, remote_conf_t *config); +static int heartbeat_timeout_parser(struct nv_pair *nv, int line, + remote_conf_t *config); static int network_retry_time_parser(struct nv_pair *nv, int line, remote_conf_t *config); static int max_tries_per_record_parser(struct nv_pair *nv, int line, @@ -100,8 +102,9 @@ {"queue_depth", depth_parser, 0 }, {"format", format_parser, 0 }, {"network_retry_time", network_retry_time_parser, 0 }, - {"max_tries_per_record", max_tries_per_record_parser, 0 }, - {"max_time_per_record", max_time_per_record_parser, 0 }, + {"max_tries_per_record", max_tries_per_record_parser, 0 }, + {"max_time_per_record", max_time_per_record_parser, 0 }, + {"heartbeat_timeout", heartbeat_timeout_parser, 0 }, {"network_failure_action", network_failure_action_parser, 0 }, {"disk_low_action", disk_low_action_parser, 0 }, {"disk_full_action", disk_full_action_parser, 0 }, @@ -161,6 +164,8 @@ config->max_tries_per_record = 3; config->max_time_per_record = 5; + config->heartbeat_timeout = 0; + #define IA(x,f) config->x##_action = f; config->x##_exe = NULL IA(network_failure, FA_STOP); IA(disk_low, FA_IGNORE); @@ -562,6 +567,12 @@ return parse_uint (nv, line, &(config->max_time_per_record), 1, INT_MAX); } +static int heartbeat_timeout_parser(struct nv_pair *nv, int line, + remote_conf_t *config) +{ + return parse_uint (nv, line, &(config->heartbeat_timeout), 0, INT_MAX); +} + /* * This function is where we do the integrated check of the audispd config * options. At this point, all fields have been read. Returns 0 if no diff -x .svn -U 3 -r pristine/audisp/plugins/remote/remote-config.h trunk/audisp/plugins/remote/remote-config.h --- pristine/audisp/plugins/remote/remote-config.h 2008-08-29 11:53:55.000000000 -0400 +++ trunk/audisp/plugins/remote/remote-config.h 2008-08-29 13:40:15.000000000 -0400 @@ -42,6 +42,7 @@ unsigned int network_retry_time; unsigned int max_tries_per_record; unsigned int max_time_per_record; + unsigned int heartbeat_timeout; failure_action_t network_failure_action; const char *network_failure_exe; diff -x .svn -U 3 -r pristine/docs/auditd.conf.5 trunk/docs/auditd.conf.5 --- pristine/docs/auditd.conf.5 2008-08-29 16:58:23.000000000 -0400 +++ trunk/docs/auditd.conf.5 2008-09-02 17:14:07.000000000 -0400 @@ -235,6 +235,13 @@ client use a priviledged port, specify .I 1-1023 for this parameter. +.TP +.I tcp_client_max_idle +This parameter indicates the number of seconds that a client may be +idle (i.e. no data from them at all) before auditd complains. Note +that this is a global setting, and must be higher than any individual +client heartbeat setting, preferably by a factor of two. The default +is zero, which disables this check. .SH NOTES In a CAPP environment, the audit trail is considered so important that access to system resources must be denied if an audit trail cannot be created. In this environment, it would be suggested that /var/log/audit be on its own partition. This is to ensure that space detection is accurate and that no other process comes along and consumes part of it. diff -x .svn -U 3 -r pristine/lib/private.h trunk/lib/private.h --- pristine/lib/private.h 2008-08-15 15:52:05.000000000 -0400 +++ trunk/lib/private.h 2008-08-29 16:08:33.000000000 -0400 @@ -91,6 +91,7 @@ /* Version 0 messages. */ #define AUDIT_RMW_TYPE_MESSAGE 0x00000000 +#define AUDIT_RMW_TYPE_HEARTBEAT 0x00000001 #define AUDIT_RMW_TYPE_ACK 0x40000000 #define AUDIT_RMW_TYPE_ENDING 0x40000001 #define AUDIT_RMW_TYPE_DISKLOW 0x50000001 diff -x .svn -U 3 -r pristine/src/auditd-config.c trunk/src/auditd-config.c --- pristine/src/auditd-config.c 2008-08-27 18:55:42.000000000 -0400 +++ trunk/src/auditd-config.c 2008-08-29 19:17:21.000000000 -0400 @@ -111,6 +111,8 @@ struct daemon_conf *config); static int tcp_client_ports_parser(struct nv_pair *nv, int line, struct daemon_conf *config); +static int tcp_client_max_idle_parser(struct nv_pair *nv, int line, + struct daemon_conf *config); static int sanity_check(struct daemon_conf *config); static const struct kw_pair keywords[] = @@ -138,6 +140,7 @@ {"tcp_listen_port", tcp_listen_port_parser, 0 }, {"tcp_listen_queue", tcp_listen_queue_parser, 0 }, {"tcp_client_ports", tcp_client_ports_parser, 0 }, + {"tcp_client_max_idle", tcp_client_max_idle_parser, 0 }, { NULL, NULL } }; @@ -242,6 +245,7 @@ config->tcp_listen_queue = 5; config->tcp_client_min_port = 0; config->tcp_client_max_port = TCP_PORT_MAX; + config->tcp_client_max_idle = 0; } static log_test_t log_test = TEST_AUDITD; @@ -1290,6 +1294,47 @@ return 0; } +static int tcp_client_max_idle_parser(struct nv_pair *nv, int line, + struct daemon_conf *config) +{ + const char *ptr = nv->value; + unsigned long i; + + audit_msg(LOG_DEBUG, "tcp_client_max_idle_parser called with: %s", + nv->value); + + /* check that all chars are numbers */ + for (i=0; ptr[i]; i++) { + if (!isdigit(ptr[i])) { + audit_msg(LOG_ERR, + "Value %s should only be numbers - line %d", + nv->value, line); + return 1; + } + } + + /* convert to unsigned int */ + errno = 0; + i = strtoul(nv->value, NULL, 10); + if (errno) { + audit_msg(LOG_ERR, + "Error converting string to a number (%s) - line %d", + strerror(errno), line); + return 1; + } + /* Check its range. While this value is technically + unlimited, it's limited by the kernel, and we limit it here + for sanity. */ + if (i > INT_MAX) { + audit_msg(LOG_ERR, + "Error - converted number (%s) is too large - line %d", + nv->value, line); + return 1; + } + config->tcp_client_max_idle = (unsigned int)i; + return 0; +} + /* * This function is where we do the integrated check of the audit config * options. At this point, all fields have been read. Returns 0 if no diff -x .svn -U 3 -r pristine/src/auditd-config.h trunk/src/auditd-config.h --- pristine/src/auditd-config.h 2008-08-15 15:52:05.000000000 -0400 +++ trunk/src/auditd-config.h 2008-09-02 22:47:33.000000000 -0400 @@ -74,6 +74,7 @@ unsigned long tcp_listen_queue; unsigned long tcp_client_min_port; unsigned long tcp_client_max_port; + unsigned long tcp_client_max_idle; }; void set_allow_links(int allow); @@ -90,6 +91,7 @@ void shutdown_config(void); void free_config(struct daemon_conf *config); +void periodic_reconfigure(void); #endif diff -x .svn -U 3 -r pristine/src/auditd-event.c trunk/src/auditd-event.c --- pristine/src/auditd-event.c 2008-08-29 11:53:55.000000000 -0400 +++ trunk/src/auditd-event.c 2008-09-02 22:40:12.000000000 -0400 @@ -154,28 +154,30 @@ rep->ack_socket = 0; rep->sequence_id = 0; - switch (consumer_data.config->log_format) - { - case LF_RAW: - buf = format_raw(&rep->reply, consumer_data.config); - break; - case LF_NOLOG: - return; - default: - audit_msg(LOG_ERR, - "Illegal log format detected %d", - consumer_data.config->log_format); - return; - } + if (rep->reply.type != AUDIT_DAEMON_RECONFIG) { + switch (consumer_data.config->log_format) + { + case LF_RAW: + buf = format_raw(&rep->reply, consumer_data.config); + break; + case LF_NOLOG: + return; + default: + audit_msg(LOG_ERR, + "Illegal log format detected %d", + consumer_data.config->log_format); + return; + } - len = strlen (buf); - if (len < MAX_AUDIT_MESSAGE_LENGTH - 1) - memcpy (rep->reply.msg.data, buf, len+1); - else - { - /* FIXME: is truncation the right thing to do? */ - memcpy (rep->reply.msg.data, buf, MAX_AUDIT_MESSAGE_LENGTH-1); - rep->reply.msg.data[MAX_AUDIT_MESSAGE_LENGTH-1] = 0; + len = strlen (buf); + if (len < MAX_AUDIT_MESSAGE_LENGTH - 1) + memcpy (rep->reply.msg.data, buf, len+1); + else + { + /* FIXME: is truncation the right thing to do? */ + memcpy (rep->reply.msg.data, buf, MAX_AUDIT_MESSAGE_LENGTH-1); + rep->reply.msg.data[MAX_AUDIT_MESSAGE_LENGTH-1] = 0; + } } rep->next = NULL; /* new packet goes at end - so zero this */ @@ -296,14 +298,29 @@ static unsigned int count = 0L; static void handle_event(struct auditd_consumer_data *data) { + char *buf = data->head->reply.msg.data; + if (data->head->reply.type == AUDIT_DAEMON_RECONFIG) { reconfigure(data); + switch (consumer_data.config->log_format) + { + case LF_RAW: + buf = format_raw(&data->head->reply, consumer_data.config); + break; + case LF_NOLOG: + return; + default: + audit_msg(LOG_ERR, + "Illegal log format detected %d", + consumer_data.config->log_format); + return; + } } else if (data->head->reply.type == AUDIT_DAEMON_ROTATE) { rotate_logs_now(data); } if (!logging_suspended) { - write_to_log(data->head->reply.msg.data, data); + write_to_log(buf, data); /* See if we need to flush to disk manually */ if (data->config->flush == FT_INCREMENTAL) { @@ -1203,6 +1220,11 @@ logging_suspended = saved_suspend; } + // TCP listener + + oconf->tcp_client_max_idle = nconf->tcp_client_max_idle; + periodic_reconfigure (); + // Next document the results srand(time(NULL)); seq_num = rand()%10000; diff -x .svn -U 3 -r pristine/src/auditd-listen.c trunk/src/auditd-listen.c --- pristine/src/auditd-listen.c 2008-08-29 16:58:23.000000000 -0400 +++ trunk/src/auditd-listen.c 2008-09-02 22:39:49.000000000 -0400 @@ -56,6 +56,7 @@ struct sockaddr_in addr; struct ev_tcp *next, *prev; int bufptr; + int client_active; unsigned char buffer [MAX_AUDIT_MESSAGE_LENGTH + 17]; } ev_tcp; @@ -96,6 +97,24 @@ free (client); } +static int ar_write (int sock, const void *buf, int len) +{ + int rc = 0, w; + while (len > 0) { + do { + w = write(sock, buf, len); + } while (w < 0 && errno == EINTR); + if (w < 0) + return w; + if (w == 0) + break; + rc += w; + len -= w; + buf = (const void *)((const char *)buf + w); + } + return rc; +} + static void client_message (struct ev_tcp *io, unsigned int length, unsigned char *header) { unsigned char ch; @@ -109,7 +128,12 @@ header[length] = 0; if (length > 1 && header[length-1] == '\n') header[length-1] = 0; - enqueue_formatted_event (header+AUDIT_RMW_HEADER_SIZE, io->io.fd, seq); + if (type == AUDIT_RMW_TYPE_HEARTBEAT) { + unsigned char ack[AUDIT_RMW_HEADER_SIZE]; + AUDIT_RMW_PACK_HEADER (ack, 0, AUDIT_RMW_TYPE_ACK, 0, seq); + ar_write (io->io.fd, ack, AUDIT_RMW_HEADER_SIZE); + } else + enqueue_formatted_event (header+AUDIT_RMW_HEADER_SIZE, io->io.fd, seq); header[length] = ch; } else { header[length] = 0; @@ -125,6 +149,8 @@ int i, r; int total_this_call = 0; + io->client_active = 1; + /* The socket is non-blocking, but we have a limited buffer size. In the event that we get a packet that's bigger than our buffer, we need to read it in multiple parts. Thus, we @@ -289,6 +315,8 @@ memset (client, 0, sizeof (struct ev_tcp)); + client->client_active = 1; + ev_io_init (&(client->io), auditd_tcp_client_handler, afd, EV_READ | EV_ERROR); ev_io_start (loop, &(client->io)); @@ -359,3 +387,19 @@ close_client (client_chain); } } + +void auditd_tcp_listen_check_idle (struct ev_loop *loop ) +{ + struct ev_tcp *ev; + int active; + + for (ev = client_chain; ev; ev = ev->next) { + active = ev->client_active; + ev->client_active = 0; + if (active) + continue; + + fprintf (stderr, "client %s idle too long\n", + sockaddr_to_ip (&(ev->addr))); + } +} diff -x .svn -U 3 -r pristine/src/auditd-listen.h trunk/src/auditd-listen.h --- pristine/src/auditd-listen.h 2008-08-27 18:55:42.000000000 -0400 +++ trunk/src/auditd-listen.h 2008-08-29 19:13:28.000000000 -0400 @@ -28,5 +28,6 @@ int auditd_tcp_listen_init ( struct ev_loop *loop, struct daemon_conf *config ); void auditd_tcp_listen_uninit ( struct ev_loop *loop ); +void auditd_tcp_listen_check_idle ( struct ev_loop *loop ); #endif diff -x .svn -U 3 -r pristine/src/auditd.c trunk/src/auditd.c --- pristine/src/auditd.c 2008-08-29 12:13:30.000000000 -0400 +++ trunk/src/auditd.c 2008-09-02 22:39:23.000000000 -0400 @@ -68,6 +68,7 @@ static struct auditd_reply_list *rep = NULL; static int hup_info_requested = 0, usr1_info_requested = 0; static char subj[SUBJ_LEN]; +static struct ev_periodic periodic_watcher; /* Local function prototypes */ int send_audit_event(int type, const char *str); @@ -430,6 +431,25 @@ } } +static void periodic_handler( struct ev_loop *loop, struct ev_periodic *per, int revents ) +{ + if (config.tcp_client_max_idle) + auditd_tcp_listen_check_idle (loop); +} + +void periodic_reconfigure () +{ + int i; + struct ev_loop *loop = ev_default_loop (EVFLAG_AUTO); + if (config.tcp_client_max_idle) { + ev_periodic_set (&periodic_watcher, ev_now (loop), + config.tcp_client_max_idle, NULL); + ev_periodic_start (loop, &periodic_watcher); + } else { + ev_periodic_stop (loop, &periodic_watcher); + } +} + int main(int argc, char *argv[]) { struct sigaction sa; @@ -697,6 +717,11 @@ ev_signal_init (&sigchld_watcher, child_handler, SIGCHLD); ev_signal_start (loop, &sigchld_watcher); + ev_periodic_init (&periodic_watcher, periodic_handler, + 0, config.tcp_client_max_idle, NULL); + if (config.tcp_client_max_idle) + ev_periodic_start (loop, &periodic_watcher); + if (auditd_tcp_listen_init (loop, &config)) { tell_parent (FAILURE); stop = 1;