Re: [PATCH] log rendering in real time in audit-viewer

This code extends audit-viewer _ParserEventSource to produce UpdatableEventSource (unprivileged). I have not added this option to SourceDialog, so to see how updating works command line call may be used (python main.py -p -s /path/to/log) diff -u -X ex or_src/event_source.py oav/src/event_source.py --- or_src/event_source.py 2009-06-09 23:10:41.000000000 +0400 +++ oav/src/event_source.py 2015-03-25 16:09:59.933965077 +0300 @@ -19,7 +19,7 @@ import datetime import os.path import re - +import os import auparse __all__ = ('ClientEventSource', 'ClientWithRotatedEventSource', @@ -95,7 +95,7 @@ '''A source of audit events, reading from an auparse parser.''' def read_events(self, filters, wanted_fields, want_other_fields, - keep_raw_records): + keep_raw_records, direction = None, tab = None): '''Return a sequence of audit events read from parser. Use filters to select events. Store wanted_fields in event.fields, the @@ -265,6 +265,106 @@ def _create_parser(self): return auparse.AuParser(auparse.AUSOURCE_BUFFER, self.str) +class UpdatableEventSourceReader(): + + '''A separate reader of audit events, created for each tab of main window. + To be used with UpdatableEventSource. + + ''' + + def __init__(self, event_source, tab): + self.event_source = event_source + self.tab = tab + self.chunk_size = event_source.chunk_size + self.up_file = open(event_source.base_file) + self.bottom_file = open(event_source.base_file) + self.up_file.seek(0, 2) + self.up_pos = self.down_pos = self.up_file.tell() + + def read_down(self): + ''' read older lines from file starting from self.down_pos''' + if os.path.exists(self.bottom_file.name): + if os.stat(self.bottom_file.name).st_ino != os.fstat(self.bottom_file.fileno()).st_ino: + self.bottom_file.close() + self.tab.want_read_down = False + return '' + if self.down_pos <= self.event_source.avg_line_length * self.chunk_size: + self.bottom_file.seek(self.down_pos) + lines = self.bottom_file.read(self.down_pos) + try: + files = os.listdir(os.path.dirname(self.event_source.base_file)) + files = sorted(files, key = lambda x : os.stat(x).st_mtime) + filename = self.bottom_file.name + self.bottom_file.close() + self.bottom_file = open(files[files.index(filename) + 1]) + except: + self.tab.want_read_down = False + return '' + else: + self.bottom_file.seek(self.down_pos - self.event_source.avg_line_length * self.chunk_size) + lines = self.bottom_file.read(self.event_source.avg_line_length * self.chunk_size) + lines = lines[lines.find('\n') + 1:] + self.down_pos -= len(lines) + return lines + + def read_up(self): + '''try to read new lines if there any after the previous read''' + if os.path.exists(self.up_file.name): + if os.stat(self.up_file.name).st_ino != os.fstat(self.up_file.fileno()).st_ino: + self.up_file.close() + self.up_file = open(self.event_source.base_file) + self.up_pos = 0 + else: + self.up_file.close() + self.tab.want_read_up = False + return '' + self.up_file.seek(0, 2) + file_end_pos = self.up_file.tell() + if self.up_pos != file_end_pos: + self.up_file.seek(self.up_pos) + if file_end_pos - self.up_pos > self.event_source.avg_line_length*self.chunk_size*5: + lines = self.up_file.read(self.event_source.avg_line_length*self.chunk_size) + file_end_pos = self.up_file.tell() + else: + lines = self.up_file.read() + if lines.endswith('\n'): + self.up_pos = file_end_pos + else: + lines = lines[:lines.rfind('\n') + 1] + self.up_pos += len(lines) + return lines + return '' + + +class UpdatableEventSource(_ParserEventSource): + + def __init__(self, base_file, chunk_size = 50, avg_line_length = 74): + self.chunk_size = chunk_size + self.avg_line_length = avg_line_length + self.readers = {} + self.base_file = base_file + self.current_lines = '' + + def add_tab(self, tab): + self.readers[tab] = UpdatableEventSourceReader(self, tab) + + def remove_tab(self, tab): + del self.readers[tab] + + def read_events(self, filters, wanted_fields, want_other_fields, + keep_raw_records, direction, tab): + if direction == 'up': + self.current_lines = self.readers[tab].read_up() + else: + self.current_lines = self.readers[tab].read_down() + return _ParserEventSource.read_events(self, filters, wanted_fields, want_other_fields, + keep_raw_records) + + + def _create_parser(self): + return auparse.AuParser(auparse.AUSOURCE_BUFFER, self.current_lines) + + def check_expression(expr): '''Check expr. diff -u -X ex or_src/list_tab.py oav/src/list_tab.py --- or_src/list_tab.py 2009-12-19 10:00:00.000000000 +0300 +++ oav/src/list_tab.py 2015-03-25 15:42:07.797941743 +0300 @@ -28,6 +28,7 @@ from search_entry import SearchEntry from tab import Tab import util +import event_source __all__ = ('ListTab') @@ -130,7 +131,7 @@ date_column_label = '__audit_viewer_date' __list_number = 1 - def __init__(self, filters, main_window, will_refresh = False): + def __init__(self, filters, main_window, will_refresh = True): Tab.__init__(self, filters, main_window, 'list_vbox') # date_column_label == event date, None == all other columns @@ -157,6 +158,13 @@ util.connect_and_run(self.selection, 'changed', self.__selection_changed) + if isinstance(self.main_window.event_source, event_source.UpdatableEventSource): + self.main_window.event_source.add_tab(self) + self.want_read_up = True + self.want_read_down = True + self.__refresh_dont_read_events = False + self.refresh(True, 'up', True) + return self.__refresh_dont_read_events = will_refresh self.refresh() self.__refresh_dont_read_events = False @@ -191,20 +199,21 @@ % (util.filename_to_utf8(filename), e.strerror)) - def refresh(self): - event_sequence = self.__refresh_get_event_sequence() + def refresh(self, updatable = False, direction = None, init = False): + event_sequence = self.__refresh_get_event_sequence(direction, self) if event_sequence is None: return - - if self.filters: - t = _(', ').join(f.ui_text() for f in self.filters) - else: - t = _('None') - self.list_filter_label.set_text(t) - self.__refresh_update_tree_view() + if not updatable or init: + if self.filters: + t = _(', ').join(f.ui_text() for f in self.filters) + else: + t = _('None') + self.list_filter_label.set_text(t) + self.__refresh_update_tree_view() events = self.__refresh_collect_events(event_sequence) - self.__refresh_update_store(events) + self.__refresh_update_store(events, updatable, direction) + def report_on_view(self): self.main_window.new_report_tab(self.filters) @@ -462,7 +471,7 @@ for record in event.records for (key, value) in record.fields])) - def __refresh_get_event_sequence(self): + def __refresh_get_event_sequence(self, direction = None, tab = None): '''Return an event sequence (as if from self.main_window.read_events()). Return None on error. @@ -480,7 +489,7 @@ elif title is not self.date_column_label: wanted_fields.add(title) return self.main_window.read_events(self.filters, wanted_fields, - want_other_fields, True) + want_other_fields, True, direction, tab) def __refresh_update_tree_view(self): '''Update self.list_tree_view for current configuration. @@ -560,7 +569,32 @@ events.sort(key = lambda event: event[0], reverse = self.sort_reverse) return events - def __refresh_update_store(self, events): + def __insert_row(self, event, direction): + ''' insert new row with event into self.store preserving sort order''' + it = None + if not self.sort_by: + if self.sort_reverse: + if direction == 'up': + it = self.store.insert(0, event[1]) + else: + it = self.store.append(event[1]) + else: + if direction == 'down': + it = self.store.insert(0, event[1]) + else: + it = self.store.append(event[1]) + else: + sort_field = self.__field_columns.index(self.sort_by) + 1 + for i in range(len(self.store)): + if event[1][sort_field] <= self.store[i][sort_field] and self.sort_reverse or \ + event[1][sort_field] > self.store[i][sort_field] and not self.sort_reverse: + it = self.store.insert(i, event[1]) + break + if not it: + it = self.store.append(event[1]) + return it + + def __refresh_update_store(self, events, updatable = False, direction = None): '''Update self.store and related data. events is the result of self.__refresh_collect_events(). @@ -571,11 +605,15 @@ key = pos.event_key l = positions_for_event_key.setdefault(key, []) l.append(pos) - self.store.clear() + if not updatable: + self.store.clear() if (self.text_filter is None and len(positions_for_event_key) == 0): # Fast path for event in events: - self.store.append(event[1]) + if not updatable: + self.store.append(event[1]) + else: + self.__insert_row(event, direction) else: event_to_it = {} text_filter = self.text_filter @@ -604,7 +642,10 @@ or (self.__other_column_event_text(event_tuple[0]). find(self.text_filter) == -1))): continue - it = self.store.append(event_tuple) + if not updatable: + it = self.store.append(event_tuple) + else: + it = self.__insert_row(event, direction) event_id = event_tuple[0].id key = (event_id.serial, event_id.sec, event_id.milli) if key in positions_for_event_key: diff -u -X ex or_src/main.py oav/src/main.py --- or_src/main.py 2008-06-26 00:17:59.000000000 +0400 +++ oav/src/main.py 2015-03-25 15:31:29.663932132 +0300 @@ -29,6 +29,7 @@ from main_window import MainWindow import settings import util +import event_source _ = gettext.gettext @@ -48,12 +49,20 @@ help = _('do not attempt to start the privileged backend ' 'for reading system audit logs')) parser.set_defaults(unprivileged = False) + parser.add_option('-p', '--updatable', action = 'store_true', + dest = 'updatable', + help = _('read new lines from log ')) + parser.set_defaults(updatable = False) + parser.add_option('-s', '--source', type = 'string', + dest = 'source', + help = _('path to log file ')) (options, args) = parser.parse_args() gnome.init(settings.gettext_domain, settings.version) gtk.glade.bindtextdomain(settings.gettext_domain, settings.localedir) gtk.glade.textdomain(settings.gettext_domain) + ev_source = None if options.unprivileged: cl = None else: @@ -66,7 +75,9 @@ sys.exit(1) except client.ClientNotAvailableError: cl = None + if options.updatable: + ev_source = event_source.UpdatableEventSource(options.source) - w = MainWindow(cl) + w = MainWindow(cl, ev_source) if w.setup_initial_window(args): gtk.main() diff -u -X ex or_src/main_window.py oav/src/main_window.py --- or_src/main_window.py 2008-08-19 14:38:16.000000000 +0400 +++ oav/src/main_window.py 2015-03-23 19:13:04.399266459 +0300 @@ -135,6 +135,8 @@ ''' try: + if isinstance(self.event_source, event_source.UpdatableEventSource): + self.updater = gobject.idle_add(self.__refresh_all_tabs, True) if isinstance(self.event_source, event_source.EmptyEventSource): self.__event_error_report_only_one_push() if self.client is not None: @@ -246,7 +248,7 @@ return (filename, extension) def read_events(self, filters, wanted_fields, want_other_fields, - keep_raw_records): + keep_raw_records, direction = None, tab = None): '''Read audit events. Return a sequence of events, or None on error (without throwing @@ -262,7 +264,7 @@ try: return self.event_source.read_events(filters, wanted_fields, want_other_fields, - keep_raw_records) + keep_raw_records, direction, tab) except IOError, e: if (self.__event_error_report_only_one_depth == 0 or not self.__event_error_reported): @@ -381,16 +383,24 @@ '''End a region in which only one error message should be reported.''' self.__event_error_report_only_one_depth -= 1 - def __refresh_all_tabs(self): + def __refresh_all_tabs(self, updatable = False): '''Refresh all tabs, taking care to report errors only once.''' self.__event_error_report_only_one_push() try: - for page_num in xrange(self.main_notebook.get_n_pages()): - tab = self.__tab_objects[self.main_notebook - .get_nth_page(page_num)] - tab.refresh() + if not updatable: + for page_num in xrange(self.main_notebook.get_n_pages()): + tab = self.__tab_objects[self.main_notebook + .get_nth_page(page_num)] + tab.refresh() + else: + for page_num in xrange(self.main_notebook.get_n_pages()): + tab = self.__tab_objects[self.main_notebook + .get_nth_page(page_num)] + tab.refresh(True, 'up') + tab.refresh(True, 'down') finally: self.__event_error_report_only_one_pop() + return True def __menu_new_list_activate(self, *_): self.new_list_tab([]) ----- Исходное сообщение ----- От: "mitr" <mitr(a)redhat.com&gt; Кому: "Xeniya Muratova" <muratova(a)itsirius.su&gt; Копия: "linux-audit" <linux-audit(a)redhat.com&gt; Отправленные: Среда, 4 Март 2015 г 20:50:53 Тема: Re: log rendering in real time in audit-viewer Hello, Well, write the code? Something like inotify could be useful. There isn’t any hidden switch to enable these features, if that is what you are asking. As for performance, I may have missed something but I think I have squeezed as much as can be done with Python; improving performance further would very likely require a C extension. (audit-viewer is a PyGtk2 application, and at I’m afraid I don’t currently have plans to port it to GTK+3/gobject-introspection or do any other non-trivial work on the project, at least in the near term.) Mirek