From 9bfdd1fb7a1d868c0274c1c0ee7c3648d1a9ac13 Mon Sep 17 00:00:00 2001 From: Gabor Guzmics Date: Wed, 15 Apr 2015 00:31:02 +0200 Subject: [PATCH] * logstream introduction: now log-parsing can be done in stream fashion, but the only logstream implementation is still the filereader. * analyze updated to be able to track existing or non-existing packets * updates in packets: new packets * update in log system: ability to append unprocessed lines to the last packet * chat system improvements * stacktrace capturing experimental --- analyze.py | 18 +++++--- logs/base.py | 37 +++++++++++++++ logs/chat.py | 33 +++++++++++++- logs/combat.py | 36 +++++++++++++-- logs/game.py | 7 ++- logs/logfile.py | 90 ++++++++---------------------------- logs/logstream.py | 113 +++++++++++++++++++++++++++++++++++++++++++++- logs/session.py | 12 +++-- 8 files changed, 259 insertions(+), 87 deletions(-) diff --git a/analyze.py b/analyze.py index da76dbe..f8cf778 100644 --- a/analyze.py +++ b/analyze.py @@ -30,8 +30,10 @@ if __name__ == '__main__': rex_combat = {} rex_game = {} rex_chat = {} + LOG_GOOD = True for logf in coll.sessions: logf.parse_files(['game.log', 'combat.log', 'chat.log']) + print "----- Log %s -----" % logf.idstr if logf.combat_log: for l in logf.combat_log.lines: @@ -39,10 +41,11 @@ if __name__ == '__main__': #print l rex_combat['dict'] = rex_combat.get('dict', 0) + 1 else: - if not l.unpack(): + if not l.unpack() or LOG_GOOD: rex_combat[l.__class__.__name__] = rex_combat.get(l.__class__.__name__, 0) + 1 if not isinstance(l, combat.UserEvent): - print l.values['log'] + if not LOG_GOOD: + print l.values['log'] if logf.game_log: for l in logf.game_log.lines: if isinstance(l, dict): @@ -50,11 +53,12 @@ if __name__ == '__main__': elif isinstance(l, str): print l else: - if l.unpack(): + if l.unpack() and not LOG_GOOD: pass else: rex_game[l.__class__.__name__] = rex_game.get(l.__class__.__name__, 0) + 1 - print l.values['log'] + if not LOG_GOOD: + print l.values['log'] if logf.chat_log: for l in logf.chat_log.lines: if isinstance(l, dict): @@ -62,11 +66,13 @@ if __name__ == '__main__': elif isinstance(l, str): print l else: - if l.unpack(): + if l.unpack() and not LOG_GOOD: pass else: rex_chat[l.__class__.__name__] = rex_chat.get(l.__class__.__name__, 0) + 1 - print l.values['log'] + if not LOG_GOOD: + print l.values['log'] + logf.clean(True) print 'Analysis complete:' print '#'*20+' RexCombat ' + '#' *20 print rex_combat diff --git a/logs/base.py b/logs/base.py index d8d261b..f1e80c6 100644 --- a/logs/base.py +++ b/logs/base.py @@ -21,3 +21,40 @@ class Log(object): def explain(self): ''' returns a String readable by humans explaining this Log ''' return '' + + def clean(self): + ''' tell the log to forget all non-essential data ''' + pass + + def append(self, something): + ''' returns true if this logfile wants an unrecognized log appended to it. ''' + return False + +class Stacktrace(Log): + ''' Special Log to catch error reports ''' + def __init__(self, values=None): + super(Stacktrace, self).__init__() + self.message = values or '' + if isinstance(self.message, dict): + self.message = self.message.get('log', '') + #self.trash = True + + @classmethod + def is_handler(cls, log): + # do i have a system crash report beginning here? + if isinstance(log, basestring): + l = log.strip() + elif isinstance(log, dict): + l = log.get('log', '').strip() + else: + return False + if l.startswith('Stack trace:') or l.startswith('BitStream::DbgLog'): + return True + + def clean(self): + self.message = '' + + def append(self, something): + ''' I take anything! ''' + print "EXC: %s" % something + self.message = '%s\n%s' % (self.message, something) \ No newline at end of file diff --git a/logs/chat.py b/logs/chat.py index 7269ba3..de2a7ad 100644 --- a/logs/chat.py +++ b/logs/chat.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from logs.base import Log, L_WARNING +from logs.base import Log, L_WARNING, Stacktrace import re """ Responsible for Chat Log. @@ -48,6 +48,10 @@ class ChatLog(Log): def explain(self): ''' returns a String readable by humans explaining this Log ''' return self.values.get('log', 'Unknown Chat Log') + + def clean(self): + if 'log' in self.values.keys(): + del self.values['log'] class SystemMessage(ChatLog): matcher = re.compile(r"^<\s+SYSTEM>\s(?P.*)") @@ -60,6 +64,12 @@ class SystemMessage(ChatLog): def explain(self): return '[SYSTEM]: %(message)s' % self.values + + def append(self, something): + ''' System Messages accept appends ''' + if 'message' in self.values.keys(): + self.values['message'] = '%s\n%s' % (self.values['message'], something) + return True @@ -74,6 +84,12 @@ class PrivateMessageReceived(ChatLog): def explain(self): return '[From %(nickname)s]: %(message)s' % self.values + + def append(self, something): + ''' Private Messages accept appends ''' + if 'message' in self.values.keys(): + self.values['message'] = '%s\n%s' % (self.values['message'], something) + return True class PrivateMessageSent(ChatLog): matcher = re.compile(r"^<\s\s\s\sPRIVATE To\s\s>\[\s*(?P[^\]]+)\]\s(?P.*)") @@ -86,6 +102,12 @@ class PrivateMessageSent(ChatLog): def explain(self): return '[To %(nickname)s]: %(message)s' % self.values + + def append(self, something): + ''' Private Messages accept appends ''' + if 'message' in self.values.keys(): + self.values['message'] = '%s\n%s' % (self.values['message'], something) + return True class ChatMessage(ChatLog): matcher = re.compile(r"^<\s*#(?P[^>]+)>\[\s*(?P[^\]]+)\]\s(?P.*)") @@ -98,6 +120,14 @@ class ChatMessage(ChatLog): def explain(self): return '[%(channel)s] <%(nickname)s>: %(message)s' % self.values + + def append(self, something): + ''' ChatMessages accept appends ''' + if not 'message' in self.values.keys(): + print "Missing message? %s" % self.values + self.values['message'] = '' + self.values['message'] = '%s\n%s' % (self.values['message'], something) + return True class ChatJoinChannel(ChatLog): matcher = re.compile(r"^Join\schannel\s<\s*#(?P[^>]+)>") @@ -168,4 +198,5 @@ CHAT_LOGS = [ ChatServerDisconnect, ChatJoinChannel, ChatLeaveChannel, + Stacktrace, ] diff --git a/logs/combat.py b/logs/combat.py index fd22623..f0a0eba 100644 --- a/logs/combat.py +++ b/logs/combat.py @@ -21,7 +21,7 @@ The typical log entry """ import re -from base import Log, L_CMBT +from base import Log, L_CMBT, Stacktrace class CombatLog(Log): __slots__ = Log.__slots__ + [ '_match_id', 'values'] @@ -93,7 +93,7 @@ class Spawn(CombatLog): class Spell(CombatLog): __slots__ = CombatLog.__slots__ - matcher = re.compile(r"^Spell\s'(?P\w+)'\sby\s+(?P.*)(?:\((?P\w+)\)|)\stargets\((?P\d+)\)\:(?:$|\s(?P.+))") + matcher = re.compile(r"^Spell\s'(?P\w+)'\sby\s+(?P.*)(?:\((?P\w+)\)|)\stargets\((?P\d+)\)\:(?:\s(?P.+)|\s*)") class Reward(CombatLog): __slots__ = CombatLog.__slots__ @@ -182,14 +182,40 @@ class GameEvent(CombatLog): return True # unknown? self.trash = True + + def clean(self): + if 'log' in self.values.keys(): + del self.values['log'] + +class PVE_Mission(CombatLog): + """ + PVE_Mission: 'bigship_building_normal'. start round 1/3 + PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 1/3 + PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 2/3 + PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 3/3 + """ + __slots__ = CombatLog.__slots__ + matcher = [] # @TODO: do this. + +class Looted(CombatLog): + """ + Looted 'ow_Mineral_Info_T3_1' from 'LootCrate_Crystal1' + Looted 'Junk_Fuel7' from 'LootCrate_Fuel_Dynamic' + Looted 'ow_Afterburner_catalyst' from 'LootCrate_T3_Junk' + """ + __slots__ = CombatLog.__slots__ + matcher = [] # @TODO: do this. class UserEvent(CombatLog): """ special class for combat logs that might be associated with the playing player """ __slots__ = CombatLog.__slots__ @classmethod def _log_handler(cls, log): - if log.get('log', '').strip(): + line = log.get('log', '').strip() + if line and 'earned medal' in line: return True + elif line: + print line return False # Action? @@ -197,6 +223,8 @@ COMBAT_LOGS = [ Apply, Damage, Spawn, Spell, Reward, Participant, Rocket, Heal, Gameplay, #? Scores, Killed, Captured, AddStack, Cancel, - GameEvent, UserEvent + PVE_Mission, Looted, + GameEvent, UserEvent, + Stacktrace, ] diff --git a/logs/game.py b/logs/game.py index ea6bd6b..c889384 100644 --- a/logs/game.py +++ b/logs/game.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -from logs.base import Log, L_WARNING +from logs.base import Log, L_WARNING, Stacktrace import re """ Interesting Lines: @@ -60,6 +60,10 @@ class GameLog(Log): self.values = values self.reviewed = False + def clean(self): + if 'log' in self.values.keys(): + del self.values['log'] + def unpack(self, force=False): if self.reviewed and not force: return True @@ -180,4 +184,5 @@ GAME_LOGS = [#SteamInitialization, ClientInfo, StartingLevel, #LevelStarted, + Stacktrace, ] \ No newline at end of file diff --git a/logs/logfile.py b/logs/logfile.py index 8b1d658..43f8469 100644 --- a/logs/logfile.py +++ b/logs/logfile.py @@ -8,33 +8,25 @@ Each Logfile represents a physical file parsed, however theoretically, you can also parse arbitrary data by setting the LogFile._data yourself. """ -import re -from .base import Log -RE_SCLOG = r'^(?P\d{2,2})\:(?P\d{2,2})\:(?P\d{2,2})\.(?P\d{3,3})\s(?P\s*[^\|\s]+\s*|\s+)\|\s(?P.*)' -R_SCLOG = re.compile(RE_SCLOG) +from .logstream import LogStream -class LogFile(object): + +class LogFile(LogStream): def __init__(self, fname=None, folder=None): + super(LogFile, self).__init__() self.fname = fname self.folder = folder # only for custom tagging. - self.lines = [] self._data = None def read(self, fname=None): fname = fname or self.fname try: f = open(fname, 'r') - self._data = f.read() + self.set_data(f.read()) finally: f.close() - def set_data(self, data): - self._data = data - - def _unset_data(self): - self._data = None - def filter(self, klasses): ret = [] for line in self.lines: @@ -46,65 +38,23 @@ class LogFile(object): def parse(self): # parse _data if we still have no lines. - if self._data: - data_lines = self._data.replace('\r', '\n').replace('\n\n', '\n').split('\n') - lines = [] + lines = [] + if self.has_data(): + data_lines = self.get_data( + #).replace('\r', '\n' + ).replace('\n\n', '\n' + ).split('\n' + ) for line in data_lines: + line = self.pre_parse_line(line) if not line: continue - elif not isinstance(line, basestring): + else: lines.append(line) - continue - elif line.startswith('---'): - continue - else: - # get the timecode & logtype - m = R_SCLOG.match(line) - if m: - g = m.groupdict() - if 'logtype' in g.keys(): - g['logtype'] = g['logtype'].strip() - lines.append(g) - else: - lines.append(line) - self.lines = lines - # try to identify (resolve) lines. - if self.lines: - lines = [] - for line in self.lines: - l = line - if isinstance(line, basestring): - # Unknown Log? - pass - elif isinstance(line, dict): - # Unresolved Log. - l = self.resolve(line) - elif line is None: - # dafuq? - pass - else: - # might be an object? - pass - lines.append(l) - - self.lines = lines - - def resolve(self, line): - # line is a dict. - # try to find a class that is responsible for this log. - return line - - def clean(self): - # cleans the logs by removing all non parsed packets. - lines = [] - for l in self.lines: - if isinstance(l, Log): - if l.unpack(): - if not getattr(l, 'trash', False): - lines.append(l) - else: - print type(l) - print l - self.lines = lines - self._unset_data() + elif self.lines: + lines = self.lines + if lines: + for line in lines: + self._parse_line(line) + diff --git a/logs/logstream.py b/logs/logstream.py index cd2c0a9..7305d70 100644 --- a/logs/logstream.py +++ b/logs/logstream.py @@ -25,4 +25,115 @@ combine it with the lookup for "watching files being changed", to create a program which listens to the logs live @see: monitor.py @see: watchdog https://pypi.python.org/pypi/watchdog -""" \ No newline at end of file +""" +from .base import Log +import re +from logs.base import Stacktrace +RE_SCLOG = r'^(?P\d{2,2})\:(?P\d{2,2})\:(?P\d{2,2})\.(?P\d{3,3})\s(?P\s*[^\|\s]+\s*|\s+)\|\s(?P.*)' +R_SCLOG = re.compile(RE_SCLOG) + +class LogStream(object): + def __init__(self): + self.lines = [] + self._data = None + self._last_object = None + + def add_to_queue(self, line): + # adds a line to the queue + pass + + def new_packets(self, finish=False): + # yields new packets. + # processes the queue a bit. + # yields new packets, once they are done. + # watch out not to process the last packet until it has a follow up! + # finish: override and yield all packets to finish. + pass + + def has_data(self): + if self._data: + return True + + def set_data(self, data): + self._data = data + + def get_data(self): + return self._data + + def clean(self, remove_log=True): + # cleans the logs by removing all non parsed packets. + # remove_log: should i remove the raw log entry? + lines = [] + for l in self.lines: + if isinstance(l, Log): + if l.unpack(): + if not getattr(l, 'trash', False): + if remove_log: + l.clean() + lines.append(l) + else: + print type(l) + print l + self.lines = lines + self._unset_data() + + data = property(set_data, get_data) + + def _unset_data(self): + self._data = None + + def pre_parse_line(self, line): + if not isinstance(line, basestring): + return line + elif line.startswith('---'): + return None + else: + # get the timecode & logtype + m = R_SCLOG.match(line) + if m: + g = m.groupdict() + if 'logtype' in g.keys(): + g['logtype'] = g['logtype'].strip() + return g + else: + #if line: + # print line + return line + return None + + def _parse_line(self, line): + # add the line to my lines. + if line is not None: + o = line + if isinstance(line, basestring): + # Unknown Log? + if not line: + return + if self._last_object is not None: + self._last_object.unpack() + if self._last_object.append(line): + return + # It might be a stacktrace. inject it./ + if Stacktrace.is_handler(o): + o = Stacktrace(o) + self._last_object = o + else: + o = None + elif isinstance(line, dict): + # Unresolved Log. + o = self.resolve(line) + self._last_object = o + else: + self._last_object = o + if o is None: + self._last_object = None + return + self.lines.append(o) + + def parse_line(self, line): + return self._parse_line(self.pre_parse_line(line)) + + def resolve(self, gd): + # gd is a dict. + # try to find a class that is responsible for this log. + return gd \ No newline at end of file diff --git a/logs/session.py b/logs/session.py index 194f5fc..09f2382 100644 --- a/logs/session.py +++ b/logs/session.py @@ -36,13 +36,13 @@ class LogFileSession(LogSession): self.idstr = None # id string to identify this log instance. self._error = False - def clean(self): + def clean(self, remove_log=True): if self.combat_log: - self.combat_log.clean() + self.combat_log.clean(remove_log) if self.game_log: - self.game_log.clean() + self.game_log.clean(remove_log) if self.chat_log: - self.chat_log.clean() + self.chat_log.clean(remove_log) def validate(self, contents=False): @@ -180,6 +180,10 @@ class LogSessionCollector(object): if session.idstr and not session.idstr in sessions_dict.keys(): sessions_dict[session.idstr] = session return sessions_dict + + def clean(self, remove_log=True): + for session in self.sessions: + session.clean(remove_log)