* logstream introduction: now log-parsing can be done in stream fashion,

but the only logstream implementation is still the filereader.
* analyze updated to be able to track existing or non-existing packets
* updates in packets: new packets
* update in log system: ability to append unprocessed lines to the last
packet
* chat system improvements
* stacktrace capturing experimental

analyze.py

@@ -30,8 +30,10 @@ if __name__ == '__main__':
     rex_combat = {}
     rex_game = {}
     rex_chat = {}
+    LOG_GOOD = True
     for logf in coll.sessions:
         logf.parse_files(['game.log', 'combat.log', 'chat.log'])
+        
         print "----- Log %s -----" % logf.idstr
         if logf.combat_log:
             for l in logf.combat_log.lines:
@@ -39,10 +41,11 @@ if __name__ == '__main__':
                     #print l
                     rex_combat['dict'] = rex_combat.get('dict', 0) + 1
                 else:
-                    if not l.unpack():
+                    if not l.unpack() or LOG_GOOD:
                         rex_combat[l.__class__.__name__] = rex_combat.get(l.__class__.__name__, 0) + 1
                         if not isinstance(l, combat.UserEvent):
-                            print l.values['log']
+                            if not LOG_GOOD:
+                                print l.values['log']
         if logf.game_log:
             for l in logf.game_log.lines:
                 if isinstance(l, dict):
@@ -50,11 +53,12 @@ if __name__ == '__main__':
                 elif isinstance(l, str):
                     print l
                 else:
-                    if l.unpack():
+                    if l.unpack() and not LOG_GOOD:
                         pass
                     else:
                         rex_game[l.__class__.__name__] = rex_game.get(l.__class__.__name__, 0) + 1
-                        print l.values['log']
+                        if not LOG_GOOD:
+                            print l.values['log']
         if logf.chat_log:
             for l in logf.chat_log.lines:
                 if isinstance(l, dict):
@@ -62,11 +66,13 @@ if __name__ == '__main__':
                 elif isinstance(l, str):
                     print l
                 else:
-                    if l.unpack():
+                    if l.unpack() and not LOG_GOOD:
                         pass
                     else:
                         rex_chat[l.__class__.__name__] = rex_chat.get(l.__class__.__name__, 0) + 1
-                        print l.values['log']
+                        if not LOG_GOOD:
+                            print l.values['log']
+        logf.clean(True)
     print 'Analysis complete:'
     print '#'*20+' RexCombat ' + '#' *20
     print rex_combat

logs/base.py

@@ -21,3 +21,40 @@ class Log(object):
     def explain(self):
         ''' returns a String readable by humans explaining this Log '''
         return ''
+    
+    def clean(self):
+        ''' tell the log to forget all non-essential data '''
+        pass
+    
+    def append(self, something):
+        ''' returns true if this logfile wants an unrecognized log appended to it. '''
+        return False
+
+class Stacktrace(Log):
+    ''' Special Log to catch error reports '''
+    def __init__(self, values=None):
+        super(Stacktrace, self).__init__()
+        self.message = values or ''
+        if isinstance(self.message, dict):
+            self.message = self.message.get('log', '')
+        #self.trash = True
+    
+    @classmethod
+    def is_handler(cls, log):
+        # do i have a system crash report beginning here?
+        if isinstance(log, basestring):
+            l = log.strip()
+        elif isinstance(log, dict):
+            l = log.get('log', '').strip()
+        else:
+            return False
+        if l.startswith('Stack trace:') or l.startswith('BitStream::DbgLog'):
+            return True
+    
+    def clean(self):
+        self.message = ''
+    
+    def append(self, something):
+        ''' I take anything! '''
+        print "EXC: %s" % something
+        self.message = '%s\n%s' % (self.message, something)

logs/chat.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from logs.base import Log, L_WARNING
+from logs.base import Log, L_WARNING, Stacktrace
 import re
 """
 Responsible for Chat Log.
@@ -48,6 +48,10 @@ class ChatLog(Log):
     def explain(self):
         ''' returns a String readable by humans explaining this Log '''
         return self.values.get('log', 'Unknown Chat Log')
+    
+    def clean(self):
+        if 'log' in self.values.keys():
+            del self.values['log']
 
 class SystemMessage(ChatLog):
     matcher = re.compile(r"^<\s+SYSTEM>\s(?P<message>.*)")
@@ -60,6 +64,12 @@ class SystemMessage(ChatLog):
     
     def explain(self):
         return '[SYSTEM]: %(message)s' % self.values
+    
+    def append(self, something):
+        ''' System Messages accept appends '''
+        if 'message' in self.values.keys():
+            self.values['message'] = '%s\n%s' % (self.values['message'], something)
+            return True
 
     
 
@@ -74,6 +84,12 @@ class PrivateMessageReceived(ChatLog):
     
     def explain(self):
         return '[From %(nickname)s]: %(message)s' % self.values
+    
+    def append(self, something):
+        ''' Private Messages accept appends '''
+        if 'message' in self.values.keys():
+            self.values['message'] = '%s\n%s' % (self.values['message'], something)
+            return True
 
 class PrivateMessageSent(ChatLog):
     matcher = re.compile(r"^<\s\s\s\sPRIVATE To\s\s>\[\s*(?P<nickname>[^\]]+)\]\s(?P<message>.*)")
@@ -86,6 +102,12 @@ class PrivateMessageSent(ChatLog):
     
     def explain(self):
         return '[To %(nickname)s]: %(message)s' % self.values
+    
+    def append(self, something):
+        ''' Private Messages accept appends '''
+        if 'message' in self.values.keys():
+            self.values['message'] = '%s\n%s' % (self.values['message'], something)
+            return True
 
 class ChatMessage(ChatLog):
     matcher = re.compile(r"^<\s*#(?P<channel>[^>]+)>\[\s*(?P<nickname>[^\]]+)\]\s(?P<message>.*)")
@@ -98,6 +120,14 @@ class ChatMessage(ChatLog):
     
     def explain(self):
         return '[%(channel)s] <%(nickname)s>: %(message)s' % self.values
+    
+    def append(self, something):
+        ''' ChatMessages accept appends '''
+        if not 'message' in self.values.keys():
+            print "Missing message? %s" % self.values
+            self.values['message'] = ''    
+        self.values['message'] = '%s\n%s' % (self.values['message'], something)
+        return True 
 
 class ChatJoinChannel(ChatLog):
     matcher = re.compile(r"^Join\schannel\s<\s*#(?P<channel>[^>]+)>")
@@ -168,4 +198,5 @@ CHAT_LOGS = [
         ChatServerDisconnect,
         ChatJoinChannel,
         ChatLeaveChannel,
+        Stacktrace,
              ]

logs/combat.py

@@ -21,7 +21,7 @@
     The typical log entry
 """
 import re
-from base import Log, L_CMBT
+from base import Log, L_CMBT, Stacktrace
 
 class CombatLog(Log):
     __slots__ = Log.__slots__ + [ '_match_id', 'values']
@@ -93,7 +93,7 @@ class Spawn(CombatLog):
 
 class Spell(CombatLog):
     __slots__ = CombatLog.__slots__
-    matcher = re.compile(r"^Spell\s'(?P<spell_name>\w+)'\sby\s+(?P<source_name>.*)(?:\((?P<module_name>\w+)\)|)\stargets\((?P<target_num>\d+)\)\:(?:$|\s(?P<targets>.+))")
+    matcher = re.compile(r"^Spell\s'(?P<spell_name>\w+)'\sby\s+(?P<source_name>.*)(?:\((?P<module_name>\w+)\)|)\stargets\((?P<target_num>\d+)\)\:(?:\s(?P<targets>.+)|\s*)")
 
 class Reward(CombatLog):
     __slots__ = CombatLog.__slots__
@@ -182,14 +182,40 @@ class GameEvent(CombatLog):
                     return True
         # unknown?
         self.trash = True
+    
+    def clean(self):
+        if 'log' in self.values.keys():
+            del self.values['log']
+
+class PVE_Mission(CombatLog):
+    """
+    PVE_Mission: 'bigship_building_normal'. start round 1/3
+    PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 1/3
+    PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 2/3
+    PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 3/3
+    """
+    __slots__ = CombatLog.__slots__
+    matcher = [] # @TODO: do this.
+
+class Looted(CombatLog):
+    """
+    Looted 'ow_Mineral_Info_T3_1' from 'LootCrate_Crystal1'
+    Looted 'Junk_Fuel7' from 'LootCrate_Fuel_Dynamic'
+    Looted 'ow_Afterburner_catalyst' from 'LootCrate_T3_Junk'
+    """
+    __slots__ = CombatLog.__slots__
+    matcher = [] # @TODO: do this.
 
 class UserEvent(CombatLog):
     """ special class for combat logs that might be associated with the playing player """
     __slots__ = CombatLog.__slots__
     @classmethod
     def _log_handler(cls, log):
-        if log.get('log', '').strip():
+        line =  log.get('log', '').strip()
+        if line and 'earned medal' in line:
             return True
+        elif line:
+            print line
         return False
 
 # Action?
@@ -197,6 +223,8 @@ COMBAT_LOGS = [ Apply, Damage, Spawn, Spell, Reward, Participant, Rocket, Heal,
                Gameplay, #?
                Scores,
                Killed, Captured, AddStack, Cancel,
-               GameEvent, UserEvent
+               PVE_Mission, Looted,
+               GameEvent, UserEvent,
+               Stacktrace,
                ]
 

logs/game.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-from logs.base import Log, L_WARNING
+from logs.base import Log, L_WARNING, Stacktrace
 import re
 """
 Interesting Lines:
@@ -60,6 +60,10 @@ class GameLog(Log):
         self.values = values
         self.reviewed = False
     
+    def clean(self):
+        if 'log' in self.values.keys():
+            del self.values['log']
+    
     def unpack(self, force=False):
         if self.reviewed and not force:
             return True
@@ -180,4 +184,5 @@ GAME_LOGS = [#SteamInitialization,
              ClientInfo,
              StartingLevel,
              #LevelStarted,
+             Stacktrace,
              ]

logs/logfile.py

@@ -8,33 +8,25 @@
     Each Logfile represents a physical file parsed, however theoretically, you can also parse arbitrary
     data by setting the LogFile<instance>._data yourself.
 """
-import re
+from .logstream import LogStream
-from .base import Log
-RE_SCLOG = r'^(?P<hh>\d{2,2})\:(?P<mm>\d{2,2})\:(?P<ss>\d{2,2})\.(?P<ns>\d{3,3})\s(?P<logtype>\s*[^\|\s]+\s*|\s+)\|\s(?P<log>.*)'
-R_SCLOG = re.compile(RE_SCLOG) 
 
-class LogFile(object):
+
+class LogFile(LogStream):
     def __init__(self, fname=None,
                        folder=None):
+        super(LogFile, self).__init__()
         self.fname = fname
         self.folder = folder # only for custom tagging.
-        self.lines = []
         self._data = None
     
     def read(self, fname=None):
         fname = fname or self.fname
         try:
             f = open(fname, 'r')
-            self._data = f.read()
+            self.set_data(f.read())
         finally:
             f.close()
         
-    def set_data(self, data):
-        self._data = data
-    
-    def _unset_data(self):
-        self._data = None
-        
     def filter(self, klasses):
         ret = []
         for line in self.lines:
@@ -46,65 +38,23 @@ class LogFile(object):
     
     def parse(self):
         # parse _data if we still have no lines.
-        if self._data:
+        lines = []
-            data_lines = self._data.replace('\r', '\n').replace('\n\n', '\n').split('\n')
+        if self.has_data():
-            lines = []
+            data_lines = self.get_data(
+                        #).replace('\r', '\n'
+                        ).replace('\n\n', '\n'
+                        ).split('\n'
+                        )
             for line in data_lines:
+                line = self.pre_parse_line(line)
                 if not line:
                     continue
-                elif not isinstance(line, basestring):
+                else:
                     lines.append(line)
-                    continue
+        elif self.lines:
-                elif line.startswith('---'):
+            lines = self.lines
-                    continue
+        if lines:
-                else:
+            for line in lines:
-                    # get the timecode & logtype
+                self._parse_line(line)
-                    m = R_SCLOG.match(line)
+
-                    if m:
-                        g = m.groupdict()
-                        if 'logtype' in g.keys():
-                            g['logtype'] = g['logtype'].strip()
-                        lines.append(g)
-                    else:
-                        lines.append(line)
-            self.lines = lines
-        # try to identify (resolve) lines.
-        if self.lines:
-            lines = []
-            for line in self.lines:
-                l = line
-                if isinstance(line, basestring):
-                    # Unknown Log?
-                    pass
-                elif isinstance(line, dict):
-                    # Unresolved Log.
-                    l = self.resolve(line)
-                elif line is None:
-                    # dafuq?
-                    pass
-                else:
-                    # might be an object?
-                    pass
-                lines.append(l)
-                
-            self.lines = lines
-    
-    def resolve(self, line):
-        # line is a dict.
-        # try to find a class that is responsible for this log.
-        return line
-    
-    def clean(self):
-        # cleans the logs by removing all non parsed packets.
-        lines = []
-        for l in self.lines:
-            if isinstance(l, Log):
-                if l.unpack():
-                    if not getattr(l, 'trash', False):
-                        lines.append(l)
-                    else:
-                        print type(l)
-                        print l
-        self.lines = lines
-        self._unset_data()
 

logs/logstream.py

@@ -25,4 +25,115 @@
     combine it with the lookup for "watching files being changed", to create a program which listens to the logs live
     @see: monitor.py
     @see: watchdog https://pypi.python.org/pypi/watchdog
-"""
+"""
+from .base import Log
+import re
+from logs.base import Stacktrace
+RE_SCLOG = r'^(?P<hh>\d{2,2})\:(?P<mm>\d{2,2})\:(?P<ss>\d{2,2})\.(?P<ns>\d{3,3})\s(?P<logtype>\s*[^\|\s]+\s*|\s+)\|\s(?P<log>.*)'
+R_SCLOG = re.compile(RE_SCLOG) 
+
+class LogStream(object):
+    def __init__(self):
+        self.lines = []
+        self._data = None
+        self._last_object = None
+    
+    def add_to_queue(self, line):
+        # adds a line to the queue
+        pass
+    
+    def new_packets(self, finish=False):
+        # yields new packets.
+        # processes the queue a bit.
+        # yields new packets, once they are done.
+        # watch out not to process the last packet until it has a follow up!
+        # finish: override and yield all packets to finish.
+        pass
+    
+    def has_data(self):
+        if self._data:
+            return True
+    
+    def set_data(self, data):
+        self._data = data
+    
+    def get_data(self):
+        return self._data
+    
+    def clean(self, remove_log=True):
+        # cleans the logs by removing all non parsed packets.
+        # remove_log: should i remove the raw log entry?
+        lines = []
+        for l in self.lines:
+            if isinstance(l, Log):
+                if l.unpack():
+                    if not getattr(l, 'trash', False):
+                        if remove_log:
+                            l.clean()
+                        lines.append(l)
+                    else:
+                        print type(l)
+                        print l
+        self.lines = lines
+        self._unset_data()
+
+    data = property(set_data, get_data)
+    
+    def _unset_data(self):
+        self._data = None
+        
+    def pre_parse_line(self, line):
+        if not isinstance(line, basestring):
+            return line
+        elif line.startswith('---'):
+            return None
+        else:
+            # get the timecode & logtype
+            m = R_SCLOG.match(line)
+            if m:
+                g = m.groupdict()
+                if 'logtype' in g.keys():
+                    g['logtype'] = g['logtype'].strip()
+                return g
+            else:
+                #if line:
+                #    print line
+                return line
+        return None
+    
+    def _parse_line(self, line):
+        # add the line to my lines.
+        if line is not None:
+            o = line
+            if isinstance(line, basestring):
+                # Unknown Log?
+                if not line:
+                    return
+                if self._last_object is not None:
+                    self._last_object.unpack()
+                    if self._last_object.append(line):
+                        return
+                # It might be a stacktrace. inject it./
+                if Stacktrace.is_handler(o):
+                    o = Stacktrace(o)
+                    self._last_object = o
+                else:
+                    o = None
+            elif isinstance(line, dict):
+                # Unresolved Log.
+                o = self.resolve(line)
+                self._last_object = o
+            else:
+                self._last_object = o
+            if o is None:
+                self._last_object = None
+                return
+            self.lines.append(o)
+    
+    def parse_line(self, line):
+        return self._parse_line(self.pre_parse_line(line))
+    
+    def resolve(self, gd):
+        # gd is a dict.
+        # try to find a class that is responsible for this log.
+        return gd

logs/session.py

@@ -36,13 +36,13 @@ class LogFileSession(LogSession):
         self.idstr = None # id string to identify this log instance.
         self._error = False
     
-    def clean(self):
+    def clean(self, remove_log=True):
         if self.combat_log:
-            self.combat_log.clean()
+            self.combat_log.clean(remove_log)
         if self.game_log:
-            self.game_log.clean()
+            self.game_log.clean(remove_log)
         if self.chat_log:
-            self.chat_log.clean()
+            self.chat_log.clean(remove_log)
         
     
     def validate(self, contents=False):
@@ -180,6 +180,10 @@ class LogSessionCollector(object):
             if session.idstr and not session.idstr in sessions_dict.keys():
                 sessions_dict[session.idstr] = session
         return sessions_dict
+    
+    def clean(self, remove_log=True):
+        for session in self.sessions:
+            session.clean(remove_log)