space/scon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

* logstream introduction: now log-parsing can be done in stream fashion,

Browse Source 
but the only logstream implementation is still the filereader.
* analyze updated to be able to track existing or non-existing packets
* updates in packets: new packets
* update in log system: ability to append unprocessed lines to the last
packet
* chat system improvements
* stacktrace capturing experimental
This commit is contained in:
Gabor Körber 2015-04-15 00:31:02 +02:00
parent 1016085bed
commit 9bfdd1fb7a
8 changed files with 259 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
analyze.py
View File

@ -30,8 +30,10 @@ if __name__ == '__main__':
rex_combat = {}
rex_game = {}
rex_chat = {}
LOG_GOOD = True
for logf in coll.sessions:
logf.parse_files(['game.log', 'combat.log', 'chat.log'])
print "----- Log %s -----" % logf.idstr
if logf.combat_log:
for l in logf.combat_log.lines:
@ -39,9 +41,10 @@ if __name__ == '__main__':
#print l
rex_combat['dict'] = rex_combat.get('dict', 0) + 1
else:
if not l.unpack():
if not l.unpack() or LOG_GOOD:
rex_combat[l.__class__.__name__] = rex_combat.get(l.__class__.__name__, 0) + 1
if not isinstance(l, combat.UserEvent):
if not LOG_GOOD:
print l.values['log']
if logf.game_log:
for l in logf.game_log.lines:
@ -50,10 +53,11 @@ if __name__ == '__main__':
elif isinstance(l, str):
print l
else:
if l.unpack():
if l.unpack() and not LOG_GOOD:
pass
else:
rex_game[l.__class__.__name__] = rex_game.get(l.__class__.__name__, 0) + 1
if not LOG_GOOD:
print l.values['log']
if logf.chat_log:
for l in logf.chat_log.lines:
@ -62,11 +66,13 @@ if __name__ == '__main__':
elif isinstance(l, str):
print l
else:
if l.unpack():
if l.unpack() and not LOG_GOOD:
pass
else:
rex_chat[l.__class__.__name__] = rex_chat.get(l.__class__.__name__, 0) + 1
if not LOG_GOOD:
print l.values['log']
logf.clean(True)
print 'Analysis complete:'
print '#'*20+' RexCombat ' + '#' *20
print rex_combat

37
logs/base.py
View File

@ -21,3 +21,40 @@ class Log(object):
def explain(self):
''' returns a String readable by humans explaining this Log '''
return ''
def clean(self):
''' tell the log to forget all non-essential data '''
pass
def append(self, something):
''' returns true if this logfile wants an unrecognized log appended to it. '''
return False
class Stacktrace(Log):
''' Special Log to catch error reports '''
def __init__(self, values=None):
super(Stacktrace, self).__init__()
self.message = values or ''
if isinstance(self.message, dict):
self.message = self.message.get('log', '')
#self.trash = True
@classmethod
def is_handler(cls, log):
# do i have a system crash report beginning here?
if isinstance(log, basestring):
l = log.strip()
elif isinstance(log, dict):
l = log.get('log', '').strip()
else:
return False
if l.startswith('Stack trace:') or l.startswith('BitStream::DbgLog'):
return True
def clean(self):
self.message = ''
def append(self, something):
''' I take anything! '''
print "EXC: %s" % something
self.message = '%s\n%s' % (self.message, something)

33
logs/chat.py
View File

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
from logs.base import Log, L_WARNING
from logs.base import Log, L_WARNING, Stacktrace
import re
"""
Responsible for Chat Log.
@ -49,6 +49,10 @@ class ChatLog(Log):
''' returns a String readable by humans explaining this Log '''
return self.values.get('log', 'Unknown Chat Log')
def clean(self):
if 'log' in self.values.keys():
del self.values['log']
class SystemMessage(ChatLog):
matcher = re.compile(r"^<\s+SYSTEM>\s(?P<message>.*)")
@ -61,6 +65,12 @@ class SystemMessage(ChatLog):
def explain(self):
return '[SYSTEM]: %(message)s' % self.values
def append(self, something):
''' System Messages accept appends '''
if 'message' in self.values.keys():
self.values['message'] = '%s\n%s' % (self.values['message'], something)
return True
class PrivateMessageReceived(ChatLog):
@ -75,6 +85,12 @@ class PrivateMessageReceived(ChatLog):
def explain(self):
return '[From %(nickname)s]: %(message)s' % self.values
def append(self, something):
''' Private Messages accept appends '''
if 'message' in self.values.keys():
self.values['message'] = '%s\n%s' % (self.values['message'], something)
return True
class PrivateMessageSent(ChatLog):
matcher = re.compile(r"^<\s\s\s\sPRIVATE To\s\s>\[\s*(?P<nickname>[^\]]+)\]\s(?P<message>.*)")
@ -87,6 +103,12 @@ class PrivateMessageSent(ChatLog):
def explain(self):
return '[To %(nickname)s]: %(message)s' % self.values
def append(self, something):
''' Private Messages accept appends '''
if 'message' in self.values.keys():
self.values['message'] = '%s\n%s' % (self.values['message'], something)
return True
class ChatMessage(ChatLog):
matcher = re.compile(r"^<\s*#(?P<channel>[^>]+)>\[\s*(?P<nickname>[^\]]+)\]\s(?P<message>.*)")
@ -99,6 +121,14 @@ class ChatMessage(ChatLog):
def explain(self):
return '[%(channel)s] <%(nickname)s>: %(message)s' % self.values
def append(self, something):
''' ChatMessages accept appends '''
if not 'message' in self.values.keys():
print "Missing message? %s" % self.values
self.values['message'] = ''
self.values['message'] = '%s\n%s' % (self.values['message'], something)
return True
class ChatJoinChannel(ChatLog):
matcher = re.compile(r"^Join\schannel\s<\s*#(?P<channel>[^>]+)>")
@ -168,4 +198,5 @@ CHAT_LOGS = [
ChatServerDisconnect,
ChatJoinChannel,
ChatLeaveChannel,
Stacktrace,
]

36
logs/combat.py
View File

@ -21,7 +21,7 @@
The typical log entry
"""
import re
from base import Log, L_CMBT
from base import Log, L_CMBT, Stacktrace
class CombatLog(Log):
__slots__ = Log.__slots__ + [ '_match_id', 'values']
@ -93,7 +93,7 @@ class Spawn(CombatLog):
class Spell(CombatLog):
__slots__ = CombatLog.__slots__
matcher = re.compile(r"^Spell\s'(?P<spell_name>\w+)'\sby\s+(?P<source_name>.*)(?:\((?P<module_name>\w+)\)|)\stargets\((?P<target_num>\d+)\)\:(?:$|\s(?P<targets>.+))")
matcher = re.compile(r"^Spell\s'(?P<spell_name>\w+)'\sby\s+(?P<source_name>.*)(?:\((?P<module_name>\w+)\)|)\stargets\((?P<target_num>\d+)\)\:(?:\s(?P<targets>.+)|\s*)")
class Reward(CombatLog):
__slots__ = CombatLog.__slots__
@ -183,13 +183,39 @@ class GameEvent(CombatLog):
# unknown?
self.trash = True
def clean(self):
if 'log' in self.values.keys():
del self.values['log']
class PVE_Mission(CombatLog):
"""
PVE_Mission: 'bigship_building_normal'. start round 1/3
PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 1/3
PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 2/3
PVE_Mission: 'bigship_building_normal'. round 1/3. start wave 3/3
"""
__slots__ = CombatLog.__slots__
matcher = [] # @TODO: do this.
class Looted(CombatLog):
"""
Looted 'ow_Mineral_Info_T3_1' from 'LootCrate_Crystal1'
Looted 'Junk_Fuel7' from 'LootCrate_Fuel_Dynamic'
Looted 'ow_Afterburner_catalyst' from 'LootCrate_T3_Junk'
"""
__slots__ = CombatLog.__slots__
matcher = [] # @TODO: do this.
class UserEvent(CombatLog):
""" special class for combat logs that might be associated with the playing player """
__slots__ = CombatLog.__slots__
@classmethod
def _log_handler(cls, log):
if log.get('log', '').strip():
line = log.get('log', '').strip()
if line and 'earned medal' in line:
return True
elif line:
print line
return False
# Action?
@ -197,6 +223,8 @@ COMBAT_LOGS = [ Apply, Damage, Spawn, Spell, Reward, Participant, Rocket, Heal,
Gameplay, #?
Scores,
Killed, Captured, AddStack, Cancel,
GameEvent, UserEvent
PVE_Mission, Looted,
GameEvent, UserEvent,
Stacktrace,
]

7
logs/game.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from logs.base import Log, L_WARNING
from logs.base import Log, L_WARNING, Stacktrace
import re
"""
Interesting Lines:
@ -60,6 +60,10 @@ class GameLog(Log):
self.values = values
self.reviewed = False
def clean(self):
if 'log' in self.values.keys():
del self.values['log']
def unpack(self, force=False):
if self.reviewed and not force:
return True
@ -180,4 +184,5 @@ GAME_LOGS = [#SteamInitialization,
ClientInfo,
StartingLevel,
#LevelStarted,
Stacktrace,
]

84
logs/logfile.py
View File

@ -8,33 +8,25 @@
Each Logfile represents a physical file parsed, however theoretically, you can also parse arbitrary
data by setting the LogFile<instance>._data yourself.
"""
import re
from .base import Log
RE_SCLOG = r'^(?P<hh>\d{2,2})\:(?P<mm>\d{2,2})\:(?P<ss>\d{2,2})\.(?P<ns>\d{3,3})\s(?P<logtype>\s*[^\|\s]+\s*|\s+)\|\s(?P<log>.*)'
R_SCLOG = re.compile(RE_SCLOG)
from .logstream import LogStream
class LogFile(object):
class LogFile(LogStream):
def __init__(self, fname=None,
folder=None):
super(LogFile, self).__init__()
self.fname = fname
self.folder = folder # only for custom tagging.
self.lines = []
self._data = None
def read(self, fname=None):
fname = fname or self.fname
try:
f = open(fname, 'r')
self._data = f.read()
self.set_data(f.read())
finally:
f.close()
def set_data(self, data):
self._data = data
def _unset_data(self):
self._data = None
def filter(self, klasses):
ret = []
for line in self.lines:
@ -46,65 +38,23 @@ class LogFile(object):
def parse(self):
# parse _data if we still have no lines.
if self._data:
data_lines = self._data.replace('\r', '\n').replace('\n\n', '\n').split('\n')
lines = []
if self.has_data():
data_lines = self.get_data(
#).replace('\r', '\n'
).replace('\n\n', '\n'
).split('\n'
)
for line in data_lines:
line = self.pre_parse_line(line)
if not line:
continue
elif not isinstance(line, basestring):
lines.append(line)
continue
elif line.startswith('---'):
continue
else:
# get the timecode & logtype
m = R_SCLOG.match(line)
if m:
g = m.groupdict()
if 'logtype' in g.keys():
g['logtype'] = g['logtype'].strip()
lines.append(g)
else:
lines.append(line)
self.lines = lines
# try to identify (resolve) lines.
if self.lines:
lines = []
for line in self.lines:
l = line
if isinstance(line, basestring):
# Unknown Log?
pass
elif isinstance(line, dict):
# Unresolved Log.
l = self.resolve(line)
elif line is None:
# dafuq?
pass
else:
# might be an object?
pass
lines.append(l)
elif self.lines:
lines = self.lines
if lines:
for line in lines:
self._parse_line(line)
self.lines = lines
def resolve(self, line):
# line is a dict.
# try to find a class that is responsible for this log.
return line
def clean(self):
# cleans the logs by removing all non parsed packets.
lines = []
for l in self.lines:
if isinstance(l, Log):
if l.unpack():
if not getattr(l, 'trash', False):
lines.append(l)
else:
print type(l)
print l
self.lines = lines
self._unset_data()

111
logs/logstream.py
View File

@ -26,3 +26,114 @@
@see: monitor.py
@see: watchdog https://pypi.python.org/pypi/watchdog
"""
from .base import Log
import re
from logs.base import Stacktrace
RE_SCLOG = r'^(?P<hh>\d{2,2})\:(?P<mm>\d{2,2})\:(?P<ss>\d{2,2})\.(?P<ns>\d{3,3})\s(?P<logtype>\s*[^\|\s]+\s*|\s+)\|\s(?P<log>.*)'
R_SCLOG = re.compile(RE_SCLOG)
class LogStream(object):
def __init__(self):
self.lines = []
self._data = None
self._last_object = None
def add_to_queue(self, line):
# adds a line to the queue
pass
def new_packets(self, finish=False):
# yields new packets.
# processes the queue a bit.
# yields new packets, once they are done.
# watch out not to process the last packet until it has a follow up!
# finish: override and yield all packets to finish.
pass
def has_data(self):
if self._data:
return True
def set_data(self, data):
self._data = data
def get_data(self):
return self._data
def clean(self, remove_log=True):
# cleans the logs by removing all non parsed packets.
# remove_log: should i remove the raw log entry?
lines = []
for l in self.lines:
if isinstance(l, Log):
if l.unpack():
if not getattr(l, 'trash', False):
if remove_log:
l.clean()
lines.append(l)
else:
print type(l)
print l
self.lines = lines
self._unset_data()
data = property(set_data, get_data)
def _unset_data(self):
self._data = None
def pre_parse_line(self, line):
if not isinstance(line, basestring):
return line
elif line.startswith('---'):
return None
else:
# get the timecode & logtype
m = R_SCLOG.match(line)
if m:
g = m.groupdict()
if 'logtype' in g.keys():
g['logtype'] = g['logtype'].strip()
return g
else:
#if line:
# print line
return line
return None
def _parse_line(self, line):
# add the line to my lines.
if line is not None:
o = line
if isinstance(line, basestring):
# Unknown Log?
if not line:
return
if self._last_object is not None:
self._last_object.unpack()
if self._last_object.append(line):
return
# It might be a stacktrace. inject it./
if Stacktrace.is_handler(o):
o = Stacktrace(o)
self._last_object = o
else:
o = None
elif isinstance(line, dict):
# Unresolved Log.
o = self.resolve(line)
self._last_object = o
else:
self._last_object = o
if o is None:
self._last_object = None
return
self.lines.append(o)
def parse_line(self, line):
return self._parse_line(self.pre_parse_line(line))
def resolve(self, gd):
# gd is a dict.
# try to find a class that is responsible for this log.
return gd

12
logs/session.py
View File

@ -36,13 +36,13 @@ class LogFileSession(LogSession):
self.idstr = None # id string to identify this log instance.
self._error = False
def clean(self):
def clean(self, remove_log=True):
if self.combat_log:
self.combat_log.clean()
self.combat_log.clean(remove_log)
if self.game_log:
self.game_log.clean()
self.game_log.clean(remove_log)
if self.chat_log:
self.chat_log.clean()
self.chat_log.clean(remove_log)
def validate(self, contents=False):
@ -181,6 +181,10 @@ class LogSessionCollector(object):
sessions_dict[session.idstr] = session
return sessions_dict
def clean(self, remove_log=True):
for session in self.sessions:
session.clean(remove_log)