* implemented chat protocol, explanation mechanics, reviewed marker for

already unpacked logs
* started to disect log parsers for using streams in future
* Qt Client started
* logstream planned.

logs/base.py

@@ -5,12 +5,19 @@ L_NET = 'NET'
 L_CHAT = 'CHAT'
 
 class Log(object):
+    __slots__ = ['matcher', 'trash', 'reviewed']
     matcher = None
     trash = False
+    reviewed = False
     
     @classmethod
     def is_handler(cls, log):
         return False
     
-    def unpack(self):
+    def unpack(self, force=False):
+        ''' unpacks this log from its data and saves values '''
         pass
+    
+    def explain(self):
+        ''' returns a String readable by humans explaining this Log '''
+        return ''

logs/chat.py

@@ -1 +1,156 @@
 # -*- coding: utf-8 -*-
+from logs.base import Log, L_WARNING
+import re
+"""
+Responsible for Chat Log.
+
+ColorChart:
+between 33-33-33 and FF-33 FF-33 FF-33
+
+"""
+
+class ChatLog(Log):
+    __slots__ = ['matcher', 'trash', '_match_id', 'values']
+    
+    @classmethod
+    def is_handler(cls, log):
+        if log.get('logtype', None) == 'CHAT':
+            return cls._is_handler(log)
+        return False
+    
+    @classmethod
+    def _is_handler(cls, log):
+        return False
+    
+    def __init__(self, values=None):
+        self.values = values or {}
+    
+    def unpack(self, force=False):
+        if self.reviewed and not force:
+            return True
+        self._match_id = None
+        # unpacks the data from the values.
+        if hasattr(self, 'matcher') and self.matcher:
+            matchers = self.matcher
+            if not isinstance(matchers, list):
+                matchers = [matchers,]
+            for i, matcher in enumerate(matchers):
+                m = matcher.match(self.values.get('log', ''))
+                if m:
+                    self.values.update(m.groupdict())
+                    self._match_id = i
+                    self.reviewed = True
+                    return True
+        # unknown?
+        self.trash = True
+    
+    def explain(self):
+        ''' returns a String readable by humans explaining this Log '''
+        return self.values.get('log', 'Unknown Chat Log')
+
+
+class PrivateMessageReceived(ChatLog):
+    matcher = re.compile(r"^<\s\s\s\sPRIVATE From>\[\s*(?P<nickname>[^\]]+)\]\s(?P<message>.*)")
+    
+    @classmethod
+    def _is_handler(cls, log):
+        if log.get('log', '').lstrip().startswith('<    PRIVATE From>'):
+            return True
+        return False
+    
+    def explain(self):
+        return '[From %(nickname)s]: %(message)s' % self.values
+
+class PrivateMessageSent(ChatLog):
+    matcher = re.compile(r"^<\s\s\s\sPRIVATE To\s\s>\[\s*(?P<nickname>[^\]]+)\]\s(?P<message>.*)")
+    
+    @classmethod
+    def _is_handler(cls, log):
+        if log.get('log', '').lstrip().startswith('<    PRIVATE To  >'):
+            return True
+        return False
+    
+    def explain(self):
+        return '[To %(nickname)s]: %(message)s' % self.values
+
+class ChatMessage(ChatLog):
+    matcher = re.compile(r"^<\s*#(?P<channel>[^>]+)>\[\s*(?P<nickname>[^\]]+)\]\s(?P<message>.*)")
+    
+    @classmethod
+    def _is_handler(cls, log):
+        if log.get('log', '').lstrip().startswith('<'):
+            return True
+        return False
+    
+    def explain(self):
+        return '[%(channel)s] <%(nickname)s>: %(message)s' % self.values
+
+class ChatJoinChannel(ChatLog):
+    matcher = re.compile(r"^Join\schannel\s<\s*#(?P<channel>[^>]+)>")
+
+    @classmethod
+    def _is_handler(cls, log):
+        if log.get('log', '').lstrip().startswith('Join channel'):
+            return True
+        return False
+    
+    def explain(self):
+        return '[joined %(channel)s]' % self.values
+
+class ChatLeaveChannel(ChatLog):
+    matcher = re.compile(r"^Leave\schannel\s<\s*#(?P<channel>[^>]+)>")
+    
+    @classmethod
+    def _is_handler(cls, log):
+        if log.get('log', '').lstrip().startswith('Leave channel'):
+            return True
+        return False
+
+    def explain(self):
+        return '[left %(channel)s]' % self.values
+
+
+class ChatServerConnect(ChatLog):
+    # 00:12:47.668     CHAT| Connection to chat-server established
+    matcher = []
+    
+    @classmethod
+    def _is_handler(cls, log):
+        if log.get('log', '').lstrip().startswith('Connection to'):
+            return True
+        return False
+    
+    def unpack(self, force=False):
+        self.reviewed = True
+        return True
+    
+    def explain(self):
+        return '[connected]'
+
+
+class ChatServerDisconnect(ChatLog):
+    # 00:53:03.738     CHAT| Disconnect form chat-server (reason 0)
+    matcher = []
+    
+    @classmethod
+    def _is_handler(cls, log):
+        if log.get('log', '').lstrip().startswith('Disconnect'):
+            return True
+        return False
+    
+    def unpack(self, force=False):
+        self.reviewed = True
+        return True
+
+    def explain(self):
+        return '[disconnected]'
+    
+CHAT_LOGS = [
+        PrivateMessageReceived,
+        PrivateMessageSent,
+        ChatMessage, # private messages need to be before chatmessage.
+        ChatServerConnect,
+        ChatServerDisconnect,
+        ChatJoinChannel,
+        ChatLeaveChannel,
+             ]

logs/combat.py

@@ -24,7 +24,7 @@ import re
 from base import Log, L_CMBT
 
 class CombatLog(Log):
-    __slots__ = ['matcher', 'trash', '_match_id', 'values']
+    __slots__ = Log.__slots__ + [ '_match_id', 'values']
     @classmethod
     def _log_handler(cls, log):
         if log.get('log', '').strip().startswith(cls.__name__):
@@ -38,9 +38,11 @@ class CombatLog(Log):
         return False
     
     def __init__(self, values=None):
-        self.values = values
+        self.values = values or {}
     
-    def unpack(self):
+    def unpack(self, force=False):
+        if self.reviewed and not force:
+            return True
         self._match_id = None
         # unpacks the data from the values.
         if hasattr(self, 'matcher') and self.matcher:
@@ -52,10 +54,16 @@ class CombatLog(Log):
                 if m:
                     self.values.update(m.groupdict())
                     self._match_id = i
+                    self.reviewed = True
                     return True
         # unknown?
         self.trash = True
     
+    def explain(self):
+        ''' returns a String readable by humans explaining this Log '''
+        return self.values.get('log', 'Unknown Combat Log')
+
+                
 # @todo: where does this come from?
 class Action(CombatLog):
     __slots__ = CombatLog.__slots__
@@ -88,7 +96,12 @@ class Spell(CombatLog):
 
 class Reward(CombatLog):
     __slots__ = CombatLog.__slots__
-    matcher = re.compile(r"^Reward\s+(?P<name>[^\s]+)(?:\s(?P<ship_class>\w+)\s+|\s+)(?P<amount>\d+)\s(?P<reward_type>.*)\s+for\s(?P<reward_reason>.*)")
+    matcher = [
+        # ordinary reward:
+        re.compile(r"^Reward\s+(?P<name>[^\s]+)(?:\s(?P<ship_class>\w+)\s+|\s+)(?P<amount>\d+)\s(?P<reward_type>.*)\s+for\s(?P<reward_reason>.*)"),
+        # openspace reward (karma):
+        re.compile(r"^Reward\s+(?P<name>[^\s]+)(?:\s(?P<ship_class>\w+)\s+|\s+)\s+(?P<karma>[\+\-]\d+)\skarma\spoints\s+for\s(?P<reward_reason>.*)"),
+        ]
 
 class Participant(CombatLog):
     __slots__ = CombatLog.__slots__
@@ -125,7 +138,7 @@ class AddStack(CombatLog):
 
 class Cancel(CombatLog):
     __slots__ = CombatLog.__slots__
-    matcher = re.compile(r"^Cancel\saura\s'(?P<spell_name>\w+)'\sid\s(?P<id>\d+)\stype\s(?P<type>\w+)\sfrom\s'(?P<source_name>[^']+)'")
+    matcher = re.compile(r"^Cancel\saura\s'(?P<spell_name>\w+)'\sid\s(?P<id>\d+)\stype\s(?P<type>\w+)\sfrom\s'(?P<source_name>[^']*)'")
 
 class Scores(CombatLog):
     __slots__ = CombatLog.__slots__
@@ -149,7 +162,9 @@ class GameEvent(CombatLog):
             return True
         return False
     
-    def unpack(self):
+    def unpack(self, force=False):
+        if self.reviewed and not force:
+            return True
         self._match_id = None
         # unpacks the data from the values.
         # small override to remove trailing "="s in the matching.
@@ -162,6 +177,7 @@ class GameEvent(CombatLog):
                 if m:
                     self.values.update(m.groupdict())
                     self._match_id = i
+                    self.reviewed = True
                     return True
         # unknown?
         self.trash = True

logs/game.py

@@ -59,7 +59,9 @@ class GameLog(Log):
     def __init__(self, values=None):
         self.values = values
     
-    def unpack(self):
+    def unpack(self, force=False):
+        if self.reviewed and not force:
+            return True
         self._match_id = None
         # unpacks the data from the values.
         if hasattr(self, 'matcher') and self.matcher:
@@ -71,10 +73,15 @@ class GameLog(Log):
                 if m:
                     self.values.update(m.groupdict())
                     self._match_id = i
+                    self.reviewed = True
                     return True
         # unknown?
         self.trash = True
     
+    def explain(self):
+        ''' returns a String readable by humans explaining this Log '''
+        return self.values.get('log', 'Unknown Game Log')
+
 class WarningLog(Log):
     __slots__ = ['trash',]
     trash = True

logs/logfiles.py

@@ -7,6 +7,7 @@
 from logfile import LogFile
 from combat import COMBAT_LOGS
 from game import GAME_LOGS
+from chat import CHAT_LOGS
 
 class LogFileResolver(LogFile):
     ''' dynamic logfile resolver '''
@@ -38,3 +39,11 @@ class GameLogFile(LogFile):
                 return klass(line)
         return line
 
+class ChatLogFile(LogFile):
+    ''' Chat Log '''
+    def resolve(self, line):
+        for klass in CHAT_LOGS:
+            if klass.is_handler(line):
+                return klass(line)
+        return line
+

logs/logstream.py

@@ -0,0 +1,28 @@
+
+
+# LogStream
+"""
+    A LogStream is supposed to:
+     - parse data feeded into it.
+     - yield new objects
+     - remember errors
+    
+    LogStream.Initialize:
+     - initialize the logstream in some way.
+    
+    LogStream.Next:
+     - once initialized, read your stream until you can yield a new class
+     the next function reads the read-stream ahead.
+     empty lines are omitted
+     it tries to match the data into a new class and yields it
+     if it runs into trouble, it just outputs the line for now.
+    
+    InitializeString:
+     - init with a data blob
+     - nice for trying it on files
+    
+    @TODO: look at how file streams in python are implemented and find a good generic solution
+    combine it with the lookup for "watching files being changed", to create a program which listens to the logs live
+    @see: monitor.py
+    @see: watchdog https://pypi.python.org/pypi/watchdog
+"""

logs/session.py

@@ -2,14 +2,22 @@
     Logging Session.
 """
 import zipfile, logging, os
-from logfiles import CombatLogFile, GameLogFile
+from logfiles import CombatLogFile, GameLogFile, ChatLogFile
 
 class LogSession(object):
     """
-        The Log-Session is supposed to save one directory of logs.
+        A basic logsession.
+        deal with data as it comes along, and output interpretable data for the outside world.
+        
+    """
+    pass
+
+class LogFileSession(LogSession):
+    """
+        The Log-File-Session is supposed to save one directory of logs.
         It can parse its logs, and build up its internal structure into Battle Instances etc.
     """
-    VALID_FILES = ['combat.log', 'game.log', ] # extend this to other logs.
+    VALID_FILES = ['combat.log', 'game.log', 'chat.log' ] # extend this to other logs.
     
     def __init__(self, directory):
         ''' if directory is a file, it will be handled as a compressed folder '''
@@ -78,6 +86,11 @@ class LogSession(object):
                 self.game_log.read()
                 self.game_log.parse()
                 self.files_parsed.append('game.log')
+            if 'chat.log' in files and not 'chat.log' in self.files_parsed:
+                self.chat_log = ChatLogFile(os.path.join(self.directory, 'chat.log'))
+                self.chat_log.read()
+                self.chat_log.parse()
+                self.files_parsed.append('chat.log')
     
     def determine_owner(self):
         ''' determines the user in the parsed gamelog '''
@@ -138,7 +151,7 @@ class LogSessionCollector(object):
         for f in os.listdir(self.initial_directory):
             full_dir = os.path.join(self.initial_directory, f)
             if os.path.isdir(full_dir) or full_dir.lower().endswith('.zip'):
-                self.sessions.append(LogSession(full_dir))
+                self.sessions.append(LogFileSession(full_dir))
     
     def collect(self):
         sessions = []
@@ -167,8 +180,8 @@ class LogSessionCollector(object):
 
 
 if __name__ == '__main__':
-    l_raw = LogSession('D:\\Users\\g4b\\Documents\\My Games\\sc\\2014.05.17 15.50.28')
-    l_zip = LogSession('D:\\Users\\g4b\\Documents\\My Games\\sc\\2014.05.20 23.49.19.zip')
+    l_raw = LogFileSession('D:\\Users\\g4b\\Documents\\My Games\\sc\\2014.05.17 15.50.28')
+    l_zip = LogFileSession('D:\\Users\\g4b\\Documents\\My Games\\sc\\2014.05.20 23.49.19.zip')
     
     l_zip.parse_files()
     print l_zip.combat_log.lines

monitor.py

@@ -1,8 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 """
-    Monitor StarConflict Logs
-    
+    Monitor a StarConflict Log Directory.
 """
 import sys, os
 import time
@@ -10,41 +9,141 @@ import logging
 from watchdog.observers import Observer
 from watchdog.events import LoggingEventHandler
 
+class SconEventHandler(LoggingEventHandler):
+    def __init__(self, monitor, *args, **kwargs):
+        self.monitor = monitor
+        return super(SconEventHandler, self).__init__(*args, **kwargs)
+    
+    def on_moved(self, event):
+        super(SconEventHandler, self).on_moved(event)
+        if not event.is_directory:
+            self.monitor.close(event.src_path)
+        self.monitor.notify_event('moved', {'src': event.src_path, 
+                                            'is_dir': event.is_directory, 
+                                            'dest': event.dest_path})
+
+    def on_created(self, event):
+        super(SconEventHandler, self).on_created(event)
+        if not event.is_directory:
+            self.monitor.open(event.src_path)
+        self.monitor.notify_event('created', {'src': event.src_path, 
+                                              'is_dir': event.is_directory})
+
+    def on_deleted(self, event):
+        super(SconEventHandler, self).on_deleted(event)
+        if not event.is_directory:
+            self.monitor.close(event.src_path)
+        self.monitor.notify_event('deleted', {'src': event.src_path, 
+                                              'is_dir': event.is_directory})
+
+    def on_modified(self, event):
+        super(SconEventHandler, self).on_modified(event)
+        self.monitor.notify_event('modified', {'src': event.src_path, 
+                                              'is_dir': event.is_directory})
+        
+        
+
 class SconMonitor(object):
         
+    def notify(self, filename, lines):
+        # notify somebody, filename has a few lines.
+        # this is basicly the function you want to overwrite.
+        # @see: self.run for how to integrate monitor into your main loop.
+        if self.notifier is not None:
+            self.notifier.notify(filename, lines)
+    
+    def notify_event(self, event_type, data):
+        if self.notifier is not None:
+            self.notifier.notify_event(event_type, data)
+    
+    def __init__(self, path=None, notifier=None):
+        # if you initialize path directly, you lose success boolean.
+        # albeit atm. this is always true.
+        if path is not None:
+            self.initialize(path)
+        self.notifier = notifier
+    
     def initialize(self, path):
-        # initialize the monitor.
-        self.event_handler = LoggingEventHandler()
+        # initialize the monitor with a path to observe.
+        self.files = {}
+        self.event_handler = SconEventHandler(self)
         self.observer = Observer()
         self.observer.schedule(self.event_handler, 
                                path, 
                                recursive=True)
-        
-    
-        
-        # return true if successful
         return True
     
     def open(self, filename=None):
-        # open the logs.
-        pass
+        # open a logfile and add it to the read-list...
+        f = open(filename, 'r')
+        new_file = { 'file': f,
+                     'cursor': f.tell() }
+        self.files[filename] = new_file
+        return new_file
     
-    def check_running(self):
-        # maybe check if the exe is running? 
-        return True
+    def close(self, filename):
+        # close a single file by key, does not do anything if not found.
+        if filename in self.files.keys():
+            close_file = self.files.pop(filename)
+            close_file['file'].close()
+            del close_file
     
-    def run(self):
-        # everytime the logfile is updated, print it.
+    def close_all(self):
+        """ closes all open files in the monitor """
+        for key in self.files.keys():
+            self.close(key)
+    
+    def read_line(self, afile):
+        # read a single line in a file.
+        f = afile.get('file', None)
+        if f is None:
+            return
+        afile['cursor'] = f.tell()
+        line = f.readline()
+        if not line:
+            f.seek(afile['cursor'])
+            return None
+        else:
+            return line
+    
+    def do(self):
+        ''' Monitor main task handler, call this in your mainloop in ~1 sec intervals '''
+        # read all file changes.
+        for key, value in self.files.items():
+            lines = []
+            data = self.read_line(value)
+            while data is not None:
+                lines.append(data)
+                data = self.read_line(value)
+            if lines:
+                self.notify(key, lines)
+            # 
+    
+    def initialize_loop(self):
+        ''' initializes the main loop for the monitor '''
         self.observer.start()
-        try:
-            while True:
-                time.sleep(1)
-        except KeyboardInterrupt:
-            self.observer.stop()
+    
+    def break_loop(self):
+        ''' call this if you want to break the monitors tasks '''
+        self.observer.stop()
+    
+    def end_loop(self):
+        ''' always call this before exiting your main loop '''
+        self.close_all()
         self.observer.join()
         
-    def output(self, line):
-        print line
+    def run(self):
+        ''' Basic Standalone Main Loop implementation, do not call this in your app. '''
+        # if you want to run this on its own.
+        # everytime any logfile is updated, print it.
+        self.initialize_loop()
+        try:
+            while True:
+                self.do()
+                time.sleep(1)
+        except KeyboardInterrupt:
+            self.break_loop()
+        self.end_loop()
 
 if __name__ == '__main__':
     monitor = SconMonitor()

qscon.py

@@ -0,0 +1,103 @@
+"""
+    Main Entry Point / Exe File for QScon, the main log handler / monitor / manager app for log files.
+    
+"""
+import os, sys, logging
+import sys
+import urllib2
+from PyQt4 import QtCore, QtGui
+from monitor import SconMonitor
+from PyQt4.QtCore import QObject, pyqtSignal, pyqtSlot
+
+
+class SconMonitorThread(QtCore.QThread):
+    updated = pyqtSignal(str, list)
+    created = pyqtSignal(str, bool)
+    
+    def __init__(self, path):
+        QtCore.QThread.__init__(self)
+        self.path = path
+    
+    def notify(self, filename, lines):
+        self.updated.emit(filename, lines)
+        #self.mainwindow.notify_filelines(filename, lines)
+        #self.list_widget.addItem('%s\n%s' % (filename, ''.join(lines)))
+    
+    def notify_event(self, event_type, data):
+        if event_type == 'created':
+            self.created.emit(data['src'], data['is_dir'])
+
+    def run(self):
+        monitor = SconMonitor(self.path, notifier=self)
+        #self.list_widget.addItem('Starting to monitor: %s' % self.path)
+        monitor.run()
+
+class MainWindow(QtGui.QWidget):
+    def __init__(self):
+        super(MainWindow, self).__init__()        
+        self.tab_list = QtGui.QTabWidget()
+        self.tabs = {}
+        self.button = QtGui.QPushButton("Start")
+        self.button.clicked.connect(self.start_monitor)
+        layout = QtGui.QVBoxLayout()
+        layout.addWidget(self.button)
+        layout.addWidget(self.tab_list)
+        self.setLayout(layout)
+    
+    
+    def notify_filelines(self, filename, lines):
+        if filename not in self.tabs.keys():
+            new_tab = QtGui.QWidget()
+            new_tab.list_widget = QtGui.QListWidget()
+            layout = QtGui.QVBoxLayout(new_tab)
+            layout.addWidget(new_tab.list_widget)
+            self.tabs[filename] = new_tab
+            self.tab_list.addTab(new_tab, "%s" % os.path.split(str(filename))[-1])
+        self.tabs[filename].list_widget.addItem(''.join(lines)[:-1])
+    
+    def notify_created(self, filename, is_directory):
+        if is_directory:
+            print "Created Directory %s" % filename
+        else:
+            print "Created File %s" % filename
+
+    def start_monitor(self):
+        self.button.setDisabled(True)
+        paths = [os.path.join(os.path.expanduser('~'),'Documents','My Games','StarConflict','logs'),
+                 ]
+        self.threads = []
+        for path in paths:
+            athread = SconMonitorThread(path)
+            athread.updated.connect(self.notify_filelines)
+            athread.created.connect(self.notify_created)
+            self.threads.append(athread)
+            athread.start()
+
+########################################################################
+
+def _main():
+    app = QtGui.QApplication(sys.argv)
+    window = MainWindow()
+    window.resize(640, 480)
+    window.show()
+    return app.exec_()
+
+def main():
+    r = _main()
+    try:
+        import psutil #@UnresolvedImport
+    except ImportError:
+        logging.warning('Cannot import PsUtil, terminating without cleaning up threads explicitly.')
+        sys.exit(r)
+        
+    def kill_proc_tree(pid, including_parent=True):    
+        parent = psutil.Process(pid)
+        if including_parent:
+            parent.kill()
+    me = os.getpid()
+    kill_proc_tree(me)
+    sys.exit(r)
+
+if __name__ == "__main__":
+    main()
+