Following a Log File with Twisted

Another POE component that I dearly miss in Twisted is POE::Wheel::FollowTail. FollowTail lets you monitor a growing log file, spewing events when new lines are received. It also gracefully handles log-rotated files by reopening it when it resets.

After some searching, I found a post in the Kragen-hacks mailing list that does something similar to the POE wheel.

I took that code, added line-buffering, wrapped it in a class, and added events for errors and file resets.

Here's the result:

# Twisted FollowTail  
# Mohit Muthanna   
#  
# A Twisted version of POE::Wheel::FollowTail. Adapted from  
# a post by Kragen Sitaker on the Kragen-hacks mailing list.  
#  
# http://lists.canonical.org/pipermail/kragen-hacks/2005-June/000413.html  

from twisted.internet import reactor  
from twisted.protocols import basic  
import os, stat  

class FollowTail:  
  from os import linesep as newline  
  __line_buffer = ""  

  def __init__( self, filename = None, seekend = True, delay = 1 ):  
    self.filename = filename  
    self.delay = delay  
    self.seekend = seekend  
    self.keeprunning = False  

  def fileIdentity( self, struct_stat ):  
    return struct_stat[stat.ST_DEV], struct_stat[stat.ST_INO]  

  def start( self ):  
    self.keeprunning = True  
    self.followTail()  

  def stop( self ):  
    self.keeprunning = False  

  def followTail( self, fileobj = None, fstat = None ):  
    if fileobj is None:  
      fileobj = open( self.filename )  
      if self.seekend: fileobj.seek( 0, 2 )  

    line = fileobj.read()  

    if line: self.dataReceived( line )  

    if fstat is None: fstat = os.fstat( fileobj.fileno() )  

    try: stat = os.stat( self.filename )  
    except: stat = fstat  

    if self.fileIdentity( stat ) != self.fileIdentity( fstat ):  
      fileobj = open( self.filename )  
      fstat = os.fstat( fileobj.fileno() )  
      self.fileReset()  


    if self.keeprunning:   
      reactor.callLater( self.delay, lambda: self.followTail( fileobj, fstat ) )  

  def dataReceived( self, data ):  
    # Fill buffer  
    self.__line_buffer += data  

    # Split lines  
    lines = self.__line_buffer.splitlines()  

    if not data.endswith( self.newline ):  
      self.__line_buffer = lines.pop()  
    else:  
      self.__line_buffer = ""  

    for line in lines:  
      self.lineReceived( line )  

  def lineReceived( self, line ):  
    """Override This"""  

  def fileReset( self ):  
    """Override This"""

Usage is quite straightforward. The FollowTail constructor has three parameters:

filename: Full path to file.
seekend: If set to "False", starts from beginning of file. Default "True".
delay: How often (in seconds) the file should be polled. Default 1 second.

When a new line is received, FollowTail calls the method "lineReceived" and supplies the line string. If the file has been rotated, it calls "fileReset". Both methods can be overridden in your program.

Here's some sample code that uses this class.

#!/usr/bin/env python  

from twisted.internet import reactor  
from followtail import FollowTail  

def onLine( line ):  
  print "Line: " + line  

def onReset():  
  print "File Rotated."  

filename = "/home/mohit/var/log/access-log"  
tailer = FollowTail( filename )  
tailer.lineReceived = onLine  
tailer.resetFile = onReset  
tailer.start()  

reactor.run()

Download: followtail.py.

Following a Log File with Twisted

1 comments (imported from blogger)