Commit 1467fbbf authored by James Prewett's avatar James Prewett
Browse files

initial add of LoGS code

parents
LoGS
*.core
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
(in-package :org.prewett.LoGS)
#-asdf
(when +use-sql+
(error "ASDF package is not loaded!... bailing!~%"))
#+asdf
(when +use-sql+
(use-package :asdf)
;; this is munged!
(push "/Users/dl/project-fw/clsql-3.2.1/" asdf:*central-registry*)
(push "/Users/dl/project-fw/uffi-1.5.1/" asdf:*central-registry*)
(asdf:operate 'asdf:load-op :clsql)
(use-package :clsql)
(defclass buffered-sql-Follower (Data-Source)
((message-list :accessor message-list
:initarg :message-list
:initform ())
(buffer-size :accessor buffer-size
:initarg :buffer-size
:initform 1024)
(current-row :accessor current-row
:initarg :current-row
:initform 1)
(username :accessor username
:initarg :username)
(password :accessor password
:initarg :password)
(host :accessor host
:initarg :host)
(database :accessor database
:initarg :database)
(thequery :accessor thequery
:initarg :thequery)))
(defmethod initialize-instance :after ((buffered-sql-follower buffered-sql-follower) &rest rest)
(declare (ignore rest))
(when (not *default-database*)
(connect `(
,(host buffered-sql-follower)
,(database buffered-sql-follower)
,(username buffered-sql-follower)
,(password buffered-sql-follower)))))
(defmethod get-next-chunk ((buffered-sql-follower buffered-sql-follower))
(progn
(setf (message-list buffered-sql-follower)
(query (format () "~A where id >= ~A and id < ~A"
(thequery buffered-sql-follower)
(current-row buffered-sql-follower)
(+ (current-row buffered-sql-follower)
(buffer-size buffered-sql-follower)))))
(setf (current-row buffered-sql-follower)
(+ (current-row buffered-sql-follower) (buffer-size buffered-sql-follower)))))
(defmethod print-thequery ((buffered-sql-follower buffered-sql-follower))
(format () "~A where id >= ~A and id < ~A"
(thequery buffered-sql-follower)
(current-row buffered-sql-follower)
(+ (current-row buffered-sql-follower)
(buffer-size buffered-sql-follower))))
(defmethod get-line ((Buffered-Sql-Follower Buffered-Sql-Follower))
(progn
(when
(not (message-list buffered-sql-follower))
(get-next-chunk buffered-sql-follower))
(let ((first-message (car (message-list Buffered-Sql-Follower))))
(setf (message-list Buffered-Sql-Follower)
(cdr (message-list Buffered-Sql-Follower)))
first-message)))
(defmethod get-logline ((Buffered-Sql-Follower Buffered-Sql-Follower))
(let ((line (get-line Buffered-Sql-Follower)))
(when line
(make-instance 'message :message line))))
)
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
(in-package :org.prewett.LoGS)
(defclass Data-Source () ())
(defgeneric get-logline (data-source)
(:documentation
"Get-logline returns the next message from the data source or
nil if there is no such next message."))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
(in-package :org.prewett.LoGS)
;;; Paul Graham, On Lisp, p191
(defmacro aif (test-form then-form &optional else-form)
`(let ((it ,test-form))
(if it ,then-form ,else-form)))
(defclass File-Follower (Data-Source)
((Filename :accessor Filename :initarg :Filename)
(FileStream :accessor FileStream :initform ())
(Inode :accessor Inode :initform ())
(offset :accessor offset :initform ())))
(defgeneric cleanup (FF)
(:documentation "clean up a file-follower"))
(defmethod cleanup ((FF File-Follower))
(close (filestream ff)))
(defgeneric start-file-follower (file-follower)
(:documentation "get a file-follower all set up"))
(defmethod start-file-follower ((ff file-follower))
"Associate the file follower with the file it is supposed to be following."
(and
(probe-file (Filename ff))
;; get the inode of the file we are reading
(let ((ino (get-inode-from-filename (filename ff))))
(when ino
(setf (Inode ff) ino)
(when (filestream ff)
(close (filestream ff)))
(handler-case
(if
(fifo-p (filename ff))
(setf (Filestream ff)
(open-fifo (filename ff)))
(setf (Filestream ff)
(open (Filename ff) :direction :input)))
#+cmu
(KERNEL:SIMPLE-FILE-ERROR () (warn "no access to file"))
#+sbcl
(SB-IMPL::SIMPLE-FILE-PERROR () 'NIL))))))
(defgeneric set-file-follower-position (file-follower position)
(:documentation "set the offset into the file of the file-follower"))
(defmethod set-file-follower-position ((file-follower file-follower)
(position number))
(file-position
(filestream file-follower)
position))
(defmethod set-file-follower-position ((file-follower file-follower)
(position string))
(cond ((or (eql position :end) (equal position "end"))
(set-file-follower-position
file-follower
(get-file-length-from-filename (filename file-follower))))
((or (eql position :start) (equal position "start"))
(set-file-follower-position
file-follower
0))
(t (error "unknown position keyword: ~A~%" position))))
;; make sure the new file follower is attached to the file.
(defmethod initialize-instance :after ((ff File-Follower) &rest rest)
(declare (ignore rest))
(start-file-follower ff))
(defgeneric get-line (file-follower)
(:documentation "get the next raw chunk of input from the data source"))
(defmethod get-line ((ff file-follower))
"Return the next line of this file. We refuse to read eof. When we
have reached end of the file, we check to see if there is a new inode
associated with our filename. if there is, we start following that filename."
(declare (OPTIMIZE SPEED (DEBUG 0) (SAFETY 0)))
(if (and (filestream ff) (open-stream-p (filestream ff)))
(if (peek-char nil (filestream ff) nil)
(read-line (filestream ff) nil)
(let ((stat-inode (get-inode-from-filename (filename ff))))
(and (not (eql (inode ff) stat-inode))
(read-line (start-file-follower ff) nil))))
(let ((stat-inode (get-inode-from-filename (filename ff))))
(when (or (not (eql (inode ff) stat-inode))
(not (filestream ff)))
(aif (start-file-follower ff)
(read-line it nil))))))
(defmethod get-logline ((ff file-follower))
"Wrap the next line of the file associated with the file-follower inside of
a message."
(declare (OPTIMIZE SPEED (DEBUG 0) (SAFETY 0)))
(let* ((position (file-position (filestream ff)))
(line (get-line ff)))
(when line
(setf (offset ff) (file-position (filestream ff)))
(cond ((and *remember-file* *tag-messages*)
(make-instance 'message :message line :from-file (filename ff) :file-inode (inode ff) :position-in-file position :tag ()))
(*remember-file*
(make-instance 'message :message line :from-file (filename ff) :file-inode (inode ff) :position-in-file position))
(*tag-messages*
(make-instance 'message :message line :tag ()))
(t
(make-instance 'message :message line))))))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
(in-package :org.prewett.LoGS)
(defun fifo-p (filename)
(< 0
(logand 4096
(excl.osi:stat-mode (excl.osi:stat filename)))))
(defun get-file-length-from-filename (filename)
"Given a filename, return the number of bytes currently in the file."
(excl.osi::stat-size (excl.osi::stat filename)))
(defun open-fifo (filename)
(let ((fifofd
(unix-open
filename
(logior EXCL::*O-RDONLY*
EXCL::*O-NONBLOCK*)
#o444)))
(make-fd-stream fifofd :input t)))
(defun get-inode-from-filename (Filename)
"Given a filename, return the inode associated with that filename."
(when (probe-file filename)
(excl.osi:stat-ino (excl.osi:stat filename))))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
(in-package :org.prewett.LoGS)
(defun fifo-p (filename)
(< 0
(logand 4096
(excl.osi:stat-mode (excl.osi:stat filename)))))
(defun get-file-length-from-filename (filename)
"Given a filename, return the number of bytes currently in the file."
(excl.osi::stat-size (excl.osi::stat filename)))
(defun open-fifo (filename)
(let ((fifofd
(unix-open
filename
(logior EXCL::*O-RDONLY*
EXCL::*O-NONBLOCK*)
#o444)))
(make-fd-stream fifofd :input t)))
(defun get-inode-from-filename (Filename)
"Given a filename, return the inode associated with that filename."
(when (probe-file filename)
(excl.osi:stat-ino (excl.osi:stat filename))))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
;; This file contains low-level CMUCL specific code
(in-package :org.prewett.LoGS)
(import '(posix:file-stat-ino posix:file-stat-mode posix:file-stat-size))
(defun fifo-p (filename)
(not (eql 0
(logand 4096
(car
(last
(file-stat-mode (posix:file-stat filename))))))))
(defun get-file-length-from-filename (filename)
"Given a filename, return the number of bytes currently in the file."
(file-stat-size (posix:file-stat filename)))
(defun open-fifo (filename)
(error "unimplemented"))
(defun get-inode-from-filename (Filename)
"Given a filename, return the inode associated with that filename."
(and
(probe-file filename)
(file-stat-ino (posix:file-stat filename))))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
;; This file contains low-level CMUCL specific code
(in-package :org.prewett.LoGS)
(import '(unix:unix-stat unix:unix-open unix:o_rdonly unix:o_nonblock system:make-fd-stream))
(defun fifo-p (filename)
(if
(logand 4096
(nth-value 3
(unix-stat filename)))
t ()))
(defun get-file-length-from-filename (filename)
"Given a filename, return the number of bytes currently in the file."
(nth-value 8 (unix-stat filename)))
(defun open-fifo (filename)
(let ((fifofd
(unix-open
filename
(logior o_rdonly
o_nonblock)
#o444)))
(make-fd-stream fifofd :input t)))
(defun get-inode-from-filename (Filename)
"Given a filename, return the inode associated with that filename."
(when (probe-file filename)
(nth-value 2
(unix:unix-stat filename))))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
;; This file contains low-level CMUCL specific code
(in-package :org.prewett.LoGS)
(import '(unix:unix-stat unix:unix-open unix:o_rdonly unix:o_nonblock system:make-fd-stream))
(defun fifo-p (filename)
(if
(eql 0
(logand 4096
(nth-value 3
(unix-stat filename))))
() t))
(defun get-file-length-from-filename (filename)
"Given a filename, return the number of bytes currently in the file."
(nth-value 8 (unix-stat filename)))
(defun open-fifo (filename)
(let ((fifofd
(unix-open
filename
(logior o_rdonly
o_nonblock)
#o444)))
(make-fd-stream fifofd :input t)))
(defun get-inode-from-filename (Filename)
"Given a filename, return the inode associated with that filename."
(when (probe-file filename)
(nth-value 2
(unix:unix-stat filename))))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
;; This file contains low-level OpenMCL specific code
(in-package :org.prewett.LoGS)
(defun fifo-p (filename)
"is the named file a fifo?"
(< 0
(logand 4096
(nth-value 1 (ccl::%stat filename)))))
(defun get-file-length-from-filename (filename)
"Given a filename, return the number of bytes currently in the file."
#+openmcl
(nth-value 2 (ccl::%stat filename)))
(defun open-fifo (filename)
"open the fifo with the given filename"
(let ((fifofd
(fd-open
(filename ff)
(logior
#$O_RDONLY
#$O_NONBLOCK)
#o444)))
(make-fd-stream fifofd :input t)))
(defun get-inode-from-filename (Filename)
"Given a filename, return the inode associated with that filename."
(when
(probe-file filename)
(nth-value 4
(CCL::%STAT filename))))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
;; This file contains low-level OpenMCL specific code
(in-package :org.prewett.LoGS)
(defun fifo-p (filename)
"is the named file a fifo?"
(< 0
(logand 4096
(nth-value 1 (ccl::%stat filename)))))
(defun get-file-length-from-filename (filename)
"Given a filename, return the number of bytes currently in the file."
#+openmcl
(nth-value 2 (ccl::%stat filename)))
(defun open-fifo (filename)
"open the fifo with the given filename"
(let ((fifofd
(fd-open
(filename ff)
(logior
#$O_RDONLY
#$O_NONBLOCK)
#o444)))
(make-fd-stream fifofd :input t)))
(defun get-inode-from-filename (Filename)
"Given a filename, return the inode associated with that filename."
(when
(probe-file filename)
(nth-value 4
(CCL::%STAT filename))))
;;;; Logs extensible (common-lisp based) log/event analysis engine/language
;;;; Copyright (C) 2003-2018 James Earl Prewett
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
(in-package :org.prewett.LoGS)
(require :sb-posix)