Branch 'devel' - cas-admin caslib/util.py
by Adam Stokes
cas-admin | 10 ++++++----
caslib/util.py | 2 +-
2 files changed, 7 insertions(+), 5 deletions(-)
New commits:
commit cae222a2b239319753fcd2cdae5408ccebbbc66e
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 15:53:37 2010 -0400
mimeinfo update
diff --git a/cas-admin b/cas-admin
index dd7b9e2..d5f6cc6 100755
--- a/cas-admin
+++ b/cas-admin
@@ -81,7 +81,7 @@ def queryWorker(input):
if not debug_IN_DB:
# add RPM to database
session = Session()
- debug_record = Debug(debug)
+ debug_record = Debug(debug, mime="application/x-rpm")
session.add(debug_record)
session.commit()
if util.testMimetype(debug):
@@ -100,13 +100,15 @@ def queryWorker(input):
session.add(timestamp_record)
# commit to database
session.commit()
- elif util.testMimetype(debug, "application/x-executable"):
+ elif util.testMimetype(debug, mime="application/x-executable"):
timestamp = coreTimestamp(debug, BUFFERSIZE)
timestamp_record = Timestamp(timestamp,
- debug,
+ debug[1:],
debug_record.debugId)
session.add(timestamp_record)
session.commit()
+ else:
+ genprint(_e[6])
session.close()
def queryLocalRpms():
@@ -126,7 +128,7 @@ def queryLocalRpms():
cmd = cmd + "\( -iregex %s" % (rpmfilter[0],)
if rpmfilter_len > 1:
concat_filter = ''.join([" -o -iregex %s " % (i,) for i in rpmfilter[1:]])
- cmd = cmd + + concat_filter + " \) "
+ cmd = cmd + concat_filter + " \) "
# append any file/dir exclusions
if "REGEXEXCLUDEDIR" in settings:
for i in settings["REGEXEXCLUDEDIR"].split(','):
diff --git a/caslib/util.py b/caslib/util.py
index c609519..b79e131 100644
--- a/caslib/util.py
+++ b/caslib/util.py
@@ -158,7 +158,7 @@ class Utility(object):
self.make_exe(crashExe)
return
- def testMimetype(self, fname, mime="application/x-rpm"):
+ def testMimetype(self, fname, mime):
""" tests file for mime info """
from caslib.core import casexecute
cmd = "file -i %s" % (fname,)
14 years
Branch 'devel' - cas-admin
by Adam Stokes
cas-admin | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
New commits:
commit bedb0c8620be6d77efc03cbcc5f274ceab220c9d
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 15:39:56 2010 -0400
cas-admin concat find expression properly depending on amount of syntaxes
diff --git a/cas-admin b/cas-admin
index a2eddc9..dd7b9e2 100755
--- a/cas-admin
+++ b/cas-admin
@@ -117,17 +117,16 @@ def queryLocalRpms():
"""
# create Queues
task_queue = Queue()
- cmd = "find -L %s -regextype %s" % (settings["KERNELS"],
- settings["REGEXTYPE"])
+ cmd = "find -L %s -regextype %s " % (settings["KERNELS"],
+ settings["REGEXTYPE"])
+ concat_filter = ""
if "RPMFILTER" in settings:
rpmfilter = settings["RPMFILTER"].split(',')
rpmfilter_len = len(rpmfilter)
+ cmd = cmd + "\( -iregex %s" % (rpmfilter[0],)
if rpmfilter_len > 1:
- cmd = "\( -iregex %s" % (rpmfilter[0],)
- cmd = cmd + [" -o %s " % (i,) for i in rpmfilter[1:]]
- cmd = cmd + "\)"
- else:
- cmd = "\( -iregex %s \)" % (rpmfilter[0],)
+ concat_filter = ''.join([" -o -iregex %s " % (i,) for i in rpmfilter[1:]])
+ cmd = cmd + + concat_filter + " \) "
# append any file/dir exclusions
if "REGEXEXCLUDEDIR" in settings:
for i in settings["REGEXEXCLUDEDIR"].split(','):
14 years
Branch 'devel' - cas-admin caslib/util.py
by Adam Stokes
cas-admin | 10 ++++++++--
caslib/util.py | 4 +++-
2 files changed, 11 insertions(+), 3 deletions(-)
New commits:
commit 86b2f79912ff2ec0b6eb82fa3dadf22afaa45e93
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 15:24:00 2010 -0400
fix cmd concat
diff --git a/cas-admin b/cas-admin
index d1d8a26..a2eddc9 100755
--- a/cas-admin
+++ b/cas-admin
@@ -120,8 +120,14 @@ def queryLocalRpms():
cmd = "find -L %s -regextype %s" % (settings["KERNELS"],
settings["REGEXTYPE"])
if "RPMFILTER" in settings:
- for i in settings["RPMFILTER"].split(','):
- cmd = cmd + " \( -iregex %s \) , " % (i,)
+ rpmfilter = settings["RPMFILTER"].split(',')
+ rpmfilter_len = len(rpmfilter)
+ if rpmfilter_len > 1:
+ cmd = "\( -iregex %s" % (rpmfilter[0],)
+ cmd = cmd + [" -o %s " % (i,) for i in rpmfilter[1:]]
+ cmd = cmd + "\)"
+ else:
+ cmd = "\( -iregex %s \)" % (rpmfilter[0],)
# append any file/dir exclusions
if "REGEXEXCLUDEDIR" in settings:
for i in settings["REGEXEXCLUDEDIR"].split(','):
diff --git a/caslib/util.py b/caslib/util.py
index 0be1d82..c609519 100644
--- a/caslib/util.py
+++ b/caslib/util.py
@@ -15,7 +15,8 @@ import os
import re
import sys
import logging
-from caslib.core import casexecute
+
+from caslib import error_messages as _e
if sys.version_info[:2] < (2,6):
from caslib.cas_subprocess import Popen, PIPE, call
@@ -159,6 +160,7 @@ class Utility(object):
def testMimetype(self, fname, mime="application/x-rpm"):
""" tests file for mime info """
+ from caslib.core import casexecute
cmd = "file -i %s" % (fname,)
results, err = casexecute(cmd, True)
if mime in results.strip():
14 years
Branch 'devel' - cas-admin caslib/util.py
by Adam Stokes
cas-admin | 4 ++--
caslib/util.py | 9 +++++++++
2 files changed, 11 insertions(+), 2 deletions(-)
New commits:
commit 8511a4fa98ae89f5c99ea1ce80a6bd36e960a9b6
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 15:01:44 2010 -0400
testMimetype method added
diff --git a/cas-admin b/cas-admin
index 36b2136..d1d8a26 100755
--- a/cas-admin
+++ b/cas-admin
@@ -84,7 +84,7 @@ def queryWorker(input):
debug_record = Debug(debug)
session.add(debug_record)
session.commit()
- if not coreIsCorefile(debug):
+ if util.testMimetype(debug):
genprint("(extracting) %-50s" % (os.path.basename(debug),))
tmpkernel = extractDebug(debug, dst)
# Sort through extracted debug for each type
@@ -100,7 +100,7 @@ def queryWorker(input):
session.add(timestamp_record)
# commit to database
session.commit()
- elif coreIsCorefile(debug):
+ elif util.testMimetype(debug, "application/x-executable"):
timestamp = coreTimestamp(debug, BUFFERSIZE)
timestamp_record = Timestamp(timestamp,
debug,
diff --git a/caslib/util.py b/caslib/util.py
index ab865e5..0be1d82 100644
--- a/caslib/util.py
+++ b/caslib/util.py
@@ -15,6 +15,7 @@ import os
import re
import sys
import logging
+from caslib.core import casexecute
if sys.version_info[:2] < (2,6):
from caslib.cas_subprocess import Popen, PIPE, call
@@ -156,3 +157,11 @@ class Utility(object):
self.make_exe(crashExe)
return
+ def testMimetype(self, fname, mime="application/x-rpm"):
+ """ tests file for mime info """
+ cmd = "file -i %s" % (fname,)
+ results, err = casexecute(cmd, True)
+ if mime in results.strip():
+ return True
+ return _e[6]
+
14 years
Branch 'devel' - cas-admin caslib/db.py cas.spec
by Adam Stokes
cas-admin | 66 ++++++++++++++++++++++++++++++++++-------------------------
cas.spec | 2 -
caslib/db.py | 52 ++++++++++------------------------------------
3 files changed, 52 insertions(+), 68 deletions(-)
New commits:
commit 5aa2c4583d1cfd8986b01c3a7d10ee72debe7d43
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 14:50:11 2010 -0400
support both RPM and extracted debugs
diff --git a/cas-admin b/cas-admin
index fcaac9d..36b2136 100755
--- a/cas-admin
+++ b/cas-admin
@@ -19,8 +19,13 @@
""" cas-admin - fingerprinting utility for cas
"""
import sys
-if sys.version_info[:2] < (2,6):
- raise SystemExit("Python >= 2.6 required")
+
+try:
+ from multiprocessing import Process, Queue, current_process, cpu_count, Lock
+except ImportError:
+ raise SystemExit('Unable to import multiprocessing module. This is shipped by' \
+ 'default in Python 2.6 and above. If using Python 2.4/2.5 a' \
+ 'backport is available at http://pypi.python.org/pypi/multiprocessing')
try:
from caslib.db import *
# Setup session for db interaction
@@ -35,9 +40,8 @@ import optparse
import datetime
import paramiko
import tempfile
-from multiprocessing import Process, Queue, current_process, cpu_count, Lock
-from caslib.core import casexecute, coreTimestamp
+from caslib.core import casexecute, coreTimestamp, coreIsCorefile
from caslib.util import Utility, genprint
from caslib.rpmutils import extractDebug
from caslib import error_messages as _e
@@ -70,30 +74,38 @@ util = Utility()
PROCESSES = cpu_count()
def queryWorker(input):
""" worker function to process rpm debugs """
- for rpm, dst in iter(input.get, 'STOP'):
+ for debug, dst in iter(input.get, 'STOP'):
session = Session()
- rpm_IN_DB = session.query(RPM).filter_by(rpmPath=rpm).first()
+ debug_IN_DB = session.query(Debug).filter_by(debugPath=debug).first()
session.close()
- if not rpm_IN_DB:
+ if not debug_IN_DB:
# add RPM to database
session = Session()
- rpm_record = RPM(rpm)
- session.add(rpm_record)
+ debug_record = Debug(debug)
+ session.add(debug_record)
session.commit()
- genprint("(extracting) %-50s" % (os.path.basename(rpm),))
- tmpkernel = extractDebug(rpm, dst)
- # Sort through extracted debug for each type
- # e.g. hugemem, PAE, smp, largesmp
- for item in tmpkernel:
- vmlinux = item.strip()
- debugKernel = os.path.normpath(vmlinux)
- timestamp = coreTimestamp(debugKernel, BUFFERSIZE)
- # add rpm id, debug, timestamp to database
+ if not coreIsCorefile(debug):
+ genprint("(extracting) %-50s" % (os.path.basename(debug),))
+ tmpkernel = extractDebug(debug, dst)
+ # Sort through extracted debug for each type
+ # e.g. hugemem, PAE, smp, largesmp
+ for item in tmpkernel:
+ vmlinux = item.strip()
+ debugKernel = os.path.normpath(vmlinux)
+ timestamp = coreTimestamp(debugKernel, BUFFERSIZE)
+ # add id, debug, timestamp to database
+ timestamp_record = Timestamp(timestamp,
+ debugKernel,
+ debug_record.debugId)
+ session.add(timestamp_record)
+ # commit to database
+ session.commit()
+ elif coreIsCorefile(debug):
+ timestamp = coreTimestamp(debug, BUFFERSIZE)
timestamp_record = Timestamp(timestamp,
- debugKernel,
- rpm_record.rpmId)
+ debug,
+ debug_record.debugId)
session.add(timestamp_record)
- # commit to database
session.commit()
session.close()
@@ -108,15 +120,15 @@ def queryLocalRpms():
cmd = "find -L %s -regextype %s" % (settings["KERNELS"],
settings["REGEXTYPE"])
if "RPMFILTER" in settings:
- for i in settings["RPMFILTER"].split():
- cmd = cmd + " \( -iregex %s \) " % (i,)
+ for i in settings["RPMFILTER"].split(','):
+ cmd = cmd + " \( -iregex %s \) , " % (i,)
# append any file/dir exclusions
if "REGEXEXCLUDEDIR" in settings:
- for i in settings["REGEXEXCLUDEDIR"].split():
- cmd = cmd + " \( -not -path %i \) " % (i,)
+ for i in settings["REGEXEXCLUDEDIR"].split(','):
+ cmd = cmd + " \( -not -path %s \) " % (i,)
if "REGEXEXCLUDEFILE" in settings:
- for i in settings["REGEXEXCLUDEFILE"].split():
- cmd = cmd + " \( -not -name %i \) " % (i,)
+ for i in settings["REGEXEXCLUDEFILE"].split(','):
+ cmd = cmd + " \( -not -name %s \) " % (i,)
results, err = casexecute(cmd, True)
dst = tempfile.mkdtemp(prefix='cas-')
for rpm in results.split():
diff --git a/cas.spec b/cas.spec
index 7deb8b0..dc3abe3 100644
--- a/cas.spec
+++ b/cas.spec
@@ -11,7 +11,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
BuildArch: noarch
Url: http://fedorahosted.org/cas
BuildRequires: python-devel
-%if 0%{?rhel}
+%if 0%{?rhel} <= 5
Requires: python-sqlite
%endif
Requires: python-paramiko
diff --git a/caslib/db.py b/caslib/db.py
index a3c5f2b..e3d490a 100644
--- a/caslib/db.py
+++ b/caslib/db.py
@@ -38,16 +38,16 @@ class Core(Base):
def __repr__(self):
return "<Core('%s','%s')>" % (self.coreFile, self.coreHash)
-class RPM(Base):
- __tablename__ = 'rpms'
- rpmId = Column(Integer, primary_key=True)
- rpmPath = Column(String(255))
+class Debug(Base):
+ __tablename__ = 'debugs'
+ debugId = Column(Integer, primary_key=True)
+ debugPath = Column(String(255))
- def __init__(self, rpmPath):
- self.rpmPath = rpmPath
+ def __init__(self, debugPath):
+ self.debugPath = debugPath
def __repr__(self):
- return "<RPM('%s')>" % (self.rpmPath,)
+ return "<Debug('%s')>" % (self.debugPath,)
class Job(Base):
__tablename__ = 'jobs'
@@ -72,52 +72,24 @@ class Job(Base):
self.vmcore,
self.path,
self.msg)
-class JobArchive(Base):
- """ Primarily for tracking """
- __tablename__ = 'jobarchive'
- jobId = Column(Integer, primary_key=True)
- identifier = Column(String(100))
- email = Column(String(40))
- vmcore = Column(String(255))
- path = Column(String(255))
- msg = Column(String(255))
- created = Column(DateTime)
- ipaddr = Column(Integer)
-
- def __init__(self, identifier, email, vmcore, ipaddr, path=None, msg=None):
- self.identifier = identifier
- self.email = email
- self.vmcore = vmcore
- self.path = path
- self.msg = msg
- self.ipaddr = ipaddr
-
- def __repr__(self):
- return "<Job('%s','%s','%s','%s','%s')>" % (self.identifier,
- self.email,
- self.vmcore,
- self.path,
- self.msg,
- self.ipaddr)
-
class Timestamp(Base):
__tablename__ = 'timestamps'
stampId = Column(Integer, primary_key=True)
stampKey = Column(String(255))
debugPath = Column(String(255))
- rpmId = Column(Integer, ForeignKey("rpms.rpmId"))
+ debugId = Column(Integer, ForeignKey("debugs.debugId"))
- rpm = relationship(RPM, backref=backref('timestamps'))
+ debug = relationship(Debug, backref=backref('timestamps'))
- def __init__(self, stampKey, debugPath, rpmId):
+ def __init__(self, stampKey, debugPath, debugId):
self.stampKey = stampKey
self.debugPath = debugPath
- self.rpmId = rpmId
+ self.debugId = debugId
def __repr__(self):
return "<Timestamp('%s','%s','%s')>" % (self.stampKey,
self.debugPath,
- self.rpmId)
+ self.debugId)
class Server(Base):
__tablename__ = 'servers'
14 years
Branch 'devel' - cas-admin cas.conf
by Adam Stokes
cas-admin | 20 ++++++++++++++------
cas.conf | 11 ++++++++++-
2 files changed, 24 insertions(+), 7 deletions(-)
New commits:
commit 785ae78fbe551a9437f8bc9e50bdeb800de8a6c6
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 12:31:48 2010 -0400
added feature for defining multiple searches, exclusions (directories, files)
diff --git a/cas-admin b/cas-admin
index cdc23a9..fcaac9d 100755
--- a/cas-admin
+++ b/cas-admin
@@ -68,7 +68,7 @@ util = Utility()
# Gather number of cpu's
PROCESSES = cpu_count()
-def queryWorker(input, mutex):
+def queryWorker(input):
""" worker function to process rpm debugs """
for rpm, dst in iter(input.get, 'STOP'):
session = Session()
@@ -105,17 +105,25 @@ def queryLocalRpms():
"""
# create Queues
task_queue = Queue()
- cmd = "find -L %s -regextype %s -iregex %s" % \
- (settings["KERNELS"],
- settings["REGEXTYPE"],
- settings["RPMFILTER"])
+ cmd = "find -L %s -regextype %s" % (settings["KERNELS"],
+ settings["REGEXTYPE"])
+ if "RPMFILTER" in settings:
+ for i in settings["RPMFILTER"].split():
+ cmd = cmd + " \( -iregex %s \) " % (i,)
+ # append any file/dir exclusions
+ if "REGEXEXCLUDEDIR" in settings:
+ for i in settings["REGEXEXCLUDEDIR"].split():
+ cmd = cmd + " \( -not -path %i \) " % (i,)
+ if "REGEXEXCLUDEFILE" in settings:
+ for i in settings["REGEXEXCLUDEFILE"].split():
+ cmd = cmd + " \( -not -name %i \) " % (i,)
results, err = casexecute(cmd, True)
dst = tempfile.mkdtemp(prefix='cas-')
for rpm in results.split():
task_queue.put((rpm, dst))
for i in range(PROCESSES):
- Process(target=queryWorker, args=(task_queue, mutex)).start()
+ Process(target=queryWorker, args=(task_queue,)).start()
for i in range(PROCESSES):
task_queue.put('STOP')
diff --git a/cas.conf b/cas.conf
index 72a585f..0c76527 100644
--- a/cas.conf
+++ b/cas.conf
@@ -25,9 +25,17 @@ kernels=/mnt/kernels
# set -regextype see `man find`
regexType=posix-extended
+# set directories to exclude separated by comma
+# uses shell globbing
+regexExcludeDir=*data*,*src*
+
+# set filenames to exclude separated by comma
+# uses shell globbing
+# regexExcludeFile=*tmp*,*\.*rcfiles*
+
# Compose -regextype regular expression for determining what
# kernel debug rpms you wish to search for
-rpmFilter=[^data].*kerne.+-debuginfo-[0-9].*\.rpm
+rpmFilter=.*debuginfo.*
# debug level (DEBUG, INFO)
debugLevel=DEBUG
@@ -65,3 +73,4 @@ autoPurge=no
# Valid options are {number, None, or blank}
# If set to None or blank default will be ~515M
# buffersize=None
+
14 years
Branch 'devel' - cas-admin cas.conf doc/index.rst
by Adam Stokes
cas-admin | 19 ++++++++++---------
cas.conf | 7 +++++--
doc/index.rst | 4 ++++
3 files changed, 19 insertions(+), 11 deletions(-)
New commits:
commit 3ac5745a0e08c3b9d7bb0b22c250352ae6e0c006
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 11:58:39 2010 -0400
add flexibilty to posix regular expression syntax
diff --git a/cas-admin b/cas-admin
index 75c071d..cdc23a9 100755
--- a/cas-admin
+++ b/cas-admin
@@ -35,7 +35,7 @@ import optparse
import datetime
import paramiko
import tempfile
-from multiprocessing import Process, Queue, current_process, cpu_count
+from multiprocessing import Process, Queue, current_process, cpu_count, Lock
from caslib.core import casexecute, coreTimestamp
from caslib.util import Utility, genprint
@@ -68,11 +68,12 @@ util = Utility()
# Gather number of cpu's
PROCESSES = cpu_count()
-def queryWorker(input):
+def queryWorker(input, mutex):
""" worker function to process rpm debugs """
for rpm, dst in iter(input.get, 'STOP'):
session = Session()
rpm_IN_DB = session.query(RPM).filter_by(rpmPath=rpm).first()
+ session.close()
if not rpm_IN_DB:
# add RPM to database
session = Session()
@@ -94,7 +95,7 @@ def queryWorker(input):
session.add(timestamp_record)
# commit to database
session.commit()
- session.close()
+ session.close()
def queryLocalRpms():
""" query debug kernels located on the filesystem
@@ -104,22 +105,22 @@ def queryLocalRpms():
"""
# create Queues
task_queue = Queue()
- # Uses emacs regex -- see `man find`
- cmd = "find -L %s -iregex %s" % (settings["KERNELS"],
- settings["RPMFILTER"])
+ cmd = "find -L %s -regextype %s -iregex %s" % \
+ (settings["KERNELS"],
+ settings["REGEXTYPE"],
+ settings["RPMFILTER"])
results, err = casexecute(cmd, True)
dst = tempfile.mkdtemp(prefix='cas-')
for rpm in results.split():
- task_queue.put([rpm, dst])
+ task_queue.put((rpm, dst))
for i in range(PROCESSES):
- Process(target=queryWorker, args(task_queue).start()
+ Process(target=queryWorker, args=(task_queue, mutex)).start()
for i in range(PROCESSES):
task_queue.put('STOP')
# Cleanup extracted debugs
shutil.rmtree(dst)
- genprint("CAS Database generation complete.")
return
def queryServers():
diff --git a/cas.conf b/cas.conf
index ba5efd2..72a585f 100644
--- a/cas.conf
+++ b/cas.conf
@@ -22,9 +22,12 @@ sshkey=dss
# indefinately recursive
kernels=/mnt/kernels
-# Compose emacs regular expression for determining what
+# set -regextype see `man find`
+regexType=posix-extended
+
+# Compose -regextype regular expression for determining what
# kernel debug rpms you wish to search for
-rpmFilter=.*kerne.+-debuginfo-[0-9].*\.rpm
+rpmFilter=[^data].*kerne.+-debuginfo-[0-9].*\.rpm
# debug level (DEBUG, INFO)
debugLevel=DEBUG
diff --git a/doc/index.rst b/doc/index.rst
index 3b41ede..e906a15 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -60,6 +60,7 @@ each section and describe its meaning::
casuser=root
sshkey=dss
kernels=/mnt/kernels
+ regexType=posix-extended
rpmFilter=.*kerne.+-debuginfo-[0-9].*\.rpm
debugLevel=DEBUG
workDirectory=/cores/processed
@@ -80,6 +81,9 @@ each section and describe its meaning::
stored. This can range anywhere from an nfs mount, samba share, local disk or
any other type of media the cas server can access.
+``regexType``: (**Required**) Defines the regex syntax to use within ``find``. Doing a
+``man find`` and searching for regextype will give you the allowable syntaxess.
+
``rpmFilters``: (**Required**) This is a emacs based regular expression which is essentially
passed to a find command to locate the various kernel-debuginfo packages defined
in ``kernels`` directive.
14 years
Branch 'devel' - 3 commits - cas-admin caslib/db.py overseer/command.py
by Adam Stokes
cas-admin | 44 ++++++++++++++++++++++++++++++--------------
caslib/db.py | 7 +++++--
overseer/command.py | 42 +++++++++++++++++++++++++++++++++++-------
3 files changed, 70 insertions(+), 23 deletions(-)
New commits:
commit 20ecd7bddc742b69fc884f52e681b494f6be3ddb
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 10:20:10 2010 -0400
process out db query
diff --git a/cas-admin b/cas-admin
index 35abdc4..75c071d 100755
--- a/cas-admin
+++ b/cas-admin
@@ -19,8 +19,8 @@
""" cas-admin - fingerprinting utility for cas
"""
import sys
-if sys.version_info[:2] < (2,4):
- raise SystemExit("Python >= 2.4 required")
+if sys.version_info[:2] < (2,6):
+ raise SystemExit("Python >= 2.6 required")
try:
from caslib.db import *
# Setup session for db interaction
@@ -35,6 +35,7 @@ import optparse
import datetime
import paramiko
import tempfile
+from multiprocessing import Process, Queue, current_process, cpu_count
from caslib.core import casexecute, coreTimestamp
from caslib.util import Utility, genprint
@@ -65,18 +66,11 @@ if config.has_option("advanced", "buffersize"):
util = Utility()
-def queryLocalRpms():
- """ query debug kernels located on the filesystem
-
- Arguments:
- - `path`: filesystem location of debug kernels
- """
- # Uses emacs regex -- see `man find`
- cmd = "find -L %s -iregex %s" % (settings["KERNELS"],
- settings["RPMFILTER"])
- results, err = casexecute(cmd, True)
- dst = tempfile.mkdtemp(prefix='cas-')
- for rpm in results.split():
+# Gather number of cpu's
+PROCESSES = cpu_count()
+def queryWorker(input):
+ """ worker function to process rpm debugs """
+ for rpm, dst in iter(input.get, 'STOP'):
session = Session()
rpm_IN_DB = session.query(RPM).filter_by(rpmPath=rpm).first()
if not rpm_IN_DB:
@@ -101,6 +95,28 @@ def queryLocalRpms():
# commit to database
session.commit()
session.close()
+
+def queryLocalRpms():
+ """ query debug kernels located on the filesystem
+
+ Arguments:
+ - `path`: filesystem location of debug kernels
+ """
+ # create Queues
+ task_queue = Queue()
+ # Uses emacs regex -- see `man find`
+ cmd = "find -L %s -iregex %s" % (settings["KERNELS"],
+ settings["RPMFILTER"])
+ results, err = casexecute(cmd, True)
+ dst = tempfile.mkdtemp(prefix='cas-')
+ for rpm in results.split():
+ task_queue.put([rpm, dst])
+
+ for i in range(PROCESSES):
+ Process(target=queryWorker, args(task_queue).start()
+ for i in range(PROCESSES):
+ task_queue.put('STOP')
+
# Cleanup extracted debugs
shutil.rmtree(dst)
genprint("CAS Database generation complete.")
commit 47092a53e807e2afa7c316456e0ff703018622de
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 09:45:38 2010 -0400
added remote-ip-addr tracking in jobs
diff --git a/caslib/db.py b/caslib/db.py
index ccbf840..a3c5f2b 100644
--- a/caslib/db.py
+++ b/caslib/db.py
@@ -82,20 +82,23 @@ class JobArchive(Base):
path = Column(String(255))
msg = Column(String(255))
created = Column(DateTime)
+ ipaddr = Column(Integer)
- def __init__(self, identifier, email, vmcore, path=None, msg=None):
+ def __init__(self, identifier, email, vmcore, ipaddr, path=None, msg=None):
self.identifier = identifier
self.email = email
self.vmcore = vmcore
self.path = path
self.msg = msg
+ self.ipaddr = ipaddr
def __repr__(self):
return "<Job('%s','%s','%s','%s','%s')>" % (self.identifier,
self.email,
self.vmcore,
self.path,
- self.msg)
+ self.msg,
+ self.ipaddr)
class Timestamp(Base):
__tablename__ = 'timestamps'
diff --git a/overseer/command.py b/overseer/command.py
index 5d3a3eb..02cc040 100755
--- a/overseer/command.py
+++ b/overseer/command.py
@@ -73,7 +73,8 @@ class Root:
if not session.query(Job).filter_by(identifier=data['identifier']).first():
jobRecord = Job(data['identifier'],
data['email'],
- data['vmcore'])
+ data['vmcore'],
+ cherrypy.request.headers['Remote-Addr'])
session.add(jobRecord)
session.commit()
record = jobRecord
commit e79c2485fa79e6aeaa09fc948c520571bc888087
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Thu May 13 09:28:58 2010 -0400
added sa to dict function for json exporting
diff --git a/overseer/command.py b/overseer/command.py
index 5f5f364..5d3a3eb 100755
--- a/overseer/command.py
+++ b/overseer/command.py
@@ -24,7 +24,7 @@ if sys.version_info[:2] < (2,6):
import os
import cherrypy
-import simplejson as json
+import simplejson
from caslib.db import *
from mako.template import Template
from mako.lookup import TemplateLookup
@@ -38,6 +38,24 @@ def run(cmd):
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
return p
+def gmap(obj):
+ for item in obj.__dict__.items():
+ if item[0][0] is '_':
+ continue
+ if isinstance(item[1], unicode):
+ yield [item[0], str(item[1])]
+ elif isinstance(item[1], datetime):
+ # datetime objects aren't json iterable so we convert to time string
+ yield [item[0], item[1].strftime("%I:%M%p %Y/%d/%m")]
+ else:
+ yield item
+
+def json(obj):
+ if isinstance(obj, list):
+ return simplejson.dumps( map(lambda x: dict(x), map(lambda x: gmap(x), obj)) )
+ else:
+ return simplejson.dumps( dict(gmap(obj)) )
+
class Root:
@cherrypy.expose
def index(self):
@@ -68,8 +86,11 @@ class Root:
p = Process(target=run, args=(cmd,))
p.start()
p.join()
- mytemplate = mylookup.get_template("create.html")
- return mytemplate.render(record=record)
+ if 'json' in data:
+ return json(record)
+ else:
+ mytemplate = mylookup.get_template("create.html")
+ return mytemplate.render(record=record)
else:
raise cherrypy.HTTPRedirect('/')
@@ -81,19 +102,25 @@ class Root:
mytemplate = mylookup.get_template("status.html")
session.close()
+ # check for json output first
+ if 'json' in data:
+ return json(status_query)
if status_query:
return mytemplate.render(status=status_query)
else:
return mytemplate.render(errors={'error' : 'Job Not Found'})
@cherrypy.expose
- def list(self):
+ def list(self, **data):
""" list all jobs """
session = Session()
job_query = session.query(Job).order_by(Job.jobId.desc()).all()
session.close()
- mytemplate = mylookup.get_template("list.html")
- return mytemplate.render(job_query=job_query)
+ if 'json' in data:
+ return json(job_query)
+ else:
+ mytemplate = mylookup.get_template("list.html")
+ return mytemplate.render(job_query=job_query)
def main():
root = Root()
14 years
Branch 'devel' - caslib/cas.py caslib/db.py
by Adam Stokes
caslib/cas.py | 2 +-
caslib/db.py | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
New commits:
commit cca149e3a3a4a0c4d95ca0c480b45f6583b549a2
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Wed May 12 16:06:47 2010 -0400
Attempting to fix out of bounds session handling
diff --git a/caslib/cas.py b/caslib/cas.py
index 4b9470c..e46e017 100755
--- a/caslib/cas.py
+++ b/caslib/cas.py
@@ -207,7 +207,7 @@ class CasApplication(object):
debugKernel = os.path.abspath("vmlinux")
else:
filterString = "*/%s" % (timestamp.debugPath,)
- extractDebug(timestamp.rpmPath, self.storagePath,
+ extractDebug(timestamp.rpm.rpmPath, self.storagePath,
filter=filterString,
return_results=False)
diff --git a/caslib/db.py b/caslib/db.py
index 01aeb77..ccbf840 100644
--- a/caslib/db.py
+++ b/caslib/db.py
@@ -20,7 +20,9 @@ from datetime import datetime
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime
from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker, scoped_session, relationship, backref
+from sqlalchemy.orm import sessionmaker, scoped_session, backref
+from sqlalchemy.orm import relation as relationship
+
Base = declarative_base()
class Core(Base):
""" Corefile information """
14 years
Branch 'devel' - caslib/db.py
by Adam Stokes
caslib/db.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
New commits:
commit 426b5d3dd858d1ba1913832c89bdeddb37267086
Author: adam stokes <astokes(a)fedoraproject.org>
Date: Wed May 12 15:54:18 2010 -0400
Setup many-to-one relationship
diff --git a/caslib/db.py b/caslib/db.py
index 3e74b37..01aeb77 100644
--- a/caslib/db.py
+++ b/caslib/db.py
@@ -19,6 +19,8 @@ from datetime import datetime
# Mapper Object
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, DateTime
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, scoped_session, relationship, backref
Base = declarative_base()
class Core(Base):
""" Corefile information """
@@ -100,6 +102,8 @@ class Timestamp(Base):
debugPath = Column(String(255))
rpmId = Column(Integer, ForeignKey("rpms.rpmId"))
+ rpm = relationship(RPM, backref=backref('timestamps'))
+
def __init__(self, stampKey, debugPath, rpmId):
self.stampKey = stampKey
self.debugPath = debugPath
@@ -127,8 +131,6 @@ class Server(Base):
self.port,
self.arch)
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker, scoped_session
# Read cas.conf for database path
import ConfigParser
config = ConfigParser.RawConfigParser()
14 years