r5049 - trunk/cumin/python/cumin
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-06 14:43:37 +0000 (Thu, 06 Oct 2011)
New Revision: 5049
Modified:
trunk/cumin/python/cumin/main.py
Log:
Prevent cumin-web crash on failed imports for Aviary, disable Aviary interface
with a log message and continue.
Modified: trunk/cumin/python/cumin/main.py
===================================================================
--- trunk/cumin/python/cumin/main.py 2011-10-06 14:15:37 UTC (rev 5048)
+++ trunk/cumin/python/cumin/main.py 2011-10-06 14:43:37 UTC (rev 5049)
@@ -175,27 +175,34 @@
self.remote = Catalog()
ops = [QmfOperations("qmf", self.session)]
+ imports_ok = True
+ if self.aviary_job_servers or self.aviary_query_servers:
+ try:
+ from sage.aviary.aviaryoperations import \
+ SudsLogging, AviaryOperationsFactory
+ except:
+ imports_ok = False
+ if imports_ok:
+ SudsLogging.set(self.aviary_suds_logs, self.home)
+ aviary_dir = os.path.join(self.home, "rpc-defs/aviary")
+
+ # The factory will choose an impl that gives us jobs, queries, or both
+ aviary_itf = AviaryOperationsFactory("aviary", aviary_dir,
+ self.aviary_job_servers,
+ self.aviary_query_servers,
+ self.aviary_key, self.aviary_cert,
+ self.aviary_root_cert,
+ self.aviary_domain_verify)
+ ops.insert(0, aviary_itf)
+ else:
+ log.info("Imports failed for Aviary interface, disabling")
+
log.info("%s Aviary interface for job submission and control." % \
- (self.aviary_job_servers and "Enabling" or "Disabling"))
+ ((self.aviary_job_servers and imports_ok) and "Enabled" or "Disabled"))
log.info("%s Aviary interface for query operations." % \
- (self.aviary_query_servers and "Enabling" or "Disabling"))
+ ((self.aviary_query_servers and imports_ok) and "Enabled" or "Disabled"))
- if self.aviary_job_servers or self.aviary_query_servers:
- from sage.aviary.aviaryoperations import \
- SudsLogging, AviaryOperationsFactory
- SudsLogging.set(self.aviary_suds_logs, self.home)
- aviary_dir = os.path.join(self.home, "rpc-defs/aviary")
-
- # The factory will choose an impl that gives us jobs, queries, or both
- aviary_itf = AviaryOperationsFactory("aviary", aviary_dir,
- self.aviary_job_servers,
- self.aviary_query_servers,
- self.aviary_key, self.aviary_cert,
- self.aviary_root_cert,
- self.aviary_domain_verify)
- ops.insert(0, aviary_itf)
-
self.remote.add_mechanisms(ops)
# Create RPC interface for Wallaby
12 years, 7 months
r5048 - trunk/sage/python/sage/wallaby
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-06 14:15:37 +0000 (Thu, 06 Oct 2011)
New Revision: 5048
Modified:
trunk/sage/python/sage/wallaby/wallabyoperations.py
Log:
Fix wallaby imports to work with rpm package
related to BZ733364
Modified: trunk/sage/python/sage/wallaby/wallabyoperations.py
===================================================================
--- trunk/sage/python/sage/wallaby/wallabyoperations.py 2011-10-05 21:23:47 UTC (rev 5047)
+++ trunk/sage/python/sage/wallaby/wallabyoperations.py 2011-10-06 14:15:37 UTC (rev 5048)
@@ -9,9 +9,10 @@
imports_ok = True
try:
- import wallaby
- from wallaby_collections import StorePatches
- from tagging import StorePatches, NodePatches, setup
+ import wallaby
+ import wallaby.tagging
+ import wallaby.collections
+ setup = wallaby.tagging.setup
except:
imports_ok = False
12 years, 7 months
r5047 - branches/lucidity/sage/python/sage/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-05 21:23:47 +0000 (Wed, 05 Oct 2011)
New Revision: 5047
Modified:
branches/lucidity/sage/python/sage/aviary/aviaryoperations.py
Log:
Merge from trunk.
svn merge -c 5046 svn+ssh://svn.fedorahosted.org/svn/cumin/trunk .
Modified: branches/lucidity/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- branches/lucidity/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 20:27:49 UTC (rev 5046)
+++ branches/lucidity/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 21:23:47 UTC (rev 5047)
@@ -575,14 +575,14 @@
# we have to always reset the transport here.
if scheme == "https":
if not os.path.isfile(self.key):
- raise Exception("Private key file"\
+ raise Exception("Private key file "\
"for ssl communication with Aviary not found")
if not os.path.isfile(self.cert):
- raise Exception("Client certificate file"\
+ raise Exception("Client certificate file "\
"for ssl communication with Aviary not found")
if self.root_cert != "" and self.server_validation_possible:
if not os.path.isfile(self.root_cert):
- raise Exception("Root certificate file"\
+ raise Exception("Root certificate file "\
"for Aviary server validation not found")
the_transport = HTTPSFullCertTransport(self.key,
self.cert,
12 years, 7 months
r5046 - trunk/sage/python/sage/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-05 20:27:49 +0000 (Wed, 05 Oct 2011)
New Revision: 5046
Modified:
trunk/sage/python/sage/aviary/aviaryoperations.py
Log:
Missing spaces in error messages
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 20:08:23 UTC (rev 5045)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 20:27:49 UTC (rev 5046)
@@ -575,14 +575,14 @@
# we have to always reset the transport here.
if scheme == "https":
if not os.path.isfile(self.key):
- raise Exception("Private key file"\
+ raise Exception("Private key file "\
"for ssl communication with Aviary not found")
if not os.path.isfile(self.cert):
- raise Exception("Client certificate file"\
+ raise Exception("Client certificate file "\
"for ssl communication with Aviary not found")
if self.root_cert != "" and self.server_validation_possible:
if not os.path.isfile(self.root_cert):
- raise Exception("Root certificate file"\
+ raise Exception("Root certificate file "\
"for Aviary server validation not found")
the_transport = HTTPSFullCertTransport(self.key,
self.cert,
12 years, 7 months
r5045 - in branches/lucidity/sage/python/sage: . aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-05 20:08:23 +0000 (Wed, 05 Oct 2011)
New Revision: 5045
Modified:
branches/lucidity/sage/python/sage/aviary/aviaryoperations.py
branches/lucidity/sage/python/sage/verifiedhttps_m2crypto.py
Log:
Merge from trunk.
svn merge -c 5044 svn+ssh://svn.fedorahosted.org/svn/cumin/trunk .
Modified: branches/lucidity/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- branches/lucidity/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 20:02:57 UTC (rev 5044)
+++ branches/lucidity/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 20:08:23 UTC (rev 5045)
@@ -46,22 +46,21 @@
pass
log.debug("AviaryOperations: suds logging off")
-'''
-Stuff to do:
-- Test pool/schedd/stuff with hierarchical collectors
+#Stuff to do:
-- Add a summary comment at the top like QMF opreations
+#- Test pool/schedd/stuff with hierarchical collectors
-- can we use default/timeout with suds?
-looks like the way to do this is through the Transport object that
-is set in the client. The timeout is set on the Transport itself,
-and the Transport can be changed on the client with set_options.
-Alternatively, we could retain a reference to the Transport in
-the clients that we pool so that we can call set_options on the
-transport.
-'''
+#- Add a summary comment at the top like QMF opreations
+#- can we use default/timeout with suds?
+#looks like the way to do this is through the Transport object that
+#is set in the client. The timeout is set on the Transport itself,
+#and the Transport can be changed on the client with set_options.
+#Alternatively, we could retain a reference to the Transport in
+#the clients that we pool so that we can call set_options on the
+#transport.
+
class _AviaryJobMethods(object):
# Do this here rather than __init__ so we don't have to worry about
Modified: branches/lucidity/sage/python/sage/verifiedhttps_m2crypto.py
===================================================================
--- branches/lucidity/sage/python/sage/verifiedhttps_m2crypto.py 2011-10-05 20:02:57 UTC (rev 5044)
+++ branches/lucidity/sage/python/sage/verifiedhttps_m2crypto.py 2011-10-05 20:08:23 UTC (rev 5045)
@@ -9,7 +9,15 @@
server_verify=True,
domain_verify=True):
"""
+ All params except those noted below are passed through to
+ the M2Crypto.httpslib.HTTPSConnection constructor. Check docs on that
+ class information. Note, root_cert is passed in the SSL.Context if
+ server_verify is True by means of SSL.Context.load_verify_locations.
+ @param server_verify: does server certificate verification if True
+ @param domain_verify: checks server certificate 'commonName' against host if True
+ @param timeout: timeout value is set on the socket after connection using
+ socket.settimeout() if timeout is not None.
"""
self.server_verify = server_verify
self.domain_verify = domain_verify
12 years, 7 months
r5044 - in trunk/sage/python/sage: . aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-05 20:02:57 +0000 (Wed, 05 Oct 2011)
New Revision: 5044
Modified:
trunk/sage/python/sage/aviary/aviaryoperations.py
trunk/sage/python/sage/verifiedhttps_m2crypto.py
Log:
Tweak comments.
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 19:39:28 UTC (rev 5043)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 20:02:57 UTC (rev 5044)
@@ -46,22 +46,21 @@
pass
log.debug("AviaryOperations: suds logging off")
-'''
-Stuff to do:
-- Test pool/schedd/stuff with hierarchical collectors
+#Stuff to do:
-- Add a summary comment at the top like QMF opreations
+#- Test pool/schedd/stuff with hierarchical collectors
-- can we use default/timeout with suds?
-looks like the way to do this is through the Transport object that
-is set in the client. The timeout is set on the Transport itself,
-and the Transport can be changed on the client with set_options.
-Alternatively, we could retain a reference to the Transport in
-the clients that we pool so that we can call set_options on the
-transport.
-'''
+#- Add a summary comment at the top like QMF opreations
+#- can we use default/timeout with suds?
+#looks like the way to do this is through the Transport object that
+#is set in the client. The timeout is set on the Transport itself,
+#and the Transport can be changed on the client with set_options.
+#Alternatively, we could retain a reference to the Transport in
+#the clients that we pool so that we can call set_options on the
+#transport.
+
class _AviaryJobMethods(object):
# Do this here rather than __init__ so we don't have to worry about
Modified: trunk/sage/python/sage/verifiedhttps_m2crypto.py
===================================================================
--- trunk/sage/python/sage/verifiedhttps_m2crypto.py 2011-10-05 19:39:28 UTC (rev 5043)
+++ trunk/sage/python/sage/verifiedhttps_m2crypto.py 2011-10-05 20:02:57 UTC (rev 5044)
@@ -9,7 +9,15 @@
server_verify=True,
domain_verify=True):
"""
+ All params except those noted below are passed through to
+ the M2Crypto.httpslib.HTTPSConnection constructor. Check docs on that
+ class information. Note, root_cert is passed in the SSL.Context if
+ server_verify is True by means of SSL.Context.load_verify_locations.
+ @param server_verify: does server certificate verification if True
+ @param domain_verify: checks server certificate 'commonName' against host if True
+ @param timeout: timeout value is set on the socket after connection using
+ socket.settimeout() if timeout is not None.
"""
self.server_verify = server_verify
self.domain_verify = domain_verify
12 years, 7 months
r5043 - in branches/lucidity: cumin cumin/bin cumin/etc cumin/python/cumin sage/python/sage sage/python/sage/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-05 19:39:28 +0000 (Wed, 05 Oct 2011)
New Revision: 5043
Added:
branches/lucidity/sage/python/sage/aviary/AVIARY-README
branches/lucidity/sage/python/sage/verifiedhttps_m2crypto.py
Modified:
branches/lucidity/cumin/Makefile
branches/lucidity/cumin/bin/cumin-web
branches/lucidity/cumin/etc/cumin.conf
branches/lucidity/cumin/python/cumin/config.py
branches/lucidity/cumin/python/cumin/main.py
branches/lucidity/cumin/python/cumin/stat.py
branches/lucidity/sage/python/sage/aviary/aviaryoperations.py
branches/lucidity/sage/python/sage/https.py
branches/lucidity/sage/python/sage/verifiedhttps.py
Log:
Merge changes from trunk.
BZ733447 modifications
BZ725451
BZ733677 modifications
svn merge -r 5036:HEAD svn+ssh://svn.fedorahosted.org/svn/cumin/trunk .
Modified: branches/lucidity/cumin/Makefile
===================================================================
--- branches/lucidity/cumin/Makefile 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/cumin/Makefile 2011-10-05 19:39:28 UTC (rev 5043)
@@ -23,6 +23,7 @@
install -pm 0755 bin/cumin bin/cumin-* ${CUMIN_HOME}/bin
install -d ${CUMIN_HOME}/doc
install -pm 0644 LICENSE COPYING ${CUMIN_HOME}/doc
+ install -pm 0644 ../sage/python/sage/aviary/AVIARY-README ${CUMIN_HOME}/doc
install -pm 0644 ../wooly/LICENSE-for-wsgiserver ${CUMIN_HOME}/doc
install -pm 0644 ../wooly/COPYING-for-wsgiserver ${CUMIN_HOME}/doc
install -d ${CUMIN_HOME}/model/upgrades
Modified: branches/lucidity/cumin/bin/cumin-web
===================================================================
--- branches/lucidity/cumin/bin/cumin-web 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/cumin/bin/cumin-web 2011-10-05 19:39:28 UTC (rev 5043)
@@ -22,6 +22,9 @@
cumin.aviary_root_cert = values.aviary_root_cert
cumin.aviary_domain_verify = values.aviary_domain_verify
+ # For development use. Default is False. Undocumented.
+ cumin.aviary_suds_logs = values.aviary_suds_logs
+
def set_wallaby_configs(cumin, values, brokers):
if values.wallaby_broker == "":
cumin.wallaby_broker = brokers[0]
Modified: branches/lucidity/cumin/etc/cumin.conf
===================================================================
--- branches/lucidity/cumin/etc/cumin.conf 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/cumin/etc/cumin.conf 2011-10-05 19:39:28 UTC (rev 5043)
@@ -52,8 +52,8 @@
# Optional flag to control whether Cumin checks that the server host matches
# the Common Name in the server certificate during server certificate validation.
-# Default value is False. Set to true to enable the check.
-#aviary-domain-verify: False
+# Default value is True. Set to false to disable the check.
+#aviary-domain-verify: True
# *************** Master configuration ***************
Modified: branches/lucidity/cumin/python/cumin/config.py
===================================================================
--- branches/lucidity/cumin/python/cumin/config.py 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/cumin/python/cumin/config.py 2011-10-05 19:39:28 UTC (rev 5043)
@@ -173,6 +173,11 @@
param = ConfigParameter(self, "aviary-domain-verify", bool)
param.default = True
+ # Intended for development use, not the end user.
+ # Undocumented.
+ param = ConfigParameter(self, "aviary-suds-logs", bool)
+ param.default = False
+
self.log_file = ConfigParameter(self, "log-file", str)
param = ConfigParameter(self, "log-level", str)
Modified: branches/lucidity/cumin/python/cumin/main.py
===================================================================
--- branches/lucidity/cumin/python/cumin/main.py 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/cumin/python/cumin/main.py 2011-10-05 19:39:28 UTC (rev 5043)
@@ -88,6 +88,9 @@
self.aviary_cert = ""
self.aviary_root_cert = ""
self.aviary_domain_verify=True
+
+ # For development use only
+ self.aviary_suds_logs = False
self.wallaby = None
self.wallaby_broker = None
@@ -175,7 +178,9 @@
(self.aviary_query_servers and "Enabling" or "Disabling"))
if self.aviary_job_servers or self.aviary_query_servers:
- from sage.aviary.aviaryoperations import AviaryOperationsFactory
+ from sage.aviary.aviaryoperations import \
+ SudsLogging, AviaryOperationsFactory
+ SudsLogging.set(self.aviary_suds_logs, self.home)
aviary_dir = os.path.join(self.home, "rpc-defs/aviary")
# The factory will choose an impl that gives us jobs, queries, or both
@@ -240,12 +245,15 @@
self.session.start()
self.server.start()
- self.wallaby.start()
+ if self.wallaby is not None:
+ self.wallaby.start()
def stop(self):
log.info("Stopping %s", self)
- self.wallaby.stop()
+ if self.wallaby is not None:
+ self.wallaby.stop()
+
self.server.stop()
try:
self.session.stop()
Modified: branches/lucidity/cumin/python/cumin/stat.py
===================================================================
--- branches/lucidity/cumin/python/cumin/stat.py 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/cumin/python/cumin/stat.py 2011-10-05 19:39:28 UTC (rev 5043)
@@ -556,6 +556,10 @@
min_value = float(min_value)
max_value = round(max_value * 1.1 + 1)
+ if type == "percent" and max_value > 100:
+ # since none of our percentage graphs go above 100%
+ # we limit it here to match the flash graph
+ max_value = 100
if min_value < 0:
min_value = round(min_value * 1.1 - 1)
Copied: branches/lucidity/sage/python/sage/aviary/AVIARY-README (from rev 5042, trunk/sage/python/sage/aviary/AVIARY-README)
===================================================================
--- branches/lucidity/sage/python/sage/aviary/AVIARY-README (rev 0)
+++ branches/lucidity/sage/python/sage/aviary/AVIARY-README 2011-10-05 19:39:28 UTC (rev 5043)
@@ -0,0 +1,38 @@
+Technology Preview feature CuminAviary
+
+Description:
+
+This feature allows Cumin to use the Aviary web services provided in the condor-aviary package for certain functions in the user interface. If the CuminAviary feature is enabled, Cumin will use Aviary services rather than QMF method calls where possible.
+
+The CuminAviary feature is enabled and configured through the /etc/cumin/cumin.conf file. Relevant configuration parameters with descriptive comments can be found in the default /etc/cumin/cumin.conf file by searching for a line containing "Aviary interface to condor".
+
+Aviary provides a job service and a query service; Cumin may use either, both or neither. By default, Cumin will use no Aviary services and will use QMF methods instead.
+
+To enable use of the Aviary job service, the 'aviary-job-servers' parameter must be uncommented and set (see the comments in the configuration file for details). Setting this parameter will cause Cumin to use the Aviary job service for job submission, for the hold, release, and remove job control functions, and for editing of job ads.
+
+To enable use of the Aviary query service, the 'aviary-query-servers' parameter must be uncommented and set (see the comments in the configuration file for details). Setting this parameter will cause Cumin to use the Aviary query service for retrieving job output files, retrieving job ad details, and retreiving the list of jobs in a submission.
+
+Cumin will make INFO level entries in the log file for cumin-web that indicate whether use of the job and/or query services has been enabled and what type of certificate validation will be used for servers configured for SSL (see below). These log entries will begin with "AviaryOperations:". If an Aviary operation fails, the yellow task banner associated with the operation will contain error information.
+
+By default, the Aviary services in condor will not use SSL (Secure Socket Layer) for communication and no other Cumin configuration parameters need to be set for the CuminAviary feature. However, if the Aviary services in condor have been configured to use SSL then additional Cumin configuration parameters must be set.
+
+First, note that the scheme for Aviary servers will change from "http" to "https" for any server using SSL. Failure to specify schemes correctly in the 'aviary-job-servers' or 'aviary-query-servers' parameters will prevent CuminAviary from functioning.
+
+Second, the 'aviary-key' and 'aviary-cert' parameters must be set. These parameters give the full paths to a PEM formated private key file and PEM formatted certificate file that Cumin will use as a client to access the Aviary services. The Aviary servers will validate Cumin's client certificate and allow access if validation succeeds.
+
+Optionally, the 'aviary-root-cert' parameter may be set. This is the full path to a PEM formatted file containing CA (certificate authority) certificates that Cumin will use to validate the server certificate. If this parameter is unset Cumin will NOT validate server certificates.
+
+Lastly, the 'aviary-domain-verify' parameter controls whether or not Cumin checks the hostname of the server against the server certificate during validation. This parameter has no effect unless 'aviary-root-cert' is set. The default value is True; it may be useful to set this parameter to False if the server is using a self-signed certificate with a non-matching hostname.
+
+Feedback: bug reports or requests for enhancement can be made through http://bugzilla.redhat.com. General questions about this feature can be handled through cumin-users(a)lists.fedorahosted.org
+
+Full support: This feature is intended to be fully supported in an upcoming minor release.
+
+Where to find this information: The content given here may be found in the Release Notes accompanying the software or in the file /usr/share/doc/cumin-*/AVIARY-README after the software is installed.
+
+Technology Preview Policy:
+Technology Preview features are not currently supported under Red Hat Enterprise Linux subscription services, may not be functionally complete, and are generally not suitable for production use. However, these features are included as a customer convenience and to provide the technologies with wider exposure.
+
+Customers may find these features useful in non-production environments, and can provide feedback and functionality suggestions prior to their transition to fully supported status. Erratas will be provided for high-priority security issues.
+
+During its development additional components of a Technology Preview feature may become available to the public for testing. It is the intention of Red Hat to fully support Technology Preview features in a future release.
Modified: branches/lucidity/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- branches/lucidity/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 19:39:28 UTC (rev 5043)
@@ -15,37 +15,40 @@
from sage.https import *
from datetime import datetime
-
log = logging.getLogger("sage.aviary")
-#f = open("./suds.client.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.client').setLevel(logging.DEBUG)
+class SudsLogging(object):
+ _on = False
+ sudslogs = {"suds.client": None, "suds.transport": None, "suds.xsd.schema": None, "suds.wsdl": None}
-#f = open("./suds.client.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.client').addHandler(h)
+ @classmethod
+ def set(cls, flag, home):
+ if flag:
+ SudsLogging._on = True
+ for k,v in SudsLogging.sudslogs.iteritems():
+ l = logging.getLogger(k)
+ l.setLevel(logging.DEBUG)
+ try:
+ f = open(os.path.join(home, "log/"+k+".log"), 'a+')
+ h = logging.StreamHandler(f)
+ l.addHandler(h)
+ SudsLogging.sudslogs[k] = h
+ except:
+ pass
+ log.debug("AviaryOperations: suds logging on")
-#logging.getLogger('suds.transport').setLevel(logging.DEBUG)
-#f = open("./suds.transport.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.transport').addHandler(h)
+ elif SudsLogging._on:
+ SudsLogging._on = False
+ for k,v in SudsLogging.sudslogs.iteritems():
+ try:
+ logging.getLogger(k).removeHandler(v)
+ except:
+ pass
+ log.debug("AviaryOperations: suds logging off")
-#logging.getLogger('suds.xsd.schema').setLevel(logging.DEBUG)
-#f = open("./suds.xsd.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.xsd.schema').addHandler(h)
-
-#logging.getLogger('suds.wsdl').setLevel(logging.DEBUG)
-#f = open("./suds.wsdl.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.wsdl').addHandler(h)
-
'''
Stuff to do:
-- Add logging to Aviary
-
- Test pool/schedd/stuff with hierarchical collectors
- Add a summary comment at the top like QMF opreations
@@ -517,11 +520,12 @@
elif not self.server_validation_possible:
log.info("AviaryOperations: server certificate validation not "\
- "supported (no ssl module?), using client validation "\
+ "supported, using client validation "\
"only for ssl connections.")
else:
log.info("AviaryOperations: using client and server "\
- "certificate validation for ssl connections.")
+ "certificate validation for ssl connections, "\
+ "solution is %s" % sage.https.technology)
log.info("AviaryOperations: verify server domain against "\
"certificate during validation (%s)" % self.domain_verify)
Modified: branches/lucidity/sage/python/sage/https.py
===================================================================
--- branches/lucidity/sage/python/sage/https.py 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/sage/python/sage/https.py 2011-10-05 19:39:28 UTC (rev 5043)
@@ -22,6 +22,13 @@
import httplib
import socket
+technology = "urllib2"
+
+# Provide an exception here that implementation classes
+# can use in common to raise exceptions and return messages
+class SSLVerificationError(Exception):
+ pass
+
class HTTPSClientAuthHandler(u2.HTTPSHandler):
def __init__(self, key, cert):
"""
@@ -94,8 +101,16 @@
# ...
# if hasattr(<whatever.>https, "HTTPSFullCertTransport"):
# ...
- from sage.verifiedhttps import VerifiedHTTPSConnection
+ try:
+ # Try a solution that uses the Python ssl module first
+ from sage.verifiedhttps import VerifiedHTTPSConnection
+ technology = "Python ssl"
+ except:
+ # Didn't work, try a solution based on m2crypto
+ from sage.verifiedhttps_m2crypto import VerifiedHTTPSConnection
+ technology = "M2Crypto"
+
class HTTPSFullAuthHandler(HTTPSClientAuthHandler):
"""
Add server certificate validation to HTTPSClientAuthHandler
Modified: branches/lucidity/sage/python/sage/verifiedhttps.py
===================================================================
--- branches/lucidity/sage/python/sage/verifiedhttps.py 2011-10-05 19:30:11 UTC (rev 5042)
+++ branches/lucidity/sage/python/sage/verifiedhttps.py 2011-10-05 19:39:28 UTC (rev 5043)
@@ -1,21 +1,18 @@
import httplib
import socket
import ssl
+from https import SSLVerificationError
# Note: much thanks to Joseph Turner for showing the world
# how to extend httplib using the ssl module to implement
# server certificate validation.
# https://github.com/josephturnerjr/urllib2.VerifiedHTTPS
-class SSLVerificationError(Exception):
- pass
-
# subclass of HTTPSConnection to do cert verification and domain verification
class VerifiedHTTPSConnection(httplib.HTTPSConnection):
def __init__(self, host, port=None, key_file=None, cert_file=None,
- root_cert=None, strict=None,
- timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ root_cert=None, strict=None, timeout=None,
server_verify=True, domain_verify=True):
"""
All params except those noted below are passed through to
@@ -27,7 +24,15 @@
@param domain_verify: checks server certificate 'commonName' against host if True
"""
httplib.HTTPSConnection.__init__(self, host, port, key_file, cert_file,
- strict, timeout)
+ strict)
+ # Handle difference between Python 2.4 and 2.6. Timeout was added
+ # in 2.6, and if not specified will be the global default timeout.
+ # In this routine allow None to indicate default, otherwise set the
+ # value since we can't set it in the constructor. If it's missing
+ # we will create it here.
+ if timeout is not None:
+ self.timeout = timeout
+
self.root_cert = root_cert
self.server_verify = server_verify
self.domain_verify = domain_verify
@@ -41,7 +46,8 @@
# There is no timeout attribute in earlier versions of this object.
# The only option available is to set a global default timeout for
# all socket objects.
- socket.setdefaulttimeout(10)
+ if hasattr(self, "timeout"):
+ socket.setdefaulttimeout(self.timeout)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.host, self.port))
Copied: branches/lucidity/sage/python/sage/verifiedhttps_m2crypto.py (from rev 5042, trunk/sage/python/sage/verifiedhttps_m2crypto.py)
===================================================================
--- branches/lucidity/sage/python/sage/verifiedhttps_m2crypto.py (rev 0)
+++ branches/lucidity/sage/python/sage/verifiedhttps_m2crypto.py 2011-10-05 19:39:28 UTC (rev 5043)
@@ -0,0 +1,52 @@
+from M2Crypto import httpslib, SSL
+from https import SSLVerificationError
+
+# wrap the creation of a SSL.Context, etc in a class
+class VerifiedHTTPSConnection(httpslib.HTTPSConnection):
+
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ root_cert=None, strict=None, timeout=None,
+ server_verify=True,
+ domain_verify=True):
+ """
+
+ """
+ self.server_verify = server_verify
+ self.domain_verify = domain_verify
+ self.timeout = timeout
+
+ ctx = SSL.Context()
+ ctx.load_cert(cert_file, key_file)
+ # Leaving the ctx verify mode set to 0 does not seem
+ # to turn off all the server certificate checks, not
+ # sure why. Something in M2Crypto. The hostname check
+ # is still applied and raises an exception, so we catch
+ # it as we do with domain_verify
+ if server_verify:
+ ctx.load_verify_locations(root_cert)
+ mode = SSL.verify_peer | SSL.verify_fail_if_no_peer_cert
+ ctx.set_verify(mode, depth=9)
+ httpslib.HTTPSConnection.__init__(self, host, port, strict,
+ key_file=key_file, cert_file=cert_file,
+ ssl_context=ctx)
+ def connect(self):
+ try:
+ # Best we can do with the timeout parameter is
+ # set it on the socket after the connection is
+ # created. There is no hook in M2Crypto to set
+ # this prior to the connection.
+ httpslib.HTTPSConnection.connect(self)
+ if self.timeout is not None:
+ self.sock.settimeout(timeout)
+ except SSL.Checker.WrongHost, e:
+ # Allow the host name check to fail if domain_verify is off.
+ # This is mostly for testing with self-signed certificates
+ # and to provide the same interface as verifiedhttps.py
+ # In order to squash the report of the mismatched hostnames,
+ # we replace the message -- could be considered a leak of
+ # domain and certificate information I suppose.
+ if self.server_verify and self.domain_verify:
+ raise SSLVerificationError("Server certificate doesn't match domain;"\
+ " untrusted connection")
+
+
12 years, 7 months
r5042 - in trunk: cumin cumin/etc sage/python/sage/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-05 19:30:11 +0000 (Wed, 05 Oct 2011)
New Revision: 5042
Added:
trunk/sage/python/sage/aviary/AVIARY-README
Modified:
trunk/cumin/Makefile
trunk/cumin/etc/cumin.conf
Log:
Add AVIARY-README file, include in /usr/share/doc/cumin-*, fix comment on aviary-domain-verify in cumin.conf.
BZ733677
Modified: trunk/cumin/Makefile
===================================================================
--- trunk/cumin/Makefile 2011-10-05 15:34:14 UTC (rev 5041)
+++ trunk/cumin/Makefile 2011-10-05 19:30:11 UTC (rev 5042)
@@ -23,6 +23,7 @@
install -pm 0755 bin/cumin bin/cumin-* ${CUMIN_HOME}/bin
install -d ${CUMIN_HOME}/doc
install -pm 0644 LICENSE COPYING ${CUMIN_HOME}/doc
+ install -pm 0644 ../sage/python/sage/aviary/AVIARY-README ${CUMIN_HOME}/doc
install -pm 0644 ../wooly/LICENSE-for-wsgiserver ${CUMIN_HOME}/doc
install -pm 0644 ../wooly/COPYING-for-wsgiserver ${CUMIN_HOME}/doc
install -d ${CUMIN_HOME}/model/upgrades
Modified: trunk/cumin/etc/cumin.conf
===================================================================
--- trunk/cumin/etc/cumin.conf 2011-10-05 15:34:14 UTC (rev 5041)
+++ trunk/cumin/etc/cumin.conf 2011-10-05 19:30:11 UTC (rev 5042)
@@ -52,8 +52,8 @@
# Optional flag to control whether Cumin checks that the server host matches
# the Common Name in the server certificate during server certificate validation.
-# Default value is False. Set to true to enable the check.
-#aviary-domain-verify: False
+# Default value is True. Set to false to disable the check.
+#aviary-domain-verify: True
# *************** Master configuration ***************
Added: trunk/sage/python/sage/aviary/AVIARY-README
===================================================================
--- trunk/sage/python/sage/aviary/AVIARY-README (rev 0)
+++ trunk/sage/python/sage/aviary/AVIARY-README 2011-10-05 19:30:11 UTC (rev 5042)
@@ -0,0 +1,38 @@
+Technology Preview feature CuminAviary
+
+Description:
+
+This feature allows Cumin to use the Aviary web services provided in the condor-aviary package for certain functions in the user interface. If the CuminAviary feature is enabled, Cumin will use Aviary services rather than QMF method calls where possible.
+
+The CuminAviary feature is enabled and configured through the /etc/cumin/cumin.conf file. Relevant configuration parameters with descriptive comments can be found in the default /etc/cumin/cumin.conf file by searching for a line containing "Aviary interface to condor".
+
+Aviary provides a job service and a query service; Cumin may use either, both or neither. By default, Cumin will use no Aviary services and will use QMF methods instead.
+
+To enable use of the Aviary job service, the 'aviary-job-servers' parameter must be uncommented and set (see the comments in the configuration file for details). Setting this parameter will cause Cumin to use the Aviary job service for job submission, for the hold, release, and remove job control functions, and for editing of job ads.
+
+To enable use of the Aviary query service, the 'aviary-query-servers' parameter must be uncommented and set (see the comments in the configuration file for details). Setting this parameter will cause Cumin to use the Aviary query service for retrieving job output files, retrieving job ad details, and retreiving the list of jobs in a submission.
+
+Cumin will make INFO level entries in the log file for cumin-web that indicate whether use of the job and/or query services has been enabled and what type of certificate validation will be used for servers configured for SSL (see below). These log entries will begin with "AviaryOperations:". If an Aviary operation fails, the yellow task banner associated with the operation will contain error information.
+
+By default, the Aviary services in condor will not use SSL (Secure Socket Layer) for communication and no other Cumin configuration parameters need to be set for the CuminAviary feature. However, if the Aviary services in condor have been configured to use SSL then additional Cumin configuration parameters must be set.
+
+First, note that the scheme for Aviary servers will change from "http" to "https" for any server using SSL. Failure to specify schemes correctly in the 'aviary-job-servers' or 'aviary-query-servers' parameters will prevent CuminAviary from functioning.
+
+Second, the 'aviary-key' and 'aviary-cert' parameters must be set. These parameters give the full paths to a PEM formated private key file and PEM formatted certificate file that Cumin will use as a client to access the Aviary services. The Aviary servers will validate Cumin's client certificate and allow access if validation succeeds.
+
+Optionally, the 'aviary-root-cert' parameter may be set. This is the full path to a PEM formatted file containing CA (certificate authority) certificates that Cumin will use to validate the server certificate. If this parameter is unset Cumin will NOT validate server certificates.
+
+Lastly, the 'aviary-domain-verify' parameter controls whether or not Cumin checks the hostname of the server against the server certificate during validation. This parameter has no effect unless 'aviary-root-cert' is set. The default value is True; it may be useful to set this parameter to False if the server is using a self-signed certificate with a non-matching hostname.
+
+Feedback: bug reports or requests for enhancement can be made through http://bugzilla.redhat.com. General questions about this feature can be handled through cumin-users(a)lists.fedorahosted.org
+
+Full support: This feature is intended to be fully supported in an upcoming minor release.
+
+Where to find this information: The content given here may be found in the Release Notes accompanying the software or in the file /usr/share/doc/cumin-*/AVIARY-README after the software is installed.
+
+Technology Preview Policy:
+Technology Preview features are not currently supported under Red Hat Enterprise Linux subscription services, may not be functionally complete, and are generally not suitable for production use. However, these features are included as a customer convenience and to provide the technologies with wider exposure.
+
+Customers may find these features useful in non-production environments, and can provide feedback and functionality suggestions prior to their transition to fully supported status. Erratas will be provided for high-priority security issues.
+
+During its development additional components of a Technology Preview feature may become available to the public for testing. It is the intention of Red Hat to fully support Technology Preview features in a future release.
12 years, 7 months
r5041 - in trunk: cumin/bin cumin/python/cumin sage/python/sage sage/python/sage/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2011-10-05 15:34:14 +0000 (Wed, 05 Oct 2011)
New Revision: 5041
Modified:
trunk/cumin/bin/cumin-web
trunk/cumin/python/cumin/config.py
trunk/cumin/python/cumin/main.py
trunk/sage/python/sage/aviary/aviaryoperations.py
trunk/sage/python/sage/verifiedhttps.py
Log:
Add development flag for suds logging as an aid to debug/verify Aviary.
Add a few "is not None" and "hasattr" checks.
BZ733677 modifications
Modified: trunk/cumin/bin/cumin-web
===================================================================
--- trunk/cumin/bin/cumin-web 2011-10-04 20:36:30 UTC (rev 5040)
+++ trunk/cumin/bin/cumin-web 2011-10-05 15:34:14 UTC (rev 5041)
@@ -22,6 +22,9 @@
cumin.aviary_root_cert = values.aviary_root_cert
cumin.aviary_domain_verify = values.aviary_domain_verify
+ # For development use. Default is False. Undocumented.
+ cumin.aviary_suds_logs = values.aviary_suds_logs
+
def set_wallaby_configs(cumin, values, brokers):
if values.wallaby_broker == "":
cumin.wallaby_broker = brokers[0]
Modified: trunk/cumin/python/cumin/config.py
===================================================================
--- trunk/cumin/python/cumin/config.py 2011-10-04 20:36:30 UTC (rev 5040)
+++ trunk/cumin/python/cumin/config.py 2011-10-05 15:34:14 UTC (rev 5041)
@@ -176,6 +176,11 @@
param = ConfigParameter(self, "aviary-domain-verify", bool)
param.default = True
+ # Intended for development use, not the end user.
+ # Undocumented.
+ param = ConfigParameter(self, "aviary-suds-logs", bool)
+ param.default = False
+
self.log_file = ConfigParameter(self, "log-file", str)
param = ConfigParameter(self, "log-level", str)
Modified: trunk/cumin/python/cumin/main.py
===================================================================
--- trunk/cumin/python/cumin/main.py 2011-10-04 20:36:30 UTC (rev 5040)
+++ trunk/cumin/python/cumin/main.py 2011-10-05 15:34:14 UTC (rev 5041)
@@ -92,6 +92,9 @@
self.aviary_cert = ""
self.aviary_root_cert = ""
self.aviary_domain_verify=True
+
+ # For development use only
+ self.aviary_suds_logs = False
self.wallaby = None
self.wallaby_broker = None
@@ -179,7 +182,9 @@
(self.aviary_query_servers and "Enabling" or "Disabling"))
if self.aviary_job_servers or self.aviary_query_servers:
- from sage.aviary.aviaryoperations import AviaryOperationsFactory
+ from sage.aviary.aviaryoperations import \
+ SudsLogging, AviaryOperationsFactory
+ SudsLogging.set(self.aviary_suds_logs, self.home)
aviary_dir = os.path.join(self.home, "rpc-defs/aviary")
# The factory will choose an impl that gives us jobs, queries, or both
@@ -244,12 +249,15 @@
self.session.start()
self.server.start()
- self.wallaby.start()
+ if self.wallaby is not None:
+ self.wallaby.start()
def stop(self):
log.info("Stopping %s", self)
- self.wallaby.stop()
+ if self.wallaby is not None:
+ self.wallaby.stop()
+
self.server.stop()
try:
self.session.stop()
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2011-10-04 20:36:30 UTC (rev 5040)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2011-10-05 15:34:14 UTC (rev 5041)
@@ -15,37 +15,40 @@
from sage.https import *
from datetime import datetime
-
log = logging.getLogger("sage.aviary")
-#f = open("./suds.client.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.client').setLevel(logging.DEBUG)
+class SudsLogging(object):
+ _on = False
+ sudslogs = {"suds.client": None, "suds.transport": None, "suds.xsd.schema": None, "suds.wsdl": None}
-#f = open("./suds.client.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.client').addHandler(h)
+ @classmethod
+ def set(cls, flag, home):
+ if flag:
+ SudsLogging._on = True
+ for k,v in SudsLogging.sudslogs.iteritems():
+ l = logging.getLogger(k)
+ l.setLevel(logging.DEBUG)
+ try:
+ f = open(os.path.join(home, "log/"+k+".log"), 'a+')
+ h = logging.StreamHandler(f)
+ l.addHandler(h)
+ SudsLogging.sudslogs[k] = h
+ except:
+ pass
+ log.debug("AviaryOperations: suds logging on")
-#logging.getLogger('suds.transport').setLevel(logging.DEBUG)
-#f = open("./suds.transport.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.transport').addHandler(h)
+ elif SudsLogging._on:
+ SudsLogging._on = False
+ for k,v in SudsLogging.sudslogs.iteritems():
+ try:
+ logging.getLogger(k).removeHandler(v)
+ except:
+ pass
+ log.debug("AviaryOperations: suds logging off")
-#logging.getLogger('suds.xsd.schema').setLevel(logging.DEBUG)
-#f = open("./suds.xsd.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.xsd.schema').addHandler(h)
-
-#logging.getLogger('suds.wsdl').setLevel(logging.DEBUG)
-#f = open("./suds.wsdl.log", 'a+')
-#h = logging.StreamHandler(f)
-#logging.getLogger('suds.wsdl').addHandler(h)
-
'''
Stuff to do:
-- Add logging to Aviary
-
- Test pool/schedd/stuff with hierarchical collectors
- Add a summary comment at the top like QMF opreations
Modified: trunk/sage/python/sage/verifiedhttps.py
===================================================================
--- trunk/sage/python/sage/verifiedhttps.py 2011-10-04 20:36:30 UTC (rev 5040)
+++ trunk/sage/python/sage/verifiedhttps.py 2011-10-05 15:34:14 UTC (rev 5041)
@@ -46,7 +46,8 @@
# There is no timeout attribute in earlier versions of this object.
# The only option available is to set a global default timeout for
# all socket objects.
- socket.setdefaulttimeout(self.timeout)
+ if hasattr(self, "timeout"):
+ socket.setdefaulttimeout(self.timeout)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.host, self.port))
12 years, 7 months
r5040 - trunk/cumin/python/cumin
by croberts＠fedoraproject.org
Author: croberts
Date: 2011-10-04 20:36:30 +0000 (Tue, 04 Oct 2011)
New Revision: 5040
Modified:
trunk/cumin/python/cumin/stat.py
Log:
Addressing BZ 725451 by limiting the PNG chart max y value to 100 when displaying a percentage chart. This matches the maximum value of 100 for percentage charts in the flash charts.
Modified: trunk/cumin/python/cumin/stat.py
===================================================================
--- trunk/cumin/python/cumin/stat.py 2011-10-04 19:18:55 UTC (rev 5039)
+++ trunk/cumin/python/cumin/stat.py 2011-10-04 20:36:30 UTC (rev 5040)
@@ -556,6 +556,10 @@
min_value = float(min_value)
max_value = round(max_value * 1.1 + 1)
+ if type == "percent" and max_value > 100:
+ # since none of our percentage graphs go above 100%
+ # we limit it here to match the flash graph
+ max_value = 100
if min_value < 0:
min_value = round(min_value * 1.1 - 1)
12 years, 7 months
