r5701 - branches/tmckay/sage/python/sage/wallaby
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-13 18:58:59 +0000 (Wed, 13 Feb 2013)
New Revision: 5701
Modified:
branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py
Log:
Fix nodes_by_tag
Modified: branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py
===================================================================
--- branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py 2013-02-13 18:43:57 UTC (rev 5700)
+++ branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py 2013-02-13 18:58:59 UTC (rev 5701)
@@ -609,27 +609,6 @@
t = min
return t
- def _generate_tag_data(self):
- # figure out the tag list and nodes per tag
- groups = self.get_data(WBTypes.GROUPS)
- tags = []
- nodes_by_tag = dict()
- for g in groups:
- if not g.name.startswith("+++") and \
- self._store.isTag(g):
- tags.append(g)
- nodes = g.membership()
- nodes_by_tag[g.name] = []
-
- self._lock.acquire()
- try:
- self._cache[WBTypes.TAGS].data = self._to_dict(tags)
- self._cache[WBTypes.TAGS].reset_remaining(len(tags) == 0)
- self._nodes_by_tag = nodes_by_tag
- finally:
- self._lock.release()
- log.debug("WallabyOperations: %s list updated (%s items)" % (WBTypes.TAGS, len(tags)))
-
def _set_cache(self, attr, data):
self._lock.acquire()
try:
@@ -701,7 +680,7 @@
if not g.name.startswith("+++") and \
ops._store.isLabel(g):
tags.append(g)
- nodes = [] #g.membership()
+ nodes = g.membership
nodes_by_tag[g.name] = nodes
ops._lock.acquire()
11 years, 3 months
r5700 - trunk/sage/python/sage/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-13 18:43:57 +0000 (Wed, 13 Feb 2013)
New Revision: 5700
Modified:
trunk/sage/python/sage/aviary/clients.py
Log:
Fix import error which causes server certificate validation to be disabled when communicating with Aviary services
BZ908051
Modified: trunk/sage/python/sage/aviary/clients.py
===================================================================
--- trunk/sage/python/sage/aviary/clients.py 2013-02-13 14:26:00 UTC (rev 5699)
+++ trunk/sage/python/sage/aviary/clients.py 2013-02-13 18:43:57 UTC (rev 5700)
@@ -1,4 +1,6 @@
import os
+import sage
+
from suds.client import Client
from sage.util import ObjectPool
from suds.transport.https import HttpAuthenticated
@@ -7,10 +9,16 @@
try:
from sage.https_full import HTTPSFullCertTransport
has_full_cert = True
- technology = sage.https_full.technology
except:
has_full_cert = False
+technology = "unspecified"
+if has_full_cert:
+ try:
+ technology = sage.https_full.technology
+ except:
+ pass
+
class TransportFactory(object):
def __init__(self, key="", cert="", root_cert="", domain_verify=True):
self.key = key
@@ -31,7 +39,7 @@
else:
log.info("%s: using client and server "\
"certificate validation for ssl connections, "\
- "solution is %s" % (where, clients.technology))
+ "solution is %s" % (where, technology))
log.info("%s: verify server domain against "\
"certificate during validation (%s)" \
11 years, 3 months
r5699 - branches/tmckay/sage/python/sage/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-13 14:26:00 +0000 (Wed, 13 Feb 2013)
New Revision: 5699
Modified:
branches/tmckay/sage/python/sage/aviary/aviaryoperations.py
Log:
Try to make get_endpoints_list work with COLLECTOR
Modified: branches/tmckay/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- branches/tmckay/sage/python/sage/aviary/aviaryoperations.py 2013-02-13 14:05:49 UTC (rev 5698)
+++ branches/tmckay/sage/python/sage/aviary/aviaryoperations.py 2013-02-13 14:26:00 UTC (rev 5699)
@@ -1466,12 +1466,15 @@
# This should return a list of objects with id/location
# id location
# [ ( (name,pool,resource,sub_type), [full urls] ) ]
- if self.locator:
+
+ # For now collector never uses locator
+ if self.locator and resource != "COLLECTOR":
ep = self.locator.get_endpoints(resource, subtype)
if ep.status.code in ("OK", "NO_MATCH"):
if hasattr(ep, "resources"):
return ep.resources
return []
+
else:
# hmmm, no locator to use.
# Make something that looks like locator responses.
@@ -1492,14 +1495,13 @@
if len(types) == 1:
return self.get_endpoints(types[0][0],
types[0][1])
-
+ endpoints = []
if self.locator:
- endpoints = []
for t in types:
endpoints.extend(self.get_endpoints(t[0], t[1]))
- else:
- endpoints = []
+ else:
for m in self.mechs:
endpoints.extend(m.get_endpoints(types))
- return endpoints
+
+ return endpoints
11 years, 3 months
r5698 - branches/tmckay/sage/python/sage/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-13 14:05:49 +0000 (Wed, 13 Feb 2013)
New Revision: 5698
Modified:
branches/tmckay/sage/python/sage/aviary/aviaryoperations.py
Log:
Turn off locator for collector service for now
Modified: branches/tmckay/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- branches/tmckay/sage/python/sage/aviary/aviaryoperations.py 2013-02-13 03:16:50 UTC (rev 5697)
+++ branches/tmckay/sage/python/sage/aviary/aviaryoperations.py 2013-02-13 14:05:49 UTC (rev 5698)
@@ -1448,7 +1448,8 @@
datadir, hadoop_servers))
if collector_servers:
- self.mechs.append(_AviaryCollectorMethods(self.locator,
+ # we never use locator for collector (for now)
+ self.mechs.append(_AviaryCollectorMethods(None,
transports,
datadir, collector_servers))
self.add_mechanisms(self.mechs)
11 years, 3 months
r5697 - branches/tmckay/sage/python/sage/wallaby
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-13 03:16:50 +0000 (Wed, 13 Feb 2013)
New Revision: 5697
Modified:
branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py
Log:
Turn group membership back on for nodes, hack last_checkin
Modified: branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py
===================================================================
--- branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py 2013-02-12 23:01:10 UTC (rev 5696)
+++ branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py 2013-02-13 03:16:50 UTC (rev 5697)
@@ -618,7 +618,7 @@
if not g.name.startswith("+++") and \
self._store.isTag(g):
tags.append(g)
- #nodes = g.membership()
+ nodes = g.membership()
nodes_by_tag[g.name] = []
self._lock.acquire()
@@ -675,9 +675,12 @@
class NodeData(CacheData):
def get_values(self, attr, ops):
- ops._set_cache(attr,
- [(name, ops._store.getNode(name)) for name in ops._store.nodes])
+ nodes = [(name, ops._store.getNode(name)) for name in ops._store.nodes]
+ for i in nodes:
+ i[1].last_checkin = time.time()
+ ops._set_cache(attr, nodes)
+
class GroupData(CacheData):
def get_values(self, attr, ops):
ops._set_cache(attr,
11 years, 3 months
r5696 - branches/tmckay/cumin/python/cumin/grid
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-12 23:01:10 +0000 (Tue, 12 Feb 2013)
New Revision: 5696
Modified:
branches/tmckay/cumin/python/cumin/grid/negotiator.py
Log:
Make NegotiatorReconfigTask invisible
Modified: branches/tmckay/cumin/python/cumin/grid/negotiator.py
===================================================================
--- branches/tmckay/cumin/python/cumin/grid/negotiator.py 2013-02-12 22:55:36 UTC (rev 5695)
+++ branches/tmckay/cumin/python/cumin/grid/negotiator.py 2013-02-12 23:01:10 UTC (rev 5696)
@@ -476,6 +476,10 @@
return 210
class NegotiatorReconfigTask(ObjectFrameTask):
+ def __init__(self, app, frame):
+ super(NegotiatorReconfigTask, self).__init__(app, frame)
+ self.visible = False
+
def do_invoke(self, invoc, negotiator):
invoc.description = "Reconfig"
result = self.app.remote.reconfig(negotiator)
11 years, 3 months
r5694 - in branches/tmckay: cumin/bin cumin/python/cumin cumin/python/cumin/grid cumin/python/cumin/inventory mint/python/mint/aviary sage/python/sage/wallaby
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-12 22:27:25 +0000 (Tue, 12 Feb 2013)
New Revision: 5694
Modified:
branches/tmckay/cumin/bin/cumin-web
branches/tmckay/cumin/python/cumin/config.py
branches/tmckay/cumin/python/cumin/grid/tags.py
branches/tmckay/cumin/python/cumin/inventory/system.py
branches/tmckay/cumin/python/cumin/main.py
branches/tmckay/mint/python/mint/aviary/collector.py
branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py
Log:
Change wallaby name to wallaroo in multiple places.
Fix string data types for Submitter records
Allow graceful failure on wallaroo connect
Modified: branches/tmckay/cumin/bin/cumin-web
===================================================================
--- branches/tmckay/cumin/bin/cumin-web 2013-02-12 20:44:07 UTC (rev 5693)
+++ branches/tmckay/cumin/bin/cumin-web 2013-02-12 22:27:25 UTC (rev 5694)
@@ -33,18 +33,18 @@
# Defautl is True. Undocumented.
cumin.aviary_prefer_condor = values.aviary_prefer_condor
-def set_wallaby_configs(cumin, values):
- if values.wallaby_broker == "None":
- values.wallaby_broker = None
+def set_wallaroo_configs(cumin, values):
+ if values.wallaroo_host == "None":
+ values.wallaroo_host = None
- cumin.wallaby_broker = values.wallaby_broker
+ cumin.wallaroo_host = values.wallaroo_host
# Let 0 indicate "no timeout", since the timeout
# value is an int in the config and None can't be
# specified
- cumin.wallaby_refresh = values.wallaby_refresh
- if cumin.wallaby_refresh == 0:
- cumin.wallaby_refresh = None
+ cumin.wallaroo_refresh = values.wallaroo_refresh
+ if cumin.wallaroo_refresh == 0:
+ cumin.wallaroo_refresh = None
def set_authorize_config(cumin, values, access_root):
@@ -207,7 +207,7 @@
values.persona, authmech)
set_aviary_configs(cumin, values)
- set_wallaby_configs(cumin, values)
+ set_wallaroo_configs(cumin, values)
set_ldap_configs(cumin, values)
set_kerberos_configs(cumin, values)
Modified: branches/tmckay/cumin/python/cumin/config.py
===================================================================
--- branches/tmckay/cumin/python/cumin/config.py 2013-02-12 20:44:07 UTC (rev 5693)
+++ branches/tmckay/cumin/python/cumin/config.py 2013-02-12 22:27:25 UTC (rev 5694)
@@ -223,10 +223,10 @@
# param = ConfigParameter(self, "auth-proxy", bool)
# param.default = False
- param = ConfigParameter(self, "wallaby-broker", str)
+ param = ConfigParameter(self, "wallaroo-host", str)
param.default = "http://localhost"
- param = ConfigParameter(self, "wallaby-refresh", int)
+ param = ConfigParameter(self, "wallaroo-refresh", int)
param.default = 60
# Web uses the job server, too
Modified: branches/tmckay/cumin/python/cumin/grid/tags.py
===================================================================
--- branches/tmckay/cumin/python/cumin/grid/tags.py 2013-02-12 20:44:07 UTC (rev 5693)
+++ branches/tmckay/cumin/python/cumin/grid/tags.py 2013-02-12 22:27:25 UTC (rev 5694)
@@ -100,14 +100,14 @@
def do_get_data(self, values):
data = []
try:
- wallaby_tags = self.app.wallaby.get_data(WBTypes.TAGS)
- wallaby_features = self.app.wallaby.get_data(WBTypes.FEATURES)
+ wallaby_tags = self.app.wallaroo.get_data(WBTypes.TAGS)
+ wallaby_features = self.app.wallaroo.get_data(WBTypes.FEATURES)
for i, tag in enumerate(wallaby_tags):
data.append({'Tag':str(escape_entity(tag.name)),
'Features':", ".join(tag.features),
- 'NumHosts':len(self.app.wallaby.get_node_names(tag)),
- 'Host':self.app.wallaby.get_node_names(tag)} )
+ 'NumHosts':len(self.app.wallaroo.get_node_names(tag)),
+ 'Host':self.app.wallaroo.get_node_names(tag)} )
except:
log.debug("Problem adapting wallaby response", exc_info=True)
@@ -143,11 +143,11 @@
Make the wallaby call and return a list of dicts that will be
formatted for the table by the process_record method
'''
- wallaby_nodes = self.app.wallaby.get_data(WBTypes.NODES)
+ wallaby_nodes = self.app.wallaroo.get_data(WBTypes.NODES)
data = []
for i, node in enumerate(wallaby_nodes):
data.append({'Host':node.name,
- 'Tags':self.app.wallaby.get_tag_names(node),
+ 'Tags':self.app.wallaroo.get_tag_names(node),
'Checkin':node.last_checkin})
return data
@@ -224,7 +224,7 @@
super(TagObjectFrame, self).do_process(session)
def get_object(self, session, id):
- return self.app.wallaby.get_tag_by_name(id)
+ return self.app.wallaroo.get_tag_by_name(id)
class TagOverview(Widget):
'''
@@ -252,7 +252,7 @@
retval = ""
try:
obj = self.tag.get(session)
- retval = xml_escape(", ".join(self.app.wallaby.get_node_names(obj)))
+ retval = xml_escape(", ".join(self.app.wallaroo.get_node_names(obj)))
except Exception, e:
log.debug("Exception in rendering tag hosts, tags probably not loaded yet: %s", e.message)
return retval
@@ -463,7 +463,7 @@
def validate(self, session):
return True
- results = self.app.wallaby.validate_configuration()
+ results = self.app.wallaroo.validate_configuration()
#handle the set of explanations that could come back if things didn't go well
try:
@@ -631,7 +631,7 @@
tags_string = ""
given_node = self.form.node_name.input.param.get(session)
- selected_tags = self.app.wallaby.get_tag_names(given_node)
+ selected_tags = self.app.wallaroo.get_tag_names(given_node)
for i, tag in enumerate(items):
selected = ""
@@ -764,7 +764,7 @@
nodes_string = ""
given_tag = self.form.tags.input.param.get(session)
- selected_nodes = self.app.wallaby.get_node_names(given_tag)
+ selected_nodes = self.app.wallaroo.get_node_names(given_tag)
for i, node in enumerate(items):
selected = ""
@@ -819,7 +819,7 @@
self.replace_child(self.input)
def render_inputs(self, session, *args):
- value = ", ".join(self.app.wallaby.get_node_names(self.form.tags.get(session)))
+ value = ", ".join(self.app.wallaroo.get_node_names(self.form.tags.get(session)))
if not value or value =="":
value = "No hosts currently selected"
value = truncate_text(value, 50, True)
@@ -890,7 +890,7 @@
features_string = ""
given_tag = self.form.tags.input.param.get(session)
- tag_object = self.app.wallaby.get_tag_by_name(given_tag)
+ tag_object = self.app.wallaroo.get_tag_by_name(given_tag)
selected_features = list()
if tag_object is not None:
selected_features = tag_object.features
@@ -940,7 +940,7 @@
def render_inputs(self, session, *args):
tag = self.form.tags.get(session)
value = ""
- tag_obj = self.app.wallaby.get_tag_by_name(tag)
+ tag_obj = self.app.wallaroo.get_tag_by_name(tag)
if tag_obj is not None:
value = xml_escape(", ".join(tag_obj.features))
if not value or value == "":
@@ -994,12 +994,12 @@
if result == False:
self.invoc.status = self.invoc.FAILED
self.invoc.end()
- self.app.wallaby.refresh(WBTypes.GROUPS,WBTypes.TAGS)
+ self.app.wallaroo.refresh(WBTypes.GROUPS,WBTypes.TAGS)
def do_invoke(self, invoc, negotiator, node_name, tags):
self.invoc = invoc
try:
- call_async(self.callback, self.app.wallaby.edit_tags, node_name, *tags)
+ call_async(self.callback, self.app.wallaroo.edit_tags, node_name, *tags)
except:
invoc.status = invoc.FAILED
log.debug("Edit node failed", exc_info=True)
@@ -1025,7 +1025,7 @@
self.form.tags.set(session, tag_id)
nodes = "No nodes currently assigned"
- node_list = self.app.wallaby.get_node_names(tag_id)
+ node_list = self.app.wallaroo.get_node_names(tag_id)
if len(node_list) > 0:
nodes = ", ".join(node_list)
self.form.node_name.set(session, nodes)
@@ -1037,7 +1037,7 @@
self.invoc.status = self.invoc.FAILED
if(self.call_count <= 0):
self.invoc.end()
- self.app.wallaby.refresh(WBTypes.GROUPS,WBTypes.TAGS)
+ self.app.wallaroo.refresh(WBTypes.GROUPS,WBTypes.TAGS)
def do_invoke(self, invoc, negotiator, tag, chosen_nodes):
'''
@@ -1060,20 +1060,20 @@
if the node is NOT on the current_nodes list and is NOT on the chosen_nodes passed in, nothing to do
'''
try:
- current_nodes = self.app.wallaby.get_node_names(tag)
+ current_nodes = self.app.wallaroo.get_node_names(tag)
for node in chosen_nodes:
if node not in current_nodes:
#we need to add the new tag to the existing tags for each node in the list
- current_tags = self.app.wallaby.get_tag_names(node)
+ current_tags = self.app.wallaroo.get_tag_names(node)
current_tags.append(tag)
self.call_count += 1
- call_async(self.callback, self.app.wallaby.edit_tags, node, *current_tags)
+ call_async(self.callback, self.app.wallaroo.edit_tags, node, *current_tags)
for node in current_nodes:
if node not in chosen_nodes:
- current_tags = self.app.wallaby.get_tag_names(node)
+ current_tags = self.app.wallaroo.get_tag_names(node)
current_tags.remove(tag)
self.call_count += 1
- call_async(self.callback, self.app.wallaby.edit_tags, node, *current_tags)
+ call_async(self.callback, self.app.wallaroo.edit_tags, node, *current_tags)
except:
self.call_count = 0
invoc.status = invoc.FAILED
@@ -1101,7 +1101,7 @@
tag_id = osession.values_by_path["main.grid.tag.id"]
self.form.tags.set(session, tag_id)
features = "No features currently assigned"
- tag = self.app.wallaby.get_tag_by_name(tag_id)
+ tag = self.app.wallaroo.get_tag_by_name(tag_id)
feature_list = list()
if tag is not None:
feature_list = tag.features
@@ -1123,7 +1123,7 @@
'''
self.invoc = invoc
try:
- call_async(self.callback, self.app.wallaby.edit_features, tag, *chosen_features)
+ call_async(self.callback, self.app.wallaroo.edit_features, tag, *chosen_features)
except:
invoc.status = invoc.FAILED
log.debug("Edit feature failed", exc_info=True)
@@ -1150,7 +1150,7 @@
def do_invoke(self, session, object, invoc):
self.invoc = invoc
try:
- call_async(self.callback, self.app.wallaby.activate_configuration)
+ call_async(self.callback, self.app.wallaroo.activate_configuration)
except:
invoc.status = invoc.FAILED
log.debug("Activate config failed", exc_info=True)
@@ -1172,7 +1172,7 @@
def callback(self, result):
if result == False:
self.invoc.status = self.invoc.FAILED
- self.app.wallaby.refresh(WBTypes.GROUPS,WBTypes.TAGS)
+ self.app.wallaroo.refresh(WBTypes.GROUPS,WBTypes.TAGS)
self.invoc.end()
@@ -1181,7 +1181,7 @@
tags = [x.strip() for x in tag.split(',')]
try:
- call_async(self.callback, self.app.wallaby.create_tags, tags)
+ call_async(self.callback, self.app.wallaroo.create_tags, tags)
except:
invoc.status = invoc.FAILED
log.debug("Adding tags failed", exc_info=True)
@@ -1215,13 +1215,13 @@
def callback(self, result):
if result == False:
self.invoc.status = self.invoc.FAILED
- self.app.wallaby.refresh()
+ self.app.wallaroo.refresh()
self.invoc.end()
def do_invoke(self, invoc, tags):
try:
- call_async(self.callback, self.app.wallaby.remove_tags, tags)
+ call_async(self.callback, self.app.wallaroo.remove_tags, tags)
except:
invoc.status = invoc.FAILED
log.debug("Remove node tags failed", exc_info=True)
@@ -1233,7 +1233,7 @@
'''
fetch the list of <type> from wallaby
'''
- wallaby_items = self.app.wallaby.get_data(type)
+ wallaby_items = self.app.wallaroo.get_data(type)
item_list = list()
for item in wallaby_items:
Modified: branches/tmckay/cumin/python/cumin/inventory/system.py
===================================================================
--- branches/tmckay/cumin/python/cumin/inventory/system.py 2013-02-12 20:44:07 UTC (rev 5693)
+++ branches/tmckay/cumin/python/cumin/inventory/system.py 2013-02-12 22:27:25 UTC (rev 5694)
@@ -185,7 +185,7 @@
def render_tags(self, session):
node_name = self._get_name(session)
try:
- tags = self.app.wallaby.get_tag_names(node_name)
+ tags = self.app.wallaroo.get_tag_names(node_name)
except AttributeError:
tags = []
tags_string = ", ".join(tags)
@@ -241,7 +241,7 @@
sqldata = super(WallabyAndSqlAdapter, self).get_data(values, options)
#now get the wallaby data
- wallaby_nodes = self.app.wallaby.get_data(WBTypes.NODES, values)
+ wallaby_nodes = self.app.wallaroo.get_data(WBTypes.NODES, values)
data = list()
if len(sqldata) > 0 and len(wallaby_nodes) > 0: #means that we have both sesame and wallaby data
@@ -251,12 +251,12 @@
if len(match_index) > 0:
## merge-in the wallaby data to the matched node entry in sqldata
new_record = list(sqldata[match_index[0]])
- new_record.append(", ".join(self.app.wallaby.get_tag_names(node)))
+ new_record.append(", ".join(self.app.wallaroo.get_tag_names(node)))
new_record.append(node.last_checkin)
sqldata[match_index[0]] = tuple(new_record)
else:
# there was no match found, add a wallaby-only row
- data_row = tuple([node.name, node.name, "", "", "", "", ", ".join(self.app.wallaby.get_tag_names(node)), node.last_checkin])
+ data_row = tuple([node.name, node.name, "", "", "", "", ", ".join(self.app.wallaroo.get_tag_names(node)), node.last_checkin])
sqldata.append(data_row)
data = sqldata
elif len(sqldata) > 0: # we only have sesame data
@@ -268,7 +268,7 @@
for node in wallaby_nodes:
# using the node name for the [0] element in this case will allow
#other pages to use the id as the node lookup key against the wallaby data
- data_row = tuple([node.name, node.name, "", "", "", "", ", ".join(self.app.wallaby.get_tag_names(node)), node.last_checkin])
+ data_row = tuple([node.name, node.name, "", "", "", "", ", ".join(self.app.wallaroo.get_tag_names(node)), node.last_checkin])
data.append(data_row)
options.sort_field = requested_sort_field
Modified: branches/tmckay/cumin/python/cumin/main.py
===================================================================
--- branches/tmckay/cumin/python/cumin/main.py 2013-02-12 20:44:07 UTC (rev 5693)
+++ branches/tmckay/cumin/python/cumin/main.py 2013-02-12 22:27:25 UTC (rev 5694)
@@ -106,9 +106,9 @@
self.aviary_suds_logs = False
self.aviary_prefer_condor = True
- self.wallaby = None
- self.wallaby_broker = None
- self.wallaby_refresh = 60
+ self.wallaroo = None
+ self.wallaroo_host = None
+ self.wallaroo_refresh = 60
def server_alive(self):
return self.server.server_alive()
@@ -236,9 +236,9 @@
self.remote.add_mechanisms(ops)
- # Create RPC interface for Wallaby
- self.wallaby = WallabyOperations(self.wallaby_broker,
- self.wallaby_refresh)
+ # Create RPC interface for Wallaroo
+ self.wallaroo = WallabyOperations(self.wallaroo_host,
+ self.wallaroo_refresh)
self.model.init()
self.database.init(schema_version_check)
@@ -260,14 +260,14 @@
log.info("Starting %s", self)
self.server.start()
- if self.wallaby is not None:
- self.wallaby.start()
+ if self.wallaroo is not None:
+ self.wallaroo.start()
def stop(self):
log.info("Stopping %s", self)
- if self.wallaby is not None:
- self.wallaby.stop(wait=True)
+ if self.wallaroo is not None:
+ self.wallaroo.stop(wait=True)
try:
log.info("Stopping update threads")
Modified: branches/tmckay/mint/python/mint/aviary/collector.py
===================================================================
--- branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-12 20:44:07 UTC (rev 5693)
+++ branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-12 22:27:25 UTC (rev 5694)
@@ -272,14 +272,14 @@
super(SubmitterObj, self).__init__(endpoint_id, res_id, "Submitter")
def set_from_summary(self, id, summaries):
- self.Machine = id.machine
- self.Name = id.name
+ self.Machine = str(id.machine)
+ self.Name = str(id.name)
summary = summaries[(id.name,"")]
self.HeldJobs = summary.held
self.IdleJobs = summary.idle
self.RunningJobs = summary.running
- self.ScheddName = id.scheduler
+ self.ScheddName = str(id.scheduler)
self.Owner = ""
self.JobQueueBirthdate = 0
Modified: branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py
===================================================================
--- branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py 2013-02-12 20:44:07 UTC (rev 5693)
+++ branches/tmckay/sage/python/sage/wallaby/wallabyoperations.py 2013-02-12 22:27:25 UTC (rev 5694)
@@ -128,7 +128,11 @@
# Get initinal connection and Store obect
while not self._stop:
- self._store = wallaroo.client.connect(**connect_args)
+ log.debug("WallabyOperations: attempting to get Wallaby store object")
+ try:
+ self._store = wallaroo.client.connect(**connect_args)
+ except:
+ pass
if self._store is not None:
self._partition_group = self._store.getPartitionGroup().name
log.debug("WallabyOperations: found wallaby store object")
11 years, 3 months
r5693 - in branches/tmckay/mint/python/mint: . aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-12 20:44:07 +0000 (Tue, 12 Feb 2013)
New Revision: 5693
Modified:
branches/tmckay/mint/python/mint/aviary/submissions.py
branches/tmckay/mint/python/mint/update.py
Log:
Remove debugging putlines from submissions.py, remove unnecessary code from update
Modified: branches/tmckay/mint/python/mint/aviary/submissions.py
===================================================================
--- branches/tmckay/mint/python/mint/aviary/submissions.py 2013-02-12 19:58:31 UTC (rev 5692)
+++ branches/tmckay/mint/python/mint/aviary/submissions.py 2013-02-12 20:44:07 UTC (rev 5693)
@@ -29,7 +29,7 @@
self.ep.after_index,
"AFTER")
- print ("querying after", self.ep.after_index, datetime.fromtimestamp(self.ep.after_index))
+ # print ("querying after", self.ep.after_index, datetime.fromtimestamp(self.ep.after_index))
#print (ids.status, ids.got_data)
# if data.ids is empty (remaining == 0)
@@ -38,15 +38,15 @@
# yet. This is how we will accomplish polling,
# just looping over the data continuously.
if ids.status == 0 and ids.got_data:
- print len(ids.data.ids)
+ # print len(ids.data.ids)
#print ids.data.ids
# ids.data.ids is a list of SubmissionIDs
# Each of these ids needs to be polled to
# get info out.
if ids.data.ids:
- print len(ids.data.ids)
- print ids.data.ids
+ # print len(ids.data.ids)
+ # print ids.data.ids
work = PollSubmissionIDs(ids.data.ids, self.ep)
foreman.add_work((work,))
@@ -61,14 +61,14 @@
# back an empty list but sets remaining...
# In that case just leave after_index as is
if ids.data.ids:
-# self.ep.after_index = ids.data.ids[-1].qdate
- self.ep.after_index = max(ids.data.ids)
+ self.ep.after_index = ids.data.ids[-1].qdate
+# self.ep.after_index = max(ids.data.ids)
else:
#print "query after reset"
self.ep.after_index = now_minus_duration(self.ep.poll_delta)
- else:
- print "query after failed"
+# else:
+# print "query after failed"
def completed(self):
return self.ep.abandoned or self.ep.after_queries_completed
@@ -88,7 +88,7 @@
self.ep.before_index,
"BEFORE")
- print ("querying before", self.ep.before_index, datetime.fromtimestamp(self.ep.before_index))
+ #print ("querying before", self.ep.before_index, datetime.fromtimestamp(self.ep.before_index))
#print (ids.status, ids.got_data)
# if data.ids is empty (remaining == 0)
@@ -97,7 +97,7 @@
# yet. This is how we will accomplish polling
# We need a date to qdate function.
if ids.status == 0 and ids.got_data:
- print len(ids.data.ids)
+ # print len(ids.data.ids)
# ids.data.ids is a list of SubmissionIDs
# each of these ids needs to be polled to
# get info out.
@@ -111,8 +111,8 @@
# the qdate of the last item gives us the
# starting index of the next query
if ids.data.remaining:
-# self.ep.before_index = ids.data.ids[-1].qdate
- self.ep.before_index = min(ids.data.ids)
+ self.ep.before_index = ids.data.ids[-1].qdate
+# self.ep.before_index = min(ids.data.ids)
if self.ep.before_index <= self.ep.before_cutoff:
#print ("hit before cutoff, finishing",
@@ -120,12 +120,12 @@
# datetime.fromtimestamp(self.ep.before_cutoff))
self.ep.before_queries_completed = True
- else:
- print "query before exiting"
- self.ep.before_queries_completed = True
+# else:
+# print "query before exiting"
+# self.ep.before_queries_completed = True
- else:
- print "query before failed"
+# else:
+# print "query before failed"
def completed(self):
return self.ep.abandoned or self.ep.before_queries_completed
Modified: branches/tmckay/mint/python/mint/update.py
===================================================================
--- branches/tmckay/mint/python/mint/update.py 2013-02-12 19:58:31 UTC (rev 5692)
+++ branches/tmckay/mint/python/mint/update.py 2013-02-12 20:44:07 UTC (rev 5693)
@@ -241,388 +241,6 @@
def __repr__(self):
return self.__class__.__name__
-class ObjectUpdate(Update):
- def __init__(self, model, qmf_object):
- super(ObjectUpdate, self).__init__(model)
-
- self.qmf_object = qmf_object
-
- def do_process(self, cursor, stats):
- cls = self.get_class()
- agent_id = self.get_agent_id()
- object_id = self.get_object_id()
-
- delete_time = self.qmf_object.getTimestamps()[2]
-
- try:
- agent = self.model.agents_by_id[agent_id]
- except KeyError:
- raise UpdateDropped()
-
- try:
- obj = agent.get_object(cursor, cls, object_id)
- except RosemaryNotFound:
- if not self.qmf_object.getProperties():
- raise UpdateDropped()
-
- if delete_time != 0:
- raise UpdateDropped()
-
- obj = self.create_object(cursor, stats, cls)
-
- return
-
- if delete_time != 0:
- self.delete_object(cursor, stats, obj)
-
- del agent.objects_by_id[obj._qmf_object_id]
-
- return
-
- if cls._package is self.model.org_apache_qpid_broker:
- self.maybe_drop_sample(obj)
-
- self.update_object(cursor, stats, obj)
-
- def get_agent_id(self):
- return make_agent_id(self.qmf_object.getAgent())
-
- def get_class(self):
- class_key = self.qmf_object.getClassKey()
- name = class_key.getPackageName()
-
- try:
- pkg = self.model._packages_by_name[name]
- except KeyError:
- raise UpdateDropped()
-
- name = class_key.getClassName()
-
- try:
- cls = pkg._classes_by_lowercase_name[name.lower()]
- except KeyError:
- raise UpdateDropped()
-
- return cls
-
- def get_object_id(self):
- return self.qmf_object.getObjectId().objectName
-
- def maybe_drop_sample(self, obj):
- properties = self.qmf_object.getProperties()
- statistics = self.qmf_object.getStatistics()
-
- if not properties and statistics:
- # Just stats; do we want it?
- # if stats.enqueued - stats.dequeued > 500:
-
- now = time.time()
- update = self.qmf_object.getTimestamps()[0] / 1000000000
- sample = obj._sample_time
-
- if update < now - sample_window_max:
- # The sample is too old
- raise UpdateDropped()
-
- if sample and sample > now - sample_window_min:
- # The samples are too fidelitous
- raise UpdateDropped()
-
- def create_object(self, cursor, stats, cls):
- update_time, create_time, delete_time = self.qmf_object.getTimestamps()
- create_time = datetime.fromtimestamp(create_time / 1000000000)
- update_time = datetime.fromtimestamp(update_time / 1000000000)
-
- obj = cls.create_object(cursor)
- obj._qmf_agent_id = self.get_agent_id()
- obj._qmf_object_id = self.get_object_id()
- obj._qmf_create_time = create_time
- obj._qmf_update_time = update_time
-
- object_columns = list()
- sample_columns = list()
-
- table = cls.sql_table
-
- object_columns.append(table._id)
- object_columns.append(table._qmf_agent_id)
- object_columns.append(table._qmf_object_id)
- object_columns.append(table._qmf_create_time)
- object_columns.append(table._qmf_update_time)
-
- self.process_properties(obj, object_columns, cursor)
- self.process_statistics(obj, object_columns, sample_columns)
-
- statements = list()
-
- sql = cls.sql_insert_object.emit(object_columns)
- statements.append(sql)
-
- if sample_columns:
- sample_columns.append(cls.sql_samples_table._qmf_update_time)
-
- sql = cls.sql_samples_insert.emit(sample_columns)
- statements.append(sql)
-
- obj._sample_time = time.time()
-
- sql = "; ".join(statements)
- self.execute_sql(cursor, sql, obj.__dict__)
-
- self.process_deferred_links(cursor, obj)
-
- obj._save_time = datetime.now()
-
- self.model.print_event(3, "Created %s", obj)
-
- stats.objects_created += 1
- #stats.objects_created_by_class[cls] += 1
-
- return obj
-
- def process_deferred_links(self, cursor, obj):
- agent = self.model.agents_by_id[obj._qmf_agent_id]
-
- if obj._qmf_object_id not in agent.deferred_links_by_id:
- return
-
- links = agent.deferred_links_by_id[obj._qmf_object_id]
-
- for link in links:
- link.realize(cursor, obj)
-
- del agent.deferred_links_by_id[obj._qmf_object_id]
-
- def update_object(self, cursor, stats, obj):
- update_time, create_time, delete_time = self.qmf_object.getTimestamps()
- update_time = datetime.fromtimestamp(update_time / 1000000000)
-
- obj._qmf_update_time = update_time
-
- object_columns = list()
- sample_columns = list()
-
- self.process_properties(obj, object_columns, cursor)
- self.process_statistics(obj, object_columns, sample_columns)
-
- statements = list()
- cls = obj._class
-
- # force a write if it's been too long, even if the values match
- if object_columns \
- or (obj._save_time != None and \
- obj._qmf_update_time != None and \
- obj._save_time < obj._qmf_update_time - timedelta(hours=1)):
- object_columns.append(cls.sql_table._qmf_update_time)
-
- sql = cls.sql_update_object.emit(object_columns)
- statements.append(sql)
-
- obj._save_time = datetime.now()
-
- if sample_columns:
- sample_columns.append(cls.sql_samples_table._qmf_update_time)
-
- sql = cls.sql_samples_insert.emit(sample_columns)
- statements.append(sql)
-
- obj._sample_time = time.time()
-
- if not statements:
- raise UpdateDropped()
-
- sql = "; ".join(statements)
- self.execute_sql(cursor, sql, obj.__dict__)
-
- self.model.print_event(4, "Updated %s", obj)
-
- stats.objects_updated += 1
- #stats.objects_updated_by_class[cls] += 1
-
- def delete_object(self, cursor, stats, obj):
- obj.delete(cursor)
-
- self.model.print_event(3, "Deleted %s", obj)
-
- stats.objects_deleted += 1
- #stats.objects_deleted_by_class[obj._class] += 1
-
- def process_properties(self, obj, columns, cursor):
- cls = obj._class
-
- set_scheduler_ref_id = False
- for prop, value in self.qmf_object.getProperties():
- try:
- if prop.type == 10:
-
- # Fix up submissions to hold a scheduler ref
- # remember that we saw this
- class FakeProp(object):
- def __init__(self, name):
- self.name = name
-
- class FakeOID(object):
- def __init__(self, agent, objname):
- self.isV2 = True
- self.agentName = agent
- self.objectName = objname
-
- try:
- if cls._name == "Submission" and \
- prop.name == "jobserverRef" and value.isV2:
- info = value.agentName.split(":")
- mach = info[2].split("@")
- set_sched_val = FakeOID("com.redhat.grid:scheduler:"+mach[-1], mach[-1])
- set_sched_prop = FakeProp("schedulerRef")
- set_scheduler_ref_id = True
- except:
- pass
-
- col, nvalue = self.process_reference \
- (obj, prop, value, cursor)
- else:
- col, nvalue = self.process_value(cls, prop, value)
- except MappingException, e:
- log.debug(e)
- continue
-
- # XXX This optimization will be obsolete when QMF does it
- # instead
-
- if nvalue == getattr(obj, col.name):
- continue
-
- setattr(obj, col.name, nvalue)
- columns.append(col)
-
- if set_scheduler_ref_id:
- try:
- col, nvalue = self.process_reference \
- (obj, set_sched_prop, set_sched_val, cursor)
- if nvalue != getattr(obj, col.name):
- setattr(obj, col.name, nvalue)
- columns.append(col)
- except MappingException, e:
- pass
-
- def process_reference(self, obj, prop, oid, cursor):
- try:
- ref = obj._class._references_by_name[prop.name]
- except KeyError:
- raise MappingException("Reference %s is unknown" % prop.name)
-
- if not ref.sql_column:
- raise MappingException("Reference %s has no column" % ref.name)
-
- value = None
-
- if oid:
- if oid.isV2:
- agent_id = oid.agentName
- else:
- # Not much we can do but assume same agent
- agent_id = self.get_agent_id()
-
- try:
- agent = self.model.agents_by_id[agent_id]
- except KeyError:
- raise MappingException("Agent %s is unknown" % agent_id)
-
- object_id = oid.objectName
-
- try:
- that = agent.get_object(cursor, ref.that_cls, object_id)
- except RosemaryNotFound:
- link = DeferredLink(obj, ref)
- agent.deferred_links_by_id[object_id].append(link)
-
- msg = "Deferring link to object %s %s"
- raise MappingException(msg % (ref.that_cls, object_id))
-
- value = that._id
-
- return ref.sql_column, value
-
- def process_value(self, cls, prop, value):
- try:
- col = cls._properties_by_name[prop.name].sql_column
- except KeyError:
- raise MappingException("Property %s is unknown" % prop)
-
- if value is not None:
- value = transform_value(prop, value)
-
- return col, value
-
- def process_statistics(self, obj, update_columns, insert_columns):
- build_columns = list()
- saw_change = False
- for stat, value in self.qmf_object.getStatistics():
- try:
- col = obj._class._statistics_by_name[stat.name].sql_column
- except KeyError:
- log.debug("Statistic %s is unknown", stat)
-
- continue
-
- if value is not None:
- value = transform_value(stat, value)
-
- # XXX hack workaround
- if col.name == "MonitorSelfTime":
- value = datetime.now()
-
- # Don't write unchanged values
- #
- # XXX This optimization will be obsolete when QMF does it
- # instead
- if value != getattr(obj, col.name):
- setattr(obj, col.name, value)
- update_columns.append(col)
- saw_change = True
-
- # If we do end up seeing a value change, we will
- # need to insert an entire row so build it up
- build_columns.append(col)
-
- if saw_change:
- insert_columns.extend(build_columns)
-
-
- def execute_sql(self, cursor, text, args):
- try:
- cursor.execute(text, args)
- except:
- log.debug("%s failed sql execute", self, exc_info=True)
- log.error("Sql execute failed")
- log.error("Sql text: %s", text)
- log.error("Sql values:")
-
- for item in sorted(args.items()):
- log.error(" %-34s %r", *item)
-
- log.error("Sql row count: %i", cursor.rowcount)
-
- log.error("Qmf properties:")
-
- for item in sorted(self.qmf_object.getProperties()):
- log.error(" %-34s %r", *item)
-
- log.error("Qmf statistics:")
-
- for item in sorted(self.qmf_object.getStatistics()):
- log.error(" %-34s %r", *item)
-
- raise
-
- def __repr__(self):
- name = self.__class__.__name__
- agent_id = self.get_agent_id()
- cls = self.qmf_object.getClassKey().getClassName()
- obj_id = self.get_object_id()
-
- return "%s(%s,%s,%s)" % (name, agent_id, cls, obj_id)
-
class DeferredLink(object):
def __init__(self, this, reference):
self.this = this
@@ -637,67 +255,6 @@
msg = "Realized deferred link to %s via %s"
log.debug(msg, that, self.reference)
-class AgentUpdate(Update):
- def __init__(self, model, qmf_agent):
- super(AgentUpdate, self).__init__(model)
-
- self.qmf_agent = qmf_agent
-
- def get_agent_id(self):
- return make_agent_id(self.qmf_agent)
-
- def do_process(self, cursor, stats):
- agent_id = self.get_agent_id()
-
- try:
- agent = self.model.agents_by_id[agent_id]
- except KeyError:
- agent = MintAgent(self.model, agent_id)
- stats.agents_created += 1
- return
-
- #timestamp = timestamp / 1000000000
- #agent.last_heartbeat = datetime.fromtimestamp(timestamp)
-
- agent.last_heartbeat = datetime.now()
-
- stats.agents_updated += 1
-
- def delete_agent_objects(self, cursor, stats, agent):
-
- def loop_body(cls):
- if cls._storage != "none" and cls.check_persistent() == "session":
- count = cls.delete_selection(cursor, _qmf_agent_id=agent.id)
- stats.objects_deleted += count
- #stats.objects_deleted_by_class[cls] += count
- cursor.connection.commit()
-
- classes = self.app.get_bound_classes()
- log.debug("Delete agent objects by bound classes " + str(classes))
- for cls in classes:
- loop_body(cls)
-
- def __repr__(self):
- name = self.__class__.__name__
- agent_id = self.get_agent_id()
-
- return "%s(%s)" % (name, agent_id)
-
-class AgentDelete(AgentUpdate):
- def do_process(self, cursor, stats):
- agent_id = self.get_agent_id()
-
- try:
- agent = self.model.agents_by_id[agent_id]
- except KeyError:
- raise UpdateDropped()
-
- agent.delete()
-
- stats.agents_deleted += 1
-
- self.delete_agent_objects(cursor, stats, agent)
-
class AviaryUpdate(Update):
def __init__(self, model, aviary_obj, cls):
super(AviaryUpdate, self).__init__(model)
11 years, 3 months
r5692 - in branches/tmckay: cumin/python/cumin mint/python/mint
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-12 19:58:31 +0000 (Tue, 12 Feb 2013)
New Revision: 5692
Modified:
branches/tmckay/cumin/python/cumin/main.py
branches/tmckay/mint/python/mint/main.py
branches/tmckay/mint/python/mint/update.py
Log:
more
Modified: branches/tmckay/cumin/python/cumin/main.py
===================================================================
--- branches/tmckay/cumin/python/cumin/main.py 2013-02-12 19:37:02 UTC (rev 5691)
+++ branches/tmckay/cumin/python/cumin/main.py 2013-02-12 19:58:31 UTC (rev 5692)
@@ -90,7 +90,7 @@
self._page_links = list()
- self.model.sql_logging_enabled = True
+ self.model.sql_logging_enabled = False
# Aviary interface. If server values are "",
# Aviary operations for that server type will not be used.
Modified: branches/tmckay/mint/python/mint/main.py
===================================================================
--- branches/tmckay/mint/python/mint/main.py 2013-02-12 19:37:02 UTC (rev 5691)
+++ branches/tmckay/mint/python/mint/main.py 2013-02-12 19:58:31 UTC (rev 5692)
@@ -16,7 +16,7 @@
model_dir = [os.path.join(self.home, x) for x in model_paths]
self.model = MintModel(self, model_dir)
- self.model.sql_logging_enabled = True
+ self.model.sql_logging_enabled = False
self.database = MintDatabase(self, database_dsn)
self.admin = CuminAdmin(self)
self.update_thread = UpdateThread(self)
Modified: branches/tmckay/mint/python/mint/update.py
===================================================================
--- branches/tmckay/mint/python/mint/update.py 2013-02-12 19:37:02 UTC (rev 5691)
+++ branches/tmckay/mint/python/mint/update.py 2013-02-12 19:58:31 UTC (rev 5692)
@@ -202,7 +202,7 @@
def execute(self, sql, args=None):
super(UpdateCursor, self).execute(sql, args)
- log.debug(self.mogrify(sql, args))
+# log.debug(self.mogrify(sql, args))
self.stats.sql_ops += 1
@@ -1025,8 +1025,8 @@
model.com_redhat_grid.Submission)
def get_object_id(self):
- return self.aviary_obj.id.owner + ":" + \
- self.aviary_obj.id.name
+ return str(self.aviary_obj.id.owner) + ":" + \
+ str(self.aviary_obj.id.name)
def get_scheduler_name(self):
return str(self.aviary_obj.id._scheduler)
@@ -1054,7 +1054,7 @@
super(AviarySubmitterUpdate, self).__init__(model, aviary_obj, cls)
def get_object_id(self):
- return self.aviary_obj.id.name
+ return str(self.aviary_obj.id.name)
def get_scheduler_name(self):
return str(self.aviary_obj.id.scheduler)
11 years, 3 months
r5691 - branches/tmckay/mint/python/mint/aviary
by tmckay＠fedoraproject.org
Author: tmckay
Date: 2013-02-12 19:37:02 +0000 (Tue, 12 Feb 2013)
New Revision: 5691
Modified:
branches/tmckay/mint/python/mint/aviary/collector.py
Log:
blanket cast
Modified: branches/tmckay/mint/python/mint/aviary/collector.py
===================================================================
--- branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-12 19:35:19 UTC (rev 5690)
+++ branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-12 19:37:02 UTC (rev 5691)
@@ -3,6 +3,7 @@
from datetime import datetime, timedelta
from mint.update import AviaryCollectorUpdate, AviarySubmitterUpdate
+import suds.sax.text
class BaseCollectorObj(object):
def __init__(self, endpoint_id, res_id, class_name):
@@ -15,7 +16,7 @@
# For most things we want a Pool attr which doesn't
# exist in the classads....
try:
- self.Pool = self.id.pool
+ self.Pool = str(self.id.pool)
except:
pass
@@ -23,7 +24,11 @@
for attr in attrs:
if attr.value == "UNDEFINED":
continue
- setattr(self, attr.name, attr.value)
+ if type(attr.value) == suds.sax.text.Text:
+ v = str(attr.value)
+ else:
+ v = attr.value
+ setattr(self, attr.name, v)
def set_from_summary(self, id, summaries):
pass
@@ -202,11 +207,15 @@
continue
# Strip the extra verbiage from the negotiator cycle attrs
+ if type(attr.value) == suds.sax.text.Text:
+ v = str(attr.value)
+ else:
+ v = attr.value
if attr.name.startswith(self.pref) and attr.name.endswith(self.suff):
name = attr.name[len(self.pref):-len(self.suff)]
- setattr(self, name, attr.value)
+ setattr(self, name, v)
else:
- setattr(self, attr.name, attr.value)
+ setattr(self, attr.name, v)
self.System = self.Machine
11 years, 3 months