Author: tmckay
Date: 2013-03-07 20:11:17 +0000 (Thu, 07 Mar 2013)
New Revision: 5734
Modified:
trunk/mint/python/mint/aviary/endpoints.py
trunk/mint/python/mint/aviary/locator.py
trunk/mint/python/mint/aviary/session.py
trunk/mint/python/mint/aviary/sysimage.py
trunk/sage/python/sage/aviary/aviaryoperations.py
trunk/sage/python/sage/util.py
Log:
Generalize locator thread handling of multiple endpoint providers.
Add wallaroo as an endpoint provider.
Modified: trunk/mint/python/mint/aviary/endpoints.py
===================================================================
--- trunk/mint/python/mint/aviary/endpoints.py 2013-03-06 21:55:02 UTC (rev 5733)
+++ trunk/mint/python/mint/aviary/endpoints.py 2013-03-07 20:11:17 UTC (rev 5734)
@@ -95,8 +95,8 @@
return work
class WallarooEndpoint(Endpoint):
- def __init__(self, url, factory):
- super(WallarooEndpoint, self).__init__(url, None)
+ def __init__(self, url, id, factory):
+ super(WallarooEndpoint, self).__init__(url, id)
self.url = url
self.app = factory.app
@@ -159,7 +159,7 @@
def _make_wallaroo_endpoint(self, url=None, id=None):
if not url:
url = self.app.wallaroo_host
- return WallarooEndpoint(url, self)
+ return WallarooEndpoint(url, id, self)
def make_endpoint(self, url, id):
try:
Modified: trunk/mint/python/mint/aviary/locator.py
===================================================================
--- trunk/mint/python/mint/aviary/locator.py 2013-03-06 21:55:02 UTC (rev 5733)
+++ trunk/mint/python/mint/aviary/locator.py 2013-03-07 20:11:17 UTC (rev 5734)
@@ -7,13 +7,13 @@
log = logging.getLogger("mint.aviary")
class LocatorThread(MintBlockingDaemonThread):
- def __init__(self, aviary_ops, foreman, endpoint_factory):
+ def __init__(self, data_providers, foreman, endpoint_factory):
# We don't care about app for this thread,
# at least not currently
super(LocatorThread, self).__init__(None)
- self.aviary_ops = aviary_ops
+ self.data_providers = data_providers
self.foreman = foreman
self.ep_fact = endpoint_factory
self.daemon = True
@@ -35,16 +35,16 @@
self._condition.wait(self.interval_secs)
self._condition.release()
- # There is no dynamic discovery for the wallaroo url.
- # For now, just create the endpoint here and add the
- # work list to the foreman so that it is polled
- wallaroo = self.ep_fact._make_wallaroo_endpoint()
- self.foreman.add_work(wallaroo.get_work_list())
+ def gather_endpoints(endpoint_types):
+ ends = []
+ for provider in self.data_providers:
+ ends.extend(provider.get_endpoints_list(endpoint_types))
+ return ends
endpoints = dict()
while not self.stop_requested:
try:
- ends = self.aviary_ops.get_endpoints_list(endpoint_types)
+ ends = gather_endpoints(endpoint_types)
except Exception:
import traceback
message = traceback.format_exc()
@@ -52,7 +52,9 @@
block()
continue
- locs = dict([[e.location[0], e.id] for e in ends])
+ locs = dict([[(e.id.name,
+ e.id.resource,
+ e.id.sub_type), e] for e in ends])
loc_keys = set(locs.keys())
# the endpoints that we already know about
@@ -75,7 +77,8 @@
del endpoints[e]
for e in new_endpoints:
- endpoints[e] = self.ep_fact.make_endpoint(e, locs[e])
+ endpoints[e] = self.ep_fact.make_endpoint(locs[e].location[0],
+ locs[e].id)
self.foreman.add_work(endpoints[e].get_work_list())
block()
Modified: trunk/mint/python/mint/aviary/session.py
===================================================================
--- trunk/mint/python/mint/aviary/session.py 2013-03-06 21:55:02 UTC (rev 5733)
+++ trunk/mint/python/mint/aviary/session.py 2013-03-07 20:11:17 UTC (rev 5734)
@@ -4,6 +4,7 @@
from endpoints import EndpointFactory
from locator import LocatorThread
from datetime import timedelta
+from sage.util import SynthEndpoint, ResourceID
log = logging.getLogger("mint.aviary.session")
@@ -44,12 +45,22 @@
self.foreman = Foreman(num_workers=8)
+ # This class stores which types of endpoints we want to poll
+ # and knows what queries we want to make for each endpoint we find.
+ # The locator thread uses this class and the data_providers below
+ # to find valid endpoints and generate work for the Foreman.
endpoint_factory = EndpointFactory(self.app,
classes,
poll_delta=timedelta(days=64),
history_delta=timedelta(weeks=52))
- self.locatorThread = LocatorThread(self.app.aviary_ops,
+ # These objects return the raw list of known data providers that
+ # can be polled for data. The locator thread will poll them
+ # for data providers (contrained by endpoint type)
+ data_providers = (self.app.aviary_ops,
+ WallarooEndpoints(self.app.wallaroo_host))
+
+ self.locatorThread = LocatorThread(data_providers,
self.foreman,
endpoint_factory)
@@ -63,3 +74,18 @@
self.locatorThread.stop()
self.locatorThread.join()
log.info("Stopped %s", self)
+
+class WallarooEndpoints(object):
+ def __init__(self, wallaroo_host):
+ self.wallaroo_host = wallaroo_host
+
+ def get_endpoints_list(self, endpoint_types):
+ ends = []
+ if ("WALLAROO","META") in endpoint_types:
+ res_id = ResourceID("WALLAROO",
+ sub_type="META",
+ name=self.wallaroo_host)
+ url_strs = [self.wallaroo_host]
+ ends.append(SynthEndpoint(res_id,
+ url_strs))
+ return ends
Modified: trunk/mint/python/mint/aviary/sysimage.py
===================================================================
--- trunk/mint/python/mint/aviary/sysimage.py 2013-03-06 21:55:02 UTC (rev 5733)
+++ trunk/mint/python/mint/aviary/sysimage.py 2013-03-07 20:11:17 UTC (rev 5734)
@@ -1,8 +1,12 @@
+import wallaroo
from work import Work
from datetime import datetime
from mint.update import AviaryUpdate
-import wallaroo
+from requests.exceptions import ConnectionError, HTTPError
+import logging
+log = logging.getLogger("mint.aviary.sysimage")
+
class SysimageID(object):
def __init__(self, name):
self.resource = "WALLAROO"
@@ -42,14 +46,15 @@
# transform it into a WallarooMetaUpdate
# enqueue it
if self.connection is None:
- try:
- self.connection = wallaroo.client.ConnectionMeta(host=self.ep.url)
- except:
- pass
+ self.connection = wallaroo.client.ConnectionMeta(host=self.ep.url)
- if self.connection:
+ try:
data =
self.connection.fetch_json_resource("/meta/sysinfo/grid9.lab.bos.redhat.com")
+ except (ConnectionError, HTTPError), e:
+ log.debug("Sysimage: wallaroo client fetch_json_resource failed %s"
% str(e))
+ data = None
+ if data:
obj = SysimageObj(data)
update = AviaryUpdate(self.ep.app.model,
obj,
Modified: trunk/sage/python/sage/aviary/aviaryoperations.py
===================================================================
--- trunk/sage/python/sage/aviary/aviaryoperations.py 2013-03-06 21:55:02 UTC (rev 5733)
+++ trunk/sage/python/sage/aviary/aviaryoperations.py 2013-03-07 20:11:17 UTC (rev 5734)
@@ -12,7 +12,7 @@
from threading import Lock
from suds import *
from sage.util import MethodResult, CallSync, CallThread, host_list,\
- parse_URL, get_datadir
+ parse_URL, get_datadir, SynthEndpoint, ResourceID
from sage.catalog import Catalog
from aviarylocator import AviaryLocator
from clients import ClientPool, TransportFactory
@@ -245,21 +245,6 @@
self._lock.release()
return res
-class ResourceID(object):
- def __init__(self, resource="COLLECTOR", pool="",
name="",
- address="", sub_type="", birthdate=None):
- self.resource = resource
- self.pool = pool
- self.name = name
- self.address = address
- self.sub_type = sub_type
- self.birthdate = birthdate
-
-class Fake_Endpoint(object):
- def __init__(self, resource_id, url_list):
- self.id = resource_id
- self.location = url_list
-
class FixedServerList(BaseServerList):
'''
Query a fixed server list for Aviary endpoints.
@@ -307,7 +292,7 @@
for host, urls in self.servers.iteritems():
res_id = ResourceID(self.resource, sub_type=self.subtype, name=host)
url_strs = [str(u) for u in urls]
- ep = Fake_Endpoint(res_id, url_strs)
+ ep = SynthEndpoint(res_id, url_strs)
endpoints.append(ep)
return endpoints
Modified: trunk/sage/python/sage/util.py
===================================================================
--- trunk/sage/python/sage/util.py 2013-03-06 21:55:02 UTC (rev 5733)
+++ trunk/sage/python/sage/util.py 2013-03-07 20:11:17 UTC (rev 5734)
@@ -353,3 +353,18 @@
# Hmm, well, just return the first one since we're
# going to get an error anyway.
return datadir[0]
+
+class ResourceID(object):
+ def __init__(self, resource="COLLECTOR", pool="",
name="",
+ address="", sub_type="", birthdate=None):
+ self.resource = resource
+ self.pool = pool
+ self.name = name
+ self.address = address
+ self.sub_type = sub_type
+ self.birthdate = birthdate
+
+class SynthEndpoint(object):
+ def __init__(self, resource_id, url_list):
+ self.id = resource_id
+ self.location = url_list