February 2013 - cumin-developers - Fedora Mailing-Lists

r5680 - in branches/tmckay/mint/python/mint: . aviary

by tmckay＠fedoraproject.org

Author: tmckay Date: 2013-02-08 16:48:42 +0000 (Fri, 08 Feb 2013) New Revision: 5680 Modified: branches/tmckay/mint/python/mint/aviary/collector.py branches/tmckay/mint/python/mint/update.py Log: Add scheduler reference for submitters Modified: branches/tmckay/mint/python/mint/aviary/collector.py =================================================================== --- branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-08 03:22:44 UTC (rev 5679) +++ branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-08 16:48:42 UTC (rev 5680) @@ -2,7 +2,7 @@ from util import now_minus_duration from datetime import datetime, timedelta -from mint.update import AviaryCollectorUpdate +from mint.update import AviaryCollectorUpdate, AviarySubmitterUpdate class BaseCollectorObj(object): def __init__(self, endpoint_id, res_id, class_name): @@ -44,11 +44,14 @@ self.deleted = False + def wrap_update(self, aviary_obj): + return AviaryCollectorUpdate(self.ep.app.model, aviary_obj) + def make_update(self, id, attrs, summaries): aviary_obj = self.make_collector_obj(id) aviary_obj.set_from_summary(id, summaries) aviary_obj.set_from_ad(attrs) - return AviaryCollectorUpdate(self.ep.app.model, aviary_obj) + return self.wrap_update(aviary_obj) def make_delete(self, id): aviary_obj = self.make_collector_obj(id) @@ -286,6 +289,9 @@ def get_objects(self, no_summary): return self.ep.app.aviary_ops.get_submitter(self.ep.url, no_summary=no_summary) + def wrap_update(self, aviary_obj): + return AviarySubmitterUpdate(self.ep.app.model, aviary_obj) + class SchedulerObj(BaseCollectorObj): def __init__(self, endpoint_id, res_id): super(SchedulerObj, self).__init__(endpoint_id, res_id, "Scheduler") Modified: branches/tmckay/mint/python/mint/update.py =================================================================== --- branches/tmckay/mint/python/mint/update.py 2013-02-08 03:22:44 UTC (rev 5679) +++ branches/tmckay/mint/python/mint/update.py 2013-02-08 16:48:42 UTC (rev 5680) @@ -974,23 +974,10 @@ return "%s(%s,%s,%s)" % (name, agent_id, self.cls, obj_id) -class AviarySubmissionUpdate(AviaryUpdate): - def __init__(self, model, submission): - super(AviarySubmissionUpdate, self).__init__(model, - submission, - model.com_redhat_grid.Submission) +class AviaryScheddReferencer(AviaryUpdate): + def __init__(self, model, aviary_obj, cls): + super(AviaryScheddReferencer, self).__init__(model, aviary_obj, cls) - def get_object_id(self): - return self.aviary_obj.id.owner + ":" + \ - self.aviary_obj.id.name - - def get_agent_id(self): - return "SCHEDULER" + ":" + \ - self.aviary_obj.id._scheduler - - def get_object_id(self): - return self.aviary_obj.id.name - def process_properties(self, obj, columns, cursor): cls = obj._class @@ -1006,10 +993,13 @@ def __init__(self, agent, objname): self.agentName = agent self.objectName = objname - - val = FakeOID("SCHEDULER:" + \ - self.aviary_obj.id._scheduler, - self.aviary_obj.id._scheduler) + + # Okay, we *know* how a scheduler is going to name itself + # so we invent a reference here so that it can be resolved + # using current machinery + sched_name = self.get_scheduler_name() + val = FakeOID("SCHEDULER:" + sched_name, sched_name) + prop = FakeProp("schedulerRef") try: col, nvalue = self.process_reference(obj, prop, val, cursor) @@ -1019,9 +1009,29 @@ except MappingException, e: log.debug(e) - super(AviarySubmissionUpdate, self).process_properties(obj, columns, cursor) + super(AviaryScheddReferencer, self).process_properties(obj, columns, cursor) + def get_agent_id(self): + return "SCHEDULER" + ":" + \ + self.get_scheduler_name() + def get_scheduler_name(self): + raise Exception("NotImplemented") + +class AviarySubmissionUpdate(AviaryScheddReferencer): + def __init__(self, model, submission): + super(AviarySubmissionUpdate, self).__init__(model, + submission, + model.com_redhat_grid.Submission) + + def get_object_id(self): + return self.aviary_obj.id.owner + ":" + \ + self.aviary_obj.id.name + + def get_scheduler_name(self): + return self.aviary_obj.id._scheduler + + class AviaryCollectorUpdate(AviaryUpdate): def __init__(self, model, aviary_obj): @@ -1033,6 +1043,22 @@ aviary_obj, cls) +class AviarySubmitterUpdate(AviaryScheddReferencer): + # This is very much like a AviaryCollectorUpdate but it needs to have + # a schedd reference injected so we derive from AviaryScheddReferencer + def __init__(self, model, aviary_obj): + + pkg = model._packages_by_name[aviary_obj.pkg_name] + cls = pkg._classes_by_lowercase_name[aviary_obj.class_name.lower()] + + super(AviarySubmitterUpdate, self).__init__(model, aviary_obj, cls) + + def get_object_id(self): + return self.aviary_obj.id.name + + def get_scheduler_name(self): + return self.aviary_obj.id.scheduler + class UpdateDropped(Exception): pass

11 years, 3 months

1
0
0 / 0

r5679 - in branches/tmckay: mint/python/mint mint/python/mint/aviary rosemary/python/rosemary sage/rpc-defs/aviary

by tmckay＠fedoraproject.org

Author: tmckay Date: 2013-02-08 03:22:44 +0000 (Fri, 08 Feb 2013) New Revision: 5679 Modified: branches/tmckay/mint/python/mint/aviary/collector.py branches/tmckay/mint/python/mint/aviary/endpoints.py branches/tmckay/mint/python/mint/aviary/session.py branches/tmckay/mint/python/mint/main.py branches/tmckay/mint/python/mint/update.py branches/tmckay/rosemary/python/rosemary/sqltype.py branches/tmckay/sage/rpc-defs/aviary/aviary-collector.wsdl branches/tmckay/sage/rpc-defs/aviary/aviary-common.xsd Log: Added slots, tweaks to others. Submitters needs a schedd reference. Modified: branches/tmckay/mint/python/mint/aviary/collector.py =================================================================== --- branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-07 19:52:33 UTC (rev 5678) +++ branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-08 03:22:44 UTC (rev 5679) @@ -21,6 +21,8 @@ def set_from_ad(self, attrs): for attr in attrs: + if attr.value == "UNDEFINED": + continue setattr(self, attr.name, attr.value) def set_from_summary(self, id, summaries): @@ -193,6 +195,8 @@ def set_from_ad(self, attrs): for attr in attrs: + if attr.value == "UNDEFINED": + continue # Strip the extra verbiage from the negotiator cycle attrs if attr.name.startswith(self.pref) and attr.name.endswith(self.suff): @@ -294,8 +298,6 @@ if hasattr(self, "Machine"): setattr(self, "System", self.Machine) - self.UpdateInterval = 0 - # Special stuff that maybe we can remap def remap(name, target): if not hasattr(self, name) and hasattr(self, target): @@ -357,14 +359,192 @@ def __init__(self, ep): super(QueryScheduler, self).__init__(ep) - # For now let's get them all - self.names = [] + # start with all the attributes based on condor-7.6 + self.names = ["CondorPlatform", + "CondorVersion", + "MonitorSelfAge", + "MonitorSelfCPUUsage", + "MonitorSelfImageSize", + "MonitorSelfRegisteredSocketCount", + "MonitorSelfResidentSetSize", + "MonitorSelfTime", + "DaemonStartTime", + "JobQueueBirthdate", + "MaxJobsRunning", + "Machine", + "MyAddress", + "NumUsers", + "Name", + "TotalHeldJobs", + "TotalIdleJobs", + "TotalJobAds", + "TotalRemovedJobs", + "TotalRunningJobs", + "WindowedStatWidth", + "UpdateInterval", + "JobsSubmitted", + "JobSubmissionRate", + "JobsCompleted", + "JobCompletionRate", + "JobsExited", + "ShadowExceptions", + "JobsSubmittedCum", + "JobsCompletedCum", + "JobsExitedCum", + "ShadowExceptionsCum", + "JobsStartedCum", + "JobsStarted", + "JobStartRate", + "MeanTimeToStartCum", + "MeanRunningTimeCum", + "SumTimeToStartCum", + "SumRunningTimeCum", + "MeanTimeToStart", + "MeanRunningTime", + + # Get the remap targets, too. These are new + # names in 7.8. If the 7.6 names aren't there, + # we will remap to these + "RecentStatsLifetime", + "RecentJobsExitException", + "JobsSubmitted", + "JobsCompleted", + "JobsExited", + "JobsExitException", + "JobsStarted", + "JobsAccumRunningTime", + "JobsAccumTimeToStart", + "RecentJobsSubmitted", + "RecentStatsLifetime", + "RecentJobsCompleted", + "RecentStatsLifetime", + "RecentJobsStarted", + "RecentStatsLifetime", + "JobsAccumTimeToStart", + "JobsStarted", + "JobsAccumRunningTime", + "JobsCompleted", + "RecentJobsAccumTimeToStart", + "RecentJobsStarted", + "RecentJobsAccumRunningTime", + "RecentJobsCompleted"] + def make_collector_obj(self, id): return SchedulerObj(self.ep.id, id) def get_objects(self, no_summary): return self.ep.app.aviary_ops.get_scheduler(self.ep.url, no_summary=no_summary) +class SlotObj(BaseCollectorObj): + def __init__(self, endpoint_id, res_id): + super(SlotObj, self).__init__(endpoint_id, res_id, "Slot") + def set_from_ad(self, attrs): + super(SlotObj, self).set_from_ad(attrs) + # Have to adapt because the cumin db + # refers to Machine as System + if hasattr(self, "Machine"): + setattr(self, "System", self.Machine) + +class QuerySlot(BaseCollectorQuery): + def __init__(self, ep): + super(QuerySlot, self).__init__(ep) + + self.names = ["CondorPlatform", + "CondorVersion", + "DaemonStartTime", + "MonitorSelfAge", + "MonitorSelfCPUUsage", + "MonitorSelfImageSize", + "MonitorSelfRegisteredSocketCount", + "MonitorSelfResidentSetSize", + "MonitorSelfTime", + "AccountingGroup", + "Activity", + "Arch", + "CheckpointPlatform", + "ClientMachine", + "ClockDay", + "ClockMin", + "ConcurrencyLimits", + "CondorLoadAvg", + "ConsoleIdle", + "Cpus", + "CurrentRank", + "Disk", + "EnteredCurrentActivity", + "EnteredCurrentState", + "FileSystemDomain", + "GlobalJobId", + "ImageSize", + "IsValidCheckpointPlatform", + "JobId", + "JobStart", + "KeyboardIdle", + "KFlops", + "LastBenchmark", + "LastFetchWorkCompleted", + "LastFetchWorkSpawned", + "LastPeriodicCheckpoint", + "LoadAvg", + "Machine", + "MaxJobRetirementTime", + "Memory", + "Mips", + "MyAddress", + "MyCurrentTime", + "Name", + "NextFetchWorkDelay", + "OpSys", + "PreemptingConcurrencyLimits", + "PreemptingOwner", + "PreemptingUser", + "PreemptingRank", + "RemoteOwner", + "RemoteUser", + "Requirements", + "Rank", + "SlotID", + "Start", + "StarterAbilityList", + "State", + "TimeToLive", + "TotalClaimRunTime", + "TotalClaimSuspendTime", + "TotalCondorLoadAvg", + "TotalCpus", + "TotalDisk", + "TotalJobRunTime", + "TotalJobSuspendTime", + "TotalLoadAvg", + "TotalMemory", + "TotalSlots", + "TotalTimeBackfillBusy", + "TotalTimeBackfillIdle", + "TotalTimeBackfillKilling", + "TotalTimeClaimedBusy", + "TotalTimeClaimedIdle", + "TotalTimeClaimedRetiring", + "TotalTimeClaimedSuspended", + "TotalTimeMatchedIdle", + "TotalTimeOwnerIdle", + "TotalTimePreemptingKilling", + "TotalTimePreemptingVacating", + "TotalTimeUnclaimedBenchmarking", + "TotalTimeUnclaimedIdle", + "TotalVirtualMemory", + "UidDomain", + "UpdateInterval", + "VirtualMemory", + "WindowsBuildNumber", + "WindowsMajorVersion", + "WindowsMinorVersion"] + + def make_collector_obj(self, id): + return SlotObj(self.ep.id, id) + + def get_objects(self, no_summary): + return self.ep.app.aviary_ops.get_slot(self.ep.url, no_summary=no_summary) + Modified: branches/tmckay/mint/python/mint/aviary/endpoints.py =================================================================== --- branches/tmckay/mint/python/mint/aviary/endpoints.py 2013-02-07 19:52:33 UTC (rev 5678) +++ branches/tmckay/mint/python/mint/aviary/endpoints.py 2013-02-08 03:22:44 UTC (rev 5679) @@ -2,7 +2,7 @@ from datetime import datetime from submissions import QuerySubmissionsBefore, QuerySubmissionsAfter from collector import QueryCollector, QueryNegotiator, QuerySubmitter, \ -QueryScheduler +QueryScheduler, QuerySlot class Endpoint(object): def __init__(self, url, id): @@ -70,7 +70,7 @@ self.app = factory.app - my_classes = ("Collector", "Negotiator", "Submitter", "Scheduler") + my_classes = ("Collector", "Negotiator", "Submitter", "Scheduler", "Slot") self.classes = [c for c in factory.abbrev_classes \ if c in my_classes] @@ -88,6 +88,9 @@ if "Scheduler" in self.classes: work.append(QueryScheduler(self)) + if "Slot" in self.classes: + work.append(QuerySlot(self)) + return work class EndpointFactory(object): @@ -124,6 +127,7 @@ "Negotiator": ("COLLECTOR", ""), "Scheduler": ("COLLECTOR", ""), "Submitter": ("COLLECTOR", ""), + "Slot": ("COLLECTOR", ""), "Submission": ("CUSTOM", "QUERY_SERVER")} endpoints = set() Modified: branches/tmckay/mint/python/mint/aviary/session.py =================================================================== --- branches/tmckay/mint/python/mint/aviary/session.py 2013-02-07 19:52:33 UTC (rev 5678) +++ branches/tmckay/mint/python/mint/aviary/session.py 2013-02-08 03:22:44 UTC (rev 5679) @@ -42,7 +42,7 @@ log.info("Querying for Aviary class %s.%s" % (pname, cname)) - self.foreman = Foreman() + self.foreman = Foreman(num_workers=8) endpoint_factory = EndpointFactory(self.app, classes, Modified: branches/tmckay/mint/python/mint/main.py =================================================================== --- branches/tmckay/mint/python/mint/main.py 2013-02-07 19:52:33 UTC (rev 5678) +++ branches/tmckay/mint/python/mint/main.py 2013-02-08 03:22:44 UTC (rev 5679) @@ -91,7 +91,8 @@ self.model.com_redhat_grid.Collector, self.model.com_redhat_grid.Negotiator, self.model.com_redhat_grid.Submitter, - self.model.com_redhat_grid.Scheduler] + self.model.com_redhat_grid.Scheduler, + self.model.com_redhat_grid.Slot] for c in supported_classes: if c in self._qmf_classes: Modified: branches/tmckay/mint/python/mint/update.py =================================================================== --- branches/tmckay/mint/python/mint/update.py 2013-02-07 19:52:33 UTC (rev 5678) +++ branches/tmckay/mint/python/mint/update.py 2013-02-08 03:22:44 UTC (rev 5679) @@ -5,7 +5,7 @@ from psycopg2 import IntegrityError, TimestampFromTicks from psycopg2.extensions import cursor as Cursor from rosemary.model import * -from rosemary.sqltype import qmf_type_code_by_string +from rosemary.sqltype import qmf_type_code_by_string, qmf_default_by_string from model import * from util import * @@ -736,9 +736,8 @@ self.update_object(cursor, stats, obj) def get_agent_id(self): - return self.aviary_obj.endpoint_id.resource + ":" + \ - self.aviary_obj.endpoint_id.sub_type + ":" + \ - self.aviary_obj.endpoint_id.name + return self.aviary_obj.id.resource + ":" + \ + self.aviary_obj.id.name def get_object_id(self): return self.aviary_obj.id.name @@ -788,6 +787,8 @@ sql = "; ".join(statements) self.execute_sql(cursor, sql, obj.__dict__) + self.process_deferred_links(cursor, obj) + obj._save_time = datetime.now() self.model.print_event(3, "Created %s", obj) @@ -852,11 +853,34 @@ t = qmf_type_code_by_string[prop.type] return transformers[t](value) + def get_default(self, prop): + if prop.type in qmf_default_by_string: + return qmf_default_by_string[prop.type] + return None + + def process_deferred_links(self, cursor, obj): + agent = self.model.agents_by_id[obj._qmf_agent_id] + + if obj._qmf_object_id not in agent.deferred_links_by_id: + return + + links = agent.deferred_links_by_id[obj._qmf_object_id] + + for link in links: + link.realize(cursor, obj) + + del agent.deferred_links_by_id[obj._qmf_object_id] + + def process_properties(self, obj, columns, cursor): cls = obj._class for name, prop in cls._properties_by_name.iteritems(): - value = getattr(self.aviary_obj, name) + try: + value = getattr(self.aviary_obj, name) + except AttributeError: + value = self.get_default(prop) + if value is not None: value = self.transform_value(prop, value) if value == getattr(obj, prop.sql_column.name): @@ -902,7 +926,11 @@ # loop through the list of properties, look for # the value in the object for name, stat in self.cls._statistics_by_name.iteritems(): - value = getattr(self.aviary_obj, name) + try: + value = getattr(self.aviary_obj, name) + except AttributeError: + value = self.get_default(stat) + if value is not None: value = self.transform_value(stat, value) @@ -933,16 +961,10 @@ log.error("Sql row count: %i", cursor.rowcount) - log.error("Qmf properties:") + log.error("attributes:") - for item in sorted(self.qmf_object.getProperties()): - log.error(" %-34s %r", *item) - - log.error("Qmf statistics:") - - for item in sorted(self.qmf_object.getStatistics()): - log.error(" %-34s %r", *item) - + for item in sorted(self.aviary_obj.__dict__.items()): + log.error(" %-34s %r" % item) raise def __repr__(self): @@ -958,6 +980,17 @@ submission, model.com_redhat_grid.Submission) + def get_object_id(self): + return self.aviary_obj.id.owner + ":" + \ + self.aviary_obj.id.name + + def get_agent_id(self): + return "SCHEDULER" + ":" + \ + self.aviary_obj.id._scheduler + + def get_object_id(self): + return self.aviary_obj.id.name + def process_properties(self, obj, columns, cursor): cls = obj._class @@ -974,7 +1007,7 @@ self.agentName = agent self.objectName = objname - val = FakeOID("com.redhat.grid:scheduler:" + \ + val = FakeOID("SCHEDULER:" + \ self.aviary_obj.id._scheduler, self.aviary_obj.id._scheduler) prop = FakeProp("schedulerRef") Modified: branches/tmckay/rosemary/python/rosemary/sqltype.py =================================================================== --- branches/tmckay/rosemary/python/rosemary/sqltype.py 2013-02-07 19:52:33 UTC (rev 5678) +++ branches/tmckay/rosemary/python/rosemary/sqltype.py 2013-02-08 03:22:44 UTC (rev 5679) @@ -43,37 +43,39 @@ sql_string_id = StringIdType("text", str) __mappings = ( - (sql_bool, 11, "bool"), - (sql_float4, 12, "float"), - (sql_float8, 13, "double"), - (sql_int2, 1, "count8"), - (sql_int2, 1, "hilo8"), - (sql_int2, 1, "uint8"), - (sql_int2, 16,"int8"), - (sql_int2, 17, "int16"), - (sql_int4, 18, "int32"), - (sql_int4, 2, "count16"), - (sql_int4, 2, "hilo16"), - (sql_int4, 2, "uint16"), - (sql_int8, 19, "int64"), - (sql_int8, 3, "count32"), - (sql_int8, 3, "hilo32"), - (sql_int8, 3, "mma32"), - (sql_int8, 3, "uint32"), + (sql_bool, 11, "bool", True), + (sql_float4, 12, "float", 0), + (sql_float8, 13, "double", 0), + (sql_int2, 1, "count8", 0), + (sql_int2, 1, "hilo8", 0), + (sql_int2, 1, "uint8", 0), + (sql_int2, 16,"int8", 0), + (sql_int2, 17, "int16", 0), + (sql_int4, 18, "int32", 0), + (sql_int4, 2, "count16", 0), + (sql_int4, 2, "hilo16", 0), + (sql_int4, 2, "uint16", 0), + (sql_int8, 19, "int64", 0), + (sql_int8, 3, "count32", 0), + (sql_int8, 3, "hilo32", 0), + (sql_int8, 3, "mma32", 0), + (sql_int8, 3, "uint32", 0), (sql_pickled_map, 15, "map"), (sql_string_id, 10, "objId"), (sql_string_id, 14, "uuid"), - (sql_text, 6, "sstr"), - (sql_text, 7, "lstr"), - (sql_timestamp, 8, "absTime"), - (sql_uint8, 4, "count64"), - (sql_uint8, 4, "hilo64"), - (sql_uint8, 4, "mma64"), - (sql_uint8, 4, "mmaTime"), - (sql_uint8, 4, "uint64"), - (sql_uint8, 9, "deltaTime"), + (sql_text, 6, "sstr", ""), + (sql_text, 7, "lstr", ""), + (sql_timestamp, 8, "absTime", 0), + (sql_uint8, 4, "count64", 0), + (sql_uint8, 4, "hilo64", 0), + (sql_uint8, 4, "mma64", 0), + (sql_uint8, 4, "mmaTime", 0), + (sql_uint8, 4, "uint64", 0), + (sql_uint8, 9, "deltaTime", 0), ) sql_types_by_qmf_type_code = dict([(x[1], x[0]) for x in __mappings]) sql_types_by_qmf_type_string = dict([(x[2], x[0]) for x in __mappings]) qmf_type_code_by_string = dict([(x[2], x[1]) for x in __mappings]) + +qmf_default_by_string = dict([(x[2], x[3]) for x in __mappings if len(x) > 3]) Modified: branches/tmckay/sage/rpc-defs/aviary/aviary-collector.wsdl =================================================================== --- branches/tmckay/sage/rpc-defs/aviary/aviary-collector.wsdl 2013-02-07 19:52:33 UTC (rev 5678) +++ branches/tmckay/sage/rpc-defs/aviary/aviary-collector.wsdl 2013-02-08 03:22:44 UTC (rev 5679) @@ -34,12 +34,12 @@ <wsdl:message name="GetMasterResponse"> <wsdl:part name="parameters" element="coll:GetMasterResponse"/> </wsdl:message> - <wsdl:message name="GetMasterIDRequest"> - <wsdl:part name="parameters" element="coll:GetMasterID"/> - </wsdl:message> - <wsdl:message name="GetMasterIDResponse"> - <wsdl:part name="parameters" element="coll:GetMasterIDResponse"/> - </wsdl:message> + <wsdl:message name="GetMasterIDRequest"> + <wsdl:part name="parameters" element="coll:GetMasterID"/> + </wsdl:message> + <wsdl:message name="GetMasterIDResponse"> + <wsdl:part name="parameters" element="coll:GetMasterIDResponse"/> + </wsdl:message> <wsdl:message name="GetNegotiatorRequest"> <wsdl:part name="parameters" element="coll:GetNegotiator"/> </wsdl:message> @@ -58,12 +58,12 @@ <wsdl:message name="GetSlotResponse"> <wsdl:part name="parameters" element="coll:GetSlotResponse"/> </wsdl:message> - <wsdl:message name="GetSlotIDRequest"> - <wsdl:part name="parameters" element="coll:GetSlotID"/> - </wsdl:message> - <wsdl:message name="GetSlotIDResponse"> - <wsdl:part name="parameters" element="coll:GetSlotIDResponse"/> - </wsdl:message> + <wsdl:message name="GetSlotIDRequest"> + <wsdl:part name="parameters" element="coll:GetSlotID"/> + </wsdl:message> + <wsdl:message name="GetSlotIDResponse"> + <wsdl:part name="parameters" element="coll:GetSlotIDResponse"/> + </wsdl:message> <wsdl:message name="GetSubmitterRequest"> <wsdl:part name="parameters" element="coll:GetSubmitter"/> </wsdl:message> @@ -85,10 +85,10 @@ <wsdl:input message="tns:GetMasterRequest"/> <wsdl:output message="tns:GetMasterResponse"/> </wsdl:operation> - <wsdl:operation name="getMasterID"> - <wsdl:input message="tns:GetMasterIDRequest"/> - <wsdl:output message="tns:GetMasterIDResponse"/> - </wsdl:operation> + <wsdl:operation name="getMasterID"> + <wsdl:input message="tns:GetMasterIDRequest"/> + <wsdl:output message="tns:GetMasterIDResponse"/> + </wsdl:operation> <wsdl:operation name="getNegotiator"> <wsdl:input message="tns:GetNegotiatorRequest"/> <wsdl:output message="tns:GetNegotiatorResponse"/> @@ -101,10 +101,10 @@ <wsdl:input message="tns:GetSlotRequest"/> <wsdl:output message="tns:GetSlotResponse"/> </wsdl:operation> - <wsdl:operation name="getSlotID"> - <wsdl:input message="tns:GetSlotIDRequest"/> - <wsdl:output message="tns:GetSlotIDResponse"/> - </wsdl:operation> + <wsdl:operation name="getSlotID"> + <wsdl:input message="tns:GetSlotIDRequest"/> + <wsdl:output message="tns:GetSlotIDResponse"/> + </wsdl:operation> <wsdl:operation name="getSubmitter"> <wsdl:input message="tns:GetSubmitterRequest"/> <wsdl:output message="tns:GetSubmitterResponse"/> @@ -134,15 +134,15 @@ <soap:body use="literal"/> </wsdl:output> </wsdl:operation> - <wsdl:operation name="getMasterID"> - <soap:operation soapAction="http://grid.redhat.com/aviary-collector/master/id"/> - <wsdl:input> - <soap:body use="literal"/> - </wsdl:input> - <wsdl:output> - <soap:body use="literal"/> - </wsdl:output> - </wsdl:operation> + <wsdl:operation name="getMasterID"> + <soap:operation soapAction="http://grid.redhat.com/aviary-collector/master/id"/> + <wsdl:input> + <soap:body use="literal"/> + </wsdl:input> + <wsdl:output> + <soap:body use="literal"/> + </wsdl:output> + </wsdl:operation> <wsdl:operation name="getNegotiator"> <soap:operation soapAction="http://grid.redhat.com/aviary-collector/negotiator"/> <wsdl:input> @@ -170,15 +170,15 @@ <soap:body use="literal"/> </wsdl:output> </wsdl:operation> - <wsdl:operation name="getSlotID"> - <soap:operation soapAction="http://grid.redhat.com/aviary-collector/slot/id"/> - <wsdl:input> - <soap:body use="literal"/> - </wsdl:input> - <wsdl:output> - <soap:body use="literal"/> - </wsdl:output> - </wsdl:operation> + <wsdl:operation name="getSlotID"> + <soap:operation soapAction="http://grid.redhat.com/aviary-collector/slot/id"/> + <wsdl:input> + <soap:body use="literal"/> + </wsdl:input> + <wsdl:output> + <soap:body use="literal"/> + </wsdl:output> + </wsdl:operation> <wsdl:operation name="getSubmitter"> <soap:operation soapAction="http://grid.redhat.com/aviary-collector/submitter"/> <wsdl:input> Modified: branches/tmckay/sage/rpc-defs/aviary/aviary-common.xsd =================================================================== --- branches/tmckay/sage/rpc-defs/aviary/aviary-common.xsd 2013-02-07 19:52:33 UTC (rev 5678) +++ branches/tmckay/sage/rpc-defs/aviary/aviary-common.xsd 2013-02-08 03:22:44 UTC (rev 5679) @@ -111,10 +111,10 @@ <xs:element name="owner" type="xs:string" minOccurs="0"/> <xs:element name="qdate" type="xs:int" minOccurs="0"/> </xs:sequence> -  -  - <xs:attribute name="pool" type="xs:string"/> - <xs:attribute name="scheduler" type="xs:string"/> +  +  + <xs:attribute name="pool" type="xs:string"/> + <xs:attribute name="scheduler" type="xs:string"/> </xs:complexType> <xs:complexType name="JobID"> <xs:sequence> @@ -260,6 +260,7 @@ </xs:complexType> <xs:complexType name="NegotiatorSummary"> <xs:sequence> + <xs:element name="latest_cycle" type="xs:dateTime"/> <xs:element name="match_rate" type="xs:double"/> <xs:element name="matches" type="xs:int"/> <xs:element name="duration" type="xs:int"/> @@ -343,6 +344,8 @@ <xs:element name="running" type="xs:int"/> <xs:element name="held" type="xs:int"/> <xs:element name="idle" type="xs:int"/> + <xs:element name="created" type="xs:dateTime"/> + <xs:element name="owner" type="xs:string"/> </xs:sequence> </xs:complexType> <xs:complexType name="Submitter">

11 years, 3 months

1
0
0 / 0

r5678 - branches/elephant/cumin/model

by croberts＠fedoraproject.org

Author: croberts Date: 2013-02-07 19:52:33 +0000 (Thu, 07 Feb 2013) New Revision: 5678 Modified: branches/elephant/cumin/model/cumin.xml Log: Making gridhadoop package "generic" rather than qmf, which is the default. Modified: branches/elephant/cumin/model/cumin.xml =================================================================== --- branches/elephant/cumin/model/cumin.xml 2013-02-07 17:34:14 UTC (rev 5677) +++ branches/elephant/cumin/model/cumin.xml 2013-02-07 19:52:33 UTC (rev 5678) @@ -45,7 +45,7 @@ </class> </package> - <package name="com.redhat.cumin.grid.hadoop"> + <package name="com.redhat.cumin.grid.hadoop" package_type="generic"> <class name="NameNode" storage="none"> <property name="Id" type="sstr"/> <property name="Ipc" type="sstr"/>

11 years, 3 months

1
0
0 / 0

r5677 - branches/elephant/cumin/python/cumin/gridhadoop

by croberts＠fedoraproject.org

Author: croberts Date: 2013-02-07 17:34:14 +0000 (Thu, 07 Feb 2013) New Revision: 5677 Modified: branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py Log: Removing checkboxes from scheduler page, fixing-up the searching capability. Modified: branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py =================================================================== --- branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py 2013-02-06 20:03:28 UTC (rev 5676) +++ branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py 2013-02-07 17:34:14 UTC (rev 5677) @@ -2,6 +2,7 @@ from hadoop import * from hadoopobjectframe import * +from cumin.objectselector import ObjectTable from jobtracker import JobTrackerFrame, JobTrackerSelector from namenode import NameNodeFrame, NameNodeSelector @@ -31,8 +32,8 @@ super(HadoopSchedulerSelector, self).__init__(app, name, cls) + self.table.adapter = HadoopSchedulerAdapter(app, cls) self.add_search_filter(self.table.id_col) - self.table.adapter = HadoopSchedulerAdapter(app, cls) self.enable_csv_export() @@ -42,13 +43,22 @@ def render_title(self, session): return "Schedulers" -class HadoopSchedulerTable(ObjectQmfSelectorTable): +class HadoopSchedulerTable(ObjectTable): def __init__(self, app, name, cls): super(HadoopSchedulerTable, self).__init__(app, name, cls) self.id_col = self.SchedNameColumn(app, "namecol", cls.Name, cls.Name, ".ghschedframe") self.add_column(self.id_col) + def get_data_values(self, session): + values = super(HadoopSchedulerTable, self).get_data_values(session) + values['session'] = session + return values + + def get_data_options(self, session): + options = super(HadoopSchedulerTable, self).get_data_options(session) + options.like_specs = self.like_specs + return options class SchedNameColumn(ObjectLinkColumn): def render_cell_href(self, session, record): @@ -57,6 +67,8 @@ # this is way less than ideal frame = self.page.page_widgets_by_path[self.path.rsplit(".", 7)[0] + self.frame_path] return frame.get_href(session, id, id) + + class HadoopSchedulerFrame(HadoopObjectFrame): def __init__(self, app, name, check_owner):

11 years, 3 months

1
0
0 / 0

r5676 - in branches/tmckay/mint/python/mint: . aviary

by tmckay＠fedoraproject.org

Author: tmckay Date: 2013-02-06 20:03:28 +0000 (Wed, 06 Feb 2013) New Revision: 5676 Added: branches/tmckay/mint/python/mint/aviary/collector.py Modified: branches/tmckay/mint/python/mint/aviary/endpoints.py branches/tmckay/mint/python/mint/main.py Log: Add scheduler support. Forgot to add collector.py Added: branches/tmckay/mint/python/mint/aviary/collector.py =================================================================== --- branches/tmckay/mint/python/mint/aviary/collector.py (rev 0) +++ branches/tmckay/mint/python/mint/aviary/collector.py 2013-02-06 20:03:28 UTC (rev 5676) @@ -0,0 +1,370 @@ +from work import Work +from util import now_minus_duration +from datetime import datetime, timedelta + +from mint.update import AviaryCollectorUpdate + +class BaseCollectorObj(object): + def __init__(self, endpoint_id, res_id, class_name): + + self.pkg_name = "com.redhat.grid" + self.class_name = class_name + self.endpoint_id = endpoint_id + self.id = res_id + + # For most things we want a Pool attr which doesn't + # exist in the classads.... + try: + self.Pool = self.id.pool + except: + pass + + def set_from_ad(self, attrs): + for attr in attrs: + setattr(self, attr.name, attr.value) + + def set_from_summary(self, id, summaries): + pass + +class BaseCollectorQuery(Work): + def __init__(self, ep): + super(BaseCollectorQuery, self).__init__() + self.ep = ep + + self.names = [] + + self.objects = dict() + + self.recycle_interval = timedelta(seconds=10) + self.max_silence = timedelta(seconds=30) + self.no_summary = True + self.no_classads = False + + self.deleted = False + + def make_update(self, id, attrs, summaries): + aviary_obj = self.make_collector_obj(id) + aviary_obj.set_from_summary(id, summaries) + aviary_obj.set_from_ad(attrs) + return AviaryCollectorUpdate(self.ep.app.model, aviary_obj) + + def make_delete(self, id): + aviary_obj = self.make_collector_obj(id) + up = AviaryCollectorUpdate(self.ep.app.model, aviary_obj) + up.delete = True + return up + + def make_collector_obj(self, id): + raise Exception("NotImplemented") + + def get_objects(self, no_summary): + raise Exception("NotImplemented") + + def do_the_job(self, foreman): + + # If the endpoint has been abandoned, then we should delete the objects + # that we've created and mark ourselves deleted. This will also prevent requeueing + if self.ep.abandoned: + for name, id in self.objects.iteritems(): + update = self.make_delete(id[0]) + self.ep.app.update_thread.enqueue(update) + + self.objects = [] + self.deleted = True + return + + # Note that if we cannot talk to this endpoint because the endpoint has gone + # away, the locator thread will eventually mark it abandoned if the + # locator is in use + stat, objs = self.get_objects(self.no_summary) + if stat == 0: + summaries = dict() + if self.no_summary: + # We're going to get everything from classads, we + # did not ask for summaries + ids = [o.id for o in objs if o.status.code == "OK"] + else: + # We'll get things from classads and/or summaries. + # Save the summaries in a dictionary so we can look them up + ids = [] + for o in objs: + if o.status.code == "OK": + ids.append(o.id) + if hasattr(o.id, "address"): + k = (o.id.name, o.id.address) + else: + k = (o.id.name, "") + summaries[k] = o.summary + + if self.no_classads: + # all we have are ids and summaries + for i in ids: + update = self.make_update(i, [], summaries) + self.ep.app.update_thread.enqueue(update) + else: + # get the classad in addition to the ids and summaries + stat, ads = self.ep.app.aviary_ops.get_attributes(self.ep.url, + ids, + self.names) + if stat == 0: + for ad in ads: + if ad.status.code == "OK": + update = self.make_update(ad.id, + ad.ad.attrs, summaries) + self.ep.app.update_thread.enqueue(update) + + # Remember the objects that we have just seen + for id in ids: + # HA names end with @ + if id.name.endswith("@"): + search = id.name + else: + search = id.name + if hasattr(id, "address"): + search += "/"+ id.address + if search in self.objects: + self.objects[search][1] = datetime.now() + else: + self.objects[search] = [id, datetime.now()] + + # Look for objects we haven't heard from in a while + deleted = [] + now = datetime.now() + for name, id in self.objects.iteritems(): + if now - id[1] >= self.max_silence: + # We need to remove this object + update = self.make_delete(id[0]) + self.ep.app.update_thread.enqueue(update) + deleted.append(name) + for d in deleted: + del self.objects[d] + del deleted + + def completed(self): + return self.ep.abandoned and self.deleted + +class CollectorObj(BaseCollectorObj): + def __init__(self, endpoint_id, res_id): + super(CollectorObj, self).__init__(endpoint_id, res_id, "Collector") + + def set_from_ad(self, attrs): + super(CollectorObj, self).set_from_ad(attrs) + + # Have to adapt because the cumin db + # refers to Machine as System + if hasattr(self, "Machine"): + setattr(self, "System", self.Machine) + +class QueryCollector(BaseCollectorQuery): + def __init__(self, ep): + super(QueryCollector, self).__init__(ep) + + self.names = ["CondorPlatform", + "CondorVersion", + "Machine", + "MonitorSelfAge", + "MonitorSelfCPUUsage", + "MonitorSelfImageSize", + "MonitorSelfRegisteredSocketCount", + "MonitorSelfResidentSetSize", + "MonitorSelfTime", + "DaemonStartTime", + "Name", + "MyAddress", + "RunningJobs", + "IdleJobs", + "HostsTotal", + "HostsClaimed", + "HostsUnclaimed", + "HostsOwner"] + + def make_collector_obj(self, id): + return CollectorObj(self.ep.id, id) + + def get_objects(self, no_summary): + return self.ep.app.aviary_ops.get_collector(self.ep.url, no_summary=no_summary) + +class NegotiatorObj(BaseCollectorObj): + def __init__(self, endpoint_id, res_id): + super(NegotiatorObj, self).__init__(endpoint_id, res_id, "Negotiator") + + self.pref = "LastNegotiationCycle" + self.suff = "0" + + def set_from_ad(self, attrs): + for attr in attrs: + + # Strip the extra verbiage from the negotiator cycle attrs + if attr.name.startswith(self.pref) and attr.name.endswith(self.suff): + name = attr.name[len(self.pref):-len(self.suff)] + setattr(self, name, attr.value) + else: + setattr(self, attr.name, attr.value) + + self.System = self.Machine + +class QueryNegotiator(BaseCollectorQuery): + def __init__(self, ep): + super(QueryNegotiator, self).__init__(ep) + + self.names = ["CondorPlatform", + "CondorVersion", + "Machine", + "MonitorSelfAge", + "MonitorSelfCPUUsage", + "MonitorSelfImageSize", + "MonitorSelfRegisteredSocketCount", + "MonitorSelfResidentSetSize", + "MonitorSelfTime", + "DaemonStartTime", + "Name", + "MyAddress", + + # the following ones will have to be remapped + "LastNegotiationCycleMatchRate0", + "LastNegotiationCycleMatches0", + "LastNegotiationCycleDuration0", + "LastNegotiationCycleMatchRateSustained0", + "LastNegotiationCyclePhase1Duration0", + "LastNegotiationCyclePhase2Duration0", + "LastNegotiationCyclePhase3Duration0", + "LastNegotiationCyclePhase4Duration0", + "LastNegotiationCycleSlotShareIter0", + "LastNegotiationCycleSubmittersFailed0", + "LastNegotiationCycleSubmittersOutOfTime0", + "LastNegotiationCycleSubmittersShareLimit0", + "LastNegotiationCycleTime0", + "LastNegotiationCycleEnd0", + "LastNegotiationCyclePeriod0", + "LastNegotiationCycleNumSchedulers0", + "LastNegotiationCycleActiveSubmitterCount0", + "LastNegotiationCycleNumIdleJobs0", + "LastNegotiationCycleNumJobsConsidered0", + "LastNegotiationCycleRejections0", + "LastNegotiationCycleTotalSlots0", + "LastNegotiationCycleCandidateSlots0", + "LastNegotiationCycleTrimmedSlots0"] + + def make_collector_obj(self, id): + return NegotiatorObj(self.ep.id, id) + + def get_objects(self, no_summary): + return self.ep.app.aviary_ops.get_negotiator(self.ep.url, no_summary=no_summary) + +class SubmitterObj(BaseCollectorObj): + def __init__(self, endpoint_id, res_id): + super(SubmitterObj, self).__init__(endpoint_id, res_id, "Submitter") + + def set_from_summary(self, id, summaries): + self.Machine = id.machine + self.Name = id.name + + summary = summaries[(id.name,"")] + self.HeldJobs = summary.held + self.IdleJobs = summary.idle + self.RunningJobs = summary.running + self.ScheddName = id.scheduler + self.Owner = "" + self.JobQueueBirthdate = 0 + +class QuerySubmitter(BaseCollectorQuery): + def __init__(self, ep): + super(QuerySubmitter, self).__init__(ep) + + # We're going to get everything from the summary + # record in this case + self.no_summary = False + self.no_classads = True + + def make_collector_obj(self, id): + return SubmitterObj(self.ep.id, id) + + def get_objects(self, no_summary): + return self.ep.app.aviary_ops.get_submitter(self.ep.url, no_summary=no_summary) + +class SchedulerObj(BaseCollectorObj): + def __init__(self, endpoint_id, res_id): + super(SchedulerObj, self).__init__(endpoint_id, res_id, "Scheduler") + + def set_from_ad(self, attrs): + super(SchedulerObj, self).set_from_ad(attrs) + + # Have to adapt because the cumin db + # refers to Machine as System + if hasattr(self, "Machine"): + setattr(self, "System", self.Machine) + + self.UpdateInterval = 0 + + # Special stuff that maybe we can remap + def remap(name, target): + if not hasattr(self, name) and hasattr(self, target): + setattr(self, name, getattr(self, target)) + + def remap_div(name, operand1, operand2): + if not hasattr(self, name) and \ + hasattr(self, operand1) and hasattr(self, operand2): + + try: + op1 = float(getattr(self, operand1)) + op2 = float(getattr(self, operand2)) + if op2 == 0: + setattr(self, name, 0) + else: + setattr(self, name, op1 / op2) + except: + pass + + remap("WindowedStatWidth", "RecentStatsLifetime") + remap("ShadowExceptions", "RecentJobsExitException") + remap("JobsSubmittedCum", "JobsSubmitted") + remap("JobsCompletedCum", "JobsCompleted") + remap("JobsExitedCum", "JobsExited") + remap("ShadowExceptionsCum", "JobsExitException") + remap("JobsStartedCum", "JobsStarted") + remap("SumRunningTimeCum", "JobsAccumRunningTime") + remap("SumTimeToStartCum", "JobsAccumTimeToStart") + + remap_div("JobSubmissionRate", + "RecentJobsSubmitted", + "RecentStatsLifetime") + + remap_div("JobCompletionRate", + "RecentJobsCompleted", + "RecentStatsLifetime") + + remap_div("JobStartRate", + "RecentJobsStarted", + "RecentStatsLifetime") + + remap_div("MeanTimeToStartCum", + "JobsAccumTimeToStart", + "JobsStarted") + + remap_div("MeanRunningTimeCum", + "JobsAccumRunningTime", + "JobsCompleted") + + remap_div("MeanTimeToStart", + "RecentJobsAccumTimeToStart", + "RecentJobsStarted") + + remap_div("MeanRunningTime", + "RecentJobsAccumRunningTime", + "RecentJobsCompleted") + +class QueryScheduler(BaseCollectorQuery): + def __init__(self, ep): + super(QueryScheduler, self).__init__(ep) + + # For now let's get them all + self.names = [] + + def make_collector_obj(self, id): + return SchedulerObj(self.ep.id, id) + + def get_objects(self, no_summary): + return self.ep.app.aviary_ops.get_scheduler(self.ep.url, no_summary=no_summary) + + + Modified: branches/tmckay/mint/python/mint/aviary/endpoints.py =================================================================== --- branches/tmckay/mint/python/mint/aviary/endpoints.py 2013-02-06 12:34:23 UTC (rev 5675) +++ branches/tmckay/mint/python/mint/aviary/endpoints.py 2013-02-06 20:03:28 UTC (rev 5676) @@ -1,7 +1,8 @@ from util import now_minus_duration from datetime import datetime from submissions import QuerySubmissionsBefore, QuerySubmissionsAfter -from collector import QueryCollector, QueryNegotiator, QuerySubmitter +from collector import QueryCollector, QueryNegotiator, QuerySubmitter, \ +QueryScheduler class Endpoint(object): def __init__(self, url, id): @@ -69,7 +70,7 @@ self.app = factory.app - my_classes = ("Collector", "Negotiator", "Submitter") + my_classes = ("Collector", "Negotiator", "Submitter", "Scheduler") self.classes = [c for c in factory.abbrev_classes \ if c in my_classes] @@ -84,6 +85,9 @@ if "Submitter" in self.classes: work.append(QuerySubmitter(self)) + if "Scheduler" in self.classes: + work.append(QueryScheduler(self)) + return work class EndpointFactory(object): @@ -118,6 +122,7 @@ classes_to_aviary = \ {"Collector": ("COLLECTOR", ""), "Negotiator": ("COLLECTOR", ""), + "Scheduler": ("COLLECTOR", ""), "Submitter": ("COLLECTOR", ""), "Submission": ("CUSTOM", "QUERY_SERVER")} Modified: branches/tmckay/mint/python/mint/main.py =================================================================== --- branches/tmckay/mint/python/mint/main.py 2013-02-06 12:34:23 UTC (rev 5675) +++ branches/tmckay/mint/python/mint/main.py 2013-02-06 20:03:28 UTC (rev 5676) @@ -90,7 +90,8 @@ supported_classes = [self.model.com_redhat_grid.Submission, self.model.com_redhat_grid.Collector, self.model.com_redhat_grid.Negotiator, - self.model.com_redhat_grid.Submitter] + self.model.com_redhat_grid.Submitter, + self.model.com_redhat_grid.Scheduler] for c in supported_classes: if c in self._qmf_classes:

11 years, 3 months

1
0
0 / 0

r5675 - branches/elephant/sage/rpc-defs/aviary

by croberts＠fedoraproject.org

Author: croberts Date: 2013-02-06 12:34:23 +0000 (Wed, 06 Feb 2013) New Revision: 5675 Modified: branches/elephant/sage/rpc-defs/aviary/aviary-hadoop.xsd Log: Updating schema for sprint drop. Modified: branches/elephant/sage/rpc-defs/aviary/aviary-hadoop.xsd =================================================================== --- branches/elephant/sage/rpc-defs/aviary/aviary-hadoop.xsd 2013-02-05 20:59:54 UTC (rev 5674) +++ branches/elephant/sage/rpc-defs/aviary/aviary-hadoop.xsd 2013-02-06 12:34:23 UTC (rev 5675) @@ -17,109 +17,116 @@ */ --> <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:hdp="http://hadoop.aviary.grid.redhat.com" xmlns:mrg="http://common.aviary.grid.redhat.com" targetNamespace="http://hadoop.aviary.grid.redhat.com"> - <xs:import namespace="http://common.aviary.grid.redhat.com" schemaLocation="aviary-common.xsd"/> -  - <xs:simpleType name="HadoopStateType"> - <xs:restriction base="xs:string"> - <xs:enumeration value="PENDING"/> - <xs:enumeration value="RUNNING"/> - <xs:enumeration value="EXITING"/> - </xs:restriction> - </xs:simpleType> - <xs:complexType name="HadoopID"> - <xs:sequence> - <xs:element name="id" type="xs:string" minOccurs="0"/> - <xs:element name="ipc" type="xs:string" minOccurs="0"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopNameNodeStart"> - <xs:sequence> -  - <xs:element name="bin_file" type="xs:string" minOccurs="0"/> - <xs:element name="owner" type="xs:string" minOccurs="0"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopStart"> - <xs:sequence> -  - <xs:element name="ref" type="hdp:HadoopID"/> -  - <xs:element name="bin_file" type="xs:string" minOccurs="0"/> - <xs:element name="owner" type="xs:string" minOccurs="0"/> - <xs:element name="count" type="xs:int" minOccurs="0" default="1"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopStartResponse"> - <xs:sequence> - <xs:element name="ref" type="hdp:HadoopID" minOccurs="0"/> - <xs:element name="status" type="mrg:Status"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopStop"> - <xs:sequence> - <xs:element name="refs" type="hdp:HadoopID" minOccurs="0" maxOccurs="unbounded"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopStopResult"> - <xs:sequence> - <xs:element name="ref" type="hdp:HadoopID" minOccurs="0"/> - <xs:element name="status" type="mrg:Status"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopStopResponse"> - <xs:sequence> - <xs:element name="results" type="hdp:HadoopStopResult" minOccurs="0" maxOccurs="unbounded"/> - <xs:element name="status" type="mrg:Status"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopQuery"> - <xs:sequence> - <xs:element name="refs" type="hdp:HadoopID" minOccurs="0" maxOccurs="unbounded"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopQueryResult"> - <xs:sequence> - <xs:element name="ref" type="hdp:HadoopID"/> - <xs:element name="owner" type="xs:string"/> - <xs:element name="submitted" type="xs:int"/> - <xs:element name="uptime" type="xs:int"/> - <xs:element name="state" type="hdp:HadoopStateType"/> - <xs:element name="status" type="mrg:Status"/> - </xs:sequence> - </xs:complexType> - <xs:complexType name="HadoopQueryResponse"> - <xs:sequence> - <xs:element name="results" type="hdp:HadoopQueryResult" minOccurs="0" maxOccurs="unbounded"/> - <xs:element name="status" type="mrg:Status"/> - </xs:sequence> - </xs:complexType> -  - <xs:element name="StartNameNode" type="hdp:HadoopNameNodeStart"/> - <xs:element name="StartNameNodeResponse" type="hdp:HadoopStartResponse"/> - <xs:element name="StopNameNode" type="hdp:HadoopStop"/> - <xs:element name="StopNameNodeResponse" type="hdp:HadoopStopResponse"/> - <xs:element name="GetNameNode" type="hdp:HadoopQuery"/> - <xs:element name="GetNameNodeResponse" type="hdp:HadoopQueryResponse"/> -  -  - <xs:element name="StartDataNode" type="hdp:HadoopStart"/> - <xs:element name="StartDataNodeResponse" type="hdp:HadoopStartResponse"/> - <xs:element name="StopDataNode" type="hdp:HadoopStop"/> - <xs:element name="StopDataNodeResponse" type="hdp:HadoopStopResponse"/> - <xs:element name="GetDataNode" type="hdp:HadoopQuery"/> - <xs:element name="GetDataNodeResponse" type="hdp:HadoopQueryResponse"/> -  - <xs:element name="StartJobTracker" type="hdp:HadoopStart"/> - <xs:element name="StartJobTrackerResponse" type="hdp:HadoopStartResponse"/> - <xs:element name="StopJobTracker" type="hdp:HadoopStop"/> - <xs:element name="StopJobTrackerResponse" type="hdp:HadoopStopResponse"/> - <xs:element name="GetJobTracker" type="hdp:HadoopQuery"/> - <xs:element name="GetJobTrackerResponse" type="hdp:HadoopQueryResponse"/> -  - <xs:element name="StartTaskTracker" type="hdp:HadoopStart"/> - <xs:element name="StartTaskTrackerResponse" type="hdp:HadoopStartResponse"/> - <xs:element name="StopTaskTracker" type="hdp:HadoopStop"/> - <xs:element name="StopTaskTrackerResponse" type="hdp:HadoopStopResponse"/> - <xs:element name="GetTaskTracker" type="hdp:HadoopQuery"/> - <xs:element name="GetTaskTrackerResponse" type="hdp:HadoopQueryResponse"/> -</xs:schema> \ No newline at end of file + <xs:import namespace="http://common.aviary.grid.redhat.com" schemaLocation="aviary-common.xsd"/> +  + <xs:simpleType name="HadoopStateType"> + <xs:restriction base="xs:string"> + <xs:enumeration value="PENDING"/> + <xs:enumeration value="RUNNING"/> + <xs:enumeration value="EXITING"/> + </xs:restriction> + </xs:simpleType> + <xs:complexType name="HadoopID"> + <xs:sequence> + <xs:element name="id" type="xs:string" minOccurs="0"/> + <xs:element name="ipc" type="xs:string" minOccurs="0"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopNameNodeStart"> + <xs:sequence> +  + <xs:element name="bin_file" type="xs:string" minOccurs="0"/> + <xs:element name="owner" type="xs:string" minOccurs="0"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopStart"> + <xs:sequence> +  + <xs:element name="ref" type="hdp:HadoopID"/> +  + <xs:element name="bin_file" type="xs:string" minOccurs="0"/> + <xs:element name="owner" type="xs:string" minOccurs="0"/> + <xs:element name="count" type="xs:int" minOccurs="0" default="1"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopStartResponse"> + <xs:sequence> + <xs:element name="ref" type="hdp:HadoopID" minOccurs="0"/> + <xs:element name="status" type="mrg:Status"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopStop"> + <xs:sequence> + <xs:element name="refs" type="hdp:HadoopID" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopStopResult"> + <xs:sequence> + <xs:element name="ref" type="hdp:HadoopID" minOccurs="0"/> + <xs:element name="status" type="mrg:Status"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopStopResponse"> + <xs:sequence> + <xs:element name="results" type="hdp:HadoopStopResult" minOccurs="0" maxOccurs="unbounded"/> + <xs:element name="status" type="mrg:Status"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopQuery"> + <xs:sequence> + <xs:element name="refs" type="hdp:HadoopID" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopQueryResult"> + <xs:sequence> + <xs:element name="ref" type="hdp:HadoopID"/> + <xs:element name="parent" type="hdp:HadoopID"/> + <xs:element name="owner" type="xs:string"/> + <xs:element name="submitted" type="xs:int"/> + <xs:element name="uptime" type="xs:int"/> + <xs:element name="state" type="hdp:HadoopStateType"/> + <xs:element name="status" type="mrg:Status"/> + <xs:element name="bin_file" type="xs:string"/> + <xs:element name="http" type="xs:anyURI"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="HadoopQueryResponse"> + <xs:sequence> + <xs:element name="results" type="hdp:HadoopQueryResult" minOccurs="0" maxOccurs="unbounded"/> + <xs:element name="status" type="mrg:Status"/> + </xs:sequence> + </xs:complexType> +  + <xs:element name="StartNameNode" type="hdp:HadoopNameNodeStart"/> + <xs:element name="StartNameNodeResponse" type="hdp:HadoopStartResponse"/> + <xs:element name="StopNameNode" type="hdp:HadoopStop"/> + <xs:element name="StopNameNodeResponse" type="hdp:HadoopStopResponse"/> + <xs:element name="GetNameNode" type="hdp:HadoopQuery"/> + <xs:element name="GetNameNodeResponse" type="hdp:HadoopQueryResponse"/> + <xs:element name="GetAssociatedDataNodes" type="hdp:HadoopQuery"/> + <xs:element name="GetAssociatedDataNodesResponse" type="hdp:HadoopQueryResponse"/> +  +  + <xs:element name="StartDataNode" type="hdp:HadoopStart"/> + <xs:element name="StartDataNodeResponse" type="hdp:HadoopStartResponse"/> + <xs:element name="StopDataNode" type="hdp:HadoopStop"/> + <xs:element name="StopDataNodeResponse" type="hdp:HadoopStopResponse"/> + <xs:element name="GetDataNode" type="hdp:HadoopQuery"/> + <xs:element name="GetDataNodeResponse" type="hdp:HadoopQueryResponse"/> +  + <xs:element name="StartJobTracker" type="hdp:HadoopStart"/> + <xs:element name="StartJobTrackerResponse" type="hdp:HadoopStartResponse"/> + <xs:element name="StopJobTracker" type="hdp:HadoopStop"/> + <xs:element name="StopJobTrackerResponse" type="hdp:HadoopStopResponse"/> + <xs:element name="GetJobTracker" type="hdp:HadoopQuery"/> + <xs:element name="GetJobTrackerResponse" type="hdp:HadoopQueryResponse"/> + <xs:element name="GetAssociatedTaskTrackers" type="hdp:HadoopQuery"/> + <xs:element name="GetAssociatedTaskTrackersResponse" type="hdp:HadoopQueryResponse"/> +  + <xs:element name="StartTaskTracker" type="hdp:HadoopStart"/> + <xs:element name="StartTaskTrackerResponse" type="hdp:HadoopStartResponse"/> + <xs:element name="StopTaskTracker" type="hdp:HadoopStop"/> + <xs:element name="StopTaskTrackerResponse" type="hdp:HadoopStopResponse"/> + <xs:element name="GetTaskTracker" type="hdp:HadoopQuery"/> + <xs:element name="GetTaskTrackerResponse" type="hdp:HadoopQueryResponse"/> +</xs:schema>

11 years, 3 months

1
0
0 / 0

r5674 - in branches/tmckay/mint/python/mint: . aviary

by tmckay＠fedoraproject.org

Author: tmckay Date: 2013-02-05 20:59:54 +0000 (Tue, 05 Feb 2013) New Revision: 5674 Modified: branches/tmckay/mint/python/mint/aviary/endpoints.py branches/tmckay/mint/python/mint/aviary/work.py branches/tmckay/mint/python/mint/main.py Log: Add submitters, delete on abandoned Modified: branches/tmckay/mint/python/mint/aviary/endpoints.py =================================================================== --- branches/tmckay/mint/python/mint/aviary/endpoints.py 2013-02-05 18:56:35 UTC (rev 5673) +++ branches/tmckay/mint/python/mint/aviary/endpoints.py 2013-02-05 20:59:54 UTC (rev 5674) @@ -1,7 +1,7 @@ from util import now_minus_duration from datetime import datetime from submissions import QuerySubmissionsBefore, QuerySubmissionsAfter -from collector import QueryCollector, QueryNegotiator +from collector import QueryCollector, QueryNegotiator, QuerySubmitter class Endpoint(object): def __init__(self, url, id): @@ -69,7 +69,7 @@ self.app = factory.app - my_classes = ("Collector","Negotiator") + my_classes = ("Collector", "Negotiator", "Submitter") self.classes = [c for c in factory.abbrev_classes \ if c in my_classes] @@ -80,6 +80,10 @@ if "Negotiator" in self.classes: work.append(QueryNegotiator(self)) + + if "Submitter" in self.classes: + work.append(QuerySubmitter(self)) + return work class EndpointFactory(object): @@ -112,8 +116,9 @@ # what types of endpoints to report classes_to_aviary = \ - {"Collector": ("COLLECTOR", ""), + {"Collector": ("COLLECTOR", ""), "Negotiator": ("COLLECTOR", ""), + "Submitter": ("COLLECTOR", ""), "Submission": ("CUSTOM", "QUERY_SERVER")} endpoints = set() Modified: branches/tmckay/mint/python/mint/aviary/work.py =================================================================== --- branches/tmckay/mint/python/mint/aviary/work.py 2013-02-05 18:56:35 UTC (rev 5673) +++ branches/tmckay/mint/python/mint/aviary/work.py 2013-02-05 20:59:54 UTC (rev 5674) @@ -91,9 +91,7 @@ if work is None: break # quittin' time work.do_the_job(self.foreman) - sleep(1) - class WaitList(Thread): def __init__(self, ready_queue): super(WaitList, self).__init__() Modified: branches/tmckay/mint/python/mint/main.py =================================================================== --- branches/tmckay/mint/python/mint/main.py 2013-02-05 18:56:35 UTC (rev 5673) +++ branches/tmckay/mint/python/mint/main.py 2013-02-05 20:59:54 UTC (rev 5674) @@ -89,7 +89,8 @@ supported_classes = [self.model.com_redhat_grid.Submission, self.model.com_redhat_grid.Collector, - self.model.com_redhat_grid.Negotiator] + self.model.com_redhat_grid.Negotiator, + self.model.com_redhat_grid.Submitter] for c in supported_classes: if c in self._qmf_classes:

11 years, 3 months

1
0
0 / 0

r5673 - branches/elephant/cumin/python/cumin/gridhadoop

by croberts＠fedoraproject.org

Author: croberts Date: 2013-02-05 18:56:35 +0000 (Tue, 05 Feb 2013) New Revision: 5673 Modified: branches/elephant/cumin/python/cumin/gridhadoop/datanode.py branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py branches/elephant/cumin/python/cumin/gridhadoop/namenode.py branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py Log: Adjusting text for frame navigation crumbs. May change again when api changes are complete. Modified: branches/elephant/cumin/python/cumin/gridhadoop/datanode.py =================================================================== --- branches/elephant/cumin/python/cumin/gridhadoop/datanode.py 2013-02-05 17:29:00 UTC (rev 5672) +++ branches/elephant/cumin/python/cumin/gridhadoop/datanode.py 2013-02-05 18:56:35 UTC (rev 5673) @@ -225,7 +225,7 @@ class Object(object): pass fake = Object() - fake.name = "fake data node object" + fake.name = id fake.id = id return fake Modified: branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py =================================================================== --- branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py 2013-02-05 17:29:00 UTC (rev 5672) +++ branches/elephant/cumin/python/cumin/gridhadoop/jobtracker.py 2013-02-05 18:56:35 UTC (rev 5673) @@ -196,7 +196,7 @@ class Object(object): pass fake = Object() - fake.name = "fake object" + fake.name = id fake.id = id return fake Modified: branches/elephant/cumin/python/cumin/gridhadoop/namenode.py =================================================================== --- branches/elephant/cumin/python/cumin/gridhadoop/namenode.py 2013-02-05 17:29:00 UTC (rev 5672) +++ branches/elephant/cumin/python/cumin/gridhadoop/namenode.py 2013-02-05 18:56:35 UTC (rev 5673) @@ -181,7 +181,7 @@ class Object(object): pass fake = Object() - fake.name = "fake object" + fake.name = id fake.id = id return fake Modified: branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py =================================================================== --- branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py 2013-02-05 17:29:00 UTC (rev 5672) +++ branches/elephant/cumin/python/cumin/gridhadoop/scheduler.py 2013-02-05 18:56:35 UTC (rev 5673) @@ -74,15 +74,15 @@ def get_object(self, session, id): class Object(object): pass - fake = Object() - fake.name = "fake scheduler" - fake.id = id - return fake + sched = Object() + sched.name = id + sched.id = id + return sched def get_title(self, session): retval = "" try: - retval = "Hadoop scheduler:'%s' (%s)" % (xml_escape(self.object.get(session).name), xml_escape(self.object.get(session).id)) + retval = "Hadoop scheduler:'%s'" % (xml_escape(self.object.get(session).name)) except Exception, e: pass return retval Modified: branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py =================================================================== --- branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py 2013-02-05 17:29:00 UTC (rev 5672) +++ branches/elephant/cumin/python/cumin/gridhadoop/tasktracker.py 2013-02-05 18:56:35 UTC (rev 5673) @@ -283,7 +283,7 @@ class Object(object): pass fake = Object() - fake.name = "fake task tracker object" + fake.name = id fake.id = id return fake

11 years, 3 months

1
0
0 / 0

r5672 - trunk/sage/python/sage/aviary

by tmckay＠fedoraproject.org

Author: tmckay Date: 2013-02-05 17:29:00 +0000 (Tue, 05 Feb 2013) New Revision: 5672 Modified: trunk/sage/python/sage/aviary/aviaryoperations.py Log: Not setting allowOverrides correctly BZ907866 Modified: trunk/sage/python/sage/aviary/aviaryoperations.py =================================================================== --- trunk/sage/python/sage/aviary/aviaryoperations.py 2013-02-05 17:08:33 UTC (rev 5671) +++ trunk/sage/python/sage/aviary/aviaryoperations.py 2013-02-05 17:29:00 UTC (rev 5672) @@ -314,7 +314,7 @@ # (otherwise, Requirements will be limited to particular # resource constraint types defined by aviary) job_client.set_enable_attributes(True) - job_client.set_attributes({"allowOverrides": True}) + job_client.set_attributes({"allowOverrides": "true"}) extras = list() for k, v in ad.iteritems(): # We don't need to send descriptors down to aviary

11 years, 3 months

1
0
0 / 0

r5671 - branches/elephant/cumin/model

by croberts＠fedoraproject.org

Author: croberts Date: 2013-02-05 17:08:33 +0000 (Tue, 05 Feb 2013) New Revision: 5671 Modified: branches/elephant/cumin/model/cumin.xml branches/elephant/cumin/model/rosemary.xml Log: Adding class for hadoop schedulers. Modified: branches/elephant/cumin/model/cumin.xml =================================================================== --- branches/elephant/cumin/model/cumin.xml 2013-02-04 18:05:45 UTC (rev 5670) +++ branches/elephant/cumin/model/cumin.xml 2013-02-05 17:08:33 UTC (rev 5671) @@ -82,5 +82,9 @@ <property name="Owner" type="sstr"/> <property name="Location" type="sstr"/> </class> + <class name="Scheduler" storage="none"> + <property name="Id" type="sstr"/> + <property name="Name" type="sstr"/> + </class> </package> </model> Modified: branches/elephant/cumin/model/rosemary.xml =================================================================== --- branches/elephant/cumin/model/rosemary.xml 2013-02-04 18:05:45 UTC (rev 5670) +++ branches/elephant/cumin/model/rosemary.xml 2013-02-05 17:08:33 UTC (rev 5671) @@ -125,6 +125,14 @@ <title>Location</title> </property> </class> + <class name="Scheduler"> + <property name="Id"> + <title>Id</title> + </property> + <property name="Name"> + <title>Name</title> + </property> + </class> </package> <package name="org.apache.qpid.broker">

11 years, 3 months

1
0
0 / 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

cumin-developers February 2013