def ListenerResults(jobID=0): if jobID == 0: print "No JobID given, using JobID = 0 as a default" else: print "Using JobID = %s" % jobID try: import apmon # #logger.warning ('Some problem prevented MonaLisa monitoring from being loaded') ml_sensor = apmon.ApMon("./destinations.conf") # This will contain just the variable and its value toSend = {} try: ListenerResults = XMLParser() except Exception,x: print "INFO: XMLParser could not parse the Listener Files: %s" % str(x) if ListenerResults: for Results in ListenerResults: # this loops through the result of each file read (could just be one file) for vars in Results.keys(): # this loops through each variable in each run for data in Results["%s" % vars]: import re # print "data is: %s" % data if re.search('@$', data): #toSend["%s" % vars] = "%s" % (Results["%s" % vars ][4]) data = data.__getslice__(0,data.__len__()-1) print "data is: %s" % data try: # dataInt = int(data.encode('latin-1')) dataToSubmit = float(data) except Exception,x: print "Error casting as an int:", str(x), " DATA: '", data, "'" dataToSubmit = data.encode('latin-1') #print "RESULTS VALUES BEING PASSED TO ML ****: key: ", vars, " value: ", Results["%s" % vars ] try: # ml_sensor.sendParameters("SimpleCluster", "Job ID: %s" % (jobID), ({vars.encode('latin-1') : data.encode('latin-1')})) ml_sensor.sendParameters("HistoCluster", "Job ID: %s" % (jobID), ({vars.encode('latin-1') : dataToSubmit})) except Exception,x: print "WARNING: Cannot send some variables to the MonaLisa server: %s" % str(x) # for vals in toSend: else: print "XMLParser output not as anticipated so cannot send data to MonaLisa" ml_sensor.free() # for vars in ListenerResults.keys(): # toSend[ "%s" % vars ] = "%s" % (ListenerResults[vars])[4] # for k in toSend.keys(): # ml_sensor.sendParameters("SimpleCluster", "SimpleNode", {k.encode('latin-1') : (toSend[k]).encode('latin-1')} ) return ListenerResults except Exception,x: print "A problem was encountered which prevented monitoring service from running from MonitorResults: %s" % str(x) ############ PARSE THE XML FILE : ################ def XMLParser(): """ Parse a results XML file according to the way it was originally defined in Scouter v1.0 and then return a map (dictionary with keys as the variables monitored and values as lists of the values for that variable) of the results """ # Using expat module, XMLParser reads in the Results.xml file and returns a hash which has any # 'VarName' tag values as keys and for values has all the values of the children tags as a list import xml.parsers.expat import re import os from time import sleep # currentValues hash will hold map of fields and values currentValues = {} # the final processing and formatting puts values in endResult to return to the user endResult = {} # List of endResults, if parsing more than one file endResultList = [] # lastReadTag is a simple way to pass variables between start_element and char_data lastReadTag = { 'Tag' : '' } # 3 handler functions def start_element(name, attrs): data = '' name = " ".join(name.split()) # remove trailing whitespace lastReadTag['Tag'] = name # add temporary value for char_data to read info passed by start_element if not currentValues.has_key(name): # if this tag has not been read before, enter it into currentValues currentValues[name] = "" # default to just add empty string if attrs : currentValues[attrs.keys()[0]] = attrs.values()[0] # if attributes currentValues[name] = data # print 'Start element:', name, attrs def end_element(name): li = [] stripPattern = re.compile('(\S+)') # if currentValues.has_key('Value'): if not re.search('@$', currentValues['Value']): currentValues['Value'] = "%s@" % currentValues['Value'] if currentValues.has_key('VarName'): # and not endResult.has_key[currentValues['VarName']]: for v in currentValues.values(): if stripPattern.search(v) and v != currentValues['VarName']: li.append(v) endResult[currentValues['VarName']] = li def char_data(data): lt = lastReadTag.values()[0] ValueMatch = '(\d+)' data = " ".join(data.split()) # remove trailing whitespace if currentValues.has_key(lt) and re.search('\S+', data): # print "%s stop %s stop %s" % (lt, currentValues[lt], data) currentValues[lt] = data if re.search(ValueMatch, data) and data: # print "FOUND!!! %s" % data try: currentValues['Value'] = data except Exception, x: print "No Value field found, continuing %s" % str(x) # print 'Character data:', repr(data) fileList = [] for f in os.listdir('./'): if re.search('log\d+', f): fileList.append(f) # Test if the last file is growing in size # MESSY!!!!!!!!!!!!!!!!!!!!!!! if fileList.__len__() != 0 : tempFile1 = fileList[(fileList.__len__())-1] tempSize1 = os.stat(tempFile1)[6] sleep(1) else : print "XMLParser found no log files, exiting for now" return {} fileList = [] for f in os.listdir('./'): if re.search('log\d+', f): fileList.append(f) tempFile2 = fileList[(fileList.__len__())-1] tempSize2 = os.stat(fileList[(fileList.__len__())-1])[6] # If nothing is currently being written and no new files created in # the last second, then consume last file, else consume last-but-one if tempSize1 == tempSize2 and tempFile1 == tempFile2 : pass else: fileList.pop() p = xml.parsers.expat.ParserCreate() p.StartElementHandler = start_element p.EndElementHandler = end_element p.CharacterDataHandler = char_data p2 = xml.parsers.expat.ParserCreate() p2.StartElementHandler = start_element p2.EndElementHandler = end_element p2.CharacterDataHandler = char_data n = 2 lif = [] lif = range(n) for i in range(n): lif[i] = xml.parsers.expat.ParserCreate() # Since each new file seems to require a new parser, this array # will contain the same amount of parser objects as there are files to be read parsers = [] fileCounter = 0 for file in fileList: print file parsers.append(xml.parsers.expat.ParserCreate()) try: (parsers[fileCounter]).StartElementHandler = start_element (parsers[fileCounter]).EndElementHandler = end_element (parsers[fileCounter]).CharacterDataHandler = char_data fsock = open(file, 'r') line = fsock.read() (parsers[fileCounter]).Parse(line) fsock.flush() fsock.close() if os.system('rm ./%s' % file): print "%s was eaten successfully, yum!" % file fileCounter += 1 except IOError: print "Failed to open XML file for parsing" endResultList.append(endResult.copy()) endResult.clear() currentValues.clear() return endResultList