From fee0ffd259d8a8bd7b69bc9b3cfe68538fb13a3e Mon Sep 17 00:00:00 2001 From: Andrew McNab Date: Fri, 2 May 2014 18:01:55 +0200 Subject: [PATCH 1/3] Add remote directory listing --- client/src/mjf/mjf.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/client/src/mjf/mjf.py b/client/src/mjf/mjf.py index 8b605de..6130405 100755 --- a/client/src/mjf/mjf.py +++ b/client/src/mjf/mjf.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os, socket, urllib2 +from HTMLParser import HTMLParser from datetime import datetime from optparse import OptionParser try: @@ -8,6 +9,26 @@ except ImportError: import json +class MJFHTMLParser(HTMLParser): + + def __init__(self, envvar, datakey, mjfObj): + HTMLParser.__init__(self) + self._envvar = envvar + self._datakey = datakey + self._mjf = mjfObj + + def handle_starttag(self, tag, attrs): + if tag =='a': + for key, f in attrs: + if key == 'href' and f[0].isalnum(): + self._mjf._debug('Checking value of URL %s/%s' % (self._envvar, f)) + fp = urllib2.urlopen(self._envvar + '/' + f) + val = fp.read() # ... read the file content + self._mjf._debug('Value of file %s is %s' % (f,val)) + fp.close() + if len(val) and val[-1] == '\n' : val = val[:-1] + self._mjf._addData(self._datakey, f, val) + class MJFException(Exception): pass class mjf: @@ -47,6 +68,7 @@ def __init__(self, ext=False, pret=False, verb=False, dbg=False, ip='0.0.0.0', t self.indent = None if self.pret : self.indent = 2 self.force = force + self._collect() def clean(self): """ @@ -140,7 +162,13 @@ def _collectViaFile(self): self._debug('Looking for variable %s' % var) datakey = var.lower() envvar = os.environ[var] - if os.path.isdir(envvar) : + if envvar[:7] == 'http://' or envvar[:8] == 'https://': + self._debug('Collecting information from HTTP URL %s' % os.environ[var]) + parser = MJFHTMLParser(envvar, datakey, self) + data = urllib2.urlopen(envvar).read() + parser.feed(data) + parser.close() + elif os.path.isdir(envvar) : self._debug('Collecting information from directory %s' % os.environ[var]) for f in os.listdir(envvar) : # for each file in the machine/job features directory self._debug('Checking value of file %s in directory %s' % (f, envvar)) From 9e95ce110f376ba7d67192dfea84be6cbef42c36 Mon Sep 17 00:00:00 2001 From: Andrew McNab Date: Fri, 2 May 2014 18:10:04 +0200 Subject: [PATCH 2/3] Ignore subdirectories, identified as containing slashes --- client/src/mjf/mjf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/mjf/mjf.py b/client/src/mjf/mjf.py index 6130405..593f581 100755 --- a/client/src/mjf/mjf.py +++ b/client/src/mjf/mjf.py @@ -20,7 +20,7 @@ def __init__(self, envvar, datakey, mjfObj): def handle_starttag(self, tag, attrs): if tag =='a': for key, f in attrs: - if key == 'href' and f[0].isalnum(): + if key == 'href' and f[0].isalnum() and not '/' in f: self._mjf._debug('Checking value of URL %s/%s' % (self._envvar, f)) fp = urllib2.urlopen(self._envvar + '/' + f) val = fp.read() # ... read the file content From 2809c6e4cfc973153ec29adbd7e7466933823f63 Mon Sep 17 00:00:00 2001 From: Andrew McNab Date: Fri, 2 May 2014 18:11:27 +0200 Subject: [PATCH 3/3] Do _collect() in _run() --- client/src/mjf/mjf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/mjf/mjf.py b/client/src/mjf/mjf.py index 593f581..184e3f7 100755 --- a/client/src/mjf/mjf.py +++ b/client/src/mjf/mjf.py @@ -68,7 +68,6 @@ def __init__(self, ext=False, pret=False, verb=False, dbg=False, ip='0.0.0.0', t self.indent = None if self.pret : self.indent = 2 self.force = force - self._collect() def clean(self): """ @@ -234,6 +233,7 @@ def _collect(self): else : self._message('ERROR', 'Cannot find job / machine features information on this node') def _run(self): + self._collect() self._print()