import Tix import datasetfinder import xml.dom from xml.dom.ext.reader import Sax2 from xml.dom.ext import PrettyPrint from xml.dom import implementation from spdeDialogs import * import sys import datasitewalk, ftplib, re, string, os.path, socket from fnmatch import fnmatch from urlparse import urlparse MSG_FNAMING = """# File Naming Codes %A - weekday name %B - month name %d - day of month (01, 31) %H - hour (00, 23) %I - hour (01, 12) %j - day of year (001, 366) %m - month number (01, 12) %M - minute (00, 59) %p - AM or PM %R - bartel rotations %S - second (00, 61) %v - variable # of characters %w - week number (00, 53) %y - short year (00, 99) %Y - year (0001, 9999) %Z - time zone %% - a literal "%" # anything else is matched literally""" class XMLTreeDisplay(Tix.Tree): def __init__(self, master = None, filelist = None): Tix.Tree.__init__(self, master, options='separator "/"') self.treelist = {} self.node_index = {} # dictionary of (tree index, element node) pairs # OPTIONS self.showinstruments = IntVar() self.showinstruments.set(1) # filelist is a list of xml files to preload if filelist == None: filelist = get_datasites().values() filelist.sort() for f in filelist: self.loadxmlfromfile(f) def loadxmlfromfile(self, filename): """Draw the XMLTreeDisplay from a filename""" self.treelist[filename] = readxml(filename) # get a display name nodelist = self.treelist[filename].getElementsByTagName('datasite') rootnode = nodelist.item(0) myname = rootnode.getAttribute('name') self.hlist.add(myname, itemtype=Tix.IMAGETEXT, text=myname, image=self.tk.call('tix', 'getimage', 'folder')) self.node_index[myname] = rootnode # recurse if rootnode.hasChildNodes(): for x in rootnode.childNodes: self.loadnode(x, myname) self.setmode(myname, 'close') self.close(myname) def loadxmlfromdocnode(self, doc): """Draw the XMLTreeDisplay from a loaded xml document""" # does not update files unless told to save nodelist = doc.getElementsByTagName('datasite') rootnode = nodelist.item(0) myname = rootnode.getAttribute('name') self.hlist.add(myname, itemtype=Tix.IMAGETEXT, text=myname, image=self.tk.call('tix', 'getimage', 'folder')) self.node_index[myname] = rootnode # recurse if rootnode.hasChildNodes(): for x in rootnode.childNodes: self.loadnode(x, myname) self.setmode(myname, 'close') self.close(myname) def reload(self): """Redraw Tree Display""" self.hlist.delete_all() for key, val in self.treelist.items(): self.loadxmlfromdocnode(val) def loadnode(self, x_node, parent): """Load an XML Node (if type is in showlist) into the display""" showlist = ['spacecraft', 'instrument', 'dataset'] if x_node.nodeType == xml.dom.Node.ELEMENT_NODE \ and x_node.nodeName in showlist: if not self.showinstruments.get() and x_node.nodeName == 'instrument': treename = parent else: myname = x_node.getAttribute('name') treename = self.hlist.add_child(parent, itemtype=Tix.IMAGETEXT, text=myname, image=self.tk.call('tix', 'getimage', 'folder')) self.node_index[treename] = x_node for x in x_node.childNodes: self.loadnode(x, treename) if x_node.nodeName != 'dataset' or not x_node.hasChildNodes(): self.setmode(treename, 'close') self.close(treename) else: pass def selectednode(self): """Get currently selected tree index and its node""" curselect = None curnode = None tuple = self.hlist.info_selection() if len(tuple) > 0: curselect = tuple[0] if curselect != None: curnode = self.node_index[curselect] return curselect, curnode def addchild(self, name = '', path = '', div = '', fname = '', ftype = ''): treeid, parent = self.selectednode() if parent.nodeName == 'dataset': return namespace = None # find the document which the parent is in lookahead = parent factory = parent while lookahead != None: factory = lookahead lookahead = factory.parentNode if parent.nodeName == 'datasite': newchild = factory.createElementNS(namespace, 'spacecraft') newchild.setAttribute('name', name) newchild.setAttribute('code', name) newchild.setAttribute('nssdcid', '?') # default description newchild.appendChild(get_text_node(factory, namespace, 'description', name.upper() + ' description')) result = self.viewattributes('Add a New Spacecraft', newchild) elif parent.nodeName == 'spacecraft' and \ self.showinstruments.get(): newchild = factory.createElementNS(namespace, 'instrument') newchild.setAttribute('name', name) newchild.setAttribute('code', path) newchild.appendChild(get_text_node(factory, namespace, 'description', name.upper() + ' description')) result = self.viewattributes('Add a New Instrument', newchild) elif parent.nodeName in ['spacecraft', 'instrument']: newchild = factory.createElementNS(namespace, 'dataset') newchild.setAttribute('name', string.join(path.split('/')[-4:], '_')) newchild.setAttribute('dataformat', ftype) newchild.setAttribute('nssdcid', '?') newchild.appendChild(get_text_node(factory, namespace, 'path', path)) newchild.appendChild(get_trange_node(factory, namespace, 'timerange')) newchild.appendChild(get_text_node(factory, namespace, 'subdividedby', div)) newchild.appendChild(get_text_node(factory, namespace, 'filename', fname)) newchild.appendChild(get_text_node(factory, namespace, 'description', name + ' description')) result = self.viewattributes('Add a New Dataset', newchild) if result == None or len(result.items()) == 0: return for key, (type, val) in result.items(): if type == ATTRIBUTE: newchild.setAttribute(key, val) elif type in [TEXT_ELEMENT, DIVISION_ELEMENT, FORMAT_ELEMENT]: for x in newchild.childNodes: if x.nodeName == key: x.firstChild.data = val break elif type == TIME_ELEMENT: for x in newchild.childNodes: if x.nodeName == key: x.setAttribute('start', val[0]) x.setAttribute('stop', val[1]) x.setAttribute('units', val[2]) break parent.appendChild(newchild) self.reload() self.openpath(treeid) def addfromftp(self, ftptree): treeid, parent = self.selectednode() ftptreeid, ftppath = ftptree.selectednode() if treeid == None or parent == None or ftptreeid == None or ftppath == None: return # analyze a dataset's remote directory for clues if parent.nodeName == 'instrument' or \ (self.showinstruments.get() == 0 and parent.nodeName == 'spacecraft'): div, fname = datasitewalk.ftpanalyzer('ftp://' + ftptree.address + ftppath) ftype = string.join(fname.split('.')[1:], '.') self.addchild(ftppath.split('/')[-1], 'ftp://' + ftptree.address + ftppath, div, fname, ftype) else: self.addchild(ftppath.split('/')[-1], 'ftp://' + ftptree.address + ftppath) def addsite(self): dt = implementation.createDocumentType("sites", "", "") doc = implementation.createDocument(None, "sites", dt) doc_elem = doc.documentElement namespace = "nasa data site" new_ds = doc.createElementNS(namespace, 'datasite') fname = getfname(self) if fname == '': return result = self.viewattributes('Add a New Datasite', new_ds, True) if result == None or len(result.items()) == 0: return for key, (type, val) in result.items(): if type == ATTRIBUTE: new_ds.setAttribute(key, val) elif type in [TEXT_ELEMENT, DIVISION_ELEMENT, FORMAT_ELEMENT]: new_ds.appendChild(get_text_node(doc, namespace, key, val)) elif type == TIME_ELEMENT: new_ds.appendChild(get_trange_node(factory, namespace, key, val)) doc_elem.appendChild(new_ds) try: f = open(fname, "w") except IOError: return else: PrettyPrint(doc, f) f.close() temp = get_datasites() temp[result['name'][1]] = fname put_datasites(temp) self.treelist[fname] = doc self.reload() def importsite(self): filename = tkFileDialog.askopenfilename(parent=self, defaultextension='.xml') if filename == '': return doc_node = readxml(filename) ds_node = doc_node.getElementsByTagName('datasite').item(0) myname = ds_node.getAttribute('name') temp = get_datasites() temp[myname] = filename put_datasites(temp) self.treelist[filename] = doc_node self.reload() def removenode(self, node = None): if node == None: treeid, node = self.selectednode() if node.nodeName == 'datasite': return self.removesite() if tkMessageBox.askyesno('Really Remove?', 'Are you sure you want to remove this ' + node.nodeName + '?', parent=self.master): oldnode = node.parentNode.removeChild(node) par_id = self.hlist.info_parent(treeid) self.reload() self.openpath(par_id) def removesite(self): treeid, node = self.selectednode() if tkMessageBox.askyesno('Really Remove?', 'Are you sure you want to remove this file from \ the list of datasites? (actual xml file is not deleted)', parent=self.master): sites = get_datasites() fname = sites[treeid] del self.treelist[fname] del sites[treeid] put_datasites(sites) self.reload() def clonenode(self, node = None): if node == None: treeid, node = self.selectednode() if node.nodeName == 'datasite': return self.clonesite() clone_node = node.cloneNode(True) clone_node.setAttribute('name', node.getAttribute('name') + ' clone') node.parentNode.appendChild(clone_node) self.reload() par_id = self.hlist.info_parent(treeid) self.openpath(par_id) def clonesite(self): treeid, old_ds_node = self.selectednode() doc_node = old_ds_node.parentNode.parentNode if doc_node == None: doc_node = old_ds_node.parentNode myname = old_ds_node.getAttribute('name') sites = get_datasites() old_file = sites[myname] new_file = tkFileDialog.asksaveasfilename(parent=self, defaultextension='.xml') if new_file == '': return if old_file != new_file and not sites.has_key(myname + ' clone'): self.treelist[new_file] = doc_node.cloneNode(True) new_doc_node = self.treelist[new_file] new_ds_node = new_doc_node.getElementsByTagName('datasite').item(0) new_ds_node.setAttribute('name', myname + ' clone') f = open(new_file, 'w') PrettyPrint(doc_node, f) f.close() sites[myname + ' clone'] = new_file put_datasites(sites) self.reload() elif old_file == new_file: tkMessageBox.showwarning('File Already Exists', 'Please try again.') else: tkMessageBox.showwarning('Clone Already Exists', 'Please rename the other clone before cloning again.') def editattributes(self, node = None): if node == None: treeid, node = self.selectednode() old_name = node.getAttribute('name') result = self.viewattributes('Edit Attributes', node) if result == None or len(result.items()) == 0: return for key, (type, val) in result.items(): if type == ATTRIBUTE: node.setAttribute(key, val) elif type in [TEXT_ELEMENT, DIVISION_ELEMENT, FORMAT_ELEMENT]: for x in node.childNodes: if x.nodeName == key: x.firstChild.data = val break elif type == TIME_ELEMENT: for x in node.childNodes: if x.nodeName == key: x.setAttribute('start', val[0]) x.setAttribute('stop', val[1]) x.setAttribute('units', val[2]) break if node.nodeName == 'datasite': sites = get_datasites() temp = sites[old_name] del sites[old_name] sites[node.getAttribute('name')] = temp put_datasites(sites) self.reload() self.openpath(treeid) def viewattributes(self, title = None, node = None, blank = False): if node == None: treeid, node = self.selectednode() if node == None: return None if title == None: title = 'View Attributes' if node.nodeName == 'datasite': title = title + ' of Datasite' fields = [("name", ATTRIBUTE, "Name", 1), ("address", ATTRIBUTE, "Address", 1), ("protocol", ATTRIBUTE, "FTP or HTTP", 1), ("datadir", ATTRIBUTE, "Data Directory", 1), ("description", TEXT_ELEMENT, "Description", 0)] elif node.nodeName == 'spacecraft': title = title + ' of Spacecraft' fields = [("name", ATTRIBUTE, "Name", 1), ("code", ATTRIBUTE, "2-3 Letter Code", 0), ("nssdcid", ATTRIBUTE, "ID on NSSDC Catalog", 0), ("description", TEXT_ELEMENT, "Description", 0)] elif node.nodeName == 'instrument': title = title + ' of Instrument' fields = [("name", ATTRIBUTE, "Name of Instrument/Type", 1), ("code", ATTRIBUTE, "Subdirectory", 1), ("description", TEXT_ELEMENT, "Description", 0)] elif node.nodeName == 'dataset': title = title + ' of Dataset' fields = [("name", ATTRIBUTE, "Dataset Descriptor", 1), ("dataformat", ATTRIBUTE, "Filename Extension", 1), ("path", TEXT_ELEMENT, "Full Remote Path List", 1), ("nssdcid", ATTRIBUTE, "ID on NSSDC Catalog", 0), ("timerange", TIME_ELEMENT, "Time Range for Data", 1), ("subdividedby", DIVISION_ELEMENT, "Directory Hierarchy", 1), ("filename", FORMAT_ELEMENT, "File Naming String", 1), ("description", TEXT_ELEMENT, "Description", 0), ("info", INFO_FRAME, MSG_FNAMING, 0)] else: return None if blank: s = XMLNodeDialog(self.master, title, None, fields) else: s = XMLNodeDialog(self.master, title, node, fields) return s.result def saveallfiles(self): for filename, xmldoc in self.treelist.items(): f = open(filename, 'w') PrettyPrint(xmldoc, f) f.close() def openpath(self, treeid, selected = None): lookahead = current = treeid if selected == None: selected = treeid while lookahead != '': self.open(lookahead) current = lookahead lookahead = self.hlist.info_parent(current) self.hlist.selection_set(selected) def get_text_node(factory, namespace, name, txt = 'nothing defined yet'): newtextelem = factory.createElementNS(namespace, name) newtextnode = factory.createTextNode(txt) newtextelem.appendChild(newtextnode) return newtextelem def get_trange_node(factory, namespace, name, times = ('BEGINNING', 'LATEST', '')): newtrangeelem = factory.createElementNS(namespace, name) newtrangeelem.setAttribute('start', times[0]) newtrangeelem.setAttribute('stop', times[1]) newtrangeelem.setAttribute('units', times[2]) return newtrangeelem def get_datasites(): try: f = open('datasites.pref') except IOError: return dict() else: lines = f.readlines() lines = map(lambda x: x.strip().split('|'), lines) f.close() return dict(lines) def put_datasites(the_dict): f = open('datasites.pref', 'w') for key, val in the_dict.items(): f.write(key + '|' + val + '\n') f.close() def getfname(master): return tkFileDialog.asksaveasfilename(parent=master, defaultextension='.xml') def readxml(filename): f = open(filename, 'r') reader = Sax2.Reader() doc = reader.fromStream(f) f.close() return doc class FTPDirTreeDisplay(Tix.Tree): def __init__(self, master = None): Tix.Tree.__init__(self, master, options='separator "/"') self.node_index = {} self.address = '' self.directory = '' self['opencmd'] = lambda x: self.loaddir(self.node_index[x], x) self.loadval = Tix.StringVar() self.showall = IntVar() self.showall.set(0) self.numfiles = IntVar() self.numfiles.set(5) def loadftpurl(self, full_path = None): if full_path == None: full_path = self.loadval.get() if full_path[:6] != 'ftp://': full_path = 'ftp://' + full_path parts = urlparse(full_path) protocol, siteadd, basedir = parts[:3] if siteadd == '': tkMessageBox.showwarning('FTP Connection Failed', 'Please type a valid address first.') return 0 try: self.ftp = ftplib.FTP(siteadd) except (socket.gaierror, EOFError): tkMessageBox.showwarning('FTP Connection Failed', 'Could not connect to the address that you typed.\n' + 'Either the address is invalid or it is currently\n' + 'unreachable. Please Try Again.') else: try: self.ftp.login() except ftplib.error_perm, msg: tkMessageBox.showwarning('FTP Connection Failed', 'Could not login to the address that you typed.\n' + 'It might require a password. Please Try Again.') else: try: self.ftp.cwd(basedir) except ftplib.error_perm, msg: tkMessageBox.showwarning('Directory Does Not Exist', 'Directory does not exist on the FTP ' + 'Server.\nStaying in root directory') self.address = siteadd self.directory = basedir self.loadbasedir() return 1 return 0 def loadbasedir(self): try: self.ftp.cwd(self.directory) except ftplib.error_perm, msg: return self.hlist.delete_all() if self.directory in ['', '/']: myname = self.address else: myname = self.directory self.hlist.add('base', itemtype=Tix.IMAGETEXT, text=myname, image=self.tk.call('tix', 'getimage', 'folder')) self.node_index['base'] = self.ftp.pwd() subdirs, files = datasitewalk.getdirlisting(self.ftp) for sub in subdirs: treeid = self.hlist.add_child('base', itemtype=Tix.IMAGETEXT, text=sub, image=self.tk.call('tix', 'getimage', 'folder')) if self.ftp.pwd() == '/': self.node_index[treeid] = '/' + sub else: self.node_index[treeid] = self.ftp.pwd() + '/' + sub self.setmode(treeid, 'close') self.close(treeid) # show the first five files in every directory if self.showall.get() == 0 and self.numfiles.get() <= len(files): filelist = files[:self.numfiles.get()] else: filelist = files for file in filelist: treeid = self.hlist.add_child('base', itemtype=Tix.IMAGETEXT, text=file, image=self.tk.call('tix', 'getimage', 'file')) self.node_index[treeid] = self.ftp.pwd() + '/' + file def loaddir(self, dir, parent): try: self.ftp.cwd(dir) except ftplib.error_perm, msg: return subdirs, files = datasitewalk.getdirlisting(self.ftp) for sub in subdirs: treeid = self.hlist.add_child(parent, itemtype=Tix.IMAGETEXT, text=sub, image=self.tk.call('tix', 'getimage', 'folder')) self.node_index[treeid] = self.ftp.pwd() + '/' + sub self.setmode(treeid, 'close') self.close(treeid) if len(subdirs) == 0 and len(files) == 0: self.setmode(parent, 'none') # show the first five files in every directory if self.showall.get() == 0 and self.numfiles.get() <= len(files): filelist = files[:self.numfiles.get()] else: filelist = files for file in filelist: treeid = self.hlist.add_child(parent, itemtype=Tix.IMAGETEXT, text=file, image=self.tk.call('tix', 'getimage', 'file')) self.node_index[treeid] = self.ftp.pwd() + '/' + file def loadselectednode(self): treeid, the_dir = self.selectednode() if the_dir != None and treeid != None: self.loadftpurl('ftp://' + self.address + the_dir) self.loadval.set('ftp://' + self.address + the_dir) def selectednode(self): curselect = None curnode = None tuple = self.hlist.info_selection() if len(tuple) > 0: curselect = tuple[0] if curselect != None: curnode = self.node_index[curselect] return curselect, curnode def getfullpath(self): treeid, the_dir = self.selectednode() if the_dir != None and treeid != None: return 'ftp://' + self.address + the_dir else: return None def directoryup(self): path = self.loadval.get() parts = urlparse(path) protocol, siteadd, basedir = parts[:3] if basedir != '': path_arr = string.split(basedir, '/') basedir = string.join(path_arr[:-1], '/') path = protocol + '://' + siteadd + basedir self.loadftpurl(path) self.loadval.set(path) if __name__ == '__main__': root = Tix.Tk() top = Tix.Frame(root, relief=Tix.RAISED, bd=1) xt = XMLTreeDisplay(top) xt.pack(expand=1, fill=Tix.BOTH, padx=10, pady=10, side=Tix.LEFT) top.pack(side=Tix.TOP, fill=Tix.BOTH, expand=1) root.mainloop()