#!/usr/bin/env python

# written by: Kuba Moscicki, 2003 CERN
# $Revision: 1.40 $
# last modified by: $Author: pfeiffer $ $Date: 2006/01/18 10:57:04 $
#
# please note: this script is a quick and dirty hack and should not be further developed
# in the current shape but it should be properly refactored instead. In particular in
# the area of recursive call to the same script which should be replaced by a recursive
# funtion call to properly inherit the original command line options.
#
import re
def usage():
    msg = """
LCG Software Distribution and Installation Management Utility
 usage:
   lcg-installation-manager.py arguments [options] commands

 arguments:
   --project=NAME  : name and version of LCG Project (example: POOL_1_0_0, SEAL_0_3_2)

 commands:
   list    : list all dependant packages of the project
   tar     : archive (tar) all dependant packages into distribution area
   html    : create html page for the distribution area
   download: download and install a project with dependant packages from central LCG repository
   diff    : list projects which need download (diff between local and main LCG repository)
             this commands does NOT require a project name
   rpm     : do the rpms for EDG/LCG1 deployment
              
   help    : instructions how to use this script, RPM building etc...
   
 options:
   --prefix=DIR    : LCG installation prefix (default: /afs/cern.ch/sw/lcg)
   --keep-going,-k : continue execution even if some errors occur
   --repository=URL: specify the repository (default: http://spi.cern.ch/extsoft)
   --arch=ARCH     : override the SCRAM_ARCH variable
   --clean-tar=no|lcg|all
                   : remove tarballs after downloading, affects only download command
                     no is default, lcg=only lcg projects,all=lcg+external packages
   --dereference   : dereference symbolic links when tarring (used to be a default)
   --distribution=DIR : change the location of tarball cache
                     ie.e use $PREFIX/external/DIR rather then $PREFIX/external/distribution

   --always-fetch  : download files even if local copies exist and have the same size as originals
   
 environment variables:
    SCRAM_ARCH - specifies platform.

 required software:
    SCRAM installation is required for html, tar and list commands.
    SCRAM is NOT required for download command.
"""
#   --commit        : unless you specify commit NO changes are done to the installation area!
    print msg

CONFIG = {}
CONFIG['lcg_prefix'] = '/afs/cern.ch/sw/lcg'
CONFIG['platf'] = 'rh73_gcc323' # default platform

# update with new projects when they arrive
CONFIG['lcg_project_list'] = ['PI','POOL', 'SEAL','GENSER','CORAL'] 

# update every time new platform is introduced
CONFIG['lcg_known_platforms'] = ['rh73_gcc32','rh73_ecc71','rh73_icc71','rh73_ecc80','rh73_icc80','win32_vc7','win32_vc71','sunos58','rh73_gcc323','cel3-i386_gcc323','slc3_ia32_gcc323','slc3_gcc323','slc3_ia64_gcc323','slc3_amd64_gcc323','slc3_amd64_gcc344','osx103_gcc33']

CONFIG['lcg_website_url'] = 'http://spi.cern.ch/lcgsoft'

CONFIG['website_dir'] = '/afs/cern.ch/project/lcg/app/www/spi/lcgsoft/'

CONFIG['rpm_prefix'] = '/usr/src/redhat/' #'/scratch/usr/src/redhat/' 

CONFIG['distribution_location'] = 'distribution' # relative location of distribution directory in external...

# configure according to prefix
def configure():
    global CONFIG
    CONFIG['external_dir'] = CONFIG['lcg_prefix']+'/external'

    
    CONFIG['distribution_dir'] = CONFIG['lcg_prefix']+'/external/'+CONFIG['distribution_location']
    CONFIG['release_dir'] = CONFIG['lcg_prefix']+'/app/releases'
    CONFIG['scram_dir'] = CONFIG['lcg_prefix']+'/app/spi/scram'

    CONFIG['lcg_repository_url'] = 'http://service-spi.web.cern.ch/service-spi/external/'+CONFIG['distribution_location']+'/'
    
    CONFIG['package_map_file_name'] =  'PACKAGE_MAP'
    CONFIG['package_map_file_dir'] = '/external/distribution'#'/external/scripts'

    CONFIG['html_template_file'] = CONFIG['distribution_dir']+'/lcgsoft_package_template.html'
    
    CONFIG['package_map_file'] = CONFIG['lcg_prefix']+CONFIG['package_map_file_dir']+'/'+CONFIG['package_map_file_name']

    CONFIG['rpm_template_file'] = CONFIG['lcg_prefix']+'/external/distribution/package.spec.template'

# this function is just internal, please refer to get_package_list()
def _get_package_list_from_scram_output(tool_list_file, package_map_file):
    """
    parse scram output file and return a list of packages required by a project as
    [scram_name, extsoft_name_string, extsoft_version_string].
    Name and version string translations are defined by the python dictionary in package_map_file
    Example of a translation dictionary:
    dict = { 'python' : ['Python', {'2.2.2': '2-2-2'} ],
             'gcc3' : None
           }
    """
    # skip first 4 lines of output    
    lines = file(tool_list_file).readlines()[4:]

    # remove blank lines and whitespaces
    splines = filter(lambda x: len(x)>0, map(lambda x: x.strip().split(), lines))

    dict = eval( reduce( lambda x,y: x+y, file(package_map_file).readlines(), "") )

    package_list = []
    
    for l in splines:
        name = l[0]
        ver = l[1]

        extsoft_name = name
        extsoft_ver = ver

        if name in dict:
            meta = dict[name]
            if meta:
                extsoft_name = meta[0]
                if meta[1] and ver in meta[1]:
                    extsoft_ver = meta[1][ver]
            else:
                extsoft_name = None

        if extsoft_name == None:
            print "### ignoring package: ", name
        else:
            if extsoft_name == 'coral' : extsoft_name = "CORAL"  # bad hack for now, needs fix in config
            package_list.append([name, extsoft_name, extsoft_ver])
    return package_list

def get_scram_version_for_project(project_name, version_name):
    return file(CONFIG['release_dir']+'/'+mk_prjpath(project_name, version_name)+'/config/scram_version').readline().strip()

def scram_tar_file_name(project_name, version_name):
    return 'SCRAM_'+get_scram_version_for_project(project_name, version_name)+'.tar.gz'

def mk_prjname(project_name, version_name):
    if version_name:
        return project_name+'_'+version_name
    else:
        return version_name

def mk_prjpath(project_name, version_name):
    return project_name+'/'+ mk_prjname(project_name,version_name)
    
def mk_tarball_name(project_name, version_name,platf=None):
    return mk_release_name(project_name, version_name,platf)+".tar.gz"

def mk_release_name(project_name, version_name,platf=None):
    if platf==None:
        platf = CONFIG['platf']
    return mk_prjname(project_name, version_name) + "__"+"LCG_"+platf

# relative path of the package in the external directory
def mk_package_path(package_name, version_name,platf):
    return package_name+"/"+version_name+"/"+platf

def mk_scram_output_name(project_name, version_name):
    return mk_prjname(project_name,version_name)+'.scram_output'

def mk_package_info_name(project_name, version_name):
    return mk_release_name(project_name, version_name)+'.info'

def mk_specfile_name(pname,vname):
    return mk_release_name(pname,vname)+'-1.spec'

# generate a spec file into SPECS subdirectory
# pinstall_dir and make_ppath allow to reuse this function for both LCG projects
# and external packages
def make_rpm_spec_file(pname,vname, pinstall_dir, make_ppath):
    prjname = mk_release_name(pname,vname)
    ppath = make_ppath(pname,vname)

    print "generating specfile for "+prjname
    
    template_file = file(CONFIG['rpm_template_file'])
    template = template_file.read()
    template = template.replace('@FULL_PACKAGE_NAME@',prjname)
    template = template.replace('@INSTALLATION_PREFIX@',pinstall_dir)

    l = list_files(pinstall_dir+'/'+ppath)

    l = l.replace('./','"%{prefix}/'+ppath+'/')
    l = l.replace('\n','"\n')
    spec_filename = CONFIG['rpm_prefix']+'/SPECS/'+mk_specfile_name(pname,vname)
    sf = file(spec_filename,'w')
    sf.write(template+l)

    
    return spec_filename

# assume gzipped tarball is in distribution dir...
def make_rpm_build(pname,vname, pinstall_dir, make_ppath, keep_going):
    prjname = mk_release_name(pname,vname)
    ppath = make_ppath(pname,vname)

    # copy tarball and add -1 suffix

    tbn = mk_release_name(pname,vname)+'-1.tar'
    rpm_tarball = CONFIG['rpm_prefix']+'/SOURCES/'+tbn
    
    import shutil
    shutil.copyfile(CONFIG['distribution_dir']+'/'+mk_tarball_name(pname,vname),rpm_tarball+'.gz')

    if not exists(rpm_tarball):
        if os.system('gunzip -f '+rpm_tarball+'.gz'):
            if not keep_going:
                sys.exit(2)

    print "building RPM for ",mk_specfile_name(pname,vname)
    cmd = 'cd '+CONFIG['rpm_prefix']+'/SPECS ; rpm -bb '+ mk_specfile_name(pname,vname)
    print cmd
    if 0: #os.system(cmd):
        if not keep_going:
            sys.exit(2)

    return 1

# get file list recursively...
def list_files(dir):
    cmd = "(cd "+dir+"; rm -f /tmp/ll; find . > /tmp/ll)"
    #print cmd
    if os.system(cmd):
        return None
    flist = file('/tmp/ll').read()
    return flist[flist.find('\n')+1:] # remove the first line

def create_scram_output_file(project_name, version_name, dest_dir):
    tmp_file = dest_dir+'/'+ mk_scram_output_name(project_name, version_name)
    cmd = 'cd '+CONFIG['release_dir']+'/'+mk_prjpath(project_name, version_name)+ ';'
    cmd += 'rm -f '+tmp_file+';'
    cmd += CONFIG['scram_dir']+'/scram tool list > '+tmp_file+';'
    if os.system('('+cmd+')'):
        print "error executing commands: "+cmd
        return None
    return tmp_file

# get all files to be tarred for the project in the release area 
# filtering out tmp and other release directories
def get_project_tar_input_files(project_name, version_name):
    prj_dir = CONFIG['release_dir']+"/"+mk_prjpath(project_name,version_name)
    flist, dlist = get_dir_list(prj_dir)

    # some projects have a lot of temporary files and directories
    if "tmp" in dlist:
        dlist.remove("tmp")

    # remove tar directories if existing
    if "tar" in dlist:
        dlist.remove("tar")

    # remove all platform directories other than current one (CONFIG['platf'])
    for p in CONFIG['lcg_known_platforms']:
        if p != CONFIG['platf']:
            for rp in get_related_platforms_list(project_name, version_name, p):
                if rp in dlist:
                    dlist.remove(rp)

    src_list = reduce(lambda x,y: x+y , map( lambda x: mk_prjpath(project_name,version_name)+"/"+x+" ", flist+dlist))
    return src_list

# return a list of directories which correspond to given platform (and all its related platforms)
# which exist in the CONFIG['release_dir']
def get_related_platforms_list(project_name, version_name, platf):
    prj_dir = CONFIG['release_dir']+"/"+mk_prjpath(project_name,version_name)
    flist, dlist = get_dir_list(prj_dir)
    rlist = []
    for d in dlist:
        if re.match('^'+platf+'(_.*$|$)',d): # directory name match with my platform name or plus _*
            rlist.append(d)
    return rlist

# scram tool info PACKAGE | grep _BASE will give the platform
def get_platform_for_package(package_name, project_name, version_name):

    if package_name.upper() in CONFIG['lcg_project_list']:
        return CONFIG['platf']
    
    cmd  = "cd "+CONFIG['release_dir']+'/'+mk_prjpath(project_name, version_name)+';'
    cmd += "unset noclobber;"
    cmd += CONFIG['scram_dir']+"/scram tool info "+package_name+' | grep _BASE > /tmp/lcgtmp2;'

    if os.system('('+cmd+')') != 0:
        print "WARNING: unable to read platform using 'scram tool info "+package_name+"'"
        print "assuming default platform:"+CONFIG['platf']
        return CONFIG['platf']

    ll = file("/tmp/lcgtmp2",'r').readlines()

    if len(ll)!=1:
        print "ERROR: missing or multiple string _BASE found in the output of scram tool info"
        print ll
        sys.exit(2)

    return ll[0].split('/')[-1].strip() # the last part of the path
    
#
# Get package dependency list
# by default run scram to obtain the list and save it in the file
# set from_file to true to read it from a file (for downloader)

def get_package_list(project_name, version_name, from_file=None):
    """
    Get package list by running scram. NEW: we may mix platforms so platform is
    specified for each package separately.
    [scram_name, extsoft_name_string, extsoft_version_string, platform]
    """

    #CONFIG['distribution_dir']+'/'+mk_scram_output_name(project_name,version_name)

    if from_file:
        # check if we use new scheme with .info files
        info_fn = CONFIG['distribution_dir']+'/'+mk_package_info_name(project_name, version_name)
        if exists(info_fn):
            #parse .info file and return the list with platform information
            return load_info_file(info_fn)

        # check if we deal with old scheme  (.scram_output files)
        info_fn = CONFIG['distribution_dir']+'/'+mk_scram_output_name(project_name, version_name)
        if exists(info_fn):
            ll =_get_package_list_from_scram_output(info_fn,CONFIG['package_map_file'])
            # add platform information
            for l in ll:
                l.append(CONFIG['platf'])
            return ll

        print "ERROR: unable to find the info file for this project"
        return []
        
    # get scram package list into a temp file
    tmp_file = create_scram_output_file(project_name, version_name,"/tmp/")

    if not tmp_file: sys.exit(2)

    package_list = _get_package_list_from_scram_output(tmp_file,CONFIG['package_map_file'])

    # read platform information for every package separately and append to package_list

    for l in package_list:
        l.append(get_platform_for_package(l[0],project_name, version_name))

    return package_list

# load and return the package dependency list from .info file (see get_package_list)
def load_info_file(info_fn):
    import xml.parsers.expat

    dep_list = []
    
    def start_element(name, attrs):
        if name == 'package':
            l = [attrs['scram_name'],attrs['spi_name'],attrs['spi_version'],attrs['arch']]
            dep_list.append(l)
        
    p = xml.parsers.expat.ParserCreate()
    p.StartElementHandler = start_element

    p.Parse(file(info_fn,'r').read())

    return dep_list
        
# save package dependency list to .info file (see get_package_list)
def save_info_file(dep_list, info_fn, project_name, version_name):
    f = file(info_fn,'w')

    print >>f, '<lcg_distribution project="'+project_name+'" version="'+version_name+'" arch="'+CONFIG['platf']+'">'

    for l in dep_list:
        print >>f, ' <package scram_name="'+l[0]+'" spi_name="'+l[1]+'" spi_version="'+l[2]+'" arch="'+l[3]+'" />'
    print >>f, '</lcg_distribution>'

# equivalence relation
def equiv(p,q):
    return (p and q) or (not p and not q)

import stat

# check if file exists
# optional arguments add more constraints on a type of the file
# directory=true|false
def exists(file,**kwds):
    try:
        mode = os.stat(file)[stat.ST_MODE]
        if "directory" in kwds:
            return equiv(stat.S_ISDIR(mode), kwds["directory"])
        return 1
    except OSError:
        return None

# return a (file_list, directory_list)
def get_dir_list(dir):
    flist, dlist = [],[]

    entry_list = None

    try:
        entry_list = os.listdir(dir)
    except Exception, x:
        return (flist,dlist)
    
    for f in entry_list: 
        pathname = '%s/%s' % (dir, f)
        try:
            mode = os.stat(pathname)[stat.ST_MODE]
        except Exception,x:
            print str(x)
            continue
        
        if stat.S_ISDIR(mode):
            if f != "CVS":
                dlist.append(f)
        elif stat.S_ISREG(mode):
            flist.append(f)
        else:
            # Unknown file type, print a message
            print "Skipping irregular file "+pathname

    return (flist,dlist)

import urllib, string, stat
# download files over http
# override == 0   : never override any files
# override == 1   : override if files differ in length
# override > 1    : always override

# download files over http
def fetch(tarFileName, targetDir=None, downloadURL = "", override=0, silent=0):

    if not targetDir:
        targetDir = CONFIG['prefix']

    if downloadURL == "":
        downloadURL = CONFIG['lcg_repository_url']

    tarFileURL = downloadURL + '/' + tarFileName
    tarFiles = []

    fileName = targetDir + '/'+tarFileName

    if not silent:
        print "fetching file: " + fileName,

    if ( os.path.exists(fileName) ) :
        # compare size here
        inf = urllib.urlopen(tarFileURL).info()
        label = 'Content-Length:'
        filesize = -1
        for h in inf.headers:
            if h.find(label) != -1:
                filesize = int(h[len(label):].strip())
                break

        if not silent:
            print 'Size:',filesize,
            
        if override > 1:
            if not silent:
                print "overriding...",
            os.unlink(fileName)
        else:
            if override:
                if filesize != os.stat(fileName)[stat.ST_SIZE]:
                    if not silent:
                        print "file has different size, overriding...",
                    os.unlink(fileName)
                else:
                    if not silent:
                        print "Local file of the same size already exists"
            else:
                if not silent:
                    print "file already fetched "
                tarFiles.append(tarFileName)
                return 1

    if not os.path.exists(fileName):
        (fileNameRet, info) = urllib.urlretrieve (tarFileURL, fileName)
    
        if (info.gettype() == "text/html") :
            tmpFile = open(fileNameRet)
            tmpTxt = tmpFile.readlines()
            tmpFile.close()
            if ( string.find( str(tmpTxt), "404 Not Found" ) != -1 ) :
                if silent < 2:
                    print "\nERROR: could not download the file :", tarFileName
                os.remove(fileName)
                return 0
    if not silent:
        print "ok"
    return 1

def HTMLMarkup(text,link=None,bold=None,tt=None,italic=None):
    buf = text

    # generate prefix
    if link:
        buf = '<a href= "'+link+'">'+buf
    if bold:
        buf = '<b>'+buf
    if tt:
        buf = '<tt>'+buf
    if italic:
        buf = '<i>'+buf        
        
    # generate suffix
    if italic:
        buf = buf+'</i>'
    if tt:
        buf = buf+'</tt>'
    if bold:
        buf = buf + '</b>'
    if link:
        buf = buf+'</a>'
        
    return buf

class HTMLWriter:
    def __init__(self):
        self.ostr = ""

    def beginDocument(self,title):
        self.ostr = '''
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<html>
<head>
   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
   <title>''' + title + '''</title>
</head>
<body bgcolor="#FAF0E6" link="#1076A6" vlink="#1076A6" alink="#FF0000">
'''

    def endDocument(self):
        self.ostr += '''
</body>
</html>
'''
    def beginTable(self,cols):
        self.ostr += '\n<table BORDER COLS='+str(len(cols))+' WIDTH="100%">'
        self.addRow(cols);

    def endTable(self):
        self.ostr += "\n</table>"

    def addRow(self, cols):
        self.ostr += "\n<tr>"
        for c in cols:
            self.ostr += "\n<td> "+str(c)+" </td>"
        self.ostr += "\n</tr>"

    def addParagraph(self,text):
        self.ostr += "\n<p> "+str(text)+ " </p>"

    def addH1(self,text):
        self.ostr += "\n<h1> "+str(text)+ " </h1>"
        
    def addH3(self,text):
        self.ostr += "\n<h3> "+str(text)+ " </h3>"
        
    def addRawText(self,text):
        self.ostr += text

    def buffer(self):
        return self.ostr

def createHTMLDocumentFromTemplateFile(body_text, template_file_name):
    tf = file(template_file_name)
    txt_out = reduce(lambda x,y: x+y, tf.readlines())
    return txt_out.replace("<!--HTML_BODY_HERE-->", body_text)
    
prj_printed = None

def check_project_name(project_name, version_name):
    global prj_printed
    if not project_name:
        usage()
        print "ERROR: missing project name"
        sys.exit(2)
    else:
        if not prj_printed:
            print "Project name: ", project_name
            print "Project version: ", version_name
            prj_printed = 1

arch_printed = None

def check_arch():
    global arch_printed

    try:
        CONFIG['platf'] = os.environ['SCRAM_ARCH']
    except:
        print "ERROR: undefined platform"
        print "ERROR: please use --arch option or set SCRAM_ARCH environment variable"
        print "hint: you do not need full SCRAM environment if you are doing download"
        sys.exit(-2)

    if not arch_printed:
        print "Architecture/platform:", CONFIG['platf']
        arch_printed = 1


        
    
## # replace all occurences of 'text' by 'replacement' in 'file_in' and write to 'file_out'
## def replace_strings(file_in, file_out, text, replacement):
##     fout = file(file_out,'w')
##     lines = file(file_in,'r').readlines()
##     for l in lines:
##         fout.write(l.replace(text,replacement))

## def mk_scram_runtime_file_name(project_name, version_name, shell):
##     return project_name+"_"+version_name+'_'+CONFIG['platf']+'.scram.runtime.'+shell

import sys, getopt, os

if __name__ == '__main__':

    os.environ['PATH'] += ":." # for sh -- recursive call to the same script
        
    try:
        opts, args = getopt.getopt(sys.argv[1:], ":vk", ["prefix=","project=", "verbose=","keep-going","arch=","clean-tar=","dereference","distribution=","always-fetch"])
    except getopt.GetoptError, x:
        print "command line syntax error"
        usage()
        sys.exit(2)

    project_name = ""
    version_name = ""
    keep_going = None

    clean_tar = "no"
    deref_tar = None
    
    inherit_options = ""

    # by default override if different length...
    override = 1
    
    for o,a in opts:
        if o in ("--project",):
            idx = a.find("_")

            if idx == -1:
                project_name = a
            else:
                project_name = a[:idx]
                version_name = a[idx+1:]

        if o in ("--prefix",):
            CONFIG['lcg_prefix'] = a
            inherit_options += ' --prefix='+a
            
        if o in ("-k","--keep-going"):
            keep_going = 1
            inherit_options += ' --keep-going'

        if o in ("--arch",):
            os.environ['SCRAM_ARCH'] = a
            inherit_options += ' --arch='+a
            
        if o in ("--clean-tar",):
            clean_tar = a
            inherit_options += ' --clean-tar='+a

        if o in ("--dereference",):
            deref_tar = 1
            inherit_options += ' --dereference'

        if o in ("--distribution",):
           CONFIG['distribution_location'] = a
           inherit_options += ' --distribution='+a

        if o == "--always-fetch":
            override = 2
            inherit_options += ' --always-fetch'
            
    
    if len(args) == 0:
        usage()
        print "ERROR: specify one or more commands"
        sys.exit(2)

    # not every command requires a project name
    #check_project_name(project_name, version_name)
    
    configure()
    
    print "LCG installation prefix: ", CONFIG['lcg_prefix']

    print "LCG distribution directory (target): ", CONFIG['distribution_dir']
    
    for a in args:
        if a == "test":
            dep_list = get_package_list(project_name, version_name)

            save_info_file(dep_list,'/tmp/lcg-deplist.info', project_name, version_name)
            dl2 = load_info_file('/tmp/lcg-deplist.info')

            print "written /tmp/lcg-deplist.info"
            if dep_list == dl2:
                print "OK"
            else:
                print "ERROR: load/save loose data"
                
            continue
        
        if a == "list":

            check_project_name(project_name, version_name)
            check_arch()
            
            print 'SCRAM VERSION REQUIRED: ', get_scram_version_for_project(project_name, version_name)

            dep_list = get_package_list(project_name, version_name)

            for l in dep_list:
                print l
            print "related platforms:", get_related_platforms_list(project_name, version_name,CONFIG['platf'])
            continue

        if a == "tar":
            check_project_name(project_name, version_name)
            check_arch()
            display_once = 1

            print "getting the dependency list...",
            dep_list = get_package_list(project_name, version_name)
            info_fn = CONFIG['distribution_dir']+'/'+mk_package_info_name(project_name, version_name)
            print "saving the .info file: ",info_fn
            save_info_file(dep_list,info_fn, project_name, version_name)
            
            #if not ok:
            #    print "FATAL: unable to get the package list"
            #    sys.exit(2)

            # verify the required version of scram is tarred in the scram directory
            scram_tar_file = CONFIG['scram_dir']+'/tarfiles/'+scram_tar_file_name(project_name, version_name)
            if not exists(scram_tar_file):
                print "ERROR: cannot find ",scram_tar_file
                if not keep_going:
                    sys.exit(2)

            # make a symlink for the scram tar file in the distribution area
            # remove the old one
            
            scram_tar_link = CONFIG['distribution_dir']+'/'+scram_tar_file_name(project_name, version_name)

            try:
                os.remove(scram_tar_link)
            except:
                pass

            try:
                os.symlink(scram_tar_file,scram_tar_link)
            except:
                pass
            
            # loop over dependant packages
            
            for l in dep_list:

                # if package is lcg-project then solve dependencies recursevly
                if l[1] in CONFIG['lcg_project_list']:
                    cmd = 'lcg-installation-manager.py '+inherit_options+" --project="+l[2]
                    if keep_going:
                        cmd += ' --keep-going '
                    cmd += " tar"
                    print '--> recursive action for: ', l[2]
                    print cmd
                    if os.system(cmd) != 0:
                        print "error during system call"
                        if not keep_going:
                            sys.exit(2)
                    continue
                
                tarname = mk_tarball_name(l[1],l[2],l[3]) #l[1]+"_"+l[2]+"__"+"LCG_"+CONFIG['platf']+".tar.gz"
                package_path = mk_package_path(l[1],l[2],l[3]) #l[1]+"/"+l[2]+"/"+CONFIG['platf']

                dest_tar = CONFIG['distribution_dir']+'/'+tarname
                
                if exists(dest_tar):
                    print "### skipping ", l[1],"... tarball already exists in external/distribution"
                else:
                    cd_cmd = "cd "+CONFIG['external_dir']
                    if deref_tar:
                        tar_cmd = "tar hczf "
                    else:
                        tar_cmd = "tar czf "
                    tar_cmd = tar_cmd+dest_tar+" "+package_path+'/*'

                    if display_once:
                        print cd_cmd
                        display_once = None
                        
                    print tar_cmd

                    if os.system("("+cd_cmd+";"+tar_cmd+")") == 0:
                        print "OK"
                    else:
                        print "FAILED: removing tarball "+dest_tar
                        os.system("("+cd_cmd+"; rm -f "+dest_tar+")")
                        if not keep_going:
                            sys.exit(3)

            tarname = mk_tarball_name(project_name, version_name)
            
            dest_tar = CONFIG['distribution_dir']+'/'+tarname

            if exists(dest_tar):
                print "### skipping ",tarname,": tarball already exists in external/distribution"
            else:
                src_list = get_project_tar_input_files(project_name,version_name)
                
                cd_cmd = 'cd '+CONFIG['release_dir']

                if deref_tar:
                    tar_cmd = "tar hczf "
                else:
                    tar_cmd = "tar czf "

                tar_cmd = tar_cmd+dest_tar+' '+src_list

                print cd_cmd
                print tar_cmd

                if os.system("("+cd_cmd+";"+tar_cmd+")") == 0:
                    print "OK"
                else:
                    print "FAILED to create tarball "+tarname

                    if not keep_going:
                        sys.exit(3)
            continue

        if a == "rpm" :
            check_project_name(project_name, version_name)
            check_arch()
            
            def success(x):
                if not x and not keep_going:
                    sys.exit(2)
                return x

            
            success(make_rpm_spec_file(project_name, version_name, CONFIG['release_dir'], mk_prjpath)) 


            success(make_rpm_build(project_name, version_name, CONFIG['release_dir'], mk_prjpath, keep_going))
            
            for l in get_package_list(project_name, version_name):

                # if package is lcg-project then solve dependencies recursevly
                if l[1] in CONFIG['lcg_project_list']:
                    cmd = 'lcg-installation-manager.py '+inherit_options+" --project="+l[2]+' rpm'

                    print '--> recursive action for: ', l[2]
                    print cmd
                    if os.system(cmd) != 0:
                        print "error during system call"
                        if not keep_going:
                            sys.exit(2)
                    continue

                success(make_rpm_spec_file(l[1],l[2],CONFIG['external_dir'],mk_package_path))
                success(make_rpm_build(l[1],l[2],CONFIG['external_dir'],mk_package_path, keep_going))

            continue
            
        if a == "download":
            check_project_name(project_name, version_name)
            check_arch()
            display_once = 1
            
            if CONFIG['lcg_prefix']== '/afs/cern.ch/sw/lcg':
                print "FATAL: cannot download into the main repository: ",CONFIG['lcg_prefix']
                print "please change the prefix!"
                sys.exit(-3)

            print "LCG repository URL: ", CONFIG['lcg_repository_url']

            if not exists(CONFIG['distribution_dir']):
                os.makedirs(CONFIG['distribution_dir'])

            if not exists(CONFIG['external_dir']):
                os.makedirs(CONFIG['external_dir'])

            if not exists(CONFIG['release_dir']):
                os.makedirs(CONFIG['release_dir'])

            if fetch('MAINTENANCE','/tmp/',override=2,silent=2): # be very silent
                print file('/tmp/MAINTENANCE').read()
                sys.exit(4)
                
            if fetch('NEWS','/tmp/',override=2,silent=2): # be very silent
                print file('/tmp/NEWS').read()
                
            # support both old and new scheme...
            if not fetch(mk_package_info_name(project_name, version_name),  CONFIG['distribution_dir']+'/',override=1):
                if not fetch(mk_scram_output_name(project_name,version_name), CONFIG['distribution_dir']+'/'):
                    print "Most probably", mk_prjname(project_name,version_name),
                    print ' is not available in the LCG software repository...'
                    print 'Please report this to LCG SPI'
                    sys.exit(3)

            # always get the newest package map
            if not fetch(CONFIG['package_map_file_name'],CONFIG['distribution_dir']+'/', override=2):
                print "FATAL: exiting..."
                sys.exit(3)

            for l in get_package_list(project_name,version_name,from_file=1):
                if l[1] in CONFIG['lcg_project_list']:
                    cmd = 'lcg-installation-manager.py '+inherit_options+" --project="+l[2]+" download"
                    print '--> recursive action for: ', l[2]
                    print cmd
                    if os.system(cmd) != 0:
                        print "error during system call"
                        if not keep_going:
                            sys.exit(2)
                    continue                
                print "downloading:  ",l

                tarname = mk_tarball_name(l[1],l[2],l[3]) #l[1]+"_"+l[2]+"__"+"LCG_"+CONFIG['platf']+".tar.gz"
                package_path = mk_package_path(l[1],l[2],l[3]) #l[1]+"/"+l[2]+"/"+CONFIG['platf']

                dest_tar = CONFIG['distribution_dir']+'/'+tarname
                
                if 0 and exists(dest_tar): # DISABLED
                    print "### skipping ", l[1],"... tarball already exists in external/distribution"
                else:
                    if not fetch(tarname,CONFIG['distribution_dir']+'/',override=override):
                        if not keep_going:
                            sys.exit(3)

                    cd_cmd = "cd "+CONFIG['external_dir']
                    tar_cmd = "tar xfz "+dest_tar

                    if display_once:
                        print cd_cmd
                        display_once = None
                        
                    print tar_cmd

                    if os.system("("+cd_cmd+";"+tar_cmd+")") == 0:
                        print "OK"
                        if clean_tar == "all":
                            os.remove(CONFIG['distribution_dir']+'/'+tarname)
                    else:
                        print "FAILED"
                        sys.exit(3)

            tarname = mk_tarball_name(project_name, version_name)
            
            dest_tar = CONFIG['distribution_dir']+'/'+tarname

            if 0 and exists(dest_tar): # DISABLED
                print "### skipping ",tarname,": tarball already exists in external/distribution"
            else:
                if not fetch(tarname,CONFIG['distribution_dir']+'/',override=override):
                    sys.exit(3)

                cd_cmd = 'cd '+CONFIG['release_dir']
                tar_cmd = 'tar xfz '+dest_tar

                print cd_cmd
                print tar_cmd

                if os.system("("+cd_cmd+";"+tar_cmd+")") == 0:
                    print "OK"
                else:
                    print "FAILED"
                    sys.exit(3)

                if clean_tar == "lcg" or clean_tar=="all":
                    os.remove(CONFIG['distribution_dir']+'/'+tarname)

            if exists( CONFIG['scram_dir']+'/'+get_scram_version_for_project(project_name, version_name) ):
                print get_scram_version_for_project(project_name, version_name)," already installed..."
            else:
                if not fetch(scram_tar_file_name(project_name, version_name),'/tmp/',override=override):
                    if not keep_going:
                        sys.exit(3)

                if not fetch("SCRAM-install.sh",CONFIG['distribution_dir']+'/',override=override):
                    if not keep_going:
                        sys.exit(3)

                print "INSTALLING SCRAM..."
                cmd  = "cd "+CONFIG['distribution_dir']+" ; "
                cmd += "chmod +x SCRAM-install.sh ;"
                cmd += "LCG_PREFIX="+CONFIG['lcg_prefix'] + " ; export LCG_PREFIX; "
                cmd += "./SCRAM-install.sh "+get_scram_version_for_project(project_name, version_name)

                if os.system('('+cmd+')') != 0:
                    if not keep_going:
                        sys.exit(3)

            if not fetch("ProcessTools.sh", CONFIG['distribution_dir']+'/',override=override):
                if not keep_going:
                    sys.exit(3)

            once = 1
            # modify SCRAM tool files with the local prefix for all our platforms
            # however need to do scram install once only!
            for platf in get_related_platforms_list(project_name,version_name,CONFIG['platf']):
                processToolsCmd = CONFIG['distribution_dir']+"/ProcessTools.sh"
                cmd = "chmod +x "+processToolsCmd+" ;"
                cmd += "cd "+CONFIG['release_dir']+"/"+mk_prjpath(project_name,version_name)+" ;"
                cmd += "chmod -R u+w .SCRAM;"
                cmd += "LCG_PREFIX="+CONFIG['lcg_prefix'] + " ; export LCG_PREFIX; "
                cmd += processToolsCmd+" "+platf+'; export SCRAM_ARCH='+platf+'; '
                if once:
                    cmd += CONFIG['scram_dir']+'/scram install'
                    once=None
                print 'executing: ' + cmd
                if os.system('('+cmd+')') != 0:
                    if not keep_going:
                        sys.exit(3)

            continue

        if a == "diff":
            check_arch()
            import glob
            locs = []
            for l in glob.glob(CONFIG['distribution_dir']+'/*.scram_output'):
                l2 = os.path.basename(l)
                idx = l2.find('.scram_output')
                locs.append(l2[:idx])

            ff = "/tmp/lcg-soft-dir-listing.html"
            if not fetch('',ff,CONFIG['lcg_repository_url'], override=2,silent=1):
                sys.exit(1)

            f = open(ff)

            print 'Local installation diff summary:'

            empty = 1
            
            for l in map(lambda x: x.split('"'),f.readlines()):
                if len(l) > 5:
                    idx = l[5].find('.scram_output')
                    if idx > 0:
                        prj = l[5][:idx]

                        if not prj in locs:
                            empty = 0
                            print prj, "needs download"

            if empty:
                print '---'
            continue
            
        if a == "html":
            check_project_name(project_name, version_name)
            check_arch()
            out_file_name = CONFIG['website_dir']+'/'+mk_prjname(project_name,version_name).strip()+'_'+CONFIG['platf']+'-download.html'

            print "writing to file: "+out_file_name,
            if exists(out_file_name):
                print " already exists, overriding..."
            else:
                print

            out_file = file(out_file_name,"w")
            
            writer = HTMLWriter()
            markup = HTMLMarkup

            t = 'Download page for '+ mk_prjname(project_name,version_name) + " / " + CONFIG['platf']

            # no header or footer -- the contents will be intersted into the template
            #writer.beginDocument(t)

            writer.addH1(t)

            writer.addH3('Automatic download')
            writer.addParagraph('On the machine where you want to install the software run the following command: ')
            writer.addParagraph(markup('./lcg-installation-manager.py --project='+mk_prjname(project_name,version_name)+' --arch='+CONFIG['platf']+' --prefix=/opt/sw/lcg download',tt=1))
            writer.addParagraph('Python 2.2 or greater is required. Download the script here: '+markup('lcg-installation-manager.py',link=CONFIG["lcg_repository_url"]+'/'+'lcg-installation-manager.py')+'    '+ markup('Make sure you read Local Installation HOWTO',bold=1,link='http://spi.cern.ch/workbook/howto/HowTo-Install-locally-lcg-software.html'))
            writer.addH3('Manual download')
            #writer.addParagraph('Release platform '+markup(CONFIG['platf'],bold=1)+" : ")
            writer.addParagraph(markup(mk_tarball_name(project_name,version_name),
                                       link=CONFIG["lcg_repository_url"]+'/'+mk_tarball_name(project_name,version_name),
                                       bold =1))

            writer.addParagraph('Required packages:')
            
            writer.beginTable(map(lambda x: markup(x,bold=1),["name","version","download"]))

            writer.addRow([markup("SCRAM",italic=1),
                           markup(get_scram_version_for_project(project_name, version_name),italic=1),
                           markup(scram_tar_file_name(project_name, version_name),link=CONFIG["lcg_repository_url"]+'/'+scram_tar_file_name(project_name, version_name))])
            
            for l in get_package_list(project_name, version_name):
                if l[1] in CONFIG['lcg_project_list']:
                    cmd = 'lcg-installation-manager.py '+inherit_options+" --project="+l[2]+" html"
                    print '--> recursive action for: ', l[2]
                    print cmd
                    if os.system(cmd) != 0:
                        print "error during system call"
                        if not keep_going:
                            sys.exit(2)
                    writer.addRow([markup(l[1],bold=1),markup(l[2],bold=1),markup('download page',link=CONFIG["lcg_website_url"]+'/'+l[2]+'_'+CONFIG['platf']+'-download.html')])
                    continue
                else:
                    tarname = mk_tarball_name(l[1],l[2],l[3])

                    if exists(CONFIG['distribution_dir']+'/'+tarname):
                        writer.addRow([l[1],l[2],markup(tarname,link=CONFIG["lcg_repository_url"]+'/'+tarname)])
                    else:
                        print "tarball "+tarname+" not found in the distribution area"
                        writer.addRow([l[1],l[2],markup("use original distribution kit",italic=1)])
                

            writer.endTable()

            #writeHTMLFile(out_file,writer.buffer(), CONFIG['html_template_file'])
            #writer.endDocument()
            out_file.write(createHTMLDocumentFromTemplateFile(writer.buffer(),CONFIG['html_template_file']))

            continue

        if a == "help":
          help_str = """
RPM Building:
These instructions as well as spec file template are based on LCG1 depolyment team recommendations.

1. Dowload the project with lcg-installation-manager.py
 % lcg-installation-manager.py  --arch=rh73_gcc32 --project=POOL_1_3_2 --prefix=/opt/sw/lcg --clean-tar=lcg download

2. LCG sofwtare is now locally installed. Make tarball which will be RPM input:
 % lcg-installation-manager.py  --arch=rh73_gcc32 --project=POOL_1_3_2 --prefix=/opt/sw/lcg --keep-going tar

3. Make sure your ~/.rpmmacros contains:
%_fixperms         true
%__os_install_post true
%__find_requires %{nil}

4. Download spec file template and copy it into:
 %  /opt/sw/lcg/external/distribution/package.spec.template

5. Generate RPM spec files and build RPMs Typically you need to be root to do that.
 % lcg-installation-manager.py   --arch=rh73_gcc32 --project=POOL_1_3_2 --prefix=/opt/sw/lcg rpm
          """

          print help_str
          continue
        
        print "command ",a," ignored..."