fetch.py

Go to the documentation of this file.

#  This file is part of repro, a Python package for automating 
#  reproducible research in scientific computing.
#
#  Copyright (C) 2008  Gilles Hennenfent, Sean Ross-Ross
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

##
# 
# @package fetch
# Provide two builders, WGET and FTP, to retrieve file(s) from remote location.
# 
# @page pg_fetchblds Retrieve file(s) from remote location
# 
# @section sec_wgetbld WGET
# 
# Defined in the fetch tool. Builder to retrieve file(s) from remote location using wget.
# 
# \verbinclude examples/FetchFiles/WGET/SConstruct
# 
# @section sec_ftpbld FTP
# 
# Defined in the fetch tool. Builder to retrieve file(s) from remote location using ftp.
# 
# \verbinclude examples/FetchFiles/FTP/SConstruct
# 

from SCons.Script import Builder, Action, COMMAND_LINE_TARGETS
from SCons.Errors import UserError
from getpass import getpass

##
# 
#     @brief Generate command to retrieve data from remote site using ftp executable.
#     @todo Make an external tool for ftp executable.
#     
def ftp_gen(source, target, env, for_signature):
    if not env.get('user'):
        raise UserError("FTP: need to specify 'user'")
    if not env.get('host'):
        raise UserError("FTP: need to specify 'host'")
    
    cmd = ['$FTPCMD']
    
    if env.get('IPV'):
        ipv = env['IPV']
        if env['IPV'] in [4,6]:
            cmd.append('-%(ipv)s' %vars() )
            
    if env.get('NETRC'):
        cmd.append('-N %s' %(env.get('NETRC')) )
    
    
    if env.get('user') == 'anonymous':
        cmd.extend( [ "-a ${host}:${str(path) + '/'}${TARGET.file} ${TARGET}" ] ) 
    else:
        cmd.extend( [ "-a ${user}@${host}:${str(path) + '/'}${TARGET.file} ${TARGET}" ] ) 
    
    return " ".join(cmd) 


##
# 
#     @brief Generate command to retrieve data from remote site using wget executable.
#     @todo Make an external tool for wget executable.
#     
def wget_gen( source, target, env, for_signature ):
    
    user = env.get('user')
    password = env.get('password')
    url = env.get('url')
    path = env.get('path')
    
    if url is None:
        raise UserError("WGET: need to specify 'url'")
    
    wgetcmd = ["cd ${TARGET.dir.abspath} && $WGETCMD"]
    wgetstr = ["cd ${TARGET.dir.abspath} && $WGETCMD"]
    
    if isinstance( path, (list,tuple) ):
        if len(path) is 1:
            path = path*len(target)
        elif len(path) != len(target):
            raise UserError( "WGET: length of list 'path' and the number of targets must be the same" )
    else:
        path = [path]*len(target)
    
    if user is not None:
        wgetcmd.append( '--user=$user' )
        wgetstr.append( '--user=$user' )
    if password is not None:
        
        if password.upper() in ["$ASK", "${ASK}"] and not ( for_signature or env.GetOption('no_exec') ):
            print "Password for user '%(user)s' at url '%(url)s':" %vars()
            password = getpass()
#            
        wgetcmd.append( '--password=%(password)s' %vars() )
        wgetstr.append( '--password=******'  )

    for u,p,t in zip([url]*len(target), path, target):
        
        wgetcmd.append( '/'.join( [u,p,t.name] ) )
        wgetstr.append( '/'.join( [u,p,t.name] ) )
     
     
    return Action( " ".join(wgetcmd)," ".join(wgetstr) )
    

##
# 
#     @brief Alias target(s) of WGET and FTP builders to 'scons fetch' command 
#     and make them Precious.
#     
#     To remove the target(s) of WGET and FTP builders you must either run 'scons -c fetch' or 
#     'scons -c targetname'.
#     
def fetch_emitter( target, source, env ):
    env.Alias( 'fetch', target)
    
    do_clean = 'fetch' in COMMAND_LINE_TARGETS
    
    for tgt in target:
#        print tgt, do_clean 
        if (str(tgt) in COMMAND_LINE_TARGETS) or do_clean:
            pass
        else:
            env.NoClean( tgt )
             
    return target, source

##
# 
#     @brief Add WGET and FTP builders to the environment. 
#     
def generate(env):
    b = Builder(generator=ftp_gen, emitter=fetch_emitter)
    env['BUILDERS']['FTP'] = b
    env['FTPCMD'] = 'ftp'
    
    wg = Builder( generator=wget_gen, emitter=fetch_emitter )
    env['BUILDERS']['WGET'] = wg
    env['WGETCMD'] = 'wget'
    
##
# 
#     @brief Return True (this tool always exists).
#     
def exists( env ):
    return 1

---