import logging
import os
import shutil
import stat
from string import Template
import sys
import tarfile
import time
import urllib2
import zipfile
import hashlib
from galaxy.util import asbool
from galaxy.util.template import fill_template
from tool_shed.util import basic_util
from tool_shed.util import tool_dependency_util
from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
# TODO: eliminate the use of fabric here.
from galaxy import eggs
eggs.require( 'paramiko' )
eggs.require( 'ssh' )
eggs.require( 'Fabric' )
from fabric.api import settings
from fabric.api import lcd
log = logging.getLogger( __name__ )
VIRTUALENV_URL = 'https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.9.1.tar.gz'
class CompressedFile( object ):
def __init__( self, file_path, mode='r' ):
if tarfile.is_tarfile( file_path ):
self.file_type = 'tar'
elif zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ):
self.file_type = 'zip'
self.file_name = os.path.splitext( os.path.basename( file_path ) )[ 0 ]
if self.file_name.endswith( '.tar' ):
self.file_name = os.path.splitext( self.file_name )[ 0 ]
self.type = self.file_type
method = 'open_%s' % self.file_type
if hasattr( self, method ):
self.archive = getattr( self, method )( file_path, mode )
else:
raise NameError( 'File type %s specified, no open method found.' % self.file_type )
def extract( self, path ):
'''Determine the path to which the archive should be extracted.'''
contents = self.getmembers()
extraction_path = path
if len( contents ) == 1:
# The archive contains a single file, return the extraction path.
if self.isfile( contents[ 0 ] ):
extraction_path = os.path.join( path, self.file_name )
if not os.path.exists( extraction_path ):
os.makedirs( extraction_path )
self.archive.extractall( extraction_path )
else:
# Get the common prefix for all the files in the archive. If the common prefix ends with a slash,
# or self.isdir() returns True, the archive contains a single directory with the desired contents.
# Otherwise, it contains multiple files and/or directories at the root of the archive.
common_prefix = os.path.commonprefix( [ self.getname( item ) for item in contents ] )
if len( common_prefix ) >= 1 and not common_prefix.endswith( os.sep ) and self.isdir( self.getmember( common_prefix ) ):
common_prefix += os.sep
if common_prefix.endswith( os.sep ):
self.archive.extractall( os.path.join( path ) )
extraction_path = os.path.join( path, common_prefix )
else:
extraction_path = os.path.join( path, self.file_name )
if not os.path.exists( extraction_path ):
os.makedirs( extraction_path )
self.archive.extractall( os.path.join( extraction_path ) )
return os.path.abspath( extraction_path )
def getmembers_tar( self ):
return self.archive.getmembers()
def getmembers_zip( self ):
return self.archive.infolist()
def getname_tar( self, item ):
return item.name
def getname_zip( self, item ):
return item.filename
def getmember( self, name ):
for member in self.getmembers():
if self.getname( member ) == name:
return member
def getmembers( self ):
return getattr( self, 'getmembers_%s' % self.type )()
def getname( self, member ):
return getattr( self, 'getname_%s' % self.type )( member )
def isdir( self, member ):
return getattr( self, 'isdir_%s' % self.type )( member )
def isdir_tar( self, member ):
return member.isdir()
def isdir_zip( self, member ):
if member.filename.endswith( os.sep ):
return True
return False
def isfile( self, member ):
if not self.isdir( member ):
return True
return False
def open_tar( self, filepath, mode ):
return tarfile.open( filepath, mode, errorlevel=0 )
def open_zip( self, filepath, mode ):
return zipfile.ZipFile( filepath, mode )
def zipfile_ok( self, path_to_archive ):
"""
This function is a bit pedantic and not functionally necessary. It checks whether there is
no file pointing outside of the extraction, because ZipFile.extractall() has some potential
security holes. See python zipfile documentation for more details.
"""
basename = os.path.realpath( os.path.dirname( path_to_archive ) )
zip_archive = zipfile.ZipFile( path_to_archive )
for member in zip_archive.namelist():
member_path = os.path.realpath( os.path.join( basename, member ) )
if not member_path.startswith( basename ):
return False
return True
class Download( object ):
def url_download( self, install_dir, downloaded_file_name, download_url, extract=True ):
"""
The given download_url can have an extension like #md5# or #sha256#.
This indicates a checksum which will be chekced after download.
If the checksum does not match an exception is thrown.
https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757
"""
file_path = os.path.join( install_dir, downloaded_file_name )
src = None
dst = None
checksum = None
sha256 = False
md5 = False
# Set a timer so we don't sit here forever.
if '#md5#' in download_url:
md5 = True
download_url, checksum = download_url.split('#md5#')
elif '#sha256#' in download_url:
sha256 = True
download_url, checksum = download_url.split('#sha256#')
start_time = time.time()
try:
src = urllib2.urlopen( download_url )
dst = open( file_path, 'wb' )
while True:
chunk = src.read( basic_util.CHUNK_SIZE )
if chunk:
dst.write( chunk )
else:
break
time_taken = time.time() - start_time
if time_taken > basic_util.NO_OUTPUT_TIMEOUT:
err_msg = 'Downloading from URL %s took longer than the defined timeout period of %.1f seconds.' % \
( str( download_url ), basic_util.NO_OUTPUT_TIMEOUT )
raise Exception( err_msg )
except Exception, e:
err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str( download_url ), str( e ) )
raise Exception( err_msg )
finally:
if src:
src.close()
if dst:
dst.close()
try:
if sha256:
downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest()
elif md5:
downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest()
if checksum and downloaded_checksum != checksum:
raise Exception( 'Given checksum does not match with the one from the downloaded file (%s).' % (downloaded_checksum) )
except Exception, e:
raise
if extract:
if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ) ):
archive = CompressedFile( file_path )
extraction_path = archive.extract( install_dir )
else:
extraction_path = os.path.abspath( install_dir )
else:
extraction_path = os.path.abspath( install_dir )
return extraction_path
class RecipeStep( object ):
"""Abstract class that defines a standard format for handling recipe steps when installing packages."""
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
raise "Unimplemented Method"
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
raise "Unimplemented Method"
class AssertDirectoryExecutable( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'assert_directory_executable'
def assert_directory_executable( self, full_path ):
"""
Return True if a symbolic link or directory exists and is executable, but if
full_path is a file, return False.
"""
if full_path is None:
return False
if os.path.isfile( full_path ):
return False
if os.path.isdir( full_path ):
# Make sure the owner has execute permission on the directory.
# See http://docs.python.org/2/library/stat.html
if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64:
return True
return False
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Make sure a symbolic link or directory on disk exists and is executable, but is not a file.
Since this class is not used in the initial download stage, no recipe step filtering is
performed here, and None values are always returned for filtered_actions and dir.
"""
if os.path.isabs( action_dict[ 'full_path' ] ):
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
if not self.assert_directory_executable( full_path=full_path ):
status = self.app.install_model.ToolDependency.installation_status.ERROR
error_message = 'The path %s is not a directory or is not executable by the owner.' % str( full_path )
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency,
status=status,
error_message=error_message,
remove_from_disk=False )
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# $INSTALL_DIR/mira/my_file
if action_elem.text:
action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment )
return action_dict
class AssertDirectoryExists( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'assert_directory_exists'
def assert_directory_exists( self, full_path ):
"""
Return True if a symbolic link or directory exists, but if full_path is a file,
return False. """
if full_path is None:
return False
if os.path.isfile( full_path ):
return False
if os.path.isdir( full_path ):
return True
return False
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Make sure a a symbolic link or directory on disk exists, but is not a file. Since this
class is not used in the initial download stage, no recipe step filtering is performed
here, and None values are always returned for filtered_actions and dir.
"""
if os.path.isabs( action_dict[ 'full_path' ] ):
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
if not self.assert_directory_exists( full_path=full_path ):
status = self.app.install_model.ToolDependency.installation_status.ERROR
error_message = 'The path %s is not a directory or does not exist.' % str( full_path )
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency,
status=status,
error_message=error_message,
remove_from_disk=False )
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# $INSTALL_DIR/mira
if action_elem.text:
action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment )
return action_dict
class AssertFileExecutable( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'assert_file_executable'
def assert_file_executable( self, full_path ):
"""
Return True if a symbolic link or file exists and is executable, but if full_path
is a directory, return False.
"""
if full_path is None:
return False
if os.path.isdir( full_path ):
return False
if os.path.exists( full_path ):
# Make sure the owner has execute permission on the file.
# See http://docs.python.org/2/library/stat.html
if stat.S_IXUSR & os.stat( full_path )[ stat.ST_MODE ] == 64:
return True
return False
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Make sure a symbolic link or file on disk exists and is executable, but is not a directory.
Since this class is not used in the initial download stage, no recipe step filtering is
performed here, and None values are always returned for filtered_actions and dir.
"""
if os.path.isabs( action_dict[ 'full_path' ] ):
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
if not self.assert_file_executable( full_path=full_path ):
status = self.app.install_model.ToolDependency.installation_status.ERROR
error_message = 'The path %s is not a file or is not executable by the owner.' % str( full_path )
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency,
status=status,
error_message=error_message,
remove_from_disk=False )
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# $INSTALL_DIR/mira/my_file
if action_elem.text:
action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment )
return action_dict
class AssertFileExists( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'assert_file_exists'
def assert_file_exists( self, full_path ):
"""
Return True if a symbolic link or file exists, but if full_path is a directory,
return False.
"""
if full_path is None:
return False
if os.path.isdir( full_path ):
return False
if os.path.exists( full_path ):
return True
return False
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Make sure a symbolic link or file on disk exists, but is not a directory. Since this
class is not used in the initial download stage, no recipe step filtering is performed
here, and None values are always returned for filtered_actions and dir.
"""
if os.path.isabs( action_dict[ 'full_path' ] ):
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
if not self.assert_file_exists( full_path=full_path ):
status = self.app.install_model.ToolDependency.installation_status.ERROR
error_message = 'The path %s is not a file or does not exist.' % str( full_path )
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
tool_dependency,
status=status,
error_message=error_message,
remove_from_disk=False )
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# $INSTALL_DIR/mira/my_file
if action_elem.text:
action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment )
return action_dict
class Autoconf( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'autoconf'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Handle configure, make and make install in a shell, allowing for configuration options. Since this
class is not used in the initial download stage, no recipe step filtering is performed here, and None
values are always returned for filtered_actions and dir.
"""
with settings( warn_only=True ):
configure_opts = action_dict.get( 'configure_opts', '' )
if 'prefix=' in configure_opts:
pre_cmd = './configure %s && make && make install' % configure_opts
else:
pre_cmd = './configure --prefix=$INSTALL_DIR %s && make && make install' % configure_opts
cmd = install_environment.build_command( basic_util.evaluate_template( pre_cmd, install_environment ) )
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
# The caller should check the status of the returned tool_dependency since this function
# does nothing with the return_code.
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# Handle configure, make and make install allow providing configuration options
if action_elem.text:
configure_opts = basic_util.evaluate_template( action_elem.text, install_environment )
action_dict[ 'configure_opts' ] = configure_opts
return action_dict
class ChangeDirectory( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'change_directory'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Change the working directory in a shell. Since this class is not used in the initial download stage,
no recipe step filtering is performed here and a None value is return for filtered_actions. However,
the new dir value is returned since it is needed for later steps.
"""
target_directory = os.path.realpath( os.path.normpath( os.path.join( current_dir, action_dict[ 'directory' ] ) ) )
if target_directory.startswith( os.path.realpath( current_dir ) ) and os.path.exists( target_directory ):
# Change directory to a directory within the current working directory.
dir = target_directory
elif target_directory.startswith( os.path.realpath( work_dir ) ) and os.path.exists( target_directory ):
# Change directory to a directory above the current working directory, but within the defined work_dir.
dir = target_directory.replace( os.path.realpath( work_dir ), '' ).lstrip( '/' )
else:
log.debug( 'Invalid or nonexistent directory %s specified, ignoring change_directory action.', target_directory )
dir = current_dir
return tool_dependency, None, dir
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# PHYLIP-3.6b
if action_elem.text:
action_dict[ 'directory' ] = action_elem.text
return action_dict
class Chmod( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'chmod'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Change the mode setting for certain files in the installation environment. Since this class is not
used in the initial download stage, no recipe step filtering is performed here, and None values are
always returned for filtered_actions and dir.
"""
for target_file, mode in action_dict[ 'change_modes' ]:
if os.path.exists( target_file ):
os.chmod( target_file, mode )
else:
log.debug( 'Invalid file %s specified, ignoring %s action.', target_file, action_type )
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# Change the read, write, and execute bits on a file.
#
# $INSTALL_DIR/bin/faToTwoBit
#
file_elems = action_elem.findall( 'file' )
chmod_actions = []
# A unix octal mode is the sum of the following values:
# Owner:
# 400 Read 200 Write 100 Execute
# Group:
# 040 Read 020 Write 010 Execute
# World:
# 004 Read 002 Write 001 Execute
for file_elem in file_elems:
# So by the above table, owner read/write/execute and group read permission would be 740.
# Python's os.chmod uses base 10 modes, convert received unix-style octal modes to base 10.
received_mode = int( file_elem.get( 'mode', 600 ), base=8 )
# For added security, ensure that the setuid and setgid bits are not set.
mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID )
file = basic_util.evaluate_template( file_elem.text, install_environment )
chmod_tuple = ( file, mode )
chmod_actions.append( chmod_tuple )
if chmod_actions:
action_dict[ 'change_modes' ] = chmod_actions
return action_dict
class DownloadBinary( Download, RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'download_binary'
def download_binary( self, url, work_dir ):
"""Download a pre-compiled binary from the specified URL."""
downloaded_filename = os.path.split( url )[ -1 ]
dir = self.url_download( work_dir, downloaded_filename, url, extract=False )
return downloaded_filename
def filter_actions_after_binary_installation( self, actions ):
'''Filter out actions that should not be processed if a binary download succeeded.'''
filtered_actions = []
for action in actions:
action_type, action_dict = action
if action_type in [ 'set_environment', 'chmod', 'download_binary' ]:
filtered_actions.append( action )
return filtered_actions
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Download a binary file. If the value of initial_download is True, the recipe steps will be
filtered and returned and the installation directory (i.e., dir) will be defined and returned.
If we're not in the initial download stage, these actions will not occur, and None values will
be returned for them.
"""
url = action_dict[ 'url' ]
# Get the target directory for this download if the user has specified one. Default to the root of $INSTALL_DIR.
target_directory = action_dict.get( 'target_directory', None )
# Attempt to download a binary from the specified URL.
log.debug( 'Attempting to download from %s to %s', url, str( target_directory ) )
downloaded_filename = None
try:
downloaded_filename = self.download_binary( url, work_dir )
if initial_download:
# Filter out any actions that are not download_binary, chmod, or set_environment.
filtered_actions = self.filter_actions_after_binary_installation( actions[ 1: ] )
# Set actions to the same, so that the current download_binary doesn't get re-run in the
# next stage. TODO: this may no longer be necessary...
actions = [ item for item in filtered_actions ]
except Exception, e:
log.exception( str( e ) )
if initial_download:
# No binary exists, or there was an error downloading the binary from the generated URL.
# Filter the actions so that stage 2 can proceed with the remaining actions.
filtered_actions = actions[ 1: ]
action_type, action_dict = filtered_actions[ 0 ]
# If the downloaded file exists, move it to $INSTALL_DIR. Put this outside the try/catch above so that
# any errors in the move step are correctly sent to the tool dependency error handler.
if downloaded_filename and os.path.exists( os.path.join( work_dir, downloaded_filename ) ):
if target_directory:
target_directory = os.path.realpath( os.path.normpath( os.path.join( install_environment.install_dir,
target_directory ) ) )
# Make sure the target directory is not outside of $INSTALL_DIR.
if target_directory.startswith( os.path.realpath( install_environment.install_dir ) ):
full_path_to_dir = os.path.abspath( os.path.join( install_environment.install_dir, target_directory ) )
else:
full_path_to_dir = os.path.abspath( install_environment.install_dir )
else:
full_path_to_dir = os.path.abspath( install_environment.install_dir )
basic_util.move_file( current_dir=work_dir,
source=downloaded_filename,
destination=full_path_to_dir )
# Not sure why dir is ignored in this method, need to investigate...
dir = None
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
platform_info_dict = tool_dependency_util.get_platform_info_dict()
platform_info_dict[ 'name' ] = str( tool_dependency.name )
platform_info_dict[ 'version' ] = str( tool_dependency.version )
url_template_elems = action_elem.findall( 'url_template' )
# Check if there are multiple url_template elements, each with attrib entries for a specific platform.
if len( url_template_elems ) > 1:
#
# http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit
#
# This method returns the url_elem that best matches the current platform as received from os.uname().
# Currently checked attributes are os and architecture. These correspond to the values sysname and
# processor from the Python documentation for os.uname().
url_template_elem = tool_dependency_util.get_download_url_for_platform( url_template_elems, platform_info_dict )
else:
url_template_elem = url_template_elems[ 0 ]
action_dict[ 'url' ] = Template( url_template_elem.text ).safe_substitute( platform_info_dict )
action_dict[ 'target_directory' ] = action_elem.get( 'target_directory', None )
return action_dict
class DownloadByUrl( Download, RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'download_by_url'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Download a file via HTTP. If the value of initial_download is True, the recipe steps will be
filtered and returned and the installation directory (i.e., dir) will be defined and returned.
If we're not in the initial download stage, these actions will not occur, and None values will
be returned for them.
"""
if initial_download:
# Eliminate the download_by_url action so remaining actions can be processed correctly.
filtered_actions = actions[ 1: ]
url = action_dict[ 'url' ]
is_binary = action_dict.get( 'is_binary', False )
log.debug( 'Attempting to download via url: %s', url )
if 'target_filename' in action_dict:
# Sometimes compressed archives extract their content to a folder other than the default
# defined file name. Using this attribute will ensure that the file name is set appropriately
# and can be located after download, decompression and extraction.
downloaded_filename = action_dict[ 'target_filename' ]
else:
downloaded_filename = os.path.split( url )[ -1 ]
dir = self.url_download( work_dir, downloaded_filename, url, extract=True )
if is_binary:
log_file = os.path.join( install_environment.install_dir, basic_util.INSTALLATION_LOG )
if os.path.exists( log_file ):
logfile = open( log_file, 'ab' )
else:
logfile = open( log_file, 'wb' )
logfile.write( 'Successfully downloaded from url: %s\n' % action_dict[ 'url' ] )
logfile.close()
log.debug( 'Successfully downloaded from url: %s' % action_dict[ 'url' ] )
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
#
# http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2
#
if is_binary_download:
action_dict[ 'is_binary' ] = True
if action_elem.text:
action_dict[ 'url' ] = action_elem.text
target_filename = action_elem.get( 'target_filename', None )
if target_filename:
action_dict[ 'target_filename' ] = target_filename
return action_dict
class DownloadFile( Download, RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'download_file'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Download a file. If the value of initial_download is True, the recipe steps will be
filtered and returned and the installation directory (i.e., dir) will be defined and returned.
If we're not in the initial download stage, these actions will not occur, and None values will
be returned for them.
"""
# http://effectors.org/download/version/TTSS_GUI-1.0.1.jar
# Download a single file to the working directory.
if initial_download:
filtered_actions = actions[ 1: ]
url = action_dict[ 'url' ]
if 'target_filename' in action_dict:
# Sometimes compressed archives extracts their content to a folder other than the default
# defined file name. Using this attribute will ensure that the file name is set appropriately
# and can be located after download, decompression and extraction.
filename = action_dict[ 'target_filename' ]
else:
filename = url.split( '/' )[ -1 ]
if current_dir is not None:
work_dir = current_dir
self.url_download( work_dir, filename, url, extract=action_dict[ 'extract' ] )
if initial_download:
dir = os.path.curdir
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# http://effectors.org/download/version/TTSS_GUI-1.0.1.jar
if action_elem.text:
action_dict[ 'url' ] = action_elem.text
target_filename = action_elem.get( 'target_filename', None )
if target_filename:
action_dict[ 'target_filename' ] = target_filename
action_dict[ 'extract' ] = asbool( action_elem.get( 'extract', False ) )
return action_dict
class MakeDirectory( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'make_directory'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Make a directory on disk. Since this class is not used in the initial download stage, no recipe step
filtering is performed here, and None values are always returned for filtered_actions and dir.
"""
if os.path.isabs( action_dict[ 'full_path' ] ):
full_path = action_dict[ 'full_path' ]
else:
full_path = os.path.join( current_dir, action_dict[ 'full_path' ] )
self.make_directory( full_path=full_path )
return tool_dependency, None, None
def make_directory( self, full_path ):
if not os.path.exists( full_path ):
os.makedirs( full_path )
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# $INSTALL_DIR/lib/python
if action_elem.text:
action_dict[ 'full_path' ] = basic_util.evaluate_template( action_elem.text, install_environment )
return action_dict
class MakeInstall( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'make_install'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Execute a make_install command in a shell. Since this class is not used in the initial download stage,
no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir.
"""
# make; make install; allow providing make options
with settings( warn_only=True ):
make_opts = action_dict.get( 'make_opts', '' )
cmd = install_environment.build_command( 'make %s && make install' % make_opts )
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
# The caller should check the status of the returned tool_dependency since this function
# does nothing with the return_code.
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# make; make install; allow providing make options
if action_elem.text:
make_opts = basic_util.evaluate_template( action_elem.text, install_environment )
action_dict[ 'make_opts' ] = make_opts
return action_dict
class MoveDirectoryFiles( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'move_directory_files'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Move a directory of files. Since this class is not used in the initial download stage, no recipe step
filtering is performed here, and None values are always returned for filtered_actions and dir.
"""
self.move_directory_files( current_dir=current_dir,
source_dir=os.path.join( action_dict[ 'source_directory' ] ),
destination_dir=os.path.join( action_dict[ 'destination_directory' ] ) )
return tool_dependency, None, None
def move_directory_files( self, current_dir, source_dir, destination_dir ):
source_directory = os.path.abspath( os.path.join( current_dir, source_dir ) )
destination_directory = os.path.join( destination_dir )
if not os.path.isdir( destination_directory ):
os.makedirs( destination_directory )
symlinks = []
regular_files = []
for file_name in os.listdir( source_directory ):
source_file = os.path.join( source_directory, file_name )
destination_file = os.path.join( destination_directory, file_name )
files_tuple = ( source_file, destination_file )
if os.path.islink( source_file ):
symlinks.append( files_tuple )
else:
regular_files.append( files_tuple )
for source_file, destination_file in symlinks:
shutil.move( source_file, destination_file )
for source_file, destination_file in regular_files:
shutil.move( source_file, destination_file )
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
#
# bin
# $INSTALL_DIR/bin
#
for move_elem in action_elem:
move_elem_text = basic_util.evaluate_template( move_elem.text, install_environment )
if move_elem_text:
action_dict[ move_elem.tag ] = move_elem_text
return action_dict
class MoveFile( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'move_file'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Move a file on disk. Since this class is not used in the initial download stage, no recipe step
filtering is performed here, and None values are always returned for filtered_actions and dir.
"""
basic_util.move_file( current_dir=current_dir,
source=os.path.join( action_dict[ 'source' ] ),
destination=os.path.join( action_dict[ 'destination' ] ),
rename_to=action_dict[ 'rename_to' ] )
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
#
# misc/some_file
# $INSTALL_DIR/bin
#
action_dict[ 'source' ] = basic_util.evaluate_template( action_elem.find( 'source' ).text, install_environment )
action_dict[ 'destination' ] = basic_util.evaluate_template( action_elem.find( 'destination' ).text, install_environment )
action_dict[ 'rename_to' ] = action_elem.get( 'rename_to' )
return action_dict
class SetEnvironment( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'set_environment'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Configure an install environment. Since this class is not used in the initial download stage,
no recipe step filtering is performed here, and None values are always returned for filtered_actions
and dir.
"""
# Currently the only action supported in this category is "environment_variable".
cmds = install_environment.environment_commands( 'set_environment' )
env_var_dicts = action_dict.get( 'environment_variable', [] )
for env_var_dict in env_var_dicts:
# Check for the presence of the $ENV[] key string and populate it if possible.
env_var_dict = self.handle_environment_variables( install_environment=install_environment,
tool_dependency=tool_dependency,
env_var_dict=env_var_dict,
set_prior_environment_commands=cmds )
env_file_builder.append_line( **env_var_dict )
# The caller should check the status of the returned tool_dependency since return_code is not
# returned by this function.
return_code = env_file_builder.return_code
return tool_dependency, None, None
def handle_environment_variables( self, install_environment, tool_dependency, env_var_dict,
set_prior_environment_commands ):
"""
This method works with with a combination of three tool dependency definition tag sets, which are defined
in the tool_dependencies.xml file in the order discussed here. The example for this discussion is the
tool_dependencies.xml file contained in the osra repository, which is available at:
http://testtoolshed.g2.bx.psu.edu/view/bgruening/osra
The first tag set defines a complex repository dependency like this. This tag set ensures that changeset
revision XXX of the repository named package_graphicsmagick_1_3 owned by YYY in the tool shed ZZZ has been
previously installed.
...
* By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks
something like this (we'll reference this file later in this discussion.
----
GRAPHICSMAGICK_ROOT_DIR=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick;
export GRAPHICSMAGICK_ROOT_DIR
----
The second tag set defines a specific package dependency that has been previously installed (guaranteed by the
tag set discussed above) and compiled, where the compiled dependency is needed by the tool dependency currently
being installed (osra version 2.0.0 in this case) and complied in order for its installation and compilation to
succeed. This tag set is contained within the tag set, which implies that
version 2.0.0 of the osra package requires version 1.3.18 of the graphicsmagick package in order to successfully
compile. When this tag set is handled, one of the effects is that the env.sh file associated with graphicsmagick
version 1.3.18 is "sourced", which undoubtedly sets or alters certain environment variables (e.g. PATH, PYTHONPATH,
etc).
The third tag set enables discovery of the same required package dependency discussed above for correctly compiling
the osra version 2.0.0 package, but in this case the package can be discovered at tool execution time. Using the
$ENV[] option as shown in this example, the value of the environment variable named GRAPHICSMAGICK_ROOT_DIR (which
was set in the environment using the second tag set described above) will be used to automatically alter the env.sh
file associated with the osra version 2.0.0 tool dependency when it is installed into Galaxy. * Refer to where we
discussed the env.sh file for version 1.3.18 of the graphicsmagick package above.
$ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/$INSTALL_DIR/potrace/build/lib/$INSTALL_DIR/bin$INSTALL_DIR/share
The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy
that looks something like this. Notice that the path to the gmagick binary is included here since it expands the
defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set.
----
LD_LIBRARY_PATH=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH;
export LD_LIBRARY_PATH
LD_LIBRARY_PATH=//osra/1.4.0/YYY/depends_on/XXX/potrace/build/lib/:$LD_LIBRARY_PATH;
export LD_LIBRARY_PATH
PATH=//osra/1.4.0/YYY/depends_on/XXX/bin:$PATH;
export PATH
OSRA_DATA_FILES=//osra/1.4.0/YYY/depends_on/XXX/share;
export OSRA_DATA_FILES
----
"""
env_var_value = env_var_dict[ 'value' ]
# env_var_value is the text of an environment variable tag like this:
# Here is an example of what env_var_value could look like: $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/
if '$ENV[' in env_var_value and ']' in env_var_value:
# Pull out the name of the environment variable to populate.
inherited_env_var_name = env_var_value.split( '[' )[1].split( ']' )[0]
to_replace = '$ENV[%s]' % inherited_env_var_name
# Build a command line that outputs VARIABLE_NAME: .
set_prior_environment_commands.append( 'echo %s: $%s' % ( inherited_env_var_name, inherited_env_var_name ) )
command = ' ; '.join( set_prior_environment_commands )
# Run the command and capture the output.
command_return = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=command,
return_output=True )
# And extract anything labeled with the name of the environment variable we're populating here.
if '%s: ' % inherited_env_var_name in command_return:
environment_variable_value = command_return.split( '\n' )
for line in environment_variable_value:
if line.startswith( inherited_env_var_name ):
inherited_env_var_value = line.replace( '%s: ' % inherited_env_var_name, '' )
log.info( 'Replacing %s with %s in env.sh for this repository.', to_replace, inherited_env_var_value )
env_var_value = env_var_value.replace( to_replace, inherited_env_var_value )
else:
# If the return is empty, replace the original $ENV[] with nothing, to avoid any shell misparsings later on.
log.debug( 'Environment variable %s not found, removing from set_environment.', inherited_env_var_name )
env_var_value = env_var_value.replace( to_replace, '$%s' % inherited_env_var_name )
env_var_dict[ 'value' ] = env_var_value
return env_var_dict
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# This function is only called for set environment actions as defined above, not within a tool
# dependency type. Here is an example of the tag set this function does handle:
#
# $INSTALL_DIR/lib/python
# $INSTALL_DIR/bin
#
# Here is an example of the tag set this function does not handle:
#
# $INSTALL_DIR
#
env_manager = EnvManager( self.app )
env_var_dicts = []
for env_elem in action_elem:
if env_elem.tag == 'environment_variable':
env_var_dict = env_manager.create_env_var_dict( elem=env_elem,
install_environment=install_environment )
if env_var_dict:
env_var_dicts.append( env_var_dict )
if env_var_dicts:
# The last child of an might be a comment, so manually set it to be 'environment_variable'.
action_dict[ 'environment_variable' ] = env_var_dicts
return action_dict
class SetEnvironmentForInstall( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'set_environment_for_install'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Configure an environment for compiling a package. Since this class is not used in the initial
download stage, no recipe step filtering is performed here, and None values are always returned
for filtered_actions and dir.
"""
# Currently the only action supported in this category is a list of paths to one or more tool
# dependency env.sh files, the environment setting in each of which will be injected into the
# environment for all tags that follow this
# tag set in the tool_dependencies.xml file.
env_shell_file_paths = action_dict.get( 'env_shell_file_paths', [] )
install_environment.add_env_shell_file_paths( env_shell_file_paths )
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
#
#
#
#
#
# This action type allows for defining an environment that will properly compile a tool dependency.
# Currently, tag set definitions like that above are supported, but in the future other approaches
# to setting environment variables or other environment attributes can be supported. The above tag
# set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling
# the current tool dependency package. See the package_matplotlib_1_2 repository in the test tool
# shed for a real-world example.
all_env_shell_file_paths = []
env_manager = EnvManager( self.app )
for env_elem in action_elem:
if env_elem.tag == 'repository':
env_shell_file_paths = env_manager.get_env_shell_file_paths( env_elem )
if env_shell_file_paths:
all_env_shell_file_paths.extend( env_shell_file_paths )
action_dict[ 'env_shell_file_paths' ] = all_env_shell_file_paths
return action_dict
class SetupPerlEnvironment( Download, RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'setup_purl_environment'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Initialize the environment for installing Perl packages. The class is called during the initial
download stage when installing packages, so the value of initial_download will generally be True.
However, the parameter value allows this class to also be used in the second stage of the installation,
although it may never be necessary. If initial_download is True, the recipe steps will be filtered
and returned and the installation directory (i.e., dir) will be defined and returned. If we're not
in the initial download stage, these actions will not occur, and None values will be returned for them.
"""
#
#
#
#
#
# XML::Parser
# http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz
#
dir = None
if initial_download:
filtered_actions = actions[ 1: ]
env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None )
if env_shell_file_paths is None:
log.debug( 'Missing Perl environment, make sure your specified Rerl installation exists.' )
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
else:
install_environment.add_env_shell_file_paths( env_shell_file_paths )
log.debug( 'Handling setup_perl_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s' % \
( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) )
dir = os.path.curdir
current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
with lcd( current_dir ):
with settings( warn_only=True ):
perl_packages = action_dict.get( 'perl_packages', [] )
for perl_package in perl_packages:
# If set to a true value then MakeMaker's prompt function will always
# return the default without waiting for user input.
cmd = '''PERL_MM_USE_DEFAULT=1; export PERL_MM_USE_DEFAULT; '''
cmd += 'export PERL5LIB=$INSTALL_DIR/lib/perl5:$PERL5LIB;'
cmd += 'export PATH=$INSTALL_DIR/bin:$PATH;'
if perl_package.find( '://' ) != -1:
# We assume a URL to a gem file.
url = perl_package
perl_package_name = url.split( '/' )[ -1 ]
dir = self.url_download( work_dir, perl_package_name, url, extract=True )
# Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build).
tmp_work_dir = os.path.join( work_dir, dir )
if os.path.exists( os.path.join( tmp_work_dir, 'Makefile.PL' ) ):
cmd += '''perl Makefile.PL INSTALL_BASE=$INSTALL_DIR && make && make install'''
elif os.path.exists( os.path.join( tmp_work_dir, 'Build.PL' ) ):
cmd += '''perl Build.PL --install_base $INSTALL_DIR && perl Build && perl Build install'''
else:
log.debug( 'No Makefile.PL or Build.PL file found in %s. Skipping installation of %s.' % \
( url, perl_package_name ) )
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
with lcd( tmp_work_dir ):
cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
else:
# perl package from CPAN without version number.
# cpanm should be installed with the parent perl distribution, otherwise this will not work.
cmd += '''cpanm --local-lib=$INSTALL_DIR %s''' % ( perl_package )
cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
# Pull in perl dependencies (runtime).
env_file_builder.handle_action_shell_file_paths( action_dict )
# Recursively add dependent PERL5LIB and PATH to env.sh & anything else needed.
env_file_builder.append_line( name="PERL5LIB",
action="prepend_to",
value=os.path.join( install_environment.install_dir, 'lib', 'perl5' ) )
env_file_builder.append_line( name="PATH",
action="prepend_to",
value=os.path.join( install_environment.install_dir, 'bin' ) )
return_code = env_file_builder.return_code
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# setup a Perl environment.
#
#
#
#
#
# XML::Parser
# http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz
#
# Discover all child repository dependency tags and define the path to an env.sh file associated
# with each repository. This will potentially update the value of the 'env_shell_file_paths' entry
# in action_dict.
all_env_shell_file_paths = []
env_manager = EnvManager( self.app )
action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( all_env_shell_file_paths,
action_elem,
action_dict )
perl_packages = []
for env_elem in action_elem:
if env_elem.tag == 'package':
# A valid package definition can be:
# XML::Parser
# http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz
# Unfortunately CPAN does not support versioning, so if you want real reproducibility you need to specify
# the tarball path and the right order of different tarballs manually.
perl_packages.append( env_elem.text.strip() )
if perl_packages:
action_dict[ 'perl_packages' ] = perl_packages
return action_dict
class SetupREnvironment( Download, RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'setup_r_environment'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Initialize the environment for installing R packages. The class is called during the initial
download stage when installing packages, so the value of initial_download will generally be True.
However, the parameter value allows this class to also be used in the second stage of the installation,
although it may never be necessary. If initial_download is True, the recipe steps will be filtered
and returned and the installation directory (i.e., dir) will be defined and returned. If we're not
in the initial download stage, these actions will not occur, and None values will be returned for them.
"""
#
#
#
#
#
# https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz
#
dir = None
if initial_download:
filtered_actions = actions[ 1: ]
env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None )
if env_shell_file_paths is None:
log.debug( 'Missing R environment. Please check your specified R installation exists.' )
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
else:
install_environment.add_env_shell_file_paths( env_shell_file_paths )
log.debug( 'Handling setup_r_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s' % \
( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) )
tarball_names = []
for url in action_dict[ 'r_packages' ]:
filename = url.split( '/' )[ -1 ]
tarball_names.append( filename )
self.url_download( work_dir, filename, url, extract=False )
dir = os.path.curdir
current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
with lcd( current_dir ):
with settings( warn_only=True ):
for tarball_name in tarball_names:
# Use raw strings so that python won't automatically unescape the quotes before passing the command
# to subprocess.Popen.
cmd = r'''PATH=$PATH:$R_HOME/bin; export PATH; R_LIBS=$INSTALL_DIR; export R_LIBS;
Rscript -e "install.packages(c('%s'),lib='$INSTALL_DIR', repos=NULL, dependencies=FALSE)"''' % \
( str( tarball_name ) )
cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
# R libraries are installed to $INSTALL_DIR (install_dir), we now set the R_LIBS path to that directory
# Pull in R environment (runtime).
env_file_builder.handle_action_shell_file_paths( action_dict )
env_file_builder.append_line( name="R_LIBS", action="prepend_to", value=install_environment.install_dir )
return_code = env_file_builder.return_code
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# setup an R environment.
#
#
#
#
#
# https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz
#
# Discover all child repository dependency tags and define the path to an env.sh file
# associated with each repository. This will potentially update the value of the
# 'env_shell_file_paths' entry in action_dict.
all_env_shell_file_paths = []
env_manager = EnvManager( self.app )
action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( all_env_shell_file_paths,
action_elem,
action_dict )
r_packages = list()
for env_elem in action_elem:
if env_elem.tag == 'package':
r_packages.append( env_elem.text.strip() )
if r_packages:
action_dict[ 'r_packages' ] = r_packages
return action_dict
class SetupRubyEnvironment( Download, RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'setup_ruby_environment'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Initialize the environment for installing Ruby packages. The class is called during the initial
download stage when installing packages, so the value of initial_download will generally be True.
However, the parameter value allows this class to also be used in the second stage of the installation,
although it may never be necessary. If initial_download is True, the recipe steps will be filtered
and returned and the installation directory (i.e., dir) will be defined and returned. If we're not
in the initial download stage, these actions will not occur, and None values will be returned for them.
"""
#
#
#
#
#
# protk
# protk=1.2.4
# http://url-to-some-gem-file.de/protk.gem
#
dir = None
if initial_download:
filtered_actions = actions[ 1: ]
env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None )
if env_shell_file_paths is None:
log.debug( 'Missing Ruby environment, make sure your specified Ruby installation exists.' )
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
else:
install_environment.add_env_shell_file_paths( env_shell_file_paths )
log.debug( 'Handling setup_ruby_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s' % \
( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) )
dir = os.path.curdir
current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
with lcd( current_dir ):
with settings( warn_only=True ):
ruby_package_tups = action_dict.get( 'ruby_package_tups', [] )
for ruby_package_tup in ruby_package_tups:
gem, gem_version = ruby_package_tup
if os.path.isfile( gem ):
# we assume a local shipped gem file
cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME;
gem install --local %s''' % ( gem )
elif gem.find( '://' ) != -1:
# We assume a URL to a gem file.
url = gem
gem_name = url.split( '/' )[ -1 ]
self.url_download( work_dir, gem_name, url, extract=False )
cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME;
gem install --local %s ''' % ( gem_name )
else:
# gem file from rubygems.org with or without version number
if gem_version:
# Specific ruby gem version was requested.
# Use raw strings so that python won't automatically unescape the quotes before passing the command
# to subprocess.Popen.
cmd = r'''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME;
gem install %s --version "=%s"''' % ( gem, gem_version)
else:
# no version number given
cmd = '''PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME;
gem install %s''' % ( gem )
cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
# Pull in ruby dependencies (runtime).
env_file_builder.handle_action_shell_file_paths( action_dict )
env_file_builder.append_line( name="GEM_PATH",
action="prepend_to",
value=install_environment.install_dir )
env_file_builder.append_line( name="PATH",
action="prepend_to",
value=os.path.join( install_environment.install_dir, 'bin' ) )
return_code = env_file_builder.return_code
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# setup a Ruby environment.
#
#
#
#
#
# protk
# protk=1.2.4
# http://url-to-some-gem-file.de/protk.gem
#
# Discover all child repository dependency tags and define the path to an env.sh file
# associated with each repository. This will potentially update the value of the
# 'env_shell_file_paths' entry in action_dict.
all_env_shell_file_paths = []
env_manager = EnvManager( self.app )
action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( all_env_shell_file_paths,
action_elem,
action_dict )
ruby_package_tups = []
for env_elem in action_elem:
if env_elem.tag == 'package':
#A valid gem definition can be:
# protk=1.2.4
# protk
# ftp://ftp.gruening.de/protk.gem
gem_token = env_elem.text.strip().split( '=' )
if len( gem_token ) == 2:
# version string
gem_name = gem_token[ 0 ]
gem_version = gem_token[ 1 ]
ruby_package_tups.append( ( gem_name, gem_version ) )
else:
# gem name for rubygems.org without version number
gem = env_elem.text.strip()
ruby_package_tups.append( ( gem, None ) )
if ruby_package_tups:
action_dict[ 'ruby_package_tups' ] = ruby_package_tups
return action_dict
class SetupPythonEnvironment( Download, RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'setup_python_environment'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Initialize the environment for installing Python packages. The class is called during the initial
download stage when installing packages, so the value of initial_download will generally be True.
However, the parameter value allows this class to also be used in the second stage of the installation,
although it may never be necessary. If initial_download is True, the recipe steps will be filtered
and returned and the installation directory (i.e., dir) will be defined and returned. If we're not
in the initial download stage, these actions will not occur, and None values will be returned for them.
Warning: easy_install is configured that it will not be install any dependency, the tool developer needs
to specify every dependency explicitly
"""
#
#
#
#
#
# pysam.tar.gz
# http://url-to-some-python-package.de/pysam.tar.gz
#
dir = None
if initial_download:
filtered_actions = actions[ 1: ]
env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None )
if env_shell_file_paths is None:
log.debug( 'Missing Python environment, make sure your specified Python installation exists.' )
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
else:
install_environment.add_env_shell_file_paths( env_shell_file_paths )
log.debug( 'Handling setup_python_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s' % \
( str( tool_dependency.name ), str( install_environment.env_shell_file_paths ) ) )
dir = os.path.curdir
current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
with lcd( current_dir ):
with settings( warn_only=True ):
python_package_tups = action_dict.get( 'python_package_tups', [] )
for python_package_tup in python_package_tups:
package, package_version = python_package_tup
package_path = os.path.join( install_environment.tool_shed_repository_install_dir, package )
if os.path.isfile( package_path ):
# we assume a local shipped python package
cmd = r'''PATH=$PATH:$PYTHONHOME/bin; export PATH;
export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR;
easy_install --no-deps --install-dir $INSTALL_DIR --script-dir $INSTALL_DIR/bin %s
''' % ( package_path )
elif package.find( '://' ) != -1:
# We assume a URL to a python package.
url = package
package_name = url.split( '/' )[ -1 ]
self.url_download( work_dir, package_name, url, extract=False )
cmd = r'''PATH=$PATH:$PYTHONHOME/bin; export PATH;
export PYTHONPATH=$PYTHONPATH:$INSTALL_DIR;
easy_install --no-deps --install-dir $INSTALL_DIR --script-dir $INSTALL_DIR/bin %s
''' % ( package_name )
else:
pass
# pypi can be implemented or for > python3.4 we can use the build-in system
cmd = install_environment.build_command( basic_util.evaluate_template( cmd, install_environment ) )
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
# Pull in python dependencies (runtime).
env_file_builder.handle_action_shell_file_paths( action_dict )
env_file_builder.append_line( name="PYTHONPATH",
action="prepend_to",
value= os.path.join( install_environment.install_dir, 'lib', 'python') )
env_file_builder.append_line( name="PATH",
action="prepend_to",
value=os.path.join( install_environment.install_dir, 'bin' ) )
return_code = env_file_builder.return_code
if return_code:
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# setup a Python environment.
#
#
#
#
#
# pysam.tar.gz
# http://url-to-some-python-package.de/pysam.tar.gz
#
# Discover all child repository dependency tags and define the path to an env.sh file
# associated with each repository. This will potentially update the value of the
# 'env_shell_file_paths' entry in action_dict.
all_env_shell_file_paths = []
env_manager = EnvManager( self.app )
action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( all_env_shell_file_paths,
action_elem,
action_dict )
python_package_tups = []
for env_elem in action_elem:
if env_elem.tag == 'package':
#A valid package definitions can be:
# pysam.tar.gz -> locally shipped tarball
# ftp://ftp.gruening.de/pysam.tar.gz -> online tarball
python_token = env_elem.text.strip().split( '=' )
if len( python_token ) == 2:
# version string
package_name = python_token[ 0 ]
package_version = python_token[ 1 ]
python_package_tups.append( ( package_name, package_version ) )
else:
# package name for pypi.org without version number
package = env_elem.text.strip()
python_package_tups.append( ( package, None ) )
if python_package_tups:
action_dict[ 'python_package_tups' ] = python_package_tups
return action_dict
class SetupVirtualEnv( Download, RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'setup_virtualenv'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Initialize a virtual environment for installing packages. If initial_download is True, the recipe
steps will be filtered and returned and the installation directory (i.e., dir) will be defined and
returned. If we're not in the initial download stage, these actions will not occur, and None values
will be returned for them.
"""
# This class is not currently used during stage 1 of the installation process, so filter_actions
# are not affected, and dir is not set. Enhancements can easily be made to this function if this
# class is needed in stage 1.
venv_src_directory = os.path.abspath( os.path.join( self.app.config.tool_dependency_dir, '__virtualenv_src' ) )
if not self.install_virtualenv( install_environment, venv_src_directory ):
log.debug( 'Unable to install virtualenv' )
return tool_dependency, None, None
requirements = action_dict[ 'requirements' ]
if os.path.exists( os.path.join( install_environment.install_dir, requirements ) ):
# requirements specified as path to a file
requirements_path = requirements
else:
# requirements specified directly in XML, create a file with these for pip.
requirements_path = os.path.join( install_environment.install_dir, "requirements.txt" )
with open( requirements_path, "w" ) as f:
f.write( requirements )
venv_directory = os.path.join( install_environment.install_dir, "venv" )
python_cmd = action_dict[ 'python' ]
# TODO: Consider making --no-site-packages optional.
setup_command = "%s %s/virtualenv.py --no-site-packages '%s'" % ( python_cmd, venv_src_directory, venv_directory )
# POSIXLY_CORRECT forces shell commands . and source to have the same
# and well defined behavior in bash/zsh.
activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join( venv_directory, "bin", "activate" )
if action_dict[ 'use_requirements_file' ]:
install_command = "python '%s' install -r '%s' --log '%s'" % \
( os.path.join( venv_directory, "bin", "pip" ),
requirements_path,
os.path.join( install_environment.install_dir, 'pip_install.log' ) )
else:
install_command = ''
with open( requirements_path, "rb" ) as f:
while True:
line = f.readline()
if not line:
break
line = line.strip()
if line:
line_install_command = "python '%s' install %s --log '%s'" % \
( os.path.join( venv_directory, "bin", "pip" ),
line,
os.path.join( install_environment.install_dir, 'pip_install_%s.log' % ( line ) ) )
if not install_command:
install_command = line_install_command
else:
install_command = "%s && %s" % ( install_command, line_install_command )
full_setup_command = "%s; %s; %s" % ( setup_command, activate_command, install_command )
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=full_setup_command,
return_output=False )
if return_code:
log.error( "Failed to do setup_virtualenv install, exit code='%s'", return_code )
# would it be better to try to set env variables anway, instead of returning here?
return tool_dependency, None, None
site_packages_directory, site_packages_directory_list = \
self.__get_site_packages_directory( install_environment,
self.app,
tool_dependency,
python_cmd,
venv_directory )
env_file_builder.append_line( name="PATH", action="prepend_to", value=os.path.join( venv_directory, "bin" ) )
if site_packages_directory is None:
log.error( "virtualenv's site-packages directory '%s' does not exist", site_packages_directory_list )
else:
env_file_builder.append_line( name="PYTHONPATH", action="prepend_to", value=site_packages_directory )
# The caller should check the status of the returned tool_dependency since this function does nothing
# with the return_code.
return_code = env_file_builder.return_code
return tool_dependency, None, None
def install_virtualenv( self, install_environment, venv_dir ):
if not os.path.exists( venv_dir ):
with install_environment.make_tmp_dir() as work_dir:
downloaded_filename = VIRTUALENV_URL.rsplit('/', 1)[-1]
try:
dir = self.url_download( work_dir, downloaded_filename, VIRTUALENV_URL )
except:
log.error( "Failed to download virtualenv: url_download( '%s', '%s', '%s' ) threw an exception",
work_dir, downloaded_filename, VIRTUALENV_URL )
return False
full_path_to_dir = os.path.abspath( os.path.join( work_dir, dir ) )
shutil.move( full_path_to_dir, venv_dir )
return True
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
#
## Install requirements from file requirements.txt of downloaded bundle - or -
# tools/requirements.txt
## Install requirements from specified file from downloaded bundle -or -
# pyyaml==3.2.0
# lxml==2.3.0
## Manually specify contents of requirements.txt file to create dynamically.
action_dict[ 'use_requirements_file' ] = asbool( action_elem.get( 'use_requirements_file', True ) )
action_dict[ 'requirements' ] = basic_util.evaluate_template( action_elem.text or 'requirements.txt', install_environment )
action_dict[ 'python' ] = action_elem.get( 'python', 'python' )
return action_dict
def __get_site_packages_directory( self, install_environment, app, tool_dependency, python_cmd, venv_directory ):
lib_dir = os.path.join( venv_directory, "lib" )
rval = os.path.join( lib_dir, python_cmd, 'site-packages' )
site_packages_directory_list = [ rval ]
if os.path.exists( rval ):
return ( rval, site_packages_directory_list )
for ( dirpath, dirnames, filenames ) in os.walk( lib_dir ):
for dirname in dirnames:
rval = os.path.join( lib_dir, dirname, 'site-packages' )
site_packages_directory_list.append( rval )
if os.path.exists( rval ):
return ( rval, site_packages_directory_list )
break
# fall back to python call to get site packages
# FIXME: This is probably more robust?, but there is currently an issue with handling the output.stdout
# preventing the entire path from being included (it gets truncated)
# Use raw strings so that python won't automatically unescape the quotes before passing the command
# to subprocess.Popen.
for site_packages_command in [ r"""%s -c 'import site; site.getsitepackages()[0]'""" % \
os.path.join( venv_directory, "bin", "python" ),
r"""%s -c 'import os, sys; print os.path.join( sys.prefix, "lib", "python" + sys.version[:3], "site-packages" )'""" % \
os.path.join( venv_directory, "bin", "python" ) ]:
output = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=site_packages_command,
return_output=True )
site_packages_directory_list.append( output.stdout )
if not output.return_code and os.path.exists( output.stdout ):
return ( output.stdout, site_packages_directory_list )
return ( None, site_packages_directory_list )
class ShellCommand( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'shell_command'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Execute a command in a shell. If the value of initial_download is True, the recipe steps will
be filtered and returned and the installation directory (i.e., dir) will be defined and returned.
If we're not in the initial download stage, these actions will not occur, and None values will
be returned for them.
"""
# git clone --recursive git://github.com/ekg/freebayes.git
# Eliminate the shell_command clone action so remaining actions can be processed correctly.
if initial_download:
# I'm not sure why we build the cmd differently in stage 1 vs stage 2. Should this process
# be the same no matter the stage?
dir = package_name
filtered_actions = actions[ 1: ]
cmd = action_dict[ 'command' ]
else:
cmd = install_environment.build_command( action_dict[ 'command' ] )
with settings( warn_only=True ):
# The caller should check the status of the returned tool_dependency since this function
# does nothing with return_code.
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
if initial_download:
return tool_dependency, filtered_actions, dir
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# make
action_elem_text = basic_util.evaluate_template( action_elem.text, install_environment )
if action_elem_text:
action_dict[ 'command' ] = action_elem_text
return action_dict
class TemplateCommand( RecipeStep ):
def __init__( self, app ):
self.app = app
self.type = 'template_command'
def execute_step( self, tool_dependency, package_name, actions, action_dict, filtered_actions, env_file_builder,
install_environment, work_dir, current_dir=None, initial_download=False ):
"""
Execute a template command in a shell. If the value of initial_download is True, the recipe steps
will be filtered and returned and the installation directory (i.e., dir) will be defined and returned.
If we're not in the initial download stage, these actions will not occur, and None values will be
returned for them.
"""
env_vars = dict()
env_vars = install_environment.environment_dict()
tool_shed_repository = tool_dependency.tool_shed_repository
env_vars.update( basic_util.get_env_var_values( install_environment ) )
language = action_dict[ 'language' ]
with settings( warn_only=True, **env_vars ):
if language == 'cheetah':
# We need to import fabric.api.env so that we can access all collected environment variables.
cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars )
# The caller should check the status of the returned tool_dependency since this function
# does nothing with return_code.
return_code = install_environment.handle_command( tool_dependency=tool_dependency,
cmd=cmd,
return_output=False )
return tool_dependency, None, None
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
# Default to Cheetah as it's the first template language supported.
language = action_elem.get( 'language', 'cheetah' ).lower()
if language == 'cheetah':
# Cheetah template syntax.
#
# #if env.PATH:
# make
# #end if
#
action_elem_text = action_elem.text.strip()
if action_elem_text:
action_dict[ 'language' ] = language
action_dict[ 'command' ] = action_elem_text
else:
log.debug( "Unsupported template language '%s'. Not proceeding." % str( language ) )
raise Exception( "Unsupported template language '%s' in tool dependency definition." % str( language ) )
return action_dict