import os
from shutil import rmtree
from string import Template
from tempfile import mkdtemp
try:
from galaxy import objectstore
except ImportError:
from lwr import objectstore
from contextlib import contextmanager
DISK_TEST_CONFIG = """
"""
def test_disk_store():
with TestConfig(DISK_TEST_CONFIG) as (directory, object_store):
# Test no dataset with id 1 exists.
absent_dataset = MockDataset(1)
assert not object_store.exists(absent_dataset)
# Write empty dataset 2 in second backend, ensure it is empty and
# exists.
empty_dataset = MockDataset(2)
directory.write(b"", "files1/000/dataset_2.dat")
assert object_store.exists(empty_dataset)
assert object_store.empty(empty_dataset)
# Write non-empty dataset in backend 1, test it is not emtpy & exists.
hello_world_dataset = MockDataset(3)
directory.write(b"Hello World!", "files1/000/dataset_3.dat")
assert object_store.exists(hello_world_dataset)
assert not object_store.empty(hello_world_dataset)
# Test get_data
data = object_store.get_data(hello_world_dataset)
assert data == b"Hello World!"
data = object_store.get_data(hello_world_dataset, start=1, count=6)
assert data == b"ello W"
# Test Size
# Test absent and empty datasets yield size of 0.
assert object_store.size(absent_dataset) == 0
assert object_store.size(empty_dataset) == 0
# Elsewise
assert object_store.size(hello_world_dataset) > 0 # Should this always be the number of bytes?
# Test percent used (to some degree)
percent_store_used = object_store.get_store_usage_percent()
assert percent_store_used > 0.0
assert percent_store_used < 100.0
# Test update_from_file test
output_dataset = MockDataset(4)
output_real_path = os.path.join(directory.temp_directory, "files1", "000", "dataset_4.dat")
assert not os.path.exists(output_real_path)
output_working_path = directory.write(b"NEW CONTENTS", "job_working_directory1/example_output")
object_store.update_from_file(output_dataset, file_name=output_working_path, create=True)
assert os.path.exists(output_real_path)
# Test delete
to_delete_dataset = MockDataset(5)
to_delete_real_path = directory.write(b"content to be deleted!", "files1/000/dataset_5.dat")
assert object_store.exists(to_delete_dataset)
assert object_store.delete(to_delete_dataset)
assert not object_store.exists(to_delete_dataset)
assert not os.path.exists(to_delete_real_path)
HIERARCHICAL_TEST_CONFIG = """
"""
def test_hierarchical_store():
with TestConfig(HIERARCHICAL_TEST_CONFIG) as (directory, object_store):
# Test no dataset with id 1 exists.
assert not object_store.exists(MockDataset(1))
# Write empty dataset 2 in second backend, ensure it is empty and
# exists.
directory.write("", "files2/000/dataset_2.dat")
assert object_store.exists(MockDataset(2))
assert object_store.empty(MockDataset(2))
# Write non-empty dataset in backend 1, test it is not emtpy & exists.
directory.write("Hello World!", "files1/000/dataset_3.dat")
assert object_store.exists(MockDataset(3))
assert not object_store.empty(MockDataset(3))
# Assert creation always happens in first backend.
for i in range(100):
dataset = MockDataset(100 + i)
object_store.create(dataset)
assert object_store.get_filename(dataset).find("files1") > 0
DISTRIBUTED_TEST_CONFIG = """
"""
def test_distributed_store():
with TestConfig(DISTRIBUTED_TEST_CONFIG) as (directory, object_store):
with __stubbed_persistence() as persisted_ids:
for i in range(100):
dataset = MockDataset(100 + i)
object_store.create(dataset)
## Test distributes datasets between backends according to weights
backend_1_count = len([v for v in persisted_ids.values() if v == "files1"])
backend_2_count = len([v for v in persisted_ids.values() if v == "files2"])
assert backend_1_count > 0
assert backend_2_count > 0
assert backend_1_count > backend_2_count
class TestConfig(object):
def __init__(self, config_xml):
self.temp_directory = mkdtemp()
self.write(config_xml, "store.xml")
config = MockConfig(self.temp_directory)
self.object_store = objectstore.build_object_store_from_config(config)
def __enter__(self):
return self, self.object_store
def __exit__(self, type, value, tb):
rmtree(self.temp_directory)
def write(self, contents, name):
path = os.path.join(self.temp_directory, name)
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
expanded_contents = Template(contents).safe_substitute(temp_directory=self.temp_directory)
open(path, "w").write(expanded_contents)
return path
class MockConfig(object):
def __init__(self, temp_directory):
self.file_path = temp_directory
self.object_store_config_file = os.path.join(temp_directory, "store.xml")
self.object_store_check_old_style = False
self.job_working_directory = temp_directory
self.new_file_path = temp_directory
self.umask = 0000
class MockDataset(object):
def __init__(self, id):
self.id = id
self.object_store_id = None
## Poor man's mocking. Need to get a real mocking library as real Galaxy development
## dependnecy.
PERSIST_METHOD_NAME = "create_object_in_session"
@contextmanager
def __stubbed_persistence():
real_method = getattr(objectstore, PERSIST_METHOD_NAME)
try:
persisted_ids = {}
def persist(object):
persisted_ids[object.id] = object.object_store_id
setattr(objectstore, PERSIST_METHOD_NAME, persist)
yield persisted_ids
finally:
setattr(objectstore, PERSIST_METHOD_NAME, real_method)