Skip to content

Commit

Permalink
Merge pull request #332 from VIDA-NYU/extensions
Browse files Browse the repository at this point in the history 
Implement EXTENSIONS directory in RPZ bundle format
  • Loading branch information
@remram44
remram44 committed Jan 5, 2022
2 parents 812071b + d431995 commit 67bbe8d
Show file tree
Hide file tree
Showing 5 changed files with 179 additions and 67 deletions.
13 changes: 13 additions & 0 deletions reprounzip/reprounzip/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,19 @@ def copy_data_tar(self, target):
copyfile(data, fp)
data.close()

def extensions(self):
"""Get a list of extensions present in this pack.
"""
extensions = set()
for m in self.tar.getmembers():
if m.name.startswith('EXTENSIONS/'):
name = m.name[11:]
if '/' in name:
name = name[:name.index('/')]
if name:
extensions.add(name)
return extensions

def close(self):
if self.data is not self.tar:
self.data.close()
Expand Down
9 changes: 9 additions & 0 deletions reprounzip/reprounzip/pack_info.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,9 @@ def get_package_info(pack, read_data=False):
'write_runs': iofile.write_runs}
for name, iofile in iteritems(inputs_outputs)}

rpz_pack = RPZPack(pack)
information['extensions'] = sorted(rpz_pack.extensions())

# Unpacker compatibility
unpacker_status = {}
for name, upk in iteritems(unpackers):
Expand Down Expand Up @@ -215,6 +218,12 @@ def _print_package_info(pack, info, verbosity=1):
t.append("out")
print(" %s (%s): %s" % (name, ' '.join(t), f['path']))

extensions = info.get('extensions')
if extensions:
print("\n----- Extensions -----")
for name in extensions:
print(name)

unpacker_status = info.get('unpacker_status')
if unpacker_status:
print("\n----- Unpackers -----")
Expand Down
13 changes: 13 additions & 0 deletions reprozip/reprozip/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,19 @@ def copy_data_tar(self, target):
copyfile(data, fp)
data.close()

def extensions(self):
"""Get a list of extensions present in this pack.
"""
extensions = set()
for m in self.tar.getmembers():
if m.name.startswith('EXTENSIONS/'):
name = m.name[11:]
if '/' in name:
name = name[:name.index('/')]
if name:
extensions.add(name)
return extensions

def close(self):
if self.data is not self.tar:
self.data.close()
Expand Down
140 changes: 74 additions & 66 deletions tests/test_reprounzip.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,17 @@

from __future__ import print_function, unicode_literals

import io
import os
import sys
import shutil
import tarfile
import tempfile
import unittest
import warnings

from reprounzip.common import RPZPack
from reprounzip.signals import Signal
import reprounzip.unpackers.common


class TestSignals(unittest.TestCase):
Expand Down Expand Up @@ -113,70 +117,74 @@ def setup_logging(tag, verbosity):


class TestCommon(unittest.TestCase):
def test_env(self):
"""Tests fixing environment variables"""
outer_env = {
'OUTONLY': 'outvalue',
'COMMON': 'commonvalueout',
'SHARED': 'sharedvalue',
'EMPTY': '',
}
inner_env = {
'INONLY': 'invalue',
'COMMON': 'commonvaluein',
'SHARED': 'sharedvalue',
}

class FakeArgs(object):
def __init__(self, pass_env, set_env):
self.pass_env = pass_env
self.set_env = set_env

old_environ, os.environ = os.environ, outer_env
def test_rpzpack(self):
def write_to_tar(tar, path, data, **kwargs):
info = tarfile.TarInfo(path)
info.size = len(data)
for key, value in kwargs.items():
if not hasattr(info, key):
raise AttributeError(key)
setattr(info, key, value)
tar.addfile(info, io.BytesIO(data))

tmp = tempfile.mkdtemp()
try:
self.assertEqual(
reprounzip.unpackers.common.fixup_environment(
inner_env,
FakeArgs([], [])),
{
'INONLY': 'invalue',
'COMMON': 'commonvaluein',
'SHARED': 'sharedvalue',
})

self.assertEqual(
reprounzip.unpackers.common.fixup_environment(
inner_env,
FakeArgs(['COMMON', 'INONLY', 'OUTONLY', 'EMPTY'], [])),
{
'INONLY': 'invalue',
'OUTONLY': 'outvalue',
'COMMON': 'commonvalueout',
'SHARED': 'sharedvalue',
'EMPTY': '',
})

self.assertEqual(
reprounzip.unpackers.common.fixup_environment(
inner_env,
FakeArgs(['OUTONLY'],
['SHARED=surprise', 'COMMON=', 'INONLY'])),
{
'OUTONLY': 'outvalue',
'COMMON': '',
'SHARED': 'surprise',
})

self.assertEqual(
reprounzip.unpackers.common.fixup_environment(
inner_env,
FakeArgs(['.*Y$'], [])),
{
'INONLY': 'invalue',
'OUTONLY': 'outvalue',
'COMMON': 'commonvaluein',
'SHARED': 'sharedvalue',
'EMPTY': '',
})
# Create data tar.gz
data = os.path.join(tmp, 'DATA.tar.gz')
tar = tarfile.open(data, 'w:gz')
write_to_tar(
tar,
'DATA/bin/init',
b'#!/bin/sh\necho "Success."\n',
mode=0o755,
)

# Create rpz
rpz = os.path.join(tmp, 'test.rpz')
tar = tarfile.open(rpz, 'w:')
write_to_tar(
tar,
'METADATA/version',
b'REPROZIP VERSION 2\n',
)
write_to_tar(
tar,
'METADATA/trace.sqlite3',
b'',
)
write_to_tar(
tar,
'METADATA/config.yml',
b'{}',
)
tar.add(
data,
'DATA.tar.gz',
)

# Add directory extension
write_to_tar(
tar,
'EXTENSIONS/foo/one.txt',
b'',
)
write_to_tar(
tar,
'EXTENSIONS/foo/two.txt',
b'',
)

# Add single file extension
write_to_tar(
tar,
'EXTENSIONS/bar',
b'',
)

tar.close()

rpz_obj = RPZPack(rpz)
self.assertEqual(rpz_obj.open_config().read(), b'{}')
self.assertEqual(rpz_obj.extensions(), {'foo', 'bar'})
finally:
os.environ = old_environ
shutil.rmtree(tmp)
71 changes: 70 additions & 1 deletion tests/test_unpackers_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@

from __future__ import print_function, unicode_literals

import os
import sys
import unittest

from reprounzip.unpackers.common import UsageError, \
unique_names, make_unique_name, get_runs
unique_names, make_unique_name, get_runs, fixup_environment
from reprounzip.utils import irange


Expand All @@ -28,6 +29,74 @@ def test_make_unique_name(self):
n[:13] == b'/some/prefix_')
self.assertEqual(len(set(names)), len(names))

def test_env(self):
"""Tests fixing environment variables"""
outer_env = {
'OUTONLY': 'outvalue',
'COMMON': 'commonvalueout',
'SHARED': 'sharedvalue',
'EMPTY': '',
}
inner_env = {
'INONLY': 'invalue',
'COMMON': 'commonvaluein',
'SHARED': 'sharedvalue',
}

class FakeArgs(object):
def __init__(self, pass_env, set_env):
self.pass_env = pass_env
self.set_env = set_env

old_environ, os.environ = os.environ, outer_env
try:
self.assertEqual(
fixup_environment(
inner_env,
FakeArgs([], [])),
{
'INONLY': 'invalue',
'COMMON': 'commonvaluein',
'SHARED': 'sharedvalue',
})

self.assertEqual(
fixup_environment(
inner_env,
FakeArgs(['COMMON', 'INONLY', 'OUTONLY', 'EMPTY'], [])),
{
'INONLY': 'invalue',
'OUTONLY': 'outvalue',
'COMMON': 'commonvalueout',
'SHARED': 'sharedvalue',
'EMPTY': '',
})

self.assertEqual(
fixup_environment(
inner_env,
FakeArgs(['OUTONLY'],
['SHARED=surprise', 'COMMON=', 'INONLY'])),
{
'OUTONLY': 'outvalue',
'COMMON': '',
'SHARED': 'surprise',
})

self.assertEqual(
fixup_environment(
inner_env,
FakeArgs(['.*Y$'], [])),
{
'INONLY': 'invalue',
'OUTONLY': 'outvalue',
'COMMON': 'commonvaluein',
'SHARED': 'sharedvalue',
'EMPTY': '',
})
finally:
os.environ = old_environ


class TestMisc(unittest.TestCase):
def do_ok(self, arg, expected, nruns=4):
Expand Down

0 comments on commit 67bbe8d

Please sign in to comment.