Merge branch '1.x' into 'master'

.github/workflows/test.yml

@@ -83,6 +83,8 @@ jobs:
       LC_ALL: C
     steps:
     - uses: actions/checkout@v2
+      with:
+        fetch-depth: 20
     - uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python }}

README.md

@@ -2,7 +2,6 @@
 [![Coverage Status](https://codecov.io/github/VIDA-NYU/reprozip/coverage.svg?branch=master)](https://codecov.io/github/VIDA-NYU/reprozip?branch=master)
 [![Documentation Status](https://readthedocs.org/projects/reprozip/badge/?version=master)](https://docs.reprozip.org/en/master/)
 [![Matrix](https://img.shields.io/badge/chat-matrix.org-blue.svg)](https://riot.im/app/#/room/#reprozip:matrix.org)
-[![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/remram44)
 [![status](https://img.shields.io/badge/JOSS-10.21105%2Fjoss.00107-green.svg)](http://joss.theoj.org/papers/b578b171263c73f64dfb9d040ca80fe0)
 [![DOI](https://img.shields.io/badge/DOI/10.5281%2Fzenodo.5081097-green.svg)](https://doi.org/10.5281/zenodo.5081097)
 

reprozip/reprozip/filters.py

@@ -4,6 +4,7 @@
 
 import logging
 import re
+from pathlib import Path
 
 from reprozip.tracer.trace import TracedFile
 
@@ -62,3 +63,53 @@ def python(files, input_files, **kwargs):
                 lst.append(path)
 
         input_files[i] = lst
+
+
+def ruby(files, input_files, **kwargs):
+    extensions = {'.rb', '.haml', '.slim', '.erb', '.js', '.html'}
+    ignored_dirs = {
+        'spec', 'test', 'tests', 'guides', 'doc-api', 'rdoc', 'doc',
+    }
+
+    gemy_path = re.compile(r'^.*/ruby[-/]\d+\.\d+\.\d+/gems')
+    appdir_paths = re.compile(r'^.*/app/(views|controllers|models|helpers)')
+
+    directories = set()
+
+    for path, fi in files.items():
+        m1 = gemy_path.match(str(path))
+        if m1:
+            directories.add(Path(m1.group(0)))
+
+        m2 = appdir_paths.match(str(path))
+        if m2:
+            app_root = Path(m2.group(0)).parent.parent
+            if (app_root / 'config/application.rb').is_file():
+                directories.add(app_root)
+
+    def add_recursive(dir_or_file):
+        if (
+            dir_or_file.is_file()
+            and dir_or_file.suffix in extensions
+        ):
+            logger.info("Adding %s", dir_or_file)
+            files[dir_or_file] = TracedFile(dir_or_file)
+        elif (
+            dir_or_file.is_dir()
+            and dir_or_file.name not in ignored_dirs
+        ):
+            for child in dir_or_file.iterdir():
+                add_recursive(child)
+
+    for directory in directories:
+        add_recursive(directory)
+
+    for i in range(len(input_files)):
+        lst = []
+        for path in input_files[i]:
+            if gemy_path.match(str(path)) or appdir_paths.match(str(path)):
+                logger.info("Removing input %s", path)
+            else:
+                lst.append(path)
+
+        input_files[i] = lst

reprozip/setup.py

@@ -48,7 +48,8 @@
               'reprozip = reprozip.main:main'],
           'reprozip.filters': [
               'python = reprozip.filters:python',
-              'builtin = reprozip.filters:builtin']},
+              'builtin = reprozip.filters:builtin',
+              'ruby = reprozip.filters:ruby']},
       install_requires=req,
       description="Linux tool enabling reproducible experiments (packer)",
       author="Remi Rampin, Fernando Chirigati, Dennis Shasha, Juliana Freire",

tests/test_rails_filter.py

@@ -0,0 +1,101 @@
+# Copyright (C) 2014-2017 New York University
+# This file is part of ReproZip which is released under the Revised BSD License
+# See file LICENSE for full license details.
+
+from __future__ import division, print_function, unicode_literals
+
+from pathlib import Path
+from reprozip.filters import ruby
+from reprozip_core.common import File
+import shutil
+import tempfile
+import unittest
+
+
+class MockTracedFile(File):
+    def __init__(self, path):
+        File.__init__(self, path, None)
+
+
+class RailsFilterTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = Path(tempfile.mkdtemp(prefix='reprozip_tests_'))
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp)
+
+    @classmethod
+    def touch(cls, test_files):
+        for fi in test_files:
+            fi.parent.mkdir(parents=True, exist_ok=True)
+            with fi.open('a'):
+                pass
+
+    def test_consuming_entire_gem(self):
+        gemdir = self.tmp / 'gems/ruby-2.2.3/gems/kaminari-0.16.3'
+        gemfiles = [
+            'app/views/kaminari/_first_page.html.erb',
+            'app/views/kaminari/_first_page.html.haml',
+            'app/views/kaminari/_first_page.html.slim',
+            'app/views/kaminari/_gap.html.erb',
+            'app/views/kaminari/_gap.html.haml',
+            'app/views/kaminari/_gap.html.slim',
+            'app/views/kaminari/_last_page.html.erb',
+            'app/views/kaminari/_last_page.html.haml',
+            'app/views/kaminari/_last_page.html.slim',
+        ]
+
+        self.touch(gemdir / f for f in gemfiles)
+
+        input_files = [[]]
+        files = {}
+
+        for path in gemdir.glob('**/*'):
+            if '_first' in path.name:
+                f = MockTracedFile(path)
+                files[f.path] = f
+                input_files[0].append(path)
+
+        ruby(files=files, input_files=input_files)
+
+        self.assertEqual(set(files.keys()), set(gemdir / f for f in gemfiles))
+
+    def test_consuming_rails_files(self):
+        # Should be recognized: has a config file
+        railsfiles = [
+            'yes/config/application.rb',
+            'yes/app/views/application.html.erb',
+            'yes/app/views/discussion-sidebar.html.erb',
+            'yes/app/views/payments_listing.html.erb',
+            'yes/app/views/print-friendly.html.erb',
+            'yes/app/views/w-sidebar.html.erb',
+            'yes/app/views/widget.html.erb',
+        ]
+        # Should NOT be: no config file
+        notrailsfiles = [
+            # 'no/config/application.rb',
+            'no/app/views/application.html.erb',
+            'no/app/views/discussion-sidebar.html.erb',
+            'no/app/views/payments_listing.html.erb',
+            'no/app/views/print-friendly.html.erb',
+            'no/app/views/w-sidebar.html.erb',
+            'no/app/views/widget.html.erb',
+        ]
+
+        self.touch(self.tmp / f for f in railsfiles)
+        self.touch(self.tmp / f for f in notrailsfiles)
+
+        input_files = [[]]
+        files = {}
+
+        viewsdir = MockTracedFile(self.tmp / railsfiles[-1])
+        files[viewsdir.path] = viewsdir
+        viewsdir = MockTracedFile(self.tmp / notrailsfiles[-1])
+        files[viewsdir.path] = viewsdir
+
+        ruby(files, input_files)
+
+        self.assertEqual(
+            set(files.keys()),
+            set(self.tmp / f for f in railsfiles + [notrailsfiles[-1]]),
+        )