Merge branch 'master' into feature-checkout

.github/workflows/ubuntu.yaml

@@ -0,0 +1,45 @@
+name: Ubuntu
+
+on:
+  push:
+  pull_request:
+
+jobs:
+
+  sciunit-test-ubuntu-20-04-with-tox:
+      runs-on: ubuntu-latest
+      container: geotrust/build-essential:ubuntu20.04
+      steps:
+        - uses: actions/checkout@v3
+          with:
+            fetch-depth: 0
+        - name: update-pip
+          run: pip3 install --upgrade pip
+        - name: install-tox
+          run: pip3 install tox
+        - name: test-with-tox
+          run: tox
+
+  sciunit-test-ubuntu-16-04-with-tox:
+      runs-on: ubuntu-latest
+      container: geotrust/build-essential:ubuntu16.04
+      steps:
+        - name: install-https
+          run:  apt-get update && apt install apt-transport-https
+        - name: update-git
+          run:  echo "deb https://ppa.launchpadcontent.net/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list
+        - name: install-key
+          run: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E1DD270288B4E6030699E45FA1715D88E1DF1F24
+        - name: install  
+          run: apt-get update && apt-get install git -y
+        - uses: actions/checkout@v3
+          with:
+            fetch-depth: 0
+        - name: git-config
+          run: git config --global --add safe.directory /__w/sciunit/sciunit
+        - name: update-pip
+          run: pip3 install --upgrade pip
+        - name: install-tox
+          run: pip3 install tox
+        - name: test-with-tox
+          run: tox -e py37

CITATION.cff

@@ -0,0 +1,19 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+title: "Sciunits: Reusable Research Objects"
+url: "https://github.com/depaul-dice/sciunit"
+preferred-citation:
+  type: article
+  authors:
+  - family-names: "That"
+    given-names: "Dai Hai Ton"
+  - family-names: "Fils"
+    given-names: "Gabriel"
+  - family-names: "Yuan"
+    given-names: "Zhihao"
+  - family-names: "Malik"
+    given-names: "Tanu"
+  title: "Sciunits: Reusable Research Objects"
+  doi: 10.48550/arXiv.1707.05731
+  url: "https://github.com/depaul-dice/sciunit"
+

CMakeLists.txt

@@ -9,21 +9,21 @@ set(_FILES_DIR ${CMAKE_SOURCE_DIR}/files)
 
 ExternalProject_Add(
   ptu
-  GIT_REPOSITORY "https://bitbucket.org/depauldbgroup/provenance-to-use"
+  GIT_REPOSITORY "https://github.com/depaul-dice/provenance-to-use"
   CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_TESTING=OFF
   INSTALL_COMMAND install -s <BINARY_DIR>/ptu ${_LIBEXEC_DIR}
 )
 
 ExternalProject_Add(
   scripter
-  GIT_REPOSITORY "https://bitbucket.org/geotrust/scripter"
+  GIT_REPOSITORY "https://github.com/depaul-dice/scripter"
   CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
   INSTALL_COMMAND install -s <BINARY_DIR>/scripter ${_LIBEXEC_DIR}
 )
 
 ExternalProject_Add(
   vvpkg
-  GIT_REPOSITORY "https://bitbucket.org/depauldbgroup/vvpkg"
+  GIT_REPOSITORY "https://github.com/depaul-dice/sciunit-dedup"
   GIT_TAG "7400941"
   PATCH_COMMAND ${GIT_EXECUTABLE} apply ${_FILES_DIR}/vv-report-size.patch
   CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}

CONTRIBUTING.md

@@ -0,0 +1,109 @@
+### Instructions for developers working with Sciunit working on Ubuntu OS
+
+1. download Sciunit source code and rename project folder:
+
+`git clone https://bitbucket.org/geotrust/sciunit2.git sciunit2-python3`
+
+`cd sciunit2-python3`
+
+2. Install python3.x (>= version 3.7). Following is a suggested link:
+
+ https://phoenixnap.com/kb/how-to-install-python-3-ubuntu
+
+3. Install git version control system:
+
+`sudo apt update`
+
+`sudo apt install git`
+
+4. install pip for python3:
+
+sudo apt-get install python3-pip
+
+5. Install these packages:
+
+`pip install --user --upgrade pytest-metadata`
+
+`pip install --user --upgrade tox`
+
+`sudo apt install cmake`
+
+6. Install the requiremenst from test-requirements.txt and requirements.txt:
+
+`pip install -r requirements.txt`
+
+`pip install -r test-requirements.txt`
+
+7. Make sure you are up-to-date with the master branch:
+
+`git pull origin master`
+
+8. Run the following command from the project folder to run all tests:
+
+`tox`
+
+All tests should pass. If there are any errors, follow error messages and resolve them first.
+
+
+9. At this point, you can start adding your code to Sciunit. DO NOT start working in the master branch. Make sure you checkout your own branch:
+
+`git checkout -b new_branch_name`
+
+10. Make sure all changes in your branch are committed locally and  pushed to the remote repository periodically:
+
+`git push origin master`
+
+Note: To push the changes, you would first need to be added as a contributor.
+
+11. After you are done making changes, run `tox` again to make sure all tests pass:
+
+12. To test the installation in production, you can install your source code locally
+
+`pip install sciunit2-python3/`
+
+13. After your changes are finalized in your branch, do a final code review and send a merge request with master. After a successful merge, package the code and upload to PyPI. You would need the credentials for the Sciunit account on PyPI:
+
+`pip install --user --upgrade twine`
+
+`python setup.py sdist bdist_wheel`
+
+`twine upload dist/*`
+
+
+### Some notes for understanding sciunit (for beginner developers):
+
+Sciunit uses application virtualization (AV) tool provenance-to-use (PTU) built on top of Code, Data, and Environment (CDE) to containerize an application as it executes.
+ptu creates a directory structure called cde-package
+A tar archive of cde-package/ is sent to vvpkg for deduplication
+
+1. Committing a package to the deduplicated storage:
+	a. take an execution directory (cde-package/) as input 
+	b. convert it into a tar archive
+	c. vvpkg performs deduplication on the archive and stores it into blocks
+2. Reconstructing an execution directory from the de-duplicated storage:
+	a. extract relevant blocks from the vvpkg storage.
+	b. create a tar archive by concatenating the blocks from the original file entries.
+	c. untar the archive to get the execution directory
+3.  'create' command does the following:
+	a. creates an empty dir for the project
+	b. opens the project
+4. 'exec' does the following:
+	a. creates a sciunit.db file in the project dir. 
+	b. creates cde-package dir in the sciunit folder
+	c. calls commit
+		At the end of exec, *.json, vvpkg.bin and vvpkg.db files have 
+		been created in the project folder.
+5. 'commit' does this:
+	a. archives the cde-package directory and writes it to vvpkg
+	b. deletes the cde-package directory
+	c. adds the new execution to sciunit db
+6. 'repeat' does this:
+	a. does ContextCheckout (see #7)
+	b. reads execution commands from cde.log
+	c. writes new cde.log with new arguments for the same execution commands
+	d. creates a Script object from cde.log and executes it as a subprocess
+7. ContextCheckout does this:
+	a. removes any existing cde-package/ present
+	b. checks out the given execution from the database using 'checkout' command.
+	c. untar the archive obtained from checkout
+	This builds cde-package directory as it was present at the time of the given execution.

LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Tanu Malik, DePaul University
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.rst

@@ -1,6 +1,15 @@
+.. image:: https://github.com/depaul-dice/sciunit/actions/workflows/ubuntu.yaml/badge.svg
+    :target: https://github.com/depaul-dice/sciunit/actions/workflows/ubuntu.yaml
+.. image:: https://img.shields.io/pypi/v/sciunit2
+   :alt: PyPI
+   :target: https://pypi.org/project/sciunit2/
+
 Sciunit
 ----------
 
+**The most updated instructions on installing Sciunit can be found at:**
+**https://sciunit.run/install**
+
 **sciunit** is a command-line tool that answers the call for a
 reusable research object that containerizes and stores applications
 simply and efficiently, facilitates sharing and collaboration, and

SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Users can report security vulnerabilities to [sciunit technical support](mailto:pr@sciunit.run)

docs/sciunit.1.rst

@@ -6,10 +6,10 @@ sciunit
 -------------------------------------------------------------
 deliver reproducibility in your research
 -------------------------------------------------------------
-:Author:    Zhihao Yuan <lichray@gmail.com>
+:Author:    Tanu Malik <tanu.malik@depaul.edu>
 :Version:   sciunit2 0.4
 :Manual section: 1
-:Copyright: Copyright 2017-2018, DePaul University
+:Copyright: Copyright 2020-2021, DePaul University
 :Licence:   FreeBSD License (BSD-2-Clause)
 
 .. raw:: manpage
@@ -28,6 +28,9 @@ SYNOPSIS
 DESCRIPTION
 ============
 
+**This page might have outdated information. The most updated Sciunit documentation can be found at:**
+**https://sciunit.run**
+
 A command line utility to create, manage, and share *sciunits*.
 A *sciunit* is a lightweight and portable unit that contains captured,
 repeatable program executions.

docs/tutorial.md

@@ -1,5 +1,8 @@
 # A Brief Introduction to Sciunit
 
+**This page might have outdated information. The most updated Sciunit documentation can be found at:**
+**https://sciunit.run/docs**
+
 From time to time, you may find that it is hard to verify or reproduce someone
 else's research, even though it is only programs, data, and output.
 Programs may be built in different ways, may behave differently if not running

sciunit2/cli.py

@@ -16,18 +16,21 @@
 from sciunit2.command.checkout import CheckoutCommand
 from sciunit2.command.post_install import PostInstallCommand
 from sciunit2.command.diff import DiffCommand
+from sciunit2.command.export import ExportCommand
+from sciunit2.command.remove import RemoveCommand
 
 import sys
 from getopt import getopt, GetoptError
 from io import StringIO
 import textwrap
 import pkg_resources
 import os
+import platform
 
 __cmds__ = [CreateCommand, OpenCommand, ExecCommand, RepeatCommand,
             ListCommand, ShowCommand, GivenCommand, CommitCommand, RmCommand,
             SortCommand, PushCommand, CopyCommand, PostInstallCommand,
-            DiffCommand, CheckoutCommand]
+            DiffCommand, RemoveCommand, ExportCommand, CheckoutCommand]
 
 
 def short_usage(out):
@@ -77,6 +80,10 @@ def main():
 
 
 def _main(args):
+    if platform.system().startswith('Linux') == False:
+        err1('Platform is not supported')
+        sys.exit(1)
+
     optlist, args = getopt(args, '', ['help', 'version', 'root='])
     if optlist:
         op, v = optlist[0]

sciunit2/command/create.py

@@ -15,15 +15,25 @@ class CreateCommand(AbstractCommand):
 
     @property
     def usage(self):
-        return [('create <name>',
-                 'Create and open a new sciunit under ~/sciunit/<name>')]
+        return [('create [-f] <name>',
+                 'Create and open a new sciunit under ~/sciunit/<name>.\n'
+                 'The -f flag overwrites the existing directory if present.')]
 
     def run(self, args):
-        optlist, args = getopt(args, '')
+        optlist, args = getopt(args, 'f')
         if len(args) != 1:
             raise CommandLineError
-        sciunit2.workspace.create(args[0])
-        return sciunit2.workspace.open(args[0])
+        else:
+            project_name = args[0].strip()
+            if optlist:
+                if optlist[0][0] == '-f':    # if force flag is provided
+                    print('Warning: This will overwrite the existing sciunit!')
+                    sciunit2.workspace.create(project_name, overwrite=True)
+                else:
+                    raise CommandLineError
+            else:
+                sciunit2.workspace.create(project_name)
+            return sciunit2.workspace.open(project_name)
 
     def note(self, project_dir):
         return quoted_format('Opened empty sciunit at {0}\n', project_dir)

sciunit2/command/diff.py

@@ -82,7 +82,9 @@ def parse_rsync(out):
         perms_changed = []
         for line in lines:
             splits = line.split()
-            assert len(splits) == 2
+            # The length of splits is two for regular files and directories 
+            # and 4 for symlinks
+#            assert len(splits) == 2
             YXcstpoguax = splits[0].strip()
             file_name = splits[1].strip()
             update_type = YXcstpoguax[0]