Skip to content

Commit

Permalink
Merge branch 'master' into feature-checkout
Browse files Browse the repository at this point in the history
  • Loading branch information
sranasir committed Jun 19, 2023
2 parents cb45a1e + 8b5e118 commit 9e32194
Show file tree
Hide file tree
Showing 26 changed files with 746 additions and 57 deletions.
45 changes: 45 additions & 0 deletions .github/workflows/ubuntu.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
name: Ubuntu

on:
push:
pull_request:

jobs:

sciunit-test-ubuntu-20-04-with-tox:
runs-on: ubuntu-latest
container: geotrust/build-essential:ubuntu20.04
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: update-pip
run: pip3 install --upgrade pip
- name: install-tox
run: pip3 install tox
- name: test-with-tox
run: tox

sciunit-test-ubuntu-16-04-with-tox:
runs-on: ubuntu-latest
container: geotrust/build-essential:ubuntu16.04
steps:
- name: install-https
run: apt-get update && apt install apt-transport-https
- name: update-git
run: echo "deb https://ppa.launchpadcontent.net/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list
- name: install-key
run: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E1DD270288B4E6030699E45FA1715D88E1DF1F24
- name: install
run: apt-get update && apt-get install git -y
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: git-config
run: git config --global --add safe.directory /__w/sciunit/sciunit
- name: update-pip
run: pip3 install --upgrade pip
- name: install-tox
run: pip3 install tox
- name: test-with-tox
run: tox -e py37
19 changes: 19 additions & 0 deletions CITATION.cff
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
cff-version: 1.2.0
message: "If you use this software, please cite it as below."
title: "Sciunits: Reusable Research Objects"
url: "https://github.com/depaul-dice/sciunit"
preferred-citation:
type: article
authors:
- family-names: "That"
given-names: "Dai Hai Ton"
- family-names: "Fils"
given-names: "Gabriel"
- family-names: "Yuan"
given-names: "Zhihao"
- family-names: "Malik"
given-names: "Tanu"
title: "Sciunits: Reusable Research Objects"
doi: 10.48550/arXiv.1707.05731
url: "https://github.com/depaul-dice/sciunit"

6 changes: 3 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,21 @@ set(_FILES_DIR ${CMAKE_SOURCE_DIR}/files)

ExternalProject_Add(
ptu
GIT_REPOSITORY "https://bitbucket.org/depauldbgroup/provenance-to-use"
GIT_REPOSITORY "https://github.com/depaul-dice/provenance-to-use"
CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_TESTING=OFF
INSTALL_COMMAND install -s <BINARY_DIR>/ptu ${_LIBEXEC_DIR}
)

ExternalProject_Add(
scripter
GIT_REPOSITORY "https://bitbucket.org/geotrust/scripter"
GIT_REPOSITORY "https://github.com/depaul-dice/scripter"
CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
INSTALL_COMMAND install -s <BINARY_DIR>/scripter ${_LIBEXEC_DIR}
)

ExternalProject_Add(
vvpkg
GIT_REPOSITORY "https://bitbucket.org/depauldbgroup/vvpkg"
GIT_REPOSITORY "https://github.com/depaul-dice/sciunit-dedup"
GIT_TAG "7400941"
PATCH_COMMAND ${GIT_EXECUTABLE} apply ${_FILES_DIR}/vv-report-size.patch
CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
Expand Down
109 changes: 109 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
### Instructions for developers working with Sciunit working on Ubuntu OS

1. download Sciunit source code and rename project folder:

`git clone https://bitbucket.org/geotrust/sciunit2.git sciunit2-python3`

`cd sciunit2-python3`

2. Install python3.x (>= version 3.7). Following is a suggested link:

https://phoenixnap.com/kb/how-to-install-python-3-ubuntu

3. Install git version control system:

`sudo apt update`

`sudo apt install git`

4. install pip for python3:

sudo apt-get install python3-pip

5. Install these packages:

`pip install --user --upgrade pytest-metadata`

`pip install --user --upgrade tox`

`sudo apt install cmake`

6. Install the requiremenst from test-requirements.txt and requirements.txt:

`pip install -r requirements.txt`

`pip install -r test-requirements.txt`

7. Make sure you are up-to-date with the master branch:

`git pull origin master`

8. Run the following command from the project folder to run all tests:

`tox`

All tests should pass. If there are any errors, follow error messages and resolve them first.


9. At this point, you can start adding your code to Sciunit. DO NOT start working in the master branch. Make sure you checkout your own branch:

`git checkout -b new_branch_name`

10. Make sure all changes in your branch are committed locally and pushed to the remote repository periodically:

`git push origin master`

Note: To push the changes, you would first need to be added as a contributor.

11. After you are done making changes, run `tox` again to make sure all tests pass:

12. To test the installation in production, you can install your source code locally

`pip install sciunit2-python3/`

13. After your changes are finalized in your branch, do a final code review and send a merge request with master. After a successful merge, package the code and upload to PyPI. You would need the credentials for the Sciunit account on PyPI:

`pip install --user --upgrade twine`

`python setup.py sdist bdist_wheel`

`twine upload dist/*`


### Some notes for understanding sciunit (for beginner developers):

Sciunit uses application virtualization (AV) tool provenance-to-use (PTU) built on top of Code, Data, and Environment (CDE) to containerize an application as it executes.
ptu creates a directory structure called cde-package
A tar archive of cde-package/ is sent to vvpkg for deduplication

1. Committing a package to the deduplicated storage:
a. take an execution directory (cde-package/) as input
b. convert it into a tar archive
c. vvpkg performs deduplication on the archive and stores it into blocks
2. Reconstructing an execution directory from the de-duplicated storage:
a. extract relevant blocks from the vvpkg storage.
b. create a tar archive by concatenating the blocks from the original file entries.
c. untar the archive to get the execution directory
3. 'create' command does the following:
a. creates an empty dir for the project
b. opens the project
4. 'exec' does the following:
a. creates a sciunit.db file in the project dir.
b. creates cde-package dir in the sciunit folder
c. calls commit
At the end of exec, *.json, vvpkg.bin and vvpkg.db files have
been created in the project folder.
5. 'commit' does this:
a. archives the cde-package directory and writes it to vvpkg
b. deletes the cde-package directory
c. adds the new execution to sciunit db
6. 'repeat' does this:
a. does ContextCheckout (see #7)
b. reads execution commands from cde.log
c. writes new cde.log with new arguments for the same execution commands
d. creates a Script object from cde.log and executes it as a subprocess
7. ContextCheckout does this:
a. removes any existing cde-package/ present
b. checks out the given execution from the database using 'checkout' command.
c. untar the archive obtained from checkout
This builds cde-package directory as it was present at the time of the given execution.
28 changes: 28 additions & 0 deletions LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
BSD 3-Clause License

Copyright (c) 2023, Tanu Malik, DePaul University

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
9 changes: 9 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
.. image:: https://github.com/depaul-dice/sciunit/actions/workflows/ubuntu.yaml/badge.svg
:target: https://github.com/depaul-dice/sciunit/actions/workflows/ubuntu.yaml
.. image:: https://img.shields.io/pypi/v/sciunit2
:alt: PyPI
:target: https://pypi.org/project/sciunit2/

Sciunit
----------

**The most updated instructions on installing Sciunit can be found at:**
**https://sciunit.run/install**

**sciunit** is a command-line tool that answers the call for a
reusable research object that containerizes and stores applications
simply and efficiently, facilitates sharing and collaboration, and
Expand Down
5 changes: 5 additions & 0 deletions SECURITY.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Security Policy

## Reporting a Vulnerability

Users can report security vulnerabilities to [sciunit technical support](mailto:pr@sciunit.run)
30 changes: 0 additions & 30 deletions bitbucket-pipelines.yml

This file was deleted.

Binary file added docs/developing_with_sciunit.pptx
Binary file not shown.
7 changes: 5 additions & 2 deletions docs/sciunit.1.rst
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ sciunit
-------------------------------------------------------------
deliver reproducibility in your research
-------------------------------------------------------------
:Author: Zhihao Yuan <lichray@gmail.com>
:Author: Tanu Malik <tanu.malik@depaul.edu>
:Version: sciunit2 0.4
:Manual section: 1
:Copyright: Copyright 2017-2018, DePaul University
:Copyright: Copyright 2020-2021, DePaul University
:Licence: FreeBSD License (BSD-2-Clause)

.. raw:: manpage
Expand All @@ -28,6 +28,9 @@ SYNOPSIS
DESCRIPTION
============

**This page might have outdated information. The most updated Sciunit documentation can be found at:**
**https://sciunit.run**

A command line utility to create, manage, and share *sciunits*.
A *sciunit* is a lightweight and portable unit that contains captured,
repeatable program executions.
Expand Down
3 changes: 3 additions & 0 deletions docs/tutorial.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# A Brief Introduction to Sciunit

**This page might have outdated information. The most updated Sciunit documentation can be found at:**
**https://sciunit.run/docs**

From time to time, you may find that it is hard to verify or reproduce someone
else's research, even though it is only programs, data, and output.
Programs may be built in different ways, may behave differently if not running
Expand Down
9 changes: 8 additions & 1 deletion sciunit2/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,21 @@
from sciunit2.command.checkout import CheckoutCommand
from sciunit2.command.post_install import PostInstallCommand
from sciunit2.command.diff import DiffCommand
from sciunit2.command.export import ExportCommand
from sciunit2.command.remove import RemoveCommand

import sys
from getopt import getopt, GetoptError
from io import StringIO
import textwrap
import pkg_resources
import os
import platform

__cmds__ = [CreateCommand, OpenCommand, ExecCommand, RepeatCommand,
ListCommand, ShowCommand, GivenCommand, CommitCommand, RmCommand,
SortCommand, PushCommand, CopyCommand, PostInstallCommand,
DiffCommand, CheckoutCommand]
DiffCommand, RemoveCommand, ExportCommand, CheckoutCommand]


def short_usage(out):
Expand Down Expand Up @@ -77,6 +80,10 @@ def main():


def _main(args):
if platform.system().startswith('Linux') == False:
err1('Platform is not supported')
sys.exit(1)

optlist, args = getopt(args, '', ['help', 'version', 'root='])
if optlist:
op, v = optlist[0]
Expand Down
20 changes: 15 additions & 5 deletions sciunit2/command/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,25 @@ class CreateCommand(AbstractCommand):

@property
def usage(self):
return [('create <name>',
'Create and open a new sciunit under ~/sciunit/<name>')]
return [('create [-f] <name>',
'Create and open a new sciunit under ~/sciunit/<name>.\n'
'The -f flag overwrites the existing directory if present.')]

def run(self, args):
optlist, args = getopt(args, '')
optlist, args = getopt(args, 'f')
if len(args) != 1:
raise CommandLineError
sciunit2.workspace.create(args[0])
return sciunit2.workspace.open(args[0])
else:
project_name = args[0].strip()
if optlist:
if optlist[0][0] == '-f': # if force flag is provided
print('Warning: This will overwrite the existing sciunit!')
sciunit2.workspace.create(project_name, overwrite=True)
else:
raise CommandLineError
else:
sciunit2.workspace.create(project_name)
return sciunit2.workspace.open(project_name)

def note(self, project_dir):
return quoted_format('Opened empty sciunit at {0}\n', project_dir)
4 changes: 3 additions & 1 deletion sciunit2/command/diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,9 @@ def parse_rsync(out):
perms_changed = []
for line in lines:
splits = line.split()
assert len(splits) == 2
# The length of splits is two for regular files and directories
# and 4 for symlinks
# assert len(splits) == 2
YXcstpoguax = splits[0].strip()
file_name = splits[1].strip()
update_type = YXcstpoguax[0]
Expand Down
Loading

0 comments on commit 9e32194

Please sign in to comment.