Permalink
Browse files

Updated CI test configuration and helper scripts

  • Loading branch information...
joachimmetz committed Apr 4, 2018
1 parent c5fa648 commit 31c7ef467e54579c026aca1275ffed7d34e84278
View
490 .pylintrc

Large diffs are not rendered by default.

Oops, something went wrong.
View
@@ -1,4 +1,7 @@
[style]
based_on_style = chromium
COALESCE_BRACKETS = True
SPLIT_BEFORE_FIRST_ARGUMENT = True
allow_split_before_dict_value = False
coalesce_brackets = True
join_multiple_lines = True
split_before_first_argument = True
split_penalty_after_opening_bracket = 0
View
@@ -1,29 +1,46 @@
language: python
matrix:
include:
- os: linux
- env: TARGET="pylint"
os: linux
dist: trusty
sudo: required
group: edge
python: 2.7
env: TOXENV=py27
- os: linux
- env: TARGET="linux-python27"
os: linux
dist: trusty
sudo: required
group: edge
python: 2.7
- env: TARGET="linux-python34"
os: linux
dist: trusty
sudo: required
group: edge
python: 3.4
- env: [TARGET="linux-python27-tox", TOXENV="py27"]
os: linux
dist: trusty
sudo: required
group: edge
python: 2.7
- env: [TARGET="linux-python34-tox", TOXENV="py34"]
os: linux
dist: trusty
sudo: required
group: edge
python: 3.4
env: TOXENV=py34
- os: osx
osx_image: xcode8.3
- env: [TARGET="macos-python27", PYTHONPATH="/Library/Python/2.7/site-packages/"]
os: osx
osx_image: xcode9.2
language: generic
env: PYTHONPATH=/Library/Python/2.7/site-packages/
allow_failures:
- env: TARGET="linux-python34"
- env: [TARGET="macos-python27", PYTHONPATH="/Library/Python/2.7/site-packages/"]
install:
- ./config/travis/install.sh
- ./config/travis/install.sh
script:
- if test ${TRAVIS_OS_NAME} = "osx"; then PYTHONPATH=/Library/Python/2.7/site-packages/ /usr/bin/python run_tests.py; elif test ${TRAVIS_OS_NAME} = "linux"; then if test ${TRAVIS_PYTHON_VERSION} = "2.7"; then coverage run --source=artifacts --omit="*_test*,*__init__*,*test_lib*" ./run_tests.py; else ./run_tests.py; fi; fi
- python setup.py build
- python setup.py sdist
- python setup.py bdist
- if test ${TRAVIS_OS_NAME} = "linux"; then mkdir -p ${PWD}/tmp/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/ && PYTHONPATH=${PWD}/tmp/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/ python setup.py install --prefix=${PWD}/tmp/; fi
- ./config/travis/runtests.sh
after_success:
- if test ${TRAVIS_OS_NAME} = "linux" && test ${TRAVIS_PYTHON_VERSION} = "2.7"; then coveralls --verbose; fi
- if test ${TARGET} = "linux-python27"; then curl -o codecov.sh -s https://codecov.io/bash && /bin/bash ./codecov.sh; fi
View
@@ -7,8 +7,9 @@ environment:
install:
- cmd: '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x86 /release'
- cmd: "%PYTHON%\\Scripts\\pip.exe install --upgrade pip"
- cmd: "%PYTHON%\\Scripts\\pip.exe install pypiwin32 WMI"
- cmd: "%PYTHON%\\python.exe -m pip install --upgrade pip"
- cmd: "%PYTHON%\\Scripts\\pip.exe install pywin32 WMI"
- cmd: "%PYTHON%\\python.exe %PYTHON%\\Scripts\\pywin32_postinstall.py -install"
- cmd: git clone https://github.com/log2timeline/l2tdevtools.git ..\l2tdevtools
- cmd: mkdir dependencies && set PYTHONPATH=..\l2tdevtools && "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type x86 --msi-targetdir "%PYTHON%" --track dev PyYAML funcsigs mock pbr six yapf
View
@@ -82,7 +82,8 @@ def AsDict(self):
for source in self.sources:
source_definition = {
'type': source.type_indicator,
'attributes': source.AsDict()}
'attributes': source.AsDict()
}
if source.supported_os:
source_definition['supported_os'] = source.supported_os
if source.conditions:
@@ -94,7 +95,8 @@ def AsDict(self):
artifact_definition = {
'name': self.name,
'doc': self.description,
'sources': sources,}
'sources': sources,
}
if self.labels:
artifact_definition['labels'] = self.labels
if self.supported_os:
View
@@ -3,7 +3,6 @@
from __future__ import unicode_literals
TYPE_INDICATOR_ARTIFACT_GROUP = 'ARTIFACT_GROUP'
TYPE_INDICATOR_COMMAND = 'COMMAND'
TYPE_INDICATOR_DIRECTORY = 'DIRECTORY'
@@ -14,54 +13,35 @@
TYPE_INDICATOR_WMI_QUERY = 'WMI'
LABELS = {
'Antivirus':
'Antivirus related artifacts, e.g. quarantine files.',
'Authentication':
'Authentication artifacts.',
'Browser':
'Web Browser artifacts.',
'Cloud':
'Cloud applications artifacts.',
'Cloud Storage':
'Cloud storage artifacts.',
'Configuration Files':
'Configuration files artifacts.',
'Execution':
'Contain execution events.',
'Antivirus': 'Antivirus related artifacts, e.g. quarantine files.',
'Authentication': 'Authentication artifacts.',
'Browser': 'Web Browser artifacts.',
'Cloud': 'Cloud applications artifacts.',
'Cloud Storage': 'Cloud storage artifacts.',
'Configuration Files': 'Configuration files artifacts.',
'Execution': 'Contain execution events.',
'ExternalAccount': (
'Information about any user accounts e.g. username, '
'account ID, etc.'),
'External Media':
'Contain external media data or events e.g. USB drives.',
'IM':
'Instant Messaging / Chat applications artifacts.',
'iOS':
'Artifacts related to iOS devices connected to the system.',
'History Files':
'History files artifacts e.g. .bash_history.',
'KnowledgeBase':
'Artifacts used in knowledge base generation.',
'Logs':
'Contain log files.',
'Mail':
'Mail client applications artifacts.',
'Memory':
'Artifacts retrieved from memory.',
'Network':
'Describe networking state.',
'Processes':
'Describe running processes.',
'Rekall':
'Artifacts using the Rekall memory forensics framework.',
'Software':
'Installed software.',
'System':
'Core system artifacts.',
'Users':
'Information about users.'}
'External Media': 'Contain external media data or events e.g. USB drives.',
'IM': 'Instant Messaging / Chat applications artifacts.',
'iOS': 'Artifacts related to iOS devices connected to the system.',
'History Files': 'History files artifacts e.g. .bash_history.',
'KnowledgeBase': 'Artifacts used in knowledge base generation.',
'Logs': 'Contain log files.',
'Mail': 'Mail client applications artifacts.',
'Memory': 'Artifacts retrieved from memory.',
'Network': 'Describe networking state.',
'Processes': 'Describe running processes.',
'Rekall': 'Artifacts using the Rekall memory forensics framework.',
'Software': 'Installed software.',
'System': 'Core system artifacts.',
'Users': 'Information about users.'
}
SUPPORTED_OS = frozenset(['Darwin', 'Linux', 'Windows'])
TOP_LEVEL_KEYS = frozenset([
'conditions', 'doc', 'labels', 'name', 'provides', 'sources',
'supported_os', 'urls'])
'supported_os', 'urls'
])
View
@@ -86,8 +86,9 @@ def DeregisterSourceType(cls, source_type_class):
indicator.
"""
if source_type_class.TYPE_INDICATOR not in cls._source_type_classes:
raise KeyError('Source type not set for type: {0:s}.'.format(
source_type_class.TYPE_INDICATOR))
raise KeyError(
'Source type not set for type: {0:s}.'.format(
source_type_class.TYPE_INDICATOR))
del cls._source_type_classes[source_type_class.TYPE_INDICATOR]
@@ -158,8 +159,9 @@ def RegisterSourceType(cls, source_type_class):
type indicator.
"""
if source_type_class.TYPE_INDICATOR in cls._source_type_classes:
raise KeyError('Source type already set for type: {0:s}.'.format(
source_type_class.TYPE_INDICATOR))
raise KeyError(
'Source type already set for type: {0:s}.'.format(
source_type_class.TYPE_INDICATOR))
cls._source_type_classes[source_type_class.TYPE_INDICATOR] = (
source_type_class)
View
@@ -226,7 +226,8 @@ class WindowsRegistryKeySourceType(SourceType):
r'HKEY_LOCAL_MACHINE',
r'HKEY_USERS',
r'HKEY_CLASSES_ROOT',
r'%%current_control_set%%',]
r'%%current_control_set%%',
]
def __init__(self, keys=None):
"""Initializes a source type.
@@ -369,22 +370,17 @@ class SourceTypeFactory(object):
"""Source type factory."""
_source_type_classes = {
definitions.TYPE_INDICATOR_ARTIFACT_GROUP:
ArtifactGroupSourceType,
definitions.TYPE_INDICATOR_COMMAND:
CommandSourceType,
definitions.TYPE_INDICATOR_DIRECTORY:
DirectorySourceType,
definitions.TYPE_INDICATOR_FILE:
FileSourceType,
definitions.TYPE_INDICATOR_PATH:
PathSourceType,
definitions.TYPE_INDICATOR_ARTIFACT_GROUP: ArtifactGroupSourceType,
definitions.TYPE_INDICATOR_COMMAND: CommandSourceType,
definitions.TYPE_INDICATOR_DIRECTORY: DirectorySourceType,
definitions.TYPE_INDICATOR_FILE: FileSourceType,
definitions.TYPE_INDICATOR_PATH: PathSourceType,
definitions.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY:
WindowsRegistryKeySourceType,
definitions.TYPE_INDICATOR_WINDOWS_REGISTRY_VALUE:
WindowsRegistryValueSourceType,
definitions.TYPE_INDICATOR_WMI_QUERY:
WMIQuerySourceType,}
definitions.TYPE_INDICATOR_WMI_QUERY: WMIQuerySourceType,
}
@classmethod
def CreateSourceType(cls, type_indicator, attributes):
View
@@ -1,36 +1,62 @@
#!/bin/bash
#
# Script to set up Travis-CI test VM.
COVERALL_DEPENDENCIES="python-coverage python-coveralls python-docopt";
#
# This file is generated by l2tdevtools update-dependencies.py any dependency
# related changes should be made in dependencies.ini.
L2TBINARIES_DEPENDENCIES="PyYAML";
L2TBINARIES_TEST_DEPENDENCIES="funcsigs mock pbr six yapf";
PYTHON2_DEPENDENCIES="python-yaml";
PYTHON2_TEST_DEPENDENCIES="python-mock python-tox python-yapf";
PYTHON2_TEST_DEPENDENCIES="python-coverage python-mock python-tox python-yapf yapf";
PYTHON3_DEPENDENCIES="python3-yaml";
PYTHON3_TEST_DEPENDENCIES="python-yapf python3-mock python3-setuptools python3-tox yapf";
# Exit on error.
set -e;
if test ${TRAVIS_OS_NAME} = "osx";
then
git clone https://github.com/log2timeline/l2tdevtools.git;
mv l2tdevtools ../;
mkdir dependencies;
PYTHONPATH=../l2tdevtools ../l2tdevtools/tools/update.py --download-directory dependencies --track dev ${L2TBINARIES_DEPENDENCIES} ${L2TBINARIES_TEST_DEPENDENCIES};
git clone https://github.com/log2timeline/l2tbinaries.git -b dev;
mv l2tbinaries ../;
for PACKAGE in ${L2TBINARIES_DEPENDENCIES};
do
echo "Installing: ${PACKAGE}";
sudo /usr/bin/hdiutil attach ../l2tbinaries/macos/${PACKAGE}-*.dmg;
sudo /usr/sbin/installer -target / -pkg /Volumes/${PACKAGE}-*.pkg/${PACKAGE}-*.pkg;
sudo /usr/bin/hdiutil detach /Volumes/${PACKAGE}-*.pkg
done
for PACKAGE in ${L2TBINARIES_TEST_DEPENDENCIES};
do
echo "Installing: ${PACKAGE}";
sudo /usr/bin/hdiutil attach ../l2tbinaries/macos/${PACKAGE}-*.dmg;
sudo /usr/sbin/installer -target / -pkg /Volumes/${PACKAGE}-*.pkg/${PACKAGE}-*.pkg;
sudo /usr/bin/hdiutil detach /Volumes/${PACKAGE}-*.pkg
done
elif test ${TRAVIS_OS_NAME} = "linux";
then
sudo rm -f /etc/apt/sources.list.d/travis_ci_zeromq3-source.list;
sudo add-apt-repository ppa:gift/dev -y;
sudo apt-get update -q;
# Only install the Python 2 dependencies.
# Also see: https://docs.travis-ci.com/user/languages/python/#Travis-CI-Uses-Isolated-virtualenvs
sudo apt-get install -y ${COVERALL_DEPENDENCIES} ${PYTHON2_DEPENDENCIES} ${PYTHON2_TEST_DEPENDENCIES};
if test ${TRAVIS_PYTHON_VERSION} = "2.7";
then
sudo apt-get install -y ${PYTHON2_DEPENDENCIES} ${PYTHON2_TEST_DEPENDENCIES};
else
sudo apt-get install -y ${PYTHON3_DEPENDENCIES} ${PYTHON3_TEST_DEPENDENCIES};
fi
if test ${TARGET} = "pylint";
then
sudo apt-get install -y pylint;
fi
fi
View
@@ -0,0 +1,58 @@
#!/bin/bash
#
# Script to run tests on Travis-CI.
#
# This file is generated by l2tdevtools update-dependencies.py, any dependency
# related changes should be made in dependencies.ini.
# Exit on error.
set -e;
if test "${TARGET}" = "pylint";
then
pylint --version
for FILE in `find setup.py artifacts tests -name \*.py`;
do
echo "Checking: ${FILE}";
pylint --rcfile=.pylintrc ${FILE};
done
elif test "${TRAVIS_OS_NAME}" = "osx";
then
PYTHONPATH=/Library/Python/2.7/site-packages/ /usr/bin/python ./run_tests.py;
python ./setup.py build
python ./setup.py sdist
python ./setup.py bdist
elif test "${TRAVIS_OS_NAME}" = "linux";
then
if test -n "${TOXENV}";
then
tox --sitepackages ${TOXENV};
elif test "${TRAVIS_PYTHON_VERSION}" = "2.7";
then
coverage erase
coverage run --source=artifacts --omit="*_test*,*__init__*,*test_lib*" ./run_tests.py
else
python ./run_tests.py
fi
python ./setup.py build
python ./setup.py sdist
python ./setup.py bdist
TMPDIR="${PWD}/tmp";
TMPSITEPACKAGES="${TMPDIR}/lib/python${TRAVIS_PYTHON_VERSION}/site-packages";
mkdir -p ${TMPSITEPACKAGES};
PYTHONPATH=${TMPSITEPACKAGES} python ./setup.py install --prefix=${TMPDIR};
fi
Oops, something went wrong.

0 comments on commit 31c7ef4

Please sign in to comment.