cleanup

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 deb-build/
 .pypirc
 .vagrant

.travis.yml

@@ -1,24 +1,31 @@
 sudo: required
 dist: trusty
 language: python
+
 python:
-    - 2.7
-    - 3.4
-    - 3.5
+  - 2.7
+  - 3.4
+  - 3.5
 
-before_install:
-    - sudo pip install pytest-cov
+install:
+  - pip install -r dev_requirements.txt
 
 script:
-    - py.test --cov=csirtg_smrt --cov-fail-under=45
-    - pip install -r dev_requirements.txt
-    - make bin
-    - ./pyinstaller-build/dist/csirtg-smrt -V
-    - sudo apt-get install build-essential virtualenvwrapper python-virtualenv devscripts debhelper cdbs
-    - make deb
-
+  - python setup.py test
+  - make bin
+  - "./pyinstaller-build/dist/csirtg-smrt -V"
+  - sudo apt-get install build-essential virtualenvwrapper python-virtualenv devscripts debhelper cdbs
+  - make deb
 
 notifications:
-    email:
-        on_success: never
-        on_failure: never
+  email:
+    on_success: never
+    on_failure: never
+
+deploy:
+  provider: pypi
+  user: wesyoung
+  password:
+    secure: diKbtf7RnRJAo7MdNW3weZDVAfP8j5XYVbcUa6iYBRpQ02zDBhQuZCtmFRms14Ls9nlj2SvBdchdk7WX5R/YbY8j3mI9VtrXdS58foetDPBMQuGJs2dAagpL/AmhssLRqTjKqTX2GXoEgvrKy/HpMiLqGmFKwvd5SlbEWEuC2BNiEL1r+wj/ZlQ36GHGKvNDRnItfwtTZxcDJ62ssdlWItYoRuZyyHEz5PlcYrnhcQHKKpmzzmeRYHHdG2HB42ynr/Plf4/bIjbK1FIMZz3UxZRujblXMkjIzxx/rngPtrUG5M5V28xva0muWrJNnqOvIoOY/A1oOVbysLsCCYR2uh6NcWfLzEFf2PJhjh3y7YZsTVX2rHrvhRKez1Qs+z6cvhvt4AVWRwilMi7TRkwgERhe0YqrRHT1EM5s1OGGN5rSE3Llqa63yvTi4EuMX0GPKZXdfkbihcY0ENexKfu36K3yBMJ39xmtGMKPiUW+aKLo3RZ+XRa+s28GFA5Tju5IYLeBTEz0QkiiovcK0Tl914jDDDE4BVcipL4DuQtF51qGO+9sgo/ppsFWKDFttIBOFYVk2X7UjY73JaXItpQofUKR1v/PRGZ7YSNfUX91bJUKtZ/nQVEysMkflWoqOv6A1296oje9KGE7cqV+8DCzPiFHQrlyLxJqtKfidMXqZC0=
+  on:
+    tags: true

Makefile

@@ -89,9 +89,9 @@ endif
 DEBUILD = $(DEBUILD_BIN) $(DEBUILD_OPTS)
 DEB_PPA ?= ppa
 # Choose the desired Ubuntu release: lucid precise saucy trusty
-DEB_DIST ?= stable
+DEB_DIST ?= unstable
 
-debian: pyinstaller-clean bin
+debian:
 	@for DIST in $(DEB_DIST) ; do \
 	    mkdir -p deb-build/$${DIST}/$(NAME)-$(VERSION)/ ; \
 	    cp -a packaging/debian deb-build/$${DIST}/$(NAME)-$(VERSION)/ ; \

Vagrantfile

@@ -9,6 +9,7 @@ $script = <<SCRIPT
 echo 'yes' | sudo add-apt-repository 'ppa:fkrull/deadsnakes-python2.7'
 sudo apt-get update && sudo apt-get install -y python2.7 python-pip python3-dev python-dev git libffi-dev libssl-dev sqlite3 build-essential virtualenvwrapper python-virtualenv devscripts debhelper cdbs
 sudo pip install versioneer
+sudo pip install -r /vagrant/dev_requirements.txt
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

csirtg_smrt/archiver.py

@@ -1,6 +1,10 @@
+try:
+    import sqlalchemy
+except ImportError:
+    raise ImportError('Requires sqlalchemy')
+
 import logging
 import os
-
 import arrow
 from sqlalchemy import Column, Integer, create_engine, DateTime, UnicodeText, Text
 from sqlalchemy.ext.declarative import declarative_base

csirtg_smrt/client/zelasticsearch.py

@@ -1,3 +1,8 @@
+try:
+    import elasticsearch
+except ImportError:
+    raise ImportError('Requires elasticsearch')
+
 from csirtg_smrt.client.plugin import Client
 from pprint import pprint
 from elasticsearch_dsl.connections import connections

csirtg_smrt/client/zzyre.py

@@ -1,6 +1,10 @@
+try:
+    from pyre import Pyre
+except ImportError as e:
+    raise ImportError('Requires pyre')
+
 from csirtg_indicator import Indicator
 from csirtg_smrt.client.plugin import Client
-from pyre import Pyre
 import logging
 from time import sleep
 logger = logging.getLogger(__name__)

dev_requirements.txt

@@ -1,2 +1,5 @@
--r requirements.txt
-https://github.com/pyinstaller/pyinstaller/archive/b78bfe530cdc2904f65ce098bdf2de08c9037abb.zip
+coverage>=4.2
+pytest-cov>=2.2.1
+pytest>=2.8.0
+https://github.com/pyinstaller/pyinstaller/archive/b78bfe530cdc2904f65ce098bdf2de08c9037abb.tar.gz
+-r extras_requirements.txt

extras_requirements.txt

@@ -0,0 +1,6 @@
+https://github.com/six8/pytailer/archive/8f78431b9d2e63077d7f7150264869506c890024.tar.gz
+https://github.com/zeromq/pyre/archive/fefc0b15803652c7422654bb53eebd8515ac51e3.tar.gz
+elasticsearch<3.0.0,>=2.0.0
+elasticsearch_dsl<3.0.0,>=2.0.0
+SQLAlchemy>=1.0.14,<1.1
+-r requirements.txt

packaging/debian/changelog

@@ -2,4 +2,4 @@ csirtg-smrt (%VERSION%-%RELEASE%~%DIST%) %DIST%; urgency=low
 
   * %VERSION% release
 
- -- CSIRT Gadgets Foundation. <https://github.com/csirtgadgets/csirtg-smrt-py/issues>  %DATE%
+ -- CSIRT Gadgets Foundation. <wes@csirtgadgets.org>  %DATE%

packaging/debian/control

@@ -1,12 +1,13 @@
 Source: csirtg-smrt
-Section: admin
 Priority: optional
 Standards-Version: 3.9.3
 Maintainer: Wes Young <wes@csirtgadgets.org>
 Build-Depends: cdbs, debhelper (>= 5.0.0)
 Homepage: https://github.com/csirtgadgets/csirtg-smrt-py
+Section: contrib/csirtg
 
 Package: csirtg-smrt
-Architecture: all
+Architecture: amd64
 Depends: ${misc:Depends}
-Description: The fastest way to consume threat intelligence
+Description: commandline utility for parsing threat intelligence
+    The fastest way to consume threat intelligence

packaging/debian/copyright

@@ -0,0 +1,30 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: csirtg-smrt
+Source: http://github.com/csirtgadgets/csirtg-smrt-py
+
+Files: *
+Copyright: 2016 CSIRT Gadgets
+ Other contributors as noted in the AUTHORS file
+License: LGPL-3.0+
+
+License: LGPL-3.0+
+ This package is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 3 of the License, or (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ .
+ On Debian systems, the complete text of the GNU Lesser General
+ Public License can be found in "/usr/share/common-licenses/LGPL-3".BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ IN THE SOFTWARE.

packaging/debian/rules

@@ -4,5 +4,8 @@ export DH_VERBOSE=1
 override_dh_auto_build override_dh_auto_install:
 	@
 
+override_dh_shlibdeps:
+    dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info
+
 %:
 	dh $@

requirements.txt

@@ -1,20 +1,13 @@
-https://github.com/csirtgadgets/csirtg-indicator-py/archive/0.0.0a11.tar.gz
-https://github.com/csirtgadgets/csirtgsdk-py/archive/0.0.a2.tar.gz
-git+https://github.com/zeromq/pyre.git@b4f3497d14ee443871ae67fe9e643a1ef9f31ef1
-git+https://github.com/csirtgadgets/py-cgmail.git@3ef0572e758ff9fcbe14905db95679fdf73f09eb
-git+https://github.com/six8/pytailer.git@8f78431b9d2e63077d7f7150264869506c890024#egg=tailer==0.4.1
-pytest-cov>=2.2.1
+csirtg_indicator<1.0
+csirtgsdk<2.0
+csirtg_mail>=0.0.0a1,<1.0.0
 ipaddress>=1.0.16
 feedparser>=5.2.1
-nltk==3.2
+nltk>=3.2,<3.3
 requests>=2.6.0
-pytest>=2.8.0
 arrow>=0.6.0
 python-magic>=0.4.6
 pyaml>=15.8.2
-SQLAlchemy==1.0.14
-chardet==2.3.0
-elasticsearch<3.0.0,>=2.0.0
-elasticsearch_dsl<3.0.0,>=2.0.0
+chardet>=2.3.0,<2.4
 # bug affecting bs4, py2 and cgmail
 html5lib==1.0b8

setup.py

@@ -1,23 +1,39 @@
 import os
 from setuptools import setup, find_packages
 import versioneer
+import sys
 
 # vagrant doesn't appreciate hard-linking
 if os.environ.get('USER') == 'vagrant' or os.path.isdir('/vagrant'):
     del os.link
 
-package_data = []
-import sys
+# https://www.pydanny.com/python-dot-py-tricks.html
+if sys.argv[-1] == 'test':
+    test_requirements = [
+        'pytest',
+        'coverage',
+        'pytest_cov',
+    ]
+    try:
+        modules = map(__import__, test_requirements)
+    except ImportError as e:
+        err_msg = e.message.replace("No module named ", "")
+        msg = "%s is not installed. Install your test requirements." % err_msg
+        raise ImportError(msg)
+    os.system('py.test test -v --cov=csirtg_smrt --cov-fail-under=45')
+    sys.exit()
+
+package_data = {}
 if sys.platform == 'nt':
-    package_data.append([os.path.join('tools', 'magic1.dll')])
+    package_data['csirtg_smrt'] = os.path.join('tools', 'magic1.dll')
 
 setup(
     name="csirtg_smrt",
     version=versioneer.get_version(),
     cmdclass=versioneer.get_cmdclass(),
     package_data=package_data,
     description="s.m.r.t",
-    long_description="",
+    long_description="the fastest way to use data",
     url="https://github.com/csirtgadgets/csirtg-smrt-py",
     license='LGPL3',
     classifiers=[
@@ -32,24 +48,17 @@
     author_email="wes@csirtgadgets.org",
     packages=find_packages(),
     install_requires=[
-        'pytest-cov>=2.2.1',
         'ipaddress>=1.0.16',
         'feedparser>=5.2.1',
-        'nltk==3.2',
+        'nltk>=3.2,<3.3',
         'requests>=2.6.0',
-        'pytest>=2.8.0',
         'arrow>=0.6.0',
         'python-magic>=0.4.6',
         'pyaml>=15.8.2',
         'chardet>=2.3.0',
-        'SQLAlchemy==1.0.14',
-        'elasticsearch<3.0.0,>=2.0.0',
-        'elasticsearch_dsl<3.0.0,>=2.0.0',
-        # bug affecting bs4, py2 and cgmail
-        'html5lib==1.0b8'
-
+        'csirtg_indicator<2.0',
+        'csirtg_mail<1.0'
     ],
-    scripts=[],
     entry_points={
         'console_scripts': [
             'csirtg-smrt=csirtg_smrt.smrt:main',

test/client/test_zyre.py

@@ -7,9 +7,10 @@
 try:
     from pyre import Pyre
     ZYRE_TEST = True
-except:
+except ImportError:
     pass
 
+
 @pytest.mark.skipif(ZYRE_TEST is False, reason='Pyre not installed')
 def test_zyre():
     with Smrt(remote=None, client='zyre') as s: