Docker integration

* Add Dockerfile

* Requested changes; moved dep install before source copy for fewer layers

* Added Cython installation to setup.py

* Add dockerfile for alpine

* Added Python version number to Dockerfiles

* Working on making tests pass

* use the docker cache, it makes things faster

* Added back requirements.txt and cleaned up Dockerfiles

* Use absolute paths for dockerfile destination paths

* Fixed my bad changes to the Dockerfiles

* Some fixes and a bunch of Dockerfiles. Still WIP

* PyPy docker builds passes tests. Ubuntu PyPy is WIP

* Improved Dockerfiles for ubuntu and more tests pass

* Deployed fix for rewind on Linux and possibly Mac

* All tests passes now

* Added dockerignore to minimize build context

* Reverted .tar.gz changes and improved .dockerignore

* Removed pyboy.tar.gz from Makefile

* Changed ADD to COPY

* adjusted and tested .dockerignore

also removed debug stuff from Dockerfile.ubuntu1804
and added absolute path to volume mounts for Makefile
docker test recipes

* Moved dockerfiles

* Added more excludes to dockerignore

* Update dockerfile paths in Makefile

Co-authored-by: Alexander Terry <mralext20@users.noreply.github.com>
Co-authored-by: Kristian Sims <kristian.sims@gmail.com>

.dockerignore

@@ -0,0 +1,24 @@
+*
+!requirements.txt
+!pyboy/
+!setup.py
+!tests/
+tests/BlarggROMs
+!examples/
+!README.md
+!README/
+**/*.pyo
+**/*.pyc
+**/*.pyd
+**/*.so
+**/*.c
+**/*.h
+**/*.dll
+**/*.lib
+**/*.exp
+**/*.html
+**/*.gb
+**/*.gbc
+**/*.bin
+**/*.rom
+!pyboy/core/bootrom.bin

.gitignore

@@ -133,3 +133,5 @@ bootrom/bootrom.sym
 bootrom/logo.asm
 bootrom/pyboy.png
 bootrom/PYBOY_ROM.bin
+
+pyboy.tar.gz

Makefile

@@ -28,6 +28,34 @@ build:
 	@echo "Building..."
 	CFLAGS=$(CFLAGS) ${PY} setup.py build_ext --inplace
 
+docker-pypy:
+	docker build -f docker/Dockerfile.pypy . -t pyboy:pypy-latest
+	docker run -v "${ROOT_DIR}/ROMs:/pyboy/ROMs" -v "${ROOT_DIR}/tests/BlarggROMs":/pyboy/tests/BlarggROMs -it pyboy:pypy-latest sh -c 'cd pyboy; TEST_DOCKER=1 TEST_NO_UI=1 pypy3 setup.py test'
+
+docker-pypy-slim:
+	docker build -f docker/Dockerfile.pypy-slim . -t pyboy:pypy-slim-latest
+	docker run -v "${ROOT_DIR}/ROMs:/pyboy/ROMs" -v "${ROOT_DIR}/tests/BlarggROMs":/pyboy/tests/BlarggROMs -it pyboy:pypy-slim-latest sh -c 'cd pyboy; TEST_DOCKER=1 TEST_NO_UI=1 pypy3 setup.py test'
+
+docker-buster:
+	docker build -f docker/Dockerfile.buster . -t pyboy:buster-latest
+	docker run -v "${ROOT_DIR}/ROMs:/pyboy/ROMs" -v "${ROOT_DIR}/tests/BlarggROMs":/pyboy/tests/BlarggROMs -it pyboy:buster-latest sh -c 'cd pyboy; TEST_DOCKER=1 TEST_NO_UI=1 python setup.py test'
+
+docker-alpine:
+	docker build -f docker/Dockerfile.alpine . -t pyboy:alpine-latest
+	docker run -v "${ROOT_DIR}/ROMs:/pyboy/ROMs" -v "${ROOT_DIR}/tests/BlarggROMs":/pyboy/tests/BlarggROMs -it pyboy:alpine-latest sh -c 'cd pyboy; TEST_NO_EXAMPLES=1 TEST_DOCKER=1 TEST_NO_UI=1 python setup.py test'
+
+docker-ubuntu1804:
+	docker build -f docker/Dockerfile.ubuntu1804 . -t pyboy:ubuntu1804-latest
+	docker run -v "${ROOT_DIR}/ROMs:/pyboy/ROMs" -v "${ROOT_DIR}/tests/BlarggROMs":/pyboy/tests/BlarggROMs -it pyboy:ubuntu1804-latest sh -c 'cd pyboy; TEST_DOCKER=1 TEST_NO_UI=1 python3 setup.py test'
+
+docker-ubuntu1804-ui:
+	docker build -f docker/Dockerfile.ubuntu1804 . -t pyboy:ubuntu1804-latest
+	docker run -v "${ROOT_DIR}/ROMs:/pyboy/ROMs" -v "${ROOT_DIR}/tests/BlarggROMs":/pyboy/tests/BlarggROMs -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=host.docker.internal:0 -e XDG_RUNTIME_DIR=/tmp -it pyboy:ubuntu1804-latest sh -c 'cd pyboy; TEST_DOCKER=1 python3 setup.py test'
+
+docker-pypy-ubuntu1804:
+	docker build -f docker/Dockerfile.pypy-ubuntu1804 . -t pyboy:pypy-ubuntu1804-latest
+	docker run -v "${ROOT_DIR}/ROMs:/pyboy/ROMs" -v "${ROOT_DIR}/tests/BlarggROMs":/pyboy/tests/BlarggROMs -it pyboy:pypy-ubuntu1804-latest sh -c 'cd pyboy; TEST_DOCKER=1 TEST_NO_UI=1 pypy3 setup.py test'
+
 clean:
 	@echo "Cleaning..."
 	CFLAGS=$(CFLAGS) ${PY} setup.py clean --inplace
@@ -45,6 +73,8 @@ test: clean build
 test_quick: clean build
 	${PY} setup.py test
 
+test_all: test docker-pypy docker-pypy-slim docker-buster docker-alpine docker-ubuntu1804 docker-pypy-ubuntu1804
+
 docs: clean
 	pdoc --html --force pyboy
 	cp html/pyboy/windowevent.html ${ROOT_DIR}/docs/

docker/Dockerfile.alpine

@@ -0,0 +1,17 @@
+FROM python:3-alpine
+
+RUN apk add \
+  build-base \
+  jpeg-dev \
+  sdl2 \
+  sdl2-dev \
+  zlib-dev
+
+COPY requirements.txt /
+RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
+
+COPY . /pyboy
+WORKDIR /pyboy
+
+RUN pip install .
+WORKDIR /

docker/Dockerfile.buster

@@ -0,0 +1,15 @@
+FROM python:3-buster
+
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  libsdl2-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /
+RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
+
+COPY . /pyboy
+WORKDIR /pyboy
+
+RUN pip install .
+WORKDIR /

docker/Dockerfile.pypy

@@ -0,0 +1,18 @@
+FROM pypy:3
+
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  libsdl2-dev \
+  libtiff5-dev \
+  libjpeg-dev \
+  zlib1g-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /
+RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
+
+COPY . /pyboy
+WORKDIR /pyboy
+
+RUN pip install .
+WORKDIR /

docker/Dockerfile.pypy-slim

@@ -0,0 +1,18 @@
+FROM pypy:3-slim
+
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  libsdl2-dev \
+  libtiff5-dev \
+  libjpeg-dev \
+  zlib1g-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /
+RUN pip install --upgrade pip && pip install --no-cache-dir -r requirements.txt
+
+COPY . /pyboy
+WORKDIR /pyboy
+
+RUN pip install .
+WORKDIR /

docker/Dockerfile.pypy-ubuntu1804

@@ -0,0 +1,22 @@
+FROM ubuntu:18.04
+RUN apt update && apt install -y \
+  build-essential \
+  libsdl2-dev \
+  libtiff5-dev \
+  libjpeg8-dev \
+  zlib1g-dev \
+  curl \
+  && rm -rf /var/lib/apt/lists/*
+
+ARG pypy_version=pypy3.6-v7.3.0-linux64
+RUN curl -OL "https://bitbucket.org/pypy/pypy/downloads/$pypy_version.tar.bz2" && tar -xvf "$pypy_version.tar.bz2"
+ENV PATH="/$pypy_version/bin:${PATH}"
+
+COPY requirements.txt /
+RUN pypy3 -m ensurepip && pypy3 -m pip install --upgrade pip && pypy3 -m pip install --no-cache-dir -r requirements.txt
+
+COPY . /pyboy
+WORKDIR /pyboy
+
+RUN pypy3 -m pip install .
+WORKDIR /

docker/Dockerfile.ubuntu1804

@@ -0,0 +1,19 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  libsdl2-dev \
+  python3 \
+  python3-pip \
+  python3-dev \
+  git \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /
+RUN pip3 install --upgrade pip && pip3 install --no-cache-dir -r requirements.txt
+
+COPY . /pyboy
+WORKDIR /pyboy
+
+RUN pip3 install .
+WORKDIR /

pyboy/pyboy.py

@@ -8,6 +8,7 @@
 """
 
 import base64
+import os
 import time
 
 import numpy as np
@@ -67,6 +68,8 @@ def __init__(
             record_input (bool): Enable input recording (internal use).
             disable_input (bool): Enable to ignore all user input.
         """
+        if not os.path.isfile(gamerom_file):
+            raise FileNotFoundError(f"ROM file {gamerom_file} was not found!")
         self.gamerom_file = gamerom_file
 
         self.window = window.window.getwindow(window_type, window_scale, debugging, hide_window)

pyboy/rewind.pxd

@@ -5,6 +5,7 @@
 
 cimport cython
 
+from libc.stdlib cimport malloc, free
 from libc.stdint cimport uint8_t, uint64_t, int64_t
 
 cdef int64_t FIXED_BUFFER_SIZE = 64*1024*128
@@ -14,6 +15,12 @@ cdef int64_t FILL_VALUE
 DEF FIXED_BUFFER_SIZE = 64*1024*128
 DEF FIXED_BUFFER_MIN_ALLOC = 64*1024
 
+cdef inline uint8_t* _malloc(size_t n):
+    return <uint8_t*> malloc(FIXED_BUFFER_SIZE)
+
+cdef inline void _free(uint8_t* pointer):
+    free(<void *> pointer)
+
 cdef class IntIOInterface:
     cdef int64_t write(self, uint8_t)
     cdef uint8_t read(self)
@@ -32,7 +39,7 @@ cdef class IntIOWrapper(IntIOInterface):
 ##############################################################
 
 cdef class FixedAllocBuffers(IntIOInterface):
-    cdef uint8_t[FIXED_BUFFER_SIZE] buffer
+    cdef uint8_t* buffer
     cdef list sections
     cdef int64_t current_section
     cdef int64_t tail_pointer

pyboy/rewind.py

@@ -5,7 +5,23 @@
 
 import array
 
-FIXED_BUFFER_SIZE = 64*1024*128
+try:
+    from cython import compiled
+    cythonmode = compiled
+except ImportError:
+    cythonmode = False
+
+if not cythonmode:
+    exec("""
+def _malloc(n):
+    return array.array('B', [0]*(FIXED_BUFFER_SIZE))
+
+def _free(_):
+    pass
+""", globals(), locals())
+
+
+FIXED_BUFFER_SIZE = 32*1024*128
 FIXED_BUFFER_MIN_ALLOC = 64*1024
 FILL_VALUE = 123
 
@@ -73,7 +89,9 @@ def flush(self):
 
 class FixedAllocBuffers(IntIOInterface):
     def __init__(self):
-        self.buffer = array.array('B', [FILL_VALUE]*(FIXED_BUFFER_SIZE))
+        self.buffer = _malloc(FIXED_BUFFER_SIZE) # NOQA: F821
+        for n in range(FIXED_BUFFER_SIZE):
+            self.buffer[n] = FILL_VALUE
         self.sections = [0]
         self.current_section = 0
         self.tail_pointer = 0
@@ -83,6 +101,9 @@ def __init__(self):
         self.section_pointer = 0
         self.avg_section_size = 0.0
 
+    def stop(self):
+        _free(self.buffer) # NOQA: F821
+
     def flush(self):
         pass
 

pyboy/window/window_opengl.py

@@ -36,7 +36,8 @@ def init(self, hide_window):
         self.color_format = u"RGB"
         self.buffer_dims = (144, 160)
 
-        glutInit()
+        if not glutInit():
+            raise Exception("OpenGL couldn't initialize!")
         glutInitDisplayMode(GLUT_SINGLE | GLUT_RGBA)
         glutInitWindowSize(*self._scaledresolution)
         glutCreateWindow("PyBoy")

requirements.txt

@@ -0,0 +1,4 @@
+cython; platform_python_implementation == 'CPython'
+numpy
+pillow
+pysdl2

setup.py

@@ -12,14 +12,30 @@
 from setuptools import Extension, find_packages, setup
 from setuptools.command.test import test
 
-CYTHON = platform.python_implementation() != "PyPy"
 
+def load_requirements(filename):
+    with open(filename, 'r') as f:
+        return [line.split(';')[0].strip() for line in f.readlines()]
+
+
+requirements = load_requirements('requirements.txt')
+
+CYTHON = platform.python_implementation() != "PyPy"
 
 if CYTHON:
+    # "Recommended" method of installing Cython: https://github.com/pypa/pip/issues/5761
+    from setuptools import dist
+    dist.Distribution().fetch_build_eggs(["cython"])
+
     from Cython.Build import cythonize
     import Cython.Compiler.Options
     from Cython.Distutils import build_ext
 else:
+    try:
+        requirements.remove('cython')
+    except ValueError:
+        pass
+
     class build_ext(distutils.cmd.Command):
 
         def initialize_options(self):
@@ -50,17 +66,18 @@ def finalize_options(self):
         self.test_args = []
 
     def run_tests(self):
-        script_path = os.path.dirname(os.path.realpath(__file__))
-        return_code = subprocess.Popen(
-            f"{sys.executable} {script_path}/examples/tetris_bot.py {script_path}/ROMs/Tetris.gb --quiet".split(' ')
-        ).wait()
-        if return_code != 0:
-            sys.exit(return_code)
-
-        return_code = subprocess.Popen(
-            f"{sys.executable} {script_path}/examples/interface_example.py --quiet".split(' ')).wait()
-        if return_code != 0:
-            sys.exit(return_code)
+        if not os.environ.get("TEST_NO_EXAMPLES"):
+            script_path = os.path.dirname(os.path.realpath(__file__))
+            return_code = subprocess.Popen(
+                f"{sys.executable} {script_path}/examples/tetris_bot.py {script_path}/ROMs/Tetris.gb --quiet".split(' ')
+            ).wait()
+            if return_code != 0:
+                sys.exit(return_code)
+
+            return_code = subprocess.Popen(
+                f"{sys.executable} {script_path}/examples/interface_example.py --quiet".split(' ')).wait()
+            if return_code != 0:
+                sys.exit(return_code)
 
         import pytest
         args = [f"-n{cpu_count()}", "-v"]
@@ -87,12 +104,12 @@ def run(self):
 
             for root, dirs, files in os.walk(ROOT_DIR):
                 if "__pycache__" in dirs:
-                    log.info(f"Removing: {os.path.join(root, '__pycache__')}")
+                    log.info(f"removing: {os.path.join(root, '__pycache__')}")
                     remove_tree(os.path.join(root, "__pycache__"))
                 for f in files:
                     if os.path.splitext(f)[1] in (".pyo", ".pyc", ".pyd", ".so", ".c", ".h",
                                                   ".dll", ".lib", ".exp", ".html"):
-                        print(f"Removing: {os.path.join(root, f)}")
+                        print(f"removing: {os.path.join(root, f)}")
                         os.remove(os.path.join(root, f))
 
 
@@ -252,11 +269,7 @@ def get_export_symbols(self, ext):
         ],
     },
     cmdclass={'build_ext': build_ext, 'clean': clean, 'test': PyTest},
-    install_requires=(["cython"] if CYTHON else []) + [
-        "pysdl2",
-        "numpy",
-        "Pillow",
-    ],
+    install_requires=requirements,
     tests_require=[
         "pytest",
         "pytest-xdist",