Merge pull request #640 from tuna-f1sh/travis

Fix Travis CI, add SAMD test support, support GNU grep on macOS

.travis.yml

@@ -1,6 +1,15 @@
-sudo: required
+os: linux
+dist: xenial
 language: c
 compiler:
   - gcc
 script: tests/script/runtests.sh
 before_install: tests/script/bootstrap.sh
+addons:
+  apt:
+    packages:
+      - "python3"
+      - "python3-pip"
+env:
+  global:
+    - ARDMK_DIR=$TRAVIS_BUILD_DIR

Arduino.mk

@@ -853,15 +853,15 @@ endif
 # Reset
 
 ifndef RESET_CMD
-  ARD_RESET_ARDUINO := $(shell which ard-reset-arduino 2> /dev/null)
+  ARD_RESET_ARDUINO := $(PYTHON_CMD) $(shell which ard-reset-arduino 2> /dev/null)
   ifndef ARD_RESET_ARDUINO
     # same level as *.mk in bin directory when checked out from git
     # or in $PATH when packaged
-    ARD_RESET_ARDUINO = $(ARDMK_DIR)/bin/ard-reset-arduino
+    ARD_RESET_ARDUINO = $(PYTHON_CMD) $(ARDMK_DIR)/bin/ard-reset-arduino
   endif
   ifneq (,$(findstring CYGWIN,$(shell uname -s)))
       # confirm user is using default cygwin unix Python (which uses ttySx) and not Windows Python (which uses COMx)
-      ifeq ($(shell which python),/usr/bin/python)
+      ifeq ($(PYTHON_CMD),/usr/bin/python)
         RESET_CMD = $(ARD_RESET_ARDUINO) $(ARD_RESET_OPTS) $(DEVICE_PATH)
       else
         RESET_CMD = $(ARD_RESET_ARDUINO) $(ARD_RESET_OPTS) $(call get_monitor_port)
@@ -1227,15 +1227,24 @@ CFLAGS        += $(CFLAGS_STD)
 CXXFLAGS      += -fpermissive -fno-exceptions $(CXXFLAGS_STD)
 ASFLAGS       += -x assembler-with-cpp
 DIAGNOSTICS_COLOR_WHEN ?= always
-ifeq ($(shell expr $(CC_VERNUM) '>' 490), 1)
-    ASFLAGS  += -flto
-    CXXFLAGS += -fno-threadsafe-statics -flto -fno-devirtualize -fdiagnostics-color=$(DIAGNOSTICS_COLOR_WHEN)
-    CFLAGS   += -flto -fno-fat-lto-objects -fdiagnostics-color=$(DIAGNOSTICS_COLOR_WHEN)
+
+# Flags for AVR
+ifeq ($(findstring avr, $(strip $(CC_NAME))), avr)
+    ifeq ($(shell expr $(CC_VERNUM) '>' 490), 1)
+        ASFLAGS  += -flto
+        CXXFLAGS += -fno-threadsafe-statics -flto -fno-devirtualize -fdiagnostics-color=$(DIAGNOSTICS_COLOR_WHEN)
+        CFLAGS   += -flto -fno-fat-lto-objects -fdiagnostics-color=$(DIAGNOSTICS_COLOR_WHEN)
+        LDFLAGS += -flto -fuse-linker-plugin
+    endif
+# Flags for ARM (most set in Sam.mk)
+else
+    ifeq ($(shell expr $(CC_VERNUM) '>' 490), 1)
+        CXXFLAGS += -fdiagnostics-color=$(DIAGNOSTICS_COLOR_WHEN)
+        CFLAGS   += -fdiagnostics-color=$(DIAGNOSTICS_COLOR_WHEN)
+    endif
 endif
+
 LDFLAGS       += -$(MCU_FLAG_NAME)=$(MCU) -Wl,--gc-sections -O$(OPTIMIZATION_LEVEL)
-ifeq ($(shell expr $(CC_VERNUM) '>' 490), 1)
-		LDFLAGS += -flto -fuse-linker-plugin
-endif
 SIZEFLAGS     ?= --mcu=$(MCU) -C
 
 # for backwards compatibility, grab ARDUINO_PORT if the user has it set

Common.mk

@@ -8,8 +8,8 @@ dir_if_exists = $(if $(wildcard $(1)$(2)),$(1))
 # result = $(call READ_BOARD_TXT, 'boardname', 'parameter')
 PARSE_BOARD = $(shell if [ -f $(BOARDS_TXT) ]; \
 then \
-  grep -Ev '^\#' $(BOARDS_TXT) | \
-  grep -E "^[ \t]*$(1).$(2)=" | \
+  $(GREP_CMD) -Ev '^\#' $(BOARDS_TXT) | \
+  $(GREP_CMD) -E "^[ \t]*$(1).$(2)=" | \
   cut -d = -f 2- | \
   cut -d : -f 2; \
 fi)
@@ -45,15 +45,24 @@ $(call arduino_output,$(call ardmk_include) Configuration:)
 ########################################################################
 #
 # Detect OS
+
 ifeq ($(OS),Windows_NT)
     CURRENT_OS = WINDOWS
+    GREP_CMD = grep
 else
     UNAME_S := $(shell uname -s)
     ifeq ($(UNAME_S),Linux)
         CURRENT_OS = LINUX
+        GREP_CMD = grep
     endif
     ifeq ($(UNAME_S),Darwin)
         CURRENT_OS = MAC
+        ifeq (, $(shell which ggrep))
+            echo $(info Using macOS BSD grep, please install GNU grep to avoid warnings)
+            GREP_CMD = grep
+        else
+            GREP_CMD = ggrep
+        endif
     endif
 endif
 $(call show_config_variable,CURRENT_OS,[AUTODETECTED])
@@ -64,12 +73,20 @@ $(call show_config_variable,CURRENT_OS,[AUTODETECTED])
 ifneq ($(TEST),)
        DEPENDENCIES_DIR = /var/tmp/Arduino-Makefile-testing-dependencies
 
-       DEPENDENCIES_MPIDE_DIR = $(DEPENDENCIES_DIR)/mpide-0023-linux64-20130817-test
+       DEPENDENCIES_MPIDE_DIR := $(shell find $(DEPENDENCIES_DIR) -name 'mpide-0023-*' -type d -exec ls -dt {} + | head -n 1)
+
        ifeq ($(MPIDE_DIR),)
               MPIDE_DIR = $(DEPENDENCIES_MPIDE_DIR)
        endif
 
-       DEPENDENCIES_ARDUINO_DIR = $(DEPENDENCIES_DIR)/arduino-1.0.6
+       ifndef ARDUINO_IDE_DIR
+              ifeq ($(CURRENT_OS),MAC)
+                  ARDUINO_IDE_DIR = Arduino.app/Contents/Resources/Java
+              else
+                  ARDUINO_IDE_DIR := $(shell basename $(basename $(basename $(lastword $(wildcard $(DEPENDENCIES_DIR)/arduino*)))))
+              endif
+       endif
+       DEPENDENCIES_ARDUINO_DIR = $(DEPENDENCIES_DIR)/$(ARDUINO_IDE_DIR)
        ifeq ($(ARDUINO_DIR),)
               ARDUINO_DIR = $(DEPENDENCIES_ARDUINO_DIR)
        endif
@@ -98,3 +115,24 @@ ifeq ($(CURRENT_OS),WINDOWS)
         echo $(error On Windows, ARDUINO_DIR and other defines must use forward slash and not contain spaces, special characters or be cygdrive relative)
     endif
 endif
+
+########################################################################
+# System Python
+
+ifndef PYTHON_CMD
+    # try for Python 3 first
+    PYTHON_CMD := $(shell which python3 2> /dev/null)
+    ifdef PYTHON_CMD
+        $(call show_config_variable,PYTHON_CMD,[AUTODETECTED])
+    else
+        # fall-back to any Python
+        PYTHON_CMD := $(shell which python 2> /dev/null)
+        ifdef PYTHON_CMD
+            $(call show_config_variable,PYTHON_CMD,[AUTODETECTED])
+        else
+            echo $(error "Unable to find system Python! Utility scipts won't work. Override this error by defining PYTHON_CMD")
+        endif
+    endif
+else
+    $(call show_config_variable,PYTHON_CMD,[USER])
+endif

HISTORY.md

@@ -12,12 +12,14 @@ I tried to give credit whenever possible. If I have missed anyone, kindly add it
 - Fix: recognize serial monitors with full path in MONITOR_CMD
 - Fix: Grab USB_PRODUCT and USB_MANUFACTURER from boards.txt for 32u4 boards (issue #594).
 - Fix: Show the configuration when ARDUINO_QUIET=0
+- Fix: Travis build and bring Arduino IDE upto date
 - Tweak: Move chip erase flag from set_fuses to ispload to prevent sketch being nuked when setting fuses
 - Tweak: Set ARDMK_VERSION to 1.6 (https://github.com/sej7278)
 - Tweak: Move non-standard-related items from CxxFLAGS_STD to CxxFLAGS (issue #523) (https://github.com/sej7278)
 - Tweak: Update Windows usage documentation and allow non-relative paths (issue #519) (https://github.com/tuna-f1sh)
 - Tweak: Support Cygwin Unix Python and Windows installation on Windows to pass correct port binding. (https://github.com/tuna-f1sh)
 - Tweak: Update how avr-size is called on Sam, also moved to gnu11 std (issue #602) (https://github.com/tuna-f1sh)
+- Tweak: Detect most recent toolchain if multiple found, add `*_VER` variable to override (issue #611) (https://github.com/tuna-f1sh)
 - New: Added -fdiagnostics-color to \*STD flags (https://github.com/sej7278)
 - New: Made -fdiagnostics-color take a variiable DIAGNOSTICS_COLOR_WHEN: never, always, auto. (https://github.com/wingunder)
 - New: Add generation of tags file using ctags, which automatically includes project libs and Arduino core. (https://github.com/tuna-f1sh)
@@ -32,6 +34,7 @@ I tried to give credit whenever possible. If I have missed anyone, kindly add it
 - New: Updated Arch instructions. (https://github.com/Akram-Chehaima)
 - New: Add support for Robotis OpenCR 1.0 boards.
 - New: Build the ArduinoCore API
+- New: Support for Python 3 and multi-os Python installation using new PYTHON_CMD variable.
 
 ### 1.6.0 (2017-07-11)
 - Fix: Allowed for SparkFun's weird usb pid/vid submenu shenanigans (issue #499). (https://github.com/sej7278)

README.md

@@ -83,38 +83,34 @@ installer or download the distribution zip file and extract it.
 The Makefile also delegates resetting the board to a short Python program.
 You'll need to install [`pySerial`](https://pypi.python.org/pypi/pyserial) to use it though.
 
-On most systems you should be able to install it using either `pip` or `easy_install`.
+On most systems you should be able to install it using either `pip3` or `easy_install3`.
 
 ```sh
-pip install pyserial
+pip3 install pyserial
 
 # or if you prefer easy_install
 
-easy_install -U pyserial
+easy_install3 -U pyserial
 ```
 
 If you prefer to install it as a package, then you can do that as well.
 
 On Debian or Ubuntu:
 
 ```sh
-apt-get install python-serial
+apt-get install python3-serial
 ```
 
 On Fedora:
 
 ```sh
-yum install pyserial
-
-# or on Fedora 22+
-
-dnf install pyserial
+dnf install python3-pyserial
 ```
 
 On openSUSE:
 
 ```sh
-zypper install python-serial
+zypper install python3-serial
 ```
 
 On Arch:
@@ -123,15 +119,16 @@ On Arch:
 sudo pacman -S python-pyserial
 ```
 
-On Mac using MacPorts:
+On macOS using Homebrew (one can install to System Python but this is not recommend or good practice):
 
 ```sh
-sudo port install py27-serial
+brew install python
+pip3 install pyserial
 ```
 
 On Windows:
 
-You need to install Cygwin and its packages for Make, Perl, Python2 and the following Serial library.
+You need to install Cygwin and its packages for Make, Perl, Python3 and the following Serial library.
 
 Assuming you included Python in your Cygwin installation:
 
@@ -141,15 +138,15 @@ Assuming you included Python in your Cygwin installation:
 4. build and install Python module: 
 
 ```
-python setup.py build
-python setup.py install
+python3 setup.py build
+python3 setup.py install
 ```
 
 Alternatively, if you have setup Cygwin to use a Windows Python installation,
 simply install using pip:
 
 ```
-pip install pyserial
+pip3 install pyserial
 ```
 
 Arduino-Makefile should automatically detect the Python installation type and

Sam.mk

@@ -31,6 +31,16 @@ ifndef COMMON_INCLUDED
     include $(ARDMK_DIR)/Common.mk
 endif
 
+ifneq ($(TEST),)
+    CORE_VER = 1.8.6
+    CMSIS_VER = 4.5.0
+    CMSIS_ATMEL_VER = 1.2.0
+    ALTERNATE_CORE_PATH = $(DEPENDENCIES_DIR)/samd
+    CMSIS_DIR = $(DEPENDENCIES_DIR)/CMSIS/CMSIS
+    CMSIS_ATMEL_DIR = $(DEPENDENCIES_DIR)/CMSIS-Atmel/CMSIS
+    ARM_TOOLS_DIR := $(basename $(basename $(firstword $(wildcard $(DEPENDENCIES_DIR)/gcc-arm-none-eabi*))))
+endif
+
 ifndef ARDUINO_PACKAGE_DIR
     # attempt to find based on Linux, macOS and Windows default
     ARDUINO_PACKAGE_DIR := $(firstword \
@@ -160,14 +170,6 @@ ifeq ($(findstring arduino_due, $(strip $(VARIANT))), arduino_due)
     SAM_CORE_C_SRCS += $(wildcard $(SAM_SYSTEM_PATH)/source/*.c)
 endif
 
-# Use arm-toolchain from Arduino install if exists and user has not defined global version
-ifndef ARM_TOOLS_DIR
-    ARM_TOOLS_DIR = $(call dir_if_exists,$(wildcard $(ARDUINO_PACKAGE_DIR)/$(ARDMK_VENDOR)/tools/$(TOOL_PREFIX)-gcc/*))
-    $(call show_config_variable,ARM_TOOLS_DIR,[COMPUTED],(from ARDUINO_PACKAGE_DIR))
-else
-    $(call show_config_variable,ARM_TOOLS_DIR,[USER])
-endif
-
 # define plaform lib dir from Arduino ARM support
 ifndef ARDUINO_PLATFORM_LIB_PATH
     ARDUINO_PLATFORM_LIB_PATH := $(ALTERNATE_CORE_PATH)/libraries
@@ -179,6 +181,20 @@ endif
 
 TOOL_PREFIX = arm-none-eabi
 
+# Use arm-toolchain from Arduino package install if exists and user has not defined global version
+# if undefined, AVR_TOOLS_DIR will resolve in Arduino.mk as a last resort as Arduino now installs with arm-gcc
+ifndef ARM_TOOLS_DIR
+    ifndef ARM_TOOLS_VER
+        ARM_TOOLS_VER := $(shell basename $(lastword $(wildcard $(ARDUINO_PACKAGE_DIR)/$(ARDMK_VENDOR)/tools/$(TOOL_PREFIX)-gcc/*)))
+    endif
+    ARM_TOOLS_DIR = $(ARDUINO_PACKAGE_DIR)/$(ARDMK_VENDOR)/tools/$(TOOL_PREFIX)-gcc/$(ARM_TOOLS_VER)
+    ifdef ARM_TOOLS_DIR
+        $(call show_config_variable,ARM_TOOLS_DIR,[COMPUTED],(from ARDUINO_PACKAGE_DIR))
+    endif
+else
+    $(call show_config_variable,ARM_TOOLS_DIR,[USER])
+endif
+
 ifndef GDB_NAME
     GDB_NAME := $(call PARSE_BOARD,$(BOARD_TAG),build.command.gdb)
     ifndef GDB_NAME
@@ -246,7 +262,9 @@ ifndef OPENOCD
     BUNDLED_OPENOCD_DIR := $(call dir_if_exists,$(ARDUINO_PACKAGE_DIR)/$(ARDMK_VENDOR)/tools/openocd)
     # Try Arduino support package first
     ifdef BUNDLED_OPENOCD_DIR
-        OPENOCD_VER := $(shell basename $(wildcard $(BUNDLED_OPENOCD_DIR)/*))
+        ifndef OPENOCD_VER
+            OPENOCD_VER := $(shell basename $(lastword $(wildcard $(BUNDLED_OPENOCD_DIR)/*)))
+        endif
         OPENOCD = $(BUNDLED_OPENOCD_DIR)/$(OPENOCD_VER)/bin/openocd -s $(BUNDLED_OPENOCD_DIR)/$(OPENOCD_VER)/share/openocd/scripts/
         $(call show_config_variable,OPENOCD,[AUTODETECTED],(from ARDUINO_PACKAGE_DIR))
     else
@@ -271,7 +289,9 @@ ifndef BOSSA
     BUNDLED_BOSSA_DIR := $(call dir_if_exists,$(ARDUINO_PACKAGE_DIR)/$(ARDMK_VENDOR)/tools/bossac)
     # Try Arduino support package first
     ifdef BUNDLED_BOSSA_DIR
-        BOSSA_VER := $(shell basename $(wildcard $(BUNDLED_BOSSA_DIR)/*))
+        ifndef BOSSA_VER
+            BOSSA_VER := $(shell basename $(lastword $(wildcard $(BUNDLED_BOSSA_DIR)/*)))
+        endif
         BOSSA = $(BUNDLED_BOSSA_DIR)/$(BOSSA_VER)/bossac
         $(call show_config_variable,BOSSA,[AUTODETECTED],(from ARDUINO_PACKAGE_DIR))
     else
@@ -394,7 +414,7 @@ CFLAGS_STD += -std=gnu11
 CPPFLAGS += -DMD -D$(USB_TYPE) '-DUSB_PRODUCT=$(USB_PRODUCT)' '-DUSB_MANUFACTURER=$(USB_MANUFACTURER)'
 
 # Get extra define flags from boards.txt
-EXFLAGS := $(shell echo $(call PARSE_BOARD,$(BOARD_TAG),build.extra_flags) | grep -oE '(-D)\w+')
+EXFLAGS := $(shell echo $(call PARSE_BOARD,$(BOARD_TAG),build.extra_flags) | $(GREP_CMD) -oE '(-D)\w+')
 
 # Strip only defines from extra flags as boards file appends user {build.usb}
 CPPFLAGS += $(EXFLAGS)