i#4000, i#3235: Simplify documentation building (#4105)

Removes build-time transformation from FIG to PNG or EPS and instead checks in the PNG files for our handful of figures. This then removes the need for Fig2Dev, Ghostscript, and ImageMagick altogether. Fixes Doxygen installation on Appveyor by directly downloading it rather than trying to use Chocolatey. Fixes identification of whether Doxygen is Cygwin or not by changing the test to try to write to a UNIX path as the old test failed on newer Doxygen versions. Fixes a few x86 INSTR_CREATE Doxygen errors reported by the new Doxygen. Suppresses documenting the many AArch64 INSTR_CREATE macros missing headers: i#4106 covers adding the headers and remove the exclusions. Issue: #4106 Fixes #3235 Fixes #4000
DynamoRIO · Feb 15, 2020 · 5935119 · 5935119
1 parent 3df8553
commit 5935119
Show file tree

Hide file tree

Showing 27 changed files with 1,799 additions and 1,656 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -74,18 +74,10 @@ install:
 
   ##################################################
   # Install packages for docs.
-  # i#2616: internet outages have caused builds to spin for tens of minutes, only to fail
-  # due to chocolatey and fig2dev download errors. We instead ignore errors here and
-  # continue to build and test anyway.
-  - choco install --timeout 600 -y doxygen.portable imagemagick.tool ghostscript || true
-  # The ghostscript package is an install that doesn't add to PATH.
-  # FIXME i#2145: fig2dev is failing to launch ghostscript, so for now
-  # we're disabling to get the initial AppVeyor to be green.
-  #- set PATH=C:\Program Files\gs\gs9.20\bin;%PATH%
-  # Install fig2dev.
-  - appveyor DownloadFile http://www.winfig.com/WinFIG-Dateien/WinFIG75.zip || true
-  - 7z x WinFIG75.zip > nul || true
-  - set PATH=c:\projects\install\WinFig;%PATH%
+  # i#4000: choco fails to download doxygen.portable so we install ourselves:
+  - appveyor DownloadFile http://doxygen.nl/files/doxygen-1.8.17.windows.x64.bin.zip
+  - 7z x doxygen-1.8.17.windows.x64.bin.zip -oc:\projects\install\doxygen > nul
+  - set PATH=c:\projects\install\doxygen;%PATH%
 
   ##################################################
   # XXX i#2145: point at Qt5 for testing drgui build.

diff --git a/.travis.yml b/.travis.yml
@@ -133,10 +133,9 @@ install:
   # FIXME: remove the "brew update" step once Travis fixes their Mac VM's
   # on 11/15/17.  Xref https://github.com/travis-ci/travis-ci/issues/8552.
   - if [[ "`uname`" == "Darwin" ]]; then brew update; brew install nasm; fi
-  # ImageMagick is present but these are not:
   - >
       if [[ "`uname`" == "Linux" ]]; then
-      sudo apt-get -y install ghostscript doxygen transfig vera++ cmake zlib1g-dev libsnappy-dev; fi
+      sudo apt-get -y install doxygen vera++ cmake zlib1g-dev libsnappy-dev; fi
   # Install multilib for non-cross-compiling Linux builds:
   - >
       if [[ "`uname`" == "Linux" && $DYNAMORIO_CROSS_AARCHXX_LINUX_ONLY == no ]]; then

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1724,25 +1724,6 @@ endif (BUILD_CLIENTS)
 
 if (CLIENT_INTERFACE OR APP_EXPORTS)
   if (BUILD_DOCS)
-    find_package(ImageMagick COMPONENTS mogrify)
-    if (NOT ImageMagick_mogrify_FOUND)
-      # cmake 2.6.0 does not set the _FOUND vars, just the exec path,
-      # and it uses all caps
-      if (EXISTS ${IMAGEMAGICK_MOGRIFY_EXECUTABLE})
-        set(ImageMagick_mogrify_EXECUTABLE ${IMAGEMAGICK_MOGRIFY_EXECUTABLE})
-      else (EXISTS ${IMAGEMAGICK_MOGRIFY_EXECUTABLE})
-        # We require on Linux pre-commit where it's trivial to install
-        if (TEST_SUITE AND UNIX AND NOT APPLE)
-          message(FATAL_ERROR "ImageMagick is required to build the documentation")
-        else ()
-          # Non-fatal for a single, un-official build, or on Windows
-          message(WARNING "ImageMagick not found: documentation will NOT be built")
-          set(BUILD_DOCS OFF)
-        endif ()
-      endif (EXISTS ${IMAGEMAGICK_MOGRIFY_EXECUTABLE})
-    endif (NOT ImageMagick_mogrify_FOUND)
-    mark_as_advanced(ImageMagick_EXECUTABLE_DIR)
-
     find_package(Doxygen)
     if (NOT DOXYGEN_FOUND)
       # We require on Linux pre-commit where it's trivial to install
@@ -1755,48 +1736,6 @@ if (CLIENT_INTERFACE OR APP_EXPORTS)
       endif ()
     endif (NOT DOXYGEN_FOUND)
 
-    include(${PROJECT_SOURCE_DIR}/make/FindFig2Dev.cmake)
-    if (NOT FIG2DEV_FOUND)
-      # We require on Linux pre-commit where it's trivial to install
-      if (TEST_SUITE AND UNIX AND NOT APPLE)
-        message(FATAL_ERROR "fig2dev (from transfig) is required to build documentation")
-      else ()
-        # Non-fatal for a single, un-official build, or on Windows
-        message(WARNING "fig2dev (from transfig) not found: docs will NOT be built")
-        set(BUILD_DOCS OFF)
-      endif ()
-    endif (NOT FIG2DEV_FOUND)
-
-    # fig2dev needs ghostscript to produce tiff files (not to produce eps)
-    include(${PROJECT_SOURCE_DIR}/make/FindGhostscript.cmake)
-    if (NOT GHOSTSCRIPT_FOUND)
-      # We require on Linux pre-commit where it's trivial to install
-      if (TEST_SUITE AND UNIX AND NOT APPLE)
-        message(FATAL_ERROR "ghostscript is required to build documentation")
-      else ()
-        # Non-fatal for a single, un-official build, or on Windows
-        message(WARNING "ghostscript not found: documentation will NOT be built")
-        set(BUILD_DOCS OFF)
-      endif ()
-    else (NOT GHOSTSCRIPT_FOUND)
-      # must be on path.  we can't just literally check the path
-      # b/c of cygwin issues: c:/cygwin/bin vs /usr/bin, etc.
-      get_filename_component(gs_name ${GHOSTSCRIPT_EXECUTABLE} NAME)
-      find_program(GHOSTSCRIPT_ON_PATH ${gs_name} PATHS ENV PATH NO_DEFAULT_PATH)
-      if (NOT GHOSTSCRIPT_ON_PATH)
-        # We require on Linux pre-commit where it's trivial to install
-        if (TEST_SUITE AND UNIX AND NOT APPLE)
-          message(FATAL_ERROR
-            "ghostscript is installed, but must be on PATH for fig2dev to find it")
-        else ()
-          # Non-fatal for a single, un-official build, or on Windows
-          message(WARNING "ghostscript not on PATH: documentation will NOT be built")
-          set(BUILD_DOCS OFF)
-        endif ()
-      endif (NOT GHOSTSCRIPT_ON_PATH)
-      mark_as_advanced(GHOSTSCRIPT_ON_PATH)
-    endif (NOT GHOSTSCRIPT_FOUND)
-
     if (BUILD_DOCS)
       add_subdirectory(api/docs)
     else (BUILD_DOCS)

diff --git a/api/docs/API.doxy b/api/docs/API.doxy
@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2012-2018 Google, Inc.  All rights reserved.
+# Copyright (c) 2012-2020 Google, Inc.  All rights reserved.
 # Copyright (c) 2007-2010 VMware, Inc.  All rights reserved.
 # **********************************************************/
 
@@ -220,7 +220,6 @@ TAGFILES               =
 GENERATE_TAGFILE       =
 ALLEXTERNALS           = NO
 EXTERNAL_GROUPS        = YES
-PERL_PATH              = /usr/bin/perl
 #---------------------------------------------------------------------------
 # Configuration options related to the dot tool
 #---------------------------------------------------------------------------

diff --git a/api/docs/CMakeLists.txt b/api/docs/CMakeLists.txt
@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2010-2017 Google, Inc.    All rights reserved.
+# Copyright (c) 2010-2020 Google, Inc.    All rights reserved.
 # Copyright (c) 2009-2010 VMware, Inc.    All rights reserved.
 # **********************************************************
 
@@ -29,36 +29,13 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 # DAMAGE.
 
-# top-level CMakeLists.txt finds the packages we need
-
-file(GLOB figures_fig *.fig)
-
-set(genimg_dirname ${CMAKE_CURRENT_BINARY_DIR}/genimages)
-file(MAKE_DIRECTORY ${genimg_dirname})
-
-if (WIN32 AND "${ImageMagick_mogrify_EXECUTABLE}" MATCHES "cygwin")
-  # i#100: cygwin mogrify mis-parses mixed paths when the drive is a,
-  # b, c, or g (haven't tracked down why those particular letters):
-  #   % mogrify -resize 33% -format png c:/client.tiff[0]
-  #   mogrify: Must specify image size `/client.tiff'.
-  find_program(CYGPATH cygpath) # XXX: should share w/ Cmake_doxyfile.cmake
-  if (NOT CYGPATH)
-    message(FATAL_ERROR "cannot find cygpath: thus cannot use cygwin ImageMagick")
-  endif (NOT CYGPATH)
-  execute_process(COMMAND
-    ${CYGPATH} -u "${genimg_dirname}"
-    RESULT_VARIABLE cygpath_result
-    ERROR_VARIABLE cygpath_err
-    OUTPUT_VARIABLE genimg_dirname_IM
-    )
-  if (cygpath_result OR cygpath_err)
-    message(FATAL_ERROR "*** ${CYGPATH} failed: ***\n${cygpath_err}")
-  endif (cygpath_result OR cygpath_err)
-  string(REGEX REPLACE "[\r\n]" "" genimg_dirname_IM "${genimg_dirname_IM}")
-else (WIN32 AND "${ImageMagick_mogrify_EXECUTABLE}" MATCHES "cygwin")
-  set(genimg_dirname_IM "${genimg_dirname}")
-endif (WIN32 AND "${ImageMagick_mogrify_EXECUTABLE}" MATCHES "cygwin")
+# We used to transform our .fig figures from FIG format to PNG or EPS as
+# part of the build process, using fig2dev, ImageMagick's mogrify, and
+# ghostscript.  However, we have not added new figures in years, and it is
+# painful to install and work with these packages on Windows (i#3235).
+# We thus instead require figures to have a checked-in PNG or EPS file.
 
+# Notes for new FIG-based figures:
 # Because fig2dev does not perform anti-aliasing, converting directly
 # to an image format can result in very poor quality lines and text.
 # This problem is compounded if that image is subsequently resized
@@ -68,42 +45,12 @@ endif (WIN32 AND "${ImageMagick_mogrify_EXECUTABLE}" MATCHES "cygwin")
 # image manipulator resize it to the final size (cannot make
 # browser-resizable vector graphics).
 # See also http://www.burningcutlery.com/derek/bargraph/
-foreach (fig ${figures_fig})
-  get_filename_component(base ${fig} NAME_WE)
-  add_custom_command(
-    OUTPUT ${genimg_dirname}/${base}.eps
-    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${base}.fig
-    # fig2dev doesn't quote gs output path; set working dir to avoid spaces
-    COMMAND ${FIG2DEV_EXECUTABLE}
-    ARGS -L eps ${CMAKE_CURRENT_SOURCE_DIR}/${base}.fig ${base}.eps
-    WORKING_DIRECTORY ${genimg_dirname}
-    VERBATIM # recommended: p260
-    )
-  # To avoid keeping the intermediate .tiff files around, which are quite
-  # large (50MB total, nearly half the space in a default build dir), we
-  # do not list as a separate custom command target but build inside
-  # a sequence of commands:
-  add_custom_command(
-    OUTPUT ${genimg_dirname}/${base}.png
-    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${base}.fig
-    # First build a tiff
-    COMMAND ${FIG2DEV_EXECUTABLE}
-      ARGS -L tiff -m 4 ${CMAKE_CURRENT_SOURCE_DIR}/${base}.fig ${base}.tiff
-    # Second, build a png from the tiff
-    COMMAND ${ImageMagick_mogrify_EXECUTABLE}
-      # fig2dev produces multi-part image so ask for just 1st
-      ARGS -resize 33% -format png ${genimg_dirname_IM}/${base}.tiff[0]
-    # Third, delete the tiff
-    COMMAND ${CMAKE_COMMAND}
-      ARGS -E remove ${genimg_dirname}/${base}.tiff
-    # fig2dev doesn't quote gs output path; set working dir to avoid spaces
-    WORKING_DIRECTORY ${genimg_dirname}
-    VERBATIM # recommended: p260
-    )
-  # FYI: for VMSAFE we did: $(SED) 's|DynamoRIO|${VMSAFE_NAME}|g'
-  set(figures_png ${figures_png} ${genimg_dirname}/${base}.png)
-  set(figures_eps ${figures_eps} ${genimg_dirname}/${base}.eps)
-endforeach (fig)
+
+file(GLOB figures_fig *.fig)
+
+set(genimg_dirname ${CMAKE_CURRENT_BINARY_DIR}/genimages)
+# This will create the genimages/ subdir for us and populate it:
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/genimages DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
 
 function (gendox_to_dox gendox_in gendox_out path master_list)
   file(GLOB dox_files "${path}/*.dox" "${path}/*.dox.in")

diff --git a/api/docs/CMake_doxyfile.cmake b/api/docs/CMake_doxyfile.cmake
@@ -154,6 +154,10 @@ file(WRITE ${outfile} "${string}")
 
 # Now update to latest doxygen.  Suppress warnings since they're misleading:
 # they say "please run doxygen -u" but we're currently doing just that.
+# Newer doxygen complains if header.html and footer.html are missing and we
+# don't create those until CMake_rundoxygen.cmake.
+file(WRITE header.html "placeholder")
+file(WRITE footer.html "placeholder")
 execute_process(COMMAND
   ${DOXYGEN_EXECUTABLE} -u
   RESULT_VARIABLE doxygen_u_result

diff --git a/api/docs/CMake_doxyutils.cmake b/api/docs/CMake_doxyutils.cmake
@@ -1,5 +1,5 @@
 # **********************************************************
-# Copyright (c) 2012 Google, Inc.    All rights reserved.
+# Copyright (c) 2012-2020 Google, Inc.    All rights reserved.
 # **********************************************************
 
 # Redistribution and use in source and binary forms, with or without
@@ -73,15 +73,15 @@ function (doxygen_path_xform DOXYGEN_EXECUTABLE list_of_path_vars)
   if (WIN32)
     # We can't rely on the presence of "cygwin" in the path to indicate this
     # is a cygwin executable, as some users place native-Windows apps in
-    # c:/cygwin/usr/local/bin.
+    # c:/cygwin/usr/local/bin.  We thus try to write to a cygwin path.
+    # Doxygen will happily clobber if our temp file already exists.
     execute_process(COMMAND
-      ${DOXYGEN_EXECUTABLE} --help
-      RESULT_VARIABLE doxygen_h_result
-      ERROR_VARIABLE doxygen_h_error
-      OUTPUT_VARIABLE doxygen_h_out
+      ${DOXYGEN_EXECUTABLE} -g /tmp/_test_doxygen_.dox
+      RESULT_VARIABLE doxygen_query_result
+      ERROR_VARIABLE doxygen_query_error
+      OUTPUT_VARIABLE doxygen_query_out
       )
-    string(REGEX MATCH " /" is_cygwin "${doxygen_h_out}")
-    if (is_cygwin)
+    if (NOT doxygen_query_result)
       # Cygwin doxygen cannot handle mixed paths!
       #    *** E:/cygwin/bin/doxygen.exe failed: ***
       #    Failed to open temporary file
@@ -93,7 +93,7 @@ function (doxygen_path_xform DOXYGEN_EXECUTABLE list_of_path_vars)
       find_program(CYGPATH cygpath)
       if (NOT CYGPATH)
         message(FATAL_ERROR "cannot find cygpath: thus cannot use cygwin doxygen")
-      endif (NOT CYGPATH)
+      endif ()
       foreach (var ${list_of_path_vars})
         execute_process(COMMAND
           ${CYGPATH} -u "${${var}}"
@@ -103,12 +103,12 @@ function (doxygen_path_xform DOXYGEN_EXECUTABLE list_of_path_vars)
           )
         if (cygpath_result OR cygpath_err)
           message(FATAL_ERROR "*** ${CYGPATH} failed: ***\n${cygpath_err}")
-        endif (cygpath_result OR cygpath_err)
+        endif ()
         string(REGEX REPLACE "[\r\n]" "" ${var} "${${var}}")
         set(${var} ${${var}} PARENT_SCOPE)
-      endforeach (var)
-    endif (is_cygwin)
-  endif (WIN32)
+      endforeach ()
+    endif ()
+  endif ()
 endfunction (doxygen_path_xform)
 
 # Modifies doxyfile to work with docs_rundoxygen.cmake