random fixes + support clang-cl on visual studio (#1129)

* delete unused shaders * hide some options in debug menu * change fullscreen logic a bit * add "all actors" toggle * borderless fix and fix alpha in direct renderer untextured (do we need a separate shader for that?) * fix fuel cell orbit icons in widescreen * fix `curve` types * refs * fix levitator task... * fix some task stuff * update font code a bit (temp) * cmake, third-party and visual studio overhaul * Update .gitmodules * update modules * clone repos * fix encoding in zydis * where did these come from * try again * add submodule * Update 11zip * Update 11zip * Update 11zip * delete * try again * clang * update compiler flags * delete 11zip. go away. * Create memory-dump-p2s.py * properly * fix minimum architecture c++ compiler flags * fix zydis * oops * Update all-types.gc * fix clang-cl tests * make "all actors" work better, entity debug qol * update game-text conversion code to be more modularized * Create vendor.txt * fix typos and minor things * update refs * clang * Attempt to add clang-cl support to vs2019 and CI * vs2022 + clang-cl * srsly? fix clang build * Update launch.vs.json * extend windows CI timer
open-goal · Feb 8, 2022 · 5b44aec · 5b44aec
1 parent 1bf6972
commit 5b44aec
Show file tree

Hide file tree

Showing 1,120 changed files with 335,783 additions and 29,283 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -5,6 +5,7 @@ indent_style = space
 indent_size = 2
 trim_trailing_whitespace = true
 insert_final_newline = true
+charset = utf-8
 
 # https://editorconfig-specification.readthedocs.io/
 # https://docs.microsoft.com/en-us/visualstudio/ide/cpp-editorconfig-properties?view=vs-2019

diff --git a/.github/workflows/windows-workflow.yaml b/.github/workflows/windows-workflow.yaml
@@ -16,15 +16,16 @@ jobs:
       # Prevent one build from failing everything (although maybe those should be included as experimental builds instead)
       fail-fast: false
       matrix:
-        os: [windows-2019]
+        os: [windows-2022]
         config: [Release]
+        compiler: [msvc, clang]
         experimental: [false]
 
-    name: ${{ matrix.config }}
+    name: ${{ matrix.config }}-${{ matrix.compiler }}
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.experimental }}
     # Set some sort of timeout in the event of run-away builds.  We are limited on concurrent jobs so, get rid of them.
-    timeout-minutes: 20
+    timeout-minutes: 25  # 5 more minutes than linux
 
     env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
       BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
@@ -53,25 +54,31 @@ jobs:
       - name: Setup Buildcache
         uses: mikehardy/buildcache-action@v1.2.2
         with:
-          cache_key: ${{ matrix.os }}-${{ matrix.config }}
+          cache_key: ${{ matrix.os }}-${{ matrix.config }}-${{ matrix.compiler }}
 
       - name: CMake Generation
         shell: cmd # ideally id like everything to be powershell but running this bat file is finicky
         run: |
-          call "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Auxiliary/Build/vcvars64.bat"
-          cmake -B build -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache.exe -DCMAKE_CXX_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache.exe .
+          if "${{ matrix.compiler }}" == "clang" (
+            call "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvars64.bat"
+            cmake -B build -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache.exe -DCMAKE_CXX_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache.exe "-DCMAKE_C_COMPILER=C:/Program Files/LLVM/bin/clang-cl.exe" "-DCMAKE_CXX_COMPILER=C:/Program Files/LLVM/bin/clang-cl.exe" .
+          ) else (
+            call "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvars64.bat"
+            cmake -B build -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache.exe -DCMAKE_CXX_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache.exe .
+          )
 
       - name: Build Project
         working-directory: ./build
         shell: cmd
         run: |
-          call "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Auxiliary/Build/vcvars64.bat"
-          set CL=/MP
+          call "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvars64.bat"
+          if "${{ matrix.compiler }}" == "msvc" (
+            set CL=/MP
+          )
           nmake
 
       - name: Run Tests
         timeout-minutes: 5
         env:
           NEXT_DIR: ${{ github.workspace }}
-          FAKE_ISO_PATH: ${{ github.workspace }}/game/fake_iso.txt
         run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_filter="-*MANUAL_TEST*"
diff --git a/.gitmodules b/.gitmodules
@@ -1,9 +0,0 @@
-[submodule "third-party/googletest"]
-	path = third-party/googletest
-	url = https://github.com/google/googletest.git
-[submodule "third-party/zydis"]
-	path = third-party/zydis
-	url = https://github.com/zyantific/zydis.git
-[submodule "third-party/glfw"]
-	path = third-party/glfw
-	url = https://github.com/glfw/glfw

diff --git a/.vs/launch.vs.json b/.vs/launch.vs.json
@@ -68,31 +68,28 @@
       "project" : "CMakeLists.txt",
       "projectTarget" : "goalc.exe (bin\\goalc.exe)",
       "name" : "Run - REPL",
-      "args" : [ "-auto-lt" ],
-      "env": {
-        "OPENGOAL_DECOMP_DIR": "jak1/"
-      }
+      "args" : [ "-auto-lt" ]
     },
     {
       "type" : "default",
       "project" : "CMakeLists.txt",
       "projectTarget" : "decompiler.exe (bin\\decompiler.exe)",
       "name" : "Run - Decompiler - Jak 1",
-      "args" : [ "${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc", "${workspaceRoot}/iso_data/jak1", "${workspaceRoot}/decompiler_out/jak1"]
+      "args" : [ "${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc", "${workspaceRoot}/iso_data", "${workspaceRoot}/decompiler_out"]
     },
     {
       "type" : "default",
       "project" : "CMakeLists.txt",
       "projectTarget" : "decompiler.exe (bin\\decompiler.exe)",
       "name" : "Run - Decompiler - Jak 1 PAL",
-      "args" : [ "${workspaceRoot}/decompiler/config/jak1_pal.jsonc", "${workspaceRoot}/iso_data/jak1pal", "${workspaceRoot}/decompiler_out/jak1pal"]
+      "args" : [ "${workspaceRoot}/decompiler/config/jak1_pal.jsonc", "${workspaceRoot}/iso_data", "${workspaceRoot}/decompiler_out"]
     },
     {
       "type" : "default",
       "project" : "CMakeLists.txt",
       "projectTarget" : "decompiler.exe (bin\\decompiler.exe)",
       "name" : "Run - Disassembler - Jak 1",
-      "args" : [ "${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc", "${workspaceRoot}/iso_data/jak1", "${workspaceRoot}/decompiler_out/jak1_ir1"]
+      "args" : [ "${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc", "${workspaceRoot}/iso_data", "${workspaceRoot}/decompiler_out/jak1_ir1"]
     },
     {
       "type" : "default",

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,18 +1,36 @@
 # Top Level CMakeLists.txt
 cmake_minimum_required(VERSION 3.10)
 
+set(CMAKE_CXX_STANDARD 17)
 project(jak)
 include(CTest)
+
 if(NOT CMAKE_BUILD_TYPE)
-    set(CMAKE_BUILD_TYPE "Debug")
+    set(CMAKE_BUILD_TYPE "Release")
 endif()
 
-set(CMAKE_CXX_STANDARD 17)
+# Set default compile flags
+# optimization level can be set here. You can overwrite these in a per-project basis if you want.
+if(MSVC AND (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
+    message(STATUS "Clang on MSVC detected! Adding compile flags")
+    set(CMAKE_CXX_FLAGS
+        "-Xclang -fcxx-exceptions \
+        -Xclang -fexceptions \
+        -Xclang -std=c++17 \
+        -mfma -mavx2 \
+        -Wno-c++11-narrowing -W3 \
+        /arch:AVX")
+
+    # additional c++ flags for release mode for our projects
+    if(CMAKE_BUILD_TYPE MATCHES "Release")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2")
+    endif()
+
+    # linker flags
+    set(CMAKE_EXE_LINKER_FLAGS "/STACK:16000000,16384")
 
-# Set default compile flags for GCC
-# optimization level can be set here. Note that game/ overwrites this for building game C++ code.
-if(UNIX)
-    message(STATUS "GCC detected, adding compile flags")
+elseif(UNIX)
+    message(STATUS "GCC detected! Adding compile flags")
     set(CMAKE_CXX_FLAGS
         "${CMAKE_CXX_FLAGS} \
         -Wall \
@@ -28,16 +46,27 @@ if(UNIX)
         -Wredundant-decls \
         -Wshadow \
         -Wsign-promo \
-        -fdiagnostics-color=always")
-else()
+        -fdiagnostics-color=always \
+        -march=haswell")
+
+    # additional c++ flags for release mode for our projects
+    if(CMAKE_BUILD_TYPE MATCHES "Release")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
+    endif()
+
+elseif(MSVC)
+    message(STATUS "MSVC detected!! Adding compile flags")
     if(CMAKE_BUILD_TYPE MATCHES "Debug")
       # This actually breaks some standard library things for some reason?
       # message("Setting Flags to Enable Edit and Continue")
       # set(CMAKE_CXX_FLAGS_DEBUG "/ZI")
     endif()
-    set(CMAKE_CXX_FLAGS "/EHsc")
-    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:16000000,16384")
-endif(UNIX)
+    # c++ flags for all build types
+    set(CMAKE_CXX_FLAGS "/EHsc /utf-8 /arch:AVX")
+
+    # linker flags
+    set(CMAKE_EXE_LINKER_FLAGS "/STACK:16000000,16384")
+endif()
 
 if(WIN32)
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
@@ -100,9 +129,6 @@ add_subdirectory(third-party/glfw)
 
 add_subdirectory(third-party/zstd)
 
-# build libzip
-add_subdirectory(third-party/11zip EXCLUDE_FROM_ALL)
-
 # build imgui
 include_directories(third-party/glad/include)
 include_directories(third-party/glfw/include)

diff --git a/CMakePresets.json b/CMakePresets.json
@@ -1,4 +1,4 @@
-{
+{
   "version": 2,
   "configurePresets": [
     {
@@ -17,11 +17,49 @@
       }
     },
     {
-      "name": "Debug",
-      "displayName": "Windows x64 Debug",
+      "name": "Debug-clang",
+      "displayName": "Windows Debug (clang-cl)",
       "description": "Target Windows with the Visual Studio development environment.",
       "generator": "Ninja",
-      "binaryDir": "${sourceDir}/out/build/${presetName}",
+      "binaryDir": "${sourceDir}/out/build/Debug",
+      "architecture": {
+        "value": "x64",
+        "strategy": "external"
+      },
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
+        "INSTALL_GTEST": "True",
+        "CMAKE_C_COMPILER": "clang-cl",
+        "CMAKE_CXX_COMPILER": "clang-cl"
+      },
+      "vendor": { "microsoft.com/VisualStudioSettings/CMake/1.0": { "hostOS": [ "Windows" ] } }
+    },
+    {
+      "name": "Release-clang",
+      "displayName": "Windows Release (clang-cl)",
+      "description": "Target Windows with the Visual Studio development environment.",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/out/build/Release",
+      "architecture": {
+        "value": "x64",
+        "strategy": "external"
+      },
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
+        "INSTALL_GTEST": "True",
+        "CMAKE_C_COMPILER": "clang-cl",
+        "CMAKE_CXX_COMPILER": "clang-cl"
+      },
+      "vendor": { "microsoft.com/VisualStudioSettings/CMake/1.0": { "hostOS": [ "Windows" ] } }
+    },
+    {
+      "name": "Debug-msvc",
+      "displayName": "Windows Debug (msvc)",
+      "description": "Target Windows with the Visual Studio development environment.",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/out/build/Debug",
       "architecture": {
         "value": "x64",
         "strategy": "external"
@@ -34,11 +72,11 @@
       "vendor": { "microsoft.com/VisualStudioSettings/CMake/1.0": { "hostOS": [ "Windows" ] } }
     },
     {
-      "name": "Release",
-      "displayName": "Windows x64 Release",
+      "name": "Release-msvc",
+      "displayName": "Windows Release (msvc)",
       "description": "Target Windows with the Visual Studio development environment.",
       "generator": "Ninja",
-      "binaryDir": "${sourceDir}/out/build/${presetName}",
+      "binaryDir": "${sourceDir}/out/build/Release",
       "architecture": {
         "value": "x64",
         "strategy": "external"

diff --git a/CMakeSettings.json b/CMakeSettings.json
@@ -1,42 +1,40 @@
 {
-	"configurations": [
-		{
-			"name": "Debug",
-			"generator": "Ninja",
-			"configurationType": "Debug",
-			"inheritEnvironments": [ "msvc_x64_x64" ],
-			"buildRoot": "${projectDir}\\out\\build\\${name}",
-			"installRoot": "${projectDir}\\out\\install\\${name}",
-			"cmakeCommandArgs": "",
-			"buildCommandArgs": "",
-			"addressSanitizerEnabled": false,
-			"ctestCommandArgs": "",
-			"variables": [
-				{
-					"name": "INSTALL_GTEST",
-					"value": "True",
-					"type": "BOOL"
-				}
-			]
-		},
-		{
-			"name": "Release",
-			"generator": "Ninja",
-			"configurationType": "Release",
-			"buildRoot": "${projectDir}\\out\\build\\${name}",
-			"installRoot": "${projectDir}\\out\\install\\${name}",
-			"cmakeCommandArgs": "",
-			"buildCommandArgs": "",
-			"addressSanitizerEnabled": false,
-			"ctestCommandArgs": "",
-			"inheritEnvironments": [ "msvc_x64_x64" ],
-			"variables": [
-				{
-					"name": "INSTALL_GTEST",
-					"value": "True",
-					"type": "BOOL"
-				}
-			]
-		}
-	]
+  "configurations": [
+    {
+      "name": "Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${configurationType}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "addressSanitizerEnabled": false
+    },
+    {
+      "name": "Release",
+      "generator": "Ninja",
+      "configurationType": "Release",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${configurationType}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "addressSanitizerEnabled": false
+    },
+    {
+      "name": "Debug (clang-cl)",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "clang_cl_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${configurationType}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "addressSanitizerEnabled": false
+    },
+    {
+      "name": "Release (clang-cl)",
+      "generator": "Ninja",
+      "configurationType": "Release",
+      "inheritEnvironments": [ "clang_cl_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${configurationType}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "addressSanitizerEnabled": false
+    }
+  ]
 }
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
@@ -51,9 +51,3 @@ if(WIN32)
 else()
     target_link_libraries(common stdc++fs)
 endif()
-
-if(UNIX)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2")
-elseif(WIN32)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2")
-endif()
diff --git a/common/goos/Reader.cpp b/common/goos/Reader.cpp
@@ -167,17 +167,19 @@ Reader::Reader() {
   m_valid_source_text_chars[(int)'\r'] = true;
 
   // allow every character that gets transformed to something else
-  for (auto& remap : g_font_large_char_remap) {
-    for (auto rc : remap.chars) {
-      m_valid_source_text_chars[(u8)rc] = true;
-    }
-  }
-  for (auto& remap : g_font_large_string_replace) {
-    for (auto rc : remap.to) {
-      m_valid_source_text_chars[(u8)rc] = true;
+  for (auto& [version, font] : g_font_banks) {
+    for (auto& remap : *font->encode_info()) {
+      for (auto rc : remap.chars) {
+        m_valid_source_text_chars[(u8)rc] = true;
+      }
     }
-    for (auto rc : remap.from) {
-      m_valid_source_text_chars[(u8)rc] = true;
+    for (auto& remap : *font->replace_info()) {
+      for (auto rc : remap.to) {
+        m_valid_source_text_chars[(u8)rc] = true;
+      }
+      for (auto rc : remap.from) {
+        m_valid_source_text_chars[(u8)rc] = true;
+      }
     }
   }
   m_valid_source_text_chars[0] = false;