fix jssrc astgen invocation on linux/aarch64 (#4671)

* fix jssrc astgen invocation on linux/aarch64 I don't like the amount of code we have there that pretends as if "ARM" is a single architecture... so I've moved things in a slightly better direction renaming some constants to ARMv8. But I'm not going to fix that all in one day. * scalafmt * add platforms to file name variables
joernio · Jun 17, 2024 · 0bd63ad · 0bd63ad
1 parent a65a3b2
commit 0bd63ad
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 51 deletions.
diff --git a/joern-cli/frontends/csharpsrc2cpg/build.sbt b/joern-cli/frontends/csharpsrc2cpg/build.sbt
@@ -20,8 +20,8 @@ lazy val astGenVersion = settingKey[String]("dotnetastgen version")
 astGenVersion := appProperties.value.getString("csharpsrc2cpg.dotnetastgen_version")
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"              %% "codepropertygraph" % Versions.cpg,
-  "org.scalatest"             %% "scalatest"         % Versions.scalatest % Test
+  "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
+  "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 
 Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value)
@@ -60,8 +60,8 @@ astGenBinaryNames := {
       case Environment.OperatingSystemType.Windows => Seq(AstgenWin)
       case Environment.OperatingSystemType.Linux =>
         Environment.architecture match {
-          case Environment.ArchitectureType.X86 => Seq(AstgenLinux)
-          case Environment.ArchitectureType.ARM => Seq(AstgenLinuxArm)
+          case Environment.ArchitectureType.X86   => Seq(AstgenLinux)
+          case Environment.ArchitectureType.ARMv8 => Seq(AstgenLinuxArm)
         }
       case Environment.OperatingSystemType.Mac => Seq(AstgenMac)
       case Environment.OperatingSystemType.Unknown =>

diff --git a/joern-cli/frontends/gosrc2cpg/build.sbt b/joern-cli/frontends/gosrc2cpg/build.sbt
@@ -9,9 +9,9 @@ name := "gosrc2cpg"
 dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"              %% "codepropertygraph" % Versions.cpg,
-  "org.scalatest"             %% "scalatest"         % Versions.scalatest % Test,
-  "com.lihaoyi"               %% "os-lib"            % Versions.osLib
+  "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
+  "org.scalatest" %% "scalatest"         % Versions.scalatest % Test,
+  "com.lihaoyi"   %% "os-lib"            % Versions.osLib
 )
 
 scalacOptions ++= Seq(
@@ -59,13 +59,13 @@ goAstGenBinaryNames := {
         Seq(GoAstgenWin)
       case Environment.OperatingSystemType.Linux =>
         Environment.architecture match {
-          case Environment.ArchitectureType.X86 => Seq(GoAstgenLinux)
-          case Environment.ArchitectureType.ARM => Seq(GoAstgenLinuxArm)
+          case Environment.ArchitectureType.X86   => Seq(GoAstgenLinux)
+          case Environment.ArchitectureType.ARMv8 => Seq(GoAstgenLinuxArm)
         }
       case Environment.OperatingSystemType.Mac =>
         Environment.architecture match {
-          case Environment.ArchitectureType.X86 => Seq(GoAstgenMac)
-          case Environment.ArchitectureType.ARM => Seq(GoAstgenMacArm)
+          case Environment.ArchitectureType.X86   => Seq(GoAstgenMac)
+          case Environment.ArchitectureType.ARMv8 => Seq(GoAstgenMacArm)
         }
       case Environment.OperatingSystemType.Unknown =>
         Seq(GoAstgenWin, GoAstgenLinux, GoAstgenLinuxArm, GoAstgenMac, GoAstgenMacArm)

diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala
@@ -34,9 +34,9 @@ class AstGenRunner(config: Config) extends AstGenRunnerBase(config) {
   override val SupportedBinaries: Set[(OperatingSystemType, ArchitectureType)] = Set(
     Environment.OperatingSystemType.Windows -> Environment.ArchitectureType.X86,
     Environment.OperatingSystemType.Linux   -> Environment.ArchitectureType.X86,
-    Environment.OperatingSystemType.Linux   -> Environment.ArchitectureType.ARM,
+    Environment.OperatingSystemType.Linux   -> Environment.ArchitectureType.ARMv8,
     Environment.OperatingSystemType.Mac     -> Environment.ArchitectureType.X86,
-    Environment.OperatingSystemType.Mac     -> Environment.ArchitectureType.ARM
+    Environment.OperatingSystemType.Mac     -> Environment.ArchitectureType.ARMv8
   )
 
   override def skippedFiles(in: File, astGenOut: List[String]): List[String] = {

diff --git a/joern-cli/frontends/jssrc2cpg/build.sbt b/joern-cli/frontends/jssrc2cpg/build.sbt
@@ -18,8 +18,8 @@ lazy val astGenVersion = settingKey[String]("astgen version")
 astGenVersion := appProperties.value.getString("jssrc2cpg.astgen_version")
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"              %% "codepropertygraph" % Versions.cpg,
-  "org.scalatest"             %% "scalatest"         % Versions.scalatest % Test
+  "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
+  "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 
 Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value)
@@ -29,10 +29,11 @@ Test / fork := false
 
 enablePlugins(JavaAppPackaging, LauncherJarPlugin)
 
-lazy val AstgenWin    = "astgen-win.exe"
-lazy val AstgenLinux  = "astgen-linux"
-lazy val AstgenMac    = "astgen-macos"
-lazy val AstgenMacArm = "astgen-macos-arm"
+lazy val AstgenWinAmd64   = "astgen-win.exe"
+lazy val AstgenLinuxAmd64 = "astgen-linux"
+lazy val AstgenLinuxArmV8 = "astgen-linux-arm"
+lazy val AstgenMacAmd64   = "astgen-macos"
+lazy val AstgenMacArmV8   = "astgen-macos-arm"
 
 lazy val astGenDlUrl = settingKey[String]("astgen download url")
 astGenDlUrl := s"https://github.com/joernio/astgen/releases/download/v${astGenVersion.value}/"
@@ -50,20 +51,15 @@ astGenBinaryNames := {
   if (hasCompatibleAstGenVersion(astGenVersion.value)) {
     Seq.empty
   } else if (sys.props.get("ALL_PLATFORMS").contains("TRUE")) {
-    Seq(AstgenWin, AstgenLinux, AstgenMac, AstgenMacArm)
+    Seq(AstgenWinAmd64, AstgenLinuxAmd64, AstgenLinuxArmV8, AstgenMacAmd64, AstgenMacArmV8)
   } else {
-    Environment.operatingSystem match {
-      case Environment.OperatingSystemType.Windows =>
-        Seq(AstgenWin)
-      case Environment.OperatingSystemType.Linux =>
-        Seq(AstgenLinux)
-      case Environment.OperatingSystemType.Mac =>
-        Environment.architecture match {
-          case Environment.ArchitectureType.X86 => Seq(AstgenMac)
-          case Environment.ArchitectureType.ARM => Seq(AstgenMacArm)
-        }
-      case Environment.OperatingSystemType.Unknown =>
-        Seq(AstgenWin, AstgenLinux, AstgenMac, AstgenMacArm)
+    (Environment.operatingSystem, Environment.architecture) match {
+      case (Environment.OperatingSystemType.Windows, _)                                => Seq(AstgenWinAmd64)
+      case (Environment.OperatingSystemType.Linux, Environment.ArchitectureType.X86)   => Seq(AstgenLinuxAmd64)
+      case (Environment.OperatingSystemType.Linux, Environment.ArchitectureType.ARMv8) => Seq(AstgenLinuxArmV8)
+      case (Environment.OperatingSystemType.Mac, Environment.ArchitectureType.X86)     => Seq(AstgenMacAmd64)
+      case (Environment.OperatingSystemType.Mac, Environment.ArchitectureType.ARMv8)   => Seq(AstgenMacArmV8)
+      case _ => Seq(AstgenWinAmd64, AstgenLinuxAmd64, AstgenLinuxArmV8, AstgenMacAmd64, AstgenMacArmV8)
     }
   }
 }

diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala
@@ -95,16 +95,14 @@ object AstGenRunner {
     case _                              => None
   }
 
-  lazy private val executableName = Environment.operatingSystem match {
-    case Environment.OperatingSystemType.Windows => "astgen-win.exe"
-    case Environment.OperatingSystemType.Linux   => "astgen-linux"
-    case Environment.OperatingSystemType.Mac =>
-      Environment.architecture match {
-        case Environment.ArchitectureType.X86 => "astgen-macos"
-        case Environment.ArchitectureType.ARM => "astgen-macos-arm"
-      }
-    case Environment.OperatingSystemType.Unknown =>
-      logger.warn("Could not detect OS version! Defaulting to 'Linux'.")
+  lazy private val executableName = (Environment.operatingSystem, Environment.architecture) match {
+    case (Environment.OperatingSystemType.Windows, _)                                => "astgen-win.exe"
+    case (Environment.OperatingSystemType.Linux, Environment.ArchitectureType.X86)   => "astgen-linux"
+    case (Environment.OperatingSystemType.Linux, Environment.ArchitectureType.ARMv8) => "astgen-linux-arm"
+    case (Environment.OperatingSystemType.Mac, Environment.ArchitectureType.X86)     => "astgen-macos"
+    case (Environment.OperatingSystemType.Mac, Environment.ArchitectureType.ARMv8)   => "astgen-macos-arm"
+    case _ =>
+      logger.warn("Could not detect OS version! Defaulting to Linux/x86_64.")
       "astgen-linux"
   }
 

diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala
@@ -98,11 +98,11 @@ trait AstGenRunnerBase(config: X2CpgConfig[?] & AstGenConfig[?]) {
     */
   protected val SupportedBinaries: Set[(OperatingSystemType, ArchitectureType)] = Set(
     Environment.OperatingSystemType.Windows -> Environment.ArchitectureType.X86,
-    Environment.OperatingSystemType.Windows -> Environment.ArchitectureType.ARM,
+    Environment.OperatingSystemType.Windows -> Environment.ArchitectureType.ARMv8,
     Environment.OperatingSystemType.Linux   -> Environment.ArchitectureType.X86,
-    Environment.OperatingSystemType.Linux   -> Environment.ArchitectureType.ARM,
+    Environment.OperatingSystemType.Linux   -> Environment.ArchitectureType.ARMv8,
     Environment.OperatingSystemType.Mac     -> Environment.ArchitectureType.X86,
-    Environment.OperatingSystemType.Mac     -> Environment.ArchitectureType.ARM
+    Environment.OperatingSystemType.Mac     -> Environment.ArchitectureType.ARMv8
   )
 
   /** Determines the name of the executable to run, based on the host system. Usually, AST GEN binaries support three
@@ -129,8 +129,8 @@ trait AstGenRunnerBase(config: X2CpgConfig[?] & AstGenConfig[?]) {
       s"${metaData.name}-$x86Suffix"
     } else {
       Environment.architecture match {
-        case Environment.ArchitectureType.X86 => s"${metaData.name}-$x86Suffix"
-        case Environment.ArchitectureType.ARM => s"${metaData.name}-$armSuffix"
+        case Environment.ArchitectureType.X86   => s"${metaData.name}-$x86Suffix"
+        case Environment.ArchitectureType.ARMv8 => s"${metaData.name}-$armSuffix"
       }
     }
   }

diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/Environment.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/Environment.scala
@@ -15,7 +15,7 @@ object Environment {
   object ArchitectureType extends Enumeration {
     type ArchitectureType = Value
 
-    val X86, ARM = Value
+    val X86, ARMv8 = Value
   }
 
   lazy val operatingSystem: OperatingSystemType.OperatingSystemType =
@@ -25,7 +25,7 @@ object Environment {
     else OperatingSystemType.Unknown
 
   lazy val architecture: ArchitectureType.ArchitectureType =
-    if (scala.util.Properties.propOrNone("os.arch").contains("aarch64")) ArchitectureType.ARM
+    if (scala.util.Properties.propOrNone("os.arch").contains("aarch64")) ArchitectureType.ARMv8
     // We do not distinguish between x86 and x64. E.g, a 64 bit Windows will always lie about
     // this and will report x86 anyway for backwards compatibility with 32 bit software.
     else ArchitectureType.X86

diff --git a/project/Environment.scala b/project/Environment.scala
@@ -9,7 +9,7 @@ object Environment {
   object ArchitectureType extends Enumeration {
     type ArchitectureType = Value
 
-    val X86, ARM = Value
+    val X86, ARMv8 = Value
   }
 
   lazy val operatingSystem: OperatingSystemType.OperatingSystemType =
@@ -19,7 +19,7 @@ object Environment {
     else OperatingSystemType.Unknown
 
   lazy val architecture: ArchitectureType.ArchitectureType =
-    if (scala.util.Properties.propOrNone("os.arch").contains("aarch64")) ArchitectureType.ARM
+    if (scala.util.Properties.propOrNone("os.arch").contains("aarch64")) ArchitectureType.ARMv8
     // We do not distinguish between x86 and x64. E.g, a 64 bit Windows will always lie about
     // this and will report x86 anyway for backwards compatibility with 32 bit software.
     else ArchitectureType.X86