From dcf4a577f7f4228321f35b566933013dd230340a Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 10 Jun 2019 13:29:02 -0400 Subject: [PATCH 01/11] Use Hackage package rather than submodules. --- .gitmodules | 3 --- .travis.yml | 2 +- cabal.project | 2 +- semantic.cabal | 21 ++++++++++----------- vendor/haskell-tree-sitter | 1 - 5 files changed, 12 insertions(+), 17 deletions(-) delete mode 160000 vendor/haskell-tree-sitter diff --git a/.gitmodules b/.gitmodules index 6dab354824..e69de29bb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "vendor/haskell-tree-sitter"] - path = vendor/haskell-tree-sitter - url = https://github.com/tree-sitter/haskell-tree-sitter.git diff --git a/.travis.yml b/.travis.yml index 0fcfe68a2b..27aa6de542 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ before_install: - cabal --version install: -- cabal new-update +- cabal new-update hackage.haskell.org,HEAD - cabal new-configure --enable-tests --write-ghc-environment-files=always - cabal new-build --only-dependencies -j diff --git a/cabal.project b/cabal.project index 7db9d1e8e2..07a0019105 100644 --- a/cabal.project +++ b/cabal.project @@ -1,4 +1,4 @@ -packages: vendor/* vendor/haskell-tree-sitter/languages/* . semantic-core +packages: . semantic-core source-repository-package type: git diff --git a/semantic.cabal b/semantic.cabal index ccb96efc0e..5442e37b19 100644 --- a/semantic.cabal +++ b/semantic.cabal @@ -55,7 +55,7 @@ common dependencies , fused-effects ^>= 0.4.0.0 , fused-effects-exceptions ^>= 0.1.1.0 , hashable ^>= 1.2.7.0 - , haskell-tree-sitter + , tree-sitter ^>= 0.1.0.0 , machines ^>= 0.6.4 , mtl ^>= 2.2.2 , network ^>= 2.8.0.0 @@ -304,7 +304,7 @@ library , reducers ^>= 3.12.3 , semigroupoids ^>= 5.3.2 , servant ^>= 0.15 - , shelly + , shelly >= 1.5 && <2 , split ^>= 0.2.3.3 , stm-chans ^>= 3.0.0.4 , template-haskell ^>= 2.14 @@ -312,16 +312,15 @@ library , unliftio-core , unordered-containers ^>= 0.2.9.0 , vector ^>= 0.12.0.2 - , haskell-tree-sitter - , tree-sitter-go + , tree-sitter-go ^>= 0.1.0.0 , tree-sitter-haskell - , tree-sitter-json - , tree-sitter-php - , tree-sitter-python - , tree-sitter-ruby - , tree-sitter-typescript - , tree-sitter-tsx - , tree-sitter-java + , tree-sitter-json ^>= 0.1.0.0 + , tree-sitter-php ^>= 0.1.0.0 + , tree-sitter-python ^>= 0.1.0.1 + , tree-sitter-ruby ^>= 0.1.0.0 + , tree-sitter-typescript ^>= 0.1.0.0 + , tree-sitter-tsx ^>= 0.1.0.0 + , tree-sitter-java ^>= 0.1.0.0 ghc-options: -Wall -Wmissing-export-lists -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-name-shadowing -j if flag(release) ghc-options: -Werror -O1 diff --git a/vendor/haskell-tree-sitter b/vendor/haskell-tree-sitter deleted file mode 160000 index aa1b3c7441..0000000000 --- a/vendor/haskell-tree-sitter +++ /dev/null @@ -1 +0,0 @@ -Subproject commit aa1b3c74410a71648a5301d6241566dfe79165a4 From 90562edd537c156846548fe5c36eb73c2b03f3ad Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 10 Jun 2019 15:03:47 -0400 Subject: [PATCH 02/11] Fixes for build time. --- cabal.project | 2 ++ semantic.cabal | 22 ++++++++-------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/cabal.project b/cabal.project index 07a0019105..02c5b3c7ce 100644 --- a/cabal.project +++ b/cabal.project @@ -1,5 +1,7 @@ packages: . semantic-core +jobs: $ncpus + source-repository-package type: git location: https://github.com/joshvera/proto3-suite.git diff --git a/semantic.cabal b/semantic.cabal index 5442e37b19..aad72ff482 100644 --- a/semantic.cabal +++ b/semantic.cabal @@ -1,8 +1,8 @@ cabal-version: 2.2 name: semantic -version: 0.6.0 -synopsis: Framework and service for analyzing and diffing untrusted code. -description: Please see README.md +version: 0.6.0.0 +synopsis: Framework and executable for analyzing and diffing untrusted code. +description: Semantic is a library for parsing, analyzing, and comparing source code across many languages. homepage: http://github.com/github/semantic#readme author: The Semantic authors maintainer: opensource+semantic@github.com @@ -224,7 +224,6 @@ library , Parsing.CMark , Parsing.Parser , Parsing.TreeSitter - , Paths_semantic -- Rendering formats , Rendering.Graph , Rendering.JSON @@ -272,7 +271,8 @@ library , Tags.Tagging -- Custom Prelude , Prologue - + autogen-modules: Paths_semantic + other-modules: Paths_semantic build-depends: base >= 4.12 && < 5 , ansi-terminal ^>= 0.8.2 , array ^>= 0.5.3.0 @@ -321,23 +321,18 @@ library , tree-sitter-typescript ^>= 0.1.0.0 , tree-sitter-tsx ^>= 0.1.0.0 , tree-sitter-java ^>= 0.1.0.0 - ghc-options: -Wall -Wmissing-export-lists -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-name-shadowing -j + ghc-options: -Wall -Wmissing-export-lists -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-name-shadowing if flag(release) - ghc-options: -Werror -O1 cpp-options: -DCOMPUTE_GIT_SHA else - ghc-options: -O0 +RTS -A128m -n2m -RTS - ghc-prof-options: -fprof-auto + ghc-options: +RTS -A128m -n2m -RTS executable semantic import: haskell, dependencies, executable-flags hs-source-dirs: app main-is: Main.hs if flag(release) - ghc-options: -O1 cpp-options: -DCOMPUTE_GIT_SHA - else - ghc-options: -O0 build-depends: base , semantic @@ -393,7 +388,6 @@ test-suite test , HUnit ^>= 1.6.0.0 , leancheck >= 0.8 && <1 , temporary - ghc-options: -O0 if flag(release) ghc-options: -dynamic @@ -413,7 +407,7 @@ benchmark evaluation hs-source-dirs: bench/evaluation type: exitcode-stdio-1.0 main-is: Main.hs - ghc-options: -static -O1 + ghc-options: -static build-depends: base , criterion , semantic From e316aec4a954d603e624db8460ea7d0019b78675 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 10 Jun 2019 17:54:36 -0400 Subject: [PATCH 03/11] see if a simpler travis invocation will help? --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27aa6de542..2a47292e6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,8 +33,9 @@ install: - cabal new-build --only-dependencies -j script: -- cabal new-build -j semantic-core semantic:exe:semantic -- cabal new-test semantic:test semantic-core:spec +- cabal new-build -j +- cabal new-run semantic:test +- cabal new-run semantic-core:spec - cabal new-run semantic:parse-examples # Any branch linked with a pull request will be built, as well as the non-PR From 0083fb481253c04e4a821462587e0dc061a5f4a1 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 10 Jun 2019 18:22:34 -0400 Subject: [PATCH 04/11] Give GHC a little more memory. --- cabal.project | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cabal.project b/cabal.project index 02c5b3c7ce..5772f08158 100644 --- a/cabal.project +++ b/cabal.project @@ -2,6 +2,9 @@ packages: . semantic-core jobs: $ncpus +package semantic + ghc-options: +RTS -A128m -n2m -RTS + source-repository-package type: git location: https://github.com/joshvera/proto3-suite.git From 7aeb88227a74b8548da92e0f320244e400312ad4 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 10 Jun 2019 18:53:20 -0400 Subject: [PATCH 05/11] Just don't optimize these specs for now. --- test/Analysis/Go/Spec.hs | 1 + test/Analysis/PHP/Spec.hs | 1 + test/Analysis/Ruby/Spec.hs | 1 + test/Analysis/TypeScript/Spec.hs | 2 ++ test/Graphing/Calls/Spec.hs | 1 - 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/Analysis/Go/Spec.hs b/test/Analysis/Go/Spec.hs index 9bdb9f7a09..a750f46bbb 100644 --- a/test/Analysis/Go/Spec.hs +++ b/test/Analysis/Go/Spec.hs @@ -1,3 +1,4 @@ +{-# OPTIONS_GHC -O0 #-} module Analysis.Go.Spec (spec) where import Data.Abstract.Evaluatable (EvalError(..)) diff --git a/test/Analysis/PHP/Spec.hs b/test/Analysis/PHP/Spec.hs index 2069e50ddb..de74db45c2 100644 --- a/test/Analysis/PHP/Spec.hs +++ b/test/Analysis/PHP/Spec.hs @@ -1,3 +1,4 @@ +{-# OPTIONS_GHC -O0 #-} module Analysis.PHP.Spec (spec) where import Control.Abstract diff --git a/test/Analysis/Ruby/Spec.hs b/test/Analysis/Ruby/Spec.hs index 6641ebfa54..1efe60f8ab 100644 --- a/test/Analysis/Ruby/Spec.hs +++ b/test/Analysis/Ruby/Spec.hs @@ -1,3 +1,4 @@ +{-# OPTIONS_GHC -O0 #-} {-# LANGUAGE TupleSections #-} module Analysis.Ruby.Spec (spec) where diff --git a/test/Analysis/TypeScript/Spec.hs b/test/Analysis/TypeScript/Spec.hs index 0e0e4b3e1a..75169a1d43 100644 --- a/test/Analysis/TypeScript/Spec.hs +++ b/test/Analysis/TypeScript/Spec.hs @@ -1,3 +1,5 @@ +{-# OPTIONS_GHC -O0 #-} + module Analysis.TypeScript.Spec (spec) where import Data.Syntax.Statement (StatementBlock(..)) diff --git a/test/Graphing/Calls/Spec.hs b/test/Graphing/Calls/Spec.hs index 4fee9ed626..d5aa71dca7 100644 --- a/test/Graphing/Calls/Spec.hs +++ b/test/Graphing/Calls/Spec.hs @@ -3,7 +3,6 @@ module Graphing.Calls.Spec ( spec ) where import Prelude hiding (readFile) -import Prologue import SpecHelpers hiding (readFile) import Algebra.Graph From 7f9b3671bfcfd1cef5ee4f08e78c08a334d4e99e Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 10 Jun 2019 19:30:11 -0400 Subject: [PATCH 06/11] Attempt to clone the vendor dir and its examples in clone script. --- script/clone-example-repos | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/clone-example-repos b/script/clone-example-repos index af2aa98fd6..513bb16a46 100755 --- a/script/clone-example-repos +++ b/script/clone-example-repos @@ -13,6 +13,9 @@ set -e cd $(dirname "$0")/.. +mkdir vendor || true +git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git vendor/haskell-tree-sitter + dir="vendor/haskell-tree-sitter/languages" # clone_repo LOCAL_PATH URL SHA From 039e187819069c58576afdec32d9cc76d6a5363e Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 11 Jun 2019 11:00:17 -0400 Subject: [PATCH 07/11] Fix travis invocation and add more bounds. --- .travis.yml | 2 +- semantic.cabal | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2a47292e6a..07f5f71ff4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ before_install: - cabal --version install: -- cabal new-update hackage.haskell.org,HEAD +- cabal new-update hackage.haskell.org - cabal new-configure --enable-tests --write-ghc-environment-files=always - cabal new-build --only-dependencies -j diff --git a/semantic.cabal b/semantic.cabal index aad72ff482..7ba07963b4 100644 --- a/semantic.cabal +++ b/semantic.cabal @@ -49,7 +49,7 @@ common dependencies , bytestring ^>= 0.10.8.2 , containers ^>= 0.6.0.1 , directory ^>= 1.3.3.0 - , fastsum + , fastsum ^>= 0.1.1.0 , filepath ^>= 1.4.2.1 , free ^>= 5.1 , fused-effects ^>= 0.4.0.0 @@ -59,11 +59,11 @@ common dependencies , machines ^>= 0.6.4 , mtl ^>= 2.2.2 , network ^>= 2.8.0.0 - , process + , process ^>= 1.6.3.0 , recursion-schemes ^>= 5.1 , scientific ^>= 0.3.6.2 - , safe-exceptions - , semilattices + , safe-exceptions ^>= 0.1.7.0 + , semilattices ^>= 0.0.0.3 , text ^>= 1.2.3.1 , these >= 0.7 && <1 , unix ^>= 2.7.2.2 @@ -309,11 +309,11 @@ library , stm-chans ^>= 3.0.0.4 , template-haskell ^>= 2.14 , time ^>= 1.8.0.2 - , unliftio-core + , unliftio-core ^>= 0.1.2.0 , unordered-containers ^>= 0.2.9.0 , vector ^>= 0.12.0.2 , tree-sitter-go ^>= 0.1.0.0 - , tree-sitter-haskell + , tree-sitter-haskell ^>= 0.1.0.0 , tree-sitter-json ^>= 0.1.0.0 , tree-sitter-php ^>= 0.1.0.0 , tree-sitter-python ^>= 0.1.0.1 From a157e4a1b4c5e27589a7623f08eabffcd0c33d03 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 11 Jun 2019 11:22:04 -0400 Subject: [PATCH 08/11] Don't clutter the workdir. --- script/clone-example-repos | 6 +++--- test/Examples.hs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/script/clone-example-repos b/script/clone-example-repos index 513bb16a46..e5ef53a080 100755 --- a/script/clone-example-repos +++ b/script/clone-example-repos @@ -13,10 +13,10 @@ set -e cd $(dirname "$0")/.. -mkdir vendor || true -git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git vendor/haskell-tree-sitter +mkdir -p test/examplerepos +git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git tmp/haskell-tree-sitter -dir="vendor/haskell-tree-sitter/languages" +dir="tmp/haskell-tree-sitter/languages" # clone_repo LOCAL_PATH URL SHA function clone_repo { diff --git a/test/Examples.hs b/test/Examples.hs index b7c5b3ab27..1f58815573 100644 --- a/test/Examples.hs +++ b/test/Examples.hs @@ -38,7 +38,7 @@ main = withOptions opts $ \ config logger statter -> hspec . parallel $ do runIO setupExampleRepos for_ languages $ \ lang@LanguageExample{..} -> do - let tsDir = languagesDir languageName ("vendor/tree-sitter-" <> languageName) + let tsDir = languagesDir languageName ("tree-sitter-" <> languageName) parallel . describe languageName $ parseExamples args lang tsDir where From 4853d9796f8a47c1f1fec5c680f532e72fc5d3a0 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 11 Jun 2019 11:23:56 -0400 Subject: [PATCH 09/11] Oops. --- test/Examples.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Examples.hs b/test/Examples.hs index 1f58815573..b7c5b3ab27 100644 --- a/test/Examples.hs +++ b/test/Examples.hs @@ -38,7 +38,7 @@ main = withOptions opts $ \ config logger statter -> hspec . parallel $ do runIO setupExampleRepos for_ languages $ \ lang@LanguageExample{..} -> do - let tsDir = languagesDir languageName ("tree-sitter-" <> languageName) + let tsDir = languagesDir languageName ("vendor/tree-sitter-" <> languageName) parallel . describe languageName $ parseExamples args lang tsDir where From 32a581f31ad425b970e28b2071c5cd8cbd1ae4be Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 11 Jun 2019 14:54:48 -0400 Subject: [PATCH 10/11] More robust example parsing. --- script/clone-example-repos | 4 ++-- test/Examples.hs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/clone-example-repos b/script/clone-example-repos index e5ef53a080..85589ad65f 100755 --- a/script/clone-example-repos +++ b/script/clone-example-repos @@ -13,8 +13,8 @@ set -e cd $(dirname "$0")/.. -mkdir -p test/examplerepos -git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git tmp/haskell-tree-sitter +mkdir -p test/examplerepos || true +git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git tmp/haskell-tree-sitter || true dir="tmp/haskell-tree-sitter/languages" diff --git a/test/Examples.hs b/test/Examples.hs index b7c5b3ab27..363f73000a 100644 --- a/test/Examples.hs +++ b/test/Examples.hs @@ -105,4 +105,4 @@ parseFilePath :: (Member (Error SomeException) sig, Member Distribute sig, Membe parseFilePath path = readBlob (fileForPath path) >>= parseTermBuilder @[] TermShow . pure >>= const (pure True) languagesDir :: FilePath -languagesDir = "vendor/haskell-tree-sitter/languages" +languagesDir = "tmp/haskell-tree-sitter/languages" From 8b3bedcd54b0e5d5006a0ae30931b80487a16521 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 11 Jun 2019 15:41:13 -0400 Subject: [PATCH 11/11] Don't try to download 2.9 GB of examples. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 07f5f71ff4..5dbd7eb182 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,8 @@ script: - cabal new-build -j - cabal new-run semantic:test - cabal new-run semantic-core:spec -- cabal new-run semantic:parse-examples +# parse-examples is disabled because it slaughters our CI +# - cabal new-run semantic:parse-examples # Any branch linked with a pull request will be built, as well as the non-PR # branches listed below: