Skip to content

Commit

Permalink
Account for git based packages
Browse files Browse the repository at this point in the history
  • Loading branch information
@jarrydlee
jarrydlee committed Mar 28, 2024
1 parent 023526f commit 11cf946
Show file tree
Hide file tree
Showing 5 changed files with 786 additions and 324 deletions.
89 changes: 59 additions & 30 deletions cli/src/semdep/parsers/mix.py
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
from pathlib import Path
from typing import Any
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple

from semdep.external.parsy import any_char
from semdep.external.parsy import Parser
from semdep.external.parsy import regex
from semdep.external.parsy import string
from semdep.external.parsy import success
from semdep.parsers.util import colon
from semdep.parsers.util import comma
from semdep.parsers.util import consume_line
from semdep.parsers.util import DependencyFileToParse
from semdep.parsers.util import filter_on_marked_lines
from semdep.parsers.util import lbrace
Expand Down Expand Up @@ -62,37 +65,63 @@
# "hexpm",
# "98767a5d1c6c3e3d20497b03293be7f83b46f89a6f3987cc1f9262d299f1eaa7"
# }
package_entry_value_block = (
whitespace
>> lbrace
>> atom # scm
>> comma
>> atom.bind( # package name
lambda package: (
comma
>> version_block.bind(
lambda version: mark_line(success((package, version)))
) # version
)
def package_entry_hex_value_block(package: str) -> Parser:
return (
whitespace
>> lbrace
>> atom
>> comma
>> atom
>> comma
>> version_block.bind(lambda version: mark_line(success((package, version))))
<< comma
<< quoted_str
<< comma
<< any_array
<< comma
<< lbrack
<< many_independency_blocks
<< rbrack
<< comma
<< quoted_str
<< comma
<< quoted_str
<< rbrace
<< comma.optional()
)
<< comma
<< quoted_str # hash
<< comma
<< any_array # options
<< comma
<< lbrack
<< many_independency_blocks # dependencies
<< rbrack
<< comma
<< quoted_str # package manager
<< comma
<< quoted_str # hash
<< rbrace
<< comma.optional()
)

# "castore": {:hex,: castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [: mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"},
package_key_value_block = whitespace >> quoted_str >> colon >> package_entry_value_block

# {:git, "https://github.com/emqx/grpc-erl.git", "31370f25643666c4be43310d62ef749ca1fc20e2", [tag: "0.6.12"]},
def package_entry_git_value_block(package: str) -> Parser:
return (
whitespace
>> lbrace
>> atom
>> comma
>> quoted_str
>> comma
>> quoted_str
>> comma
>> lbrack
>> string("tag")
>> colon
>> version_block.bind(lambda version: mark_line(success((package, version))))
<< rbrack
<< rbrace
<< comma.optional()
)


# "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [: mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"},
# "grpc": {:git, "https://github.com/emqx/grpc-erl.git", "31370f25643666c4be43310d62ef749ca1fc20e2", [tag: "0.6.12"]},
package_key_value_block = whitespace >> quoted_str.bind(
lambda package: colon
>> (
package_entry_hex_value_block(package)
| package_entry_git_value_block(package)
| consume_line
)
)

many_package_blocks = package_key_value_block.sep_by(whitespace)

Expand Down Expand Up @@ -157,7 +186,7 @@ def _parse_manifest_deps(manifest: List[Tuple]) -> Set[str]:


def _build_found_dependencies(
direct_deps: Set[str], lockfile_deps: List[Tuple]
direct_deps: Set[str], lockfile_deps: List[Any]
) -> List[FoundDependency]:
result = []
for line_number, (package, version) in lockfile_deps:
Expand Down
34 changes: 34 additions & 0 deletions cli/tests/default/e2e-pro/parsers/mix/mix.exs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
defmodule Myapp.MixProject do
use Mix.Project

def project do
[
app: :myapp,
version: "0.1.0",
elixir: "~> 1.16",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end

# Run "mix help compile.app" to learn about applications.
def application do
[
extra_applications: [:logger]
]
end

# Run "mix help deps" to learn about dependencies.
defp deps do
[
{:ex_aws_s3, "~> 2.0"},
{:ex_aws_lambda, "~> 2.0"},
{:ex_aws_ses, "~> 2.0"},
{:hackney, "~> 1.17"},
{:jason, "~> 1.0"},
{:joken, "~> 2.0"},
{:sweet_xml, "~> 0.7.1"},
{:uuid, "~> 1.1"}
]
end
end
33 changes: 33 additions & 0 deletions cli/tests/default/e2e-pro/parsers/mix/mix.lock
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
%{
"certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
"cowboy": {:git, "https://github.com/emqx/cowboy", "c8e2b9e069655ad166702a7deb935cabddc85888", [tag: "2.9.0"]},
"cowlib": {:git, "https://github.com/ninenines/cowlib", "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", [tag: "2.8.0"]},
"ecpool": {:git, "https://github.com/emqx/ecpool.git", "a9719f2d4ae9c778a8ca59f1ec9f644bcf7874a7", [tag: "0.5.7"]},
"ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "9c053a45386894ca78359783a4d242de49b230cb", [tag: "0.4.13"]},
"ex_aws": {:hex, :ex_aws, "2.5.3", "9c2d05ba0c057395b12c7b5ca6267d14cdaec1d8e65bdf6481fe1fd245accfb4", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "67115f1d399d7ec4d191812ee565c6106cb4b1bbf19a9d4db06f265fd87da97e"},
"ex_aws_lambda": {:hex, :ex_aws_lambda, "2.1.0", "f28bffae6dde34ba17ef815ef50a82a1e7f3af26d36fe31dbda3a7657e0de449", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}], "hexpm", "25608630b8b45fe22b0237696662f88be724bc89f77ee42708a5871511f531a0"},
"ex_aws_s3": {:hex, :ex_aws_s3, "2.5.3", "422468e5c3e1a4da5298e66c3468b465cfd354b842e512cb1f6fbbe4e2f5bdaf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "4f09dd372cc386550e484808c5ac5027766c8d0cd8271ccc578b82ee6ef4f3b8"},
"ex_aws_ses": {:hex, :ex_aws_ses, "2.4.1", "1aa945610121c9891054c27d0f71f5799b2e0a2062044d742d89c1cee251f9e2", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}], "hexpm", "dddac42d4d7b826f7099bbe7402a35e68eb76434d6c58bfa332002ea2b522645"},
"gproc": {:git, "https://github.com/emqx/gproc.git", "21a5995812498969bb5e47b520b47ea7c514f16b", [tag: "0.9.0.1"]},
"grpc": {:git, "https://github.com/emqx/grpc-erl.git", "31370f25643666c4be43310d62ef749ca1fc20e2", [tag: "0.6.12"]},
"gun": {:git, "https://github.com/emqx/gun", "4faea40b9a8ca1eac5288355f8202e0cea379d50", [tag: "1.3.7"]},
"hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
"jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
"joken": {:hex, :joken, "2.6.0", "b9dd9b6d52e3e6fcb6c65e151ad38bf4bc286382b5b6f97079c47ade6b1bcc6a", [:mix], [{:jose, "~> 1.11.5", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5a95b05a71cd0b54abd35378aeb1d487a23a52c324fa7efdffc512b655b5aaa7"},
"jose": {:hex, :jose, "1.11.6", "613fda82552128aa6fb804682e3a616f4bc15565a048dabd05b1ebd5827ed965", [:mix, :rebar3], [], "hexpm", "6275cb75504f9c1e60eeacb771adfeee4905a9e182103aa59b53fed651ff9738"},
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
"mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
"mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
"parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
"pbkdf2": {:git, "https://github.com/emqx/erlang-pbkdf2.git", "45d9981209ea07a83a58cf85aaf8236457da4342", [tag: "2.0.4"]},
"ranch": {:git, "https://github.com/ninenines/ranch", "a692f44567034dacf5efcaa24a24183788594eb7", [tag: "1.8.0"]},
"rocksdb": {:git, "https://github.com/emqx/erlang-rocksdb.git", "27226d982f5f74965190672bbca5e348a9d9ea94", [tag: "1.8.0-emqx-2"]},
"snabbkaffe": {:git, "https://github.com/kafka4beam/snabbkaffe.git", "67e699b4c7558d71ad3977aa5ad8916c9a8d2334", [tag: "1.0.7"]},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
"sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"},
"telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"},
"typerefl": {:git, "https://github.com/ieQu1/typerefl.git", "310b82ff02f96207c519b9556491433b6ea02d01", [tag: "0.9.1"]},
"unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
"uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm", "c790593b4c3b601f5dc2378baae7efaf5b3d73c4c6456ba85759905be792f2ac"},
}

0 comments on commit 11cf946

Please sign in to comment.