From 3539d51e059f658f6610d225a5077da03044893f Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 21 Apr 2024 15:01:41 -0700 Subject: [PATCH 01/25] Move test --- .../{invalid_annotation => invalid_annotation_exclude}/BUILD.in | 0 .../{invalid_annotation => invalid_annotation_exclude}/BUILD.out | 0 .../{invalid_annotation => invalid_annotation_exclude}/README.md | 0 .../{invalid_annotation => invalid_annotation_exclude}/WORKSPACE | 0 .../__init__.py | 0 .../{invalid_annotation => invalid_annotation_exclude}/test.yaml | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename gazelle/python/testdata/{invalid_annotation => invalid_annotation_exclude}/BUILD.in (100%) rename gazelle/python/testdata/{invalid_annotation => invalid_annotation_exclude}/BUILD.out (100%) rename gazelle/python/testdata/{invalid_annotation => invalid_annotation_exclude}/README.md (100%) rename gazelle/python/testdata/{invalid_annotation => invalid_annotation_exclude}/WORKSPACE (100%) rename gazelle/python/testdata/{invalid_annotation => invalid_annotation_exclude}/__init__.py (100%) rename gazelle/python/testdata/{invalid_annotation => invalid_annotation_exclude}/test.yaml (100%) diff --git a/gazelle/python/testdata/invalid_annotation/BUILD.in b/gazelle/python/testdata/invalid_annotation_exclude/BUILD.in similarity index 100% rename from gazelle/python/testdata/invalid_annotation/BUILD.in rename to gazelle/python/testdata/invalid_annotation_exclude/BUILD.in diff --git a/gazelle/python/testdata/invalid_annotation/BUILD.out b/gazelle/python/testdata/invalid_annotation_exclude/BUILD.out similarity index 100% rename from gazelle/python/testdata/invalid_annotation/BUILD.out rename to gazelle/python/testdata/invalid_annotation_exclude/BUILD.out diff --git a/gazelle/python/testdata/invalid_annotation/README.md b/gazelle/python/testdata/invalid_annotation_exclude/README.md similarity index 100% rename from gazelle/python/testdata/invalid_annotation/README.md rename to gazelle/python/testdata/invalid_annotation_exclude/README.md diff --git a/gazelle/python/testdata/invalid_annotation/WORKSPACE b/gazelle/python/testdata/invalid_annotation_exclude/WORKSPACE similarity index 100% rename from gazelle/python/testdata/invalid_annotation/WORKSPACE rename to gazelle/python/testdata/invalid_annotation_exclude/WORKSPACE diff --git a/gazelle/python/testdata/invalid_annotation/__init__.py b/gazelle/python/testdata/invalid_annotation_exclude/__init__.py similarity index 100% rename from gazelle/python/testdata/invalid_annotation/__init__.py rename to gazelle/python/testdata/invalid_annotation_exclude/__init__.py diff --git a/gazelle/python/testdata/invalid_annotation/test.yaml b/gazelle/python/testdata/invalid_annotation_exclude/test.yaml similarity index 100% rename from gazelle/python/testdata/invalid_annotation/test.yaml rename to gazelle/python/testdata/invalid_annotation_exclude/test.yaml From ce9ee9d77134f2cde2d7130b71a4153b4c223499 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 21 Apr 2024 15:10:40 -0700 Subject: [PATCH 02/25] Add invalid_annotation_include_dep test --- .../invalid_annotation_include_dep/BUILD.in | 0 .../invalid_annotation_include_dep/BUILD.out | 0 .../invalid_annotation_include_dep/README.md | 3 +++ .../invalid_annotation_include_dep/WORKSPACE | 1 + .../__init__.py | 15 +++++++++++++++ .../invalid_annotation_include_dep/test.yaml | 19 +++++++++++++++++++ 6 files changed, 38 insertions(+) create mode 100644 gazelle/python/testdata/invalid_annotation_include_dep/BUILD.in create mode 100644 gazelle/python/testdata/invalid_annotation_include_dep/BUILD.out create mode 100644 gazelle/python/testdata/invalid_annotation_include_dep/README.md create mode 100644 gazelle/python/testdata/invalid_annotation_include_dep/WORKSPACE create mode 100644 gazelle/python/testdata/invalid_annotation_include_dep/__init__.py create mode 100644 gazelle/python/testdata/invalid_annotation_include_dep/test.yaml diff --git a/gazelle/python/testdata/invalid_annotation_include_dep/BUILD.in b/gazelle/python/testdata/invalid_annotation_include_dep/BUILD.in new file mode 100644 index 000000000..e69de29bb diff --git a/gazelle/python/testdata/invalid_annotation_include_dep/BUILD.out b/gazelle/python/testdata/invalid_annotation_include_dep/BUILD.out new file mode 100644 index 000000000..e69de29bb diff --git a/gazelle/python/testdata/invalid_annotation_include_dep/README.md b/gazelle/python/testdata/invalid_annotation_include_dep/README.md new file mode 100644 index 000000000..2f8e02405 --- /dev/null +++ b/gazelle/python/testdata/invalid_annotation_include_dep/README.md @@ -0,0 +1,3 @@ +# Invalid annotation +This test case asserts that the parse step fails as expected due to invalid annotation format of +the `include_dep` annotation. diff --git a/gazelle/python/testdata/invalid_annotation_include_dep/WORKSPACE b/gazelle/python/testdata/invalid_annotation_include_dep/WORKSPACE new file mode 100644 index 000000000..faff6af87 --- /dev/null +++ b/gazelle/python/testdata/invalid_annotation_include_dep/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/invalid_annotation_include_dep/__init__.py b/gazelle/python/testdata/invalid_annotation_include_dep/__init__.py new file mode 100644 index 000000000..f707fd0ca --- /dev/null +++ b/gazelle/python/testdata/invalid_annotation_include_dep/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# gazelle:include_dep diff --git a/gazelle/python/testdata/invalid_annotation_include_dep/test.yaml b/gazelle/python/testdata/invalid_annotation_include_dep/test.yaml new file mode 100644 index 000000000..3325c7c1c --- /dev/null +++ b/gazelle/python/testdata/invalid_annotation_include_dep/test.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +expect: + exit_code: 1 + stderr: | + gazelle: ERROR: failed to parse annotations: `# gazelle:include_dep` requires a value From f2e42ad2d71f292ef5f84c454c42218b5f56fc56 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 21 Apr 2024 15:10:59 -0700 Subject: [PATCH 03/25] Add annotation_include_dep test --- .../testdata/annotation_include_dep/BUILD.in | 1 + .../testdata/annotation_include_dep/BUILD.out | 53 +++++++++++++++++++ .../testdata/annotation_include_dep/README.md | 4 ++ .../testdata/annotation_include_dep/WORKSPACE | 0 .../annotation_include_dep/__init__.py | 9 ++++ .../annotation_include_dep/__main__.py | 5 ++ .../gazelle_python.yaml | 18 +++++++ .../annotation_include_dep/module1.py | 0 .../annotation_include_dep/module2.py | 5 ++ .../annotation_include_dep/module2_test.py | 5 ++ .../annotation_include_dep/subpkg/BUILD.in | 1 + .../annotation_include_dep/subpkg/BUILD.out | 29 ++++++++++ .../annotation_include_dep/subpkg/__init__.py | 0 .../annotation_include_dep/subpkg/module1.py | 3 ++ .../subpkg/module1_test.py | 5 ++ .../annotation_include_dep/subpkg/module2.py | 4 ++ .../annotation_include_dep/subpkg/module3.py | 3 ++ .../testdata/annotation_include_dep/test.yaml | 17 ++++++ 18 files changed, 162 insertions(+) create mode 100644 gazelle/python/testdata/annotation_include_dep/BUILD.in create mode 100644 gazelle/python/testdata/annotation_include_dep/BUILD.out create mode 100644 gazelle/python/testdata/annotation_include_dep/README.md create mode 100644 gazelle/python/testdata/annotation_include_dep/WORKSPACE create mode 100644 gazelle/python/testdata/annotation_include_dep/__init__.py create mode 100644 gazelle/python/testdata/annotation_include_dep/__main__.py create mode 100644 gazelle/python/testdata/annotation_include_dep/gazelle_python.yaml create mode 100644 gazelle/python/testdata/annotation_include_dep/module1.py create mode 100644 gazelle/python/testdata/annotation_include_dep/module2.py create mode 100644 gazelle/python/testdata/annotation_include_dep/module2_test.py create mode 100644 gazelle/python/testdata/annotation_include_dep/subpkg/BUILD.in create mode 100644 gazelle/python/testdata/annotation_include_dep/subpkg/BUILD.out create mode 100644 gazelle/python/testdata/annotation_include_dep/subpkg/__init__.py create mode 100644 gazelle/python/testdata/annotation_include_dep/subpkg/module1.py create mode 100644 gazelle/python/testdata/annotation_include_dep/subpkg/module1_test.py create mode 100644 gazelle/python/testdata/annotation_include_dep/subpkg/module2.py create mode 100644 gazelle/python/testdata/annotation_include_dep/subpkg/module3.py create mode 100644 gazelle/python/testdata/annotation_include_dep/test.yaml diff --git a/gazelle/python/testdata/annotation_include_dep/BUILD.in b/gazelle/python/testdata/annotation_include_dep/BUILD.in new file mode 100644 index 000000000..af2c2cea4 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_generation_mode file diff --git a/gazelle/python/testdata/annotation_include_dep/BUILD.out b/gazelle/python/testdata/annotation_include_dep/BUILD.out new file mode 100644 index 000000000..1cff8f467 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/BUILD.out @@ -0,0 +1,53 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") + +# gazelle:python_generation_mode file + +py_library( + name = "__init__", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], + deps = [ + ":module1", + ":module2", + "//foo/bar:baz", + "//hello:world", + "@gazelle_python_test//foo", + "@star_wars//rebel_alliance/luke:skywalker", + ], +) + +py_library( + name = "module1", + srcs = ["module1.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "module2", + srcs = ["module2.py"], + visibility = ["//:__subpackages__"], + deps = [ + "//checking/py_binary/from/if:works", + "//foo:bar", + ], +) + +py_binary( + name = "annotation_include_dep_bin", + srcs = ["__main__.py"], + main = "__main__.py", + visibility = ["//:__subpackages__"], + deps = [ + ":module2", + "//checking/py_binary/from/__main__:works", + ], +) + +py_test( + name = "module2_test", + srcs = ["module2_test.py"], + deps = [ + ":module2", + "//checking/py_test/works:too", + ], +) diff --git a/gazelle/python/testdata/annotation_include_dep/README.md b/gazelle/python/testdata/annotation_include_dep/README.md new file mode 100644 index 000000000..a41054acd --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/README.md @@ -0,0 +1,4 @@ +# Annotation: Include Dep + +Test that the Python gazelle annotation `# gazelle:include_dep` correctly adds dependences +to the generated target even if those dependencies are not imported by the Python module. diff --git a/gazelle/python/testdata/annotation_include_dep/WORKSPACE b/gazelle/python/testdata/annotation_include_dep/WORKSPACE new file mode 100644 index 000000000..e69de29bb diff --git a/gazelle/python/testdata/annotation_include_dep/__init__.py b/gazelle/python/testdata/annotation_include_dep/__init__.py new file mode 100644 index 000000000..61015346d --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/__init__.py @@ -0,0 +1,9 @@ +import module1 +import foo # third party package + +# gazelle:include_dep //foo/bar:baz +# gazelle:include_dep //hello:world,@star_wars//rebel_alliance/luke:skywalker +# gazelle:include_dep :module2 + +del module1 +del foo diff --git a/gazelle/python/testdata/annotation_include_dep/__main__.py b/gazelle/python/testdata/annotation_include_dep/__main__.py new file mode 100644 index 000000000..7b4569aa4 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/__main__.py @@ -0,0 +1,5 @@ +# gazelle:include_dep //checking/py_binary/from/__main__:works + +import module2 + +del module2 diff --git a/gazelle/python/testdata/annotation_include_dep/gazelle_python.yaml b/gazelle/python/testdata/annotation_include_dep/gazelle_python.yaml new file mode 100644 index 000000000..4b12372b4 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/gazelle_python.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +manifest: + modules_mapping: + foo: foo + pip_deps_repository_name: gazelle_python_test diff --git a/gazelle/python/testdata/annotation_include_dep/module1.py b/gazelle/python/testdata/annotation_include_dep/module1.py new file mode 100644 index 000000000..e69de29bb diff --git a/gazelle/python/testdata/annotation_include_dep/module2.py b/gazelle/python/testdata/annotation_include_dep/module2.py new file mode 100644 index 000000000..23a75afee --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/module2.py @@ -0,0 +1,5 @@ +# gazelle:include_dep //foo:bar + +if __name__ == "__main__": + # gazelle:include_dep //checking/py_binary/from/if:works + print("hello") diff --git a/gazelle/python/testdata/annotation_include_dep/module2_test.py b/gazelle/python/testdata/annotation_include_dep/module2_test.py new file mode 100644 index 000000000..6fa18c6f5 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/module2_test.py @@ -0,0 +1,5 @@ +# gazelle:include_dep //checking/py_test/works:too + +import module2 + +del module2 diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/BUILD.in b/gazelle/python/testdata/annotation_include_dep/subpkg/BUILD.in new file mode 100644 index 000000000..421b48688 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/subpkg/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_generation_mode package diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/BUILD.out b/gazelle/python/testdata/annotation_include_dep/subpkg/BUILD.out new file mode 100644 index 000000000..921c89288 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/subpkg/BUILD.out @@ -0,0 +1,29 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +# gazelle:python_generation_mode package + +py_library( + name = "subpkg", + srcs = [ + "__init__.py", + "module1.py", + "module2.py", + "module3.py", + ], + visibility = ["//:__subpackages__"], + deps = [ + ":nonexistant_target_from_include_dep_in_module3", + "//me_from_module1", + "//other/thing:from_include_dep_in_module2", + "//you_from_module1", + ], +) + +py_test( + name = "module1_test", + srcs = ["module1_test.py"], + deps = [ + ":subpkg", + "//:bagel_from_include_dep_in_module1_test", + ], +) diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/__init__.py b/gazelle/python/testdata/annotation_include_dep/subpkg/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/module1.py b/gazelle/python/testdata/annotation_include_dep/subpkg/module1.py new file mode 100644 index 000000000..01566a07e --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/subpkg/module1.py @@ -0,0 +1,3 @@ +def hello(): + # gazelle:include_dep //you_from_module1,//me_from_module1 + pass diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/module1_test.py b/gazelle/python/testdata/annotation_include_dep/subpkg/module1_test.py new file mode 100644 index 000000000..087763a69 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/subpkg/module1_test.py @@ -0,0 +1,5 @@ +# gazelle:include_dep //:bagel_from_include_dep_in_module1_test + +import module1 + +del module1 diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/module2.py b/gazelle/python/testdata/annotation_include_dep/subpkg/module2.py new file mode 100644 index 000000000..dabeb6794 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/subpkg/module2.py @@ -0,0 +1,4 @@ +# gazelle:include_dep //other/thing:from_include_dep_in_module2 +import module1 + +del module1 diff --git a/gazelle/python/testdata/annotation_include_dep/subpkg/module3.py b/gazelle/python/testdata/annotation_include_dep/subpkg/module3.py new file mode 100644 index 000000000..899a7c4f5 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/subpkg/module3.py @@ -0,0 +1,3 @@ +def goodbye(): + # gazelle:include_dep :nonexistant_target_from_include_dep_in_module3 + pass diff --git a/gazelle/python/testdata/annotation_include_dep/test.yaml b/gazelle/python/testdata/annotation_include_dep/test.yaml new file mode 100644 index 000000000..2410223e5 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_dep/test.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +expect: + exit_code: 0 From fbc02f3664d571b94dfd4386d92046dfa5ae8fd1 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 21 Apr 2024 15:22:25 -0700 Subject: [PATCH 04/25] Add includeDep to the annotations struct but we don't use it yet. --- gazelle/python/parser.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 9b00b831e..f2af19e19 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -212,6 +212,7 @@ const ( annotationPrefix string = "gazelle:" // The ignore annotation kind. E.g. '# gazelle:ignore '. annotationKindIgnore annotationKind = "ignore" + annotationKindIncludeDep annotationKind = "include_dep" ) // comment represents a Python comment. @@ -247,12 +248,15 @@ type annotation struct { type annotations struct { // The parsed modules to be ignored by Gazelle. ignore map[string]struct{} + // Labels that Gazelle should include as deps of the generated target. + includeDep []string } // annotationsFromComments returns all the annotations parsed out of the // comments of a Python module. func annotationsFromComments(comments []comment) (*annotations, error) { ignore := make(map[string]struct{}) + includeDep := []string{} for _, comment := range comments { annotation, err := comment.asAnnotation() if err != nil { @@ -269,10 +273,21 @@ func annotationsFromComments(comments []comment) (*annotations, error) { ignore[m] = struct{}{} } } + if annotation.kind == annotationKindIncludeDep { + targets := strings.Split(annotation.value, ",") + for _, t := range targets { + if t == "" { + continue + } + t = strings.TrimSpace(t) + includeDep = append(includeDep, t) + } + } } } return &annotations{ ignore: ignore, + includeDep: includeDep, }, nil } From 0fe5750dd6e53b31def86776bd526ca616743c34 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 21 Apr 2024 15:34:10 -0700 Subject: [PATCH 05/25] Have parser.parse return a 4-tuple that includes the parsed annotations. --- gazelle/python/generate.go | 8 ++++---- gazelle/python/parser.go | 15 ++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index 1937831c4..e2a405b58 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -233,7 +233,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes collisionErrors := singlylinkedlist.New() appendPyLibrary := func(srcs *treeset.Set, pyLibraryTargetName string) { - allDeps, mainModules, err := parser.parse(srcs) + allDeps, mainModules, _, err := parser.parse(srcs) if err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -314,7 +314,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes } if hasPyBinaryEntryPointFile { - deps, _, err := parser.parseSingle(pyBinaryEntrypointFilename) + deps, _, _, err := parser.parseSingle(pyBinaryEntrypointFilename) if err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -348,7 +348,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes var conftest *rule.Rule if hasConftestFile { - deps, _, err := parser.parseSingle(conftestFilename) + deps, _, _, err := parser.parseSingle(conftestFilename) if err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -379,7 +379,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes var pyTestTargets []*targetBuilder newPyTestTargetBuilder := func(srcs *treeset.Set, pyTestTargetName string) *targetBuilder { - deps, _, err := parser.parse(srcs) + deps, _, _, err := parser.parse(srcs) if err != nil { log.Fatalf("ERROR: %v\n", err) } diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index f2af19e19..cc852e63c 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -101,7 +101,7 @@ func newPython3Parser( // parseSingle parses a single Python file and returns the extracted modules // from the import statements as well as the parsed comments. -func (p *python3Parser) parseSingle(pyFilename string) (*treeset.Set, map[string]*treeset.Set, error) { +func (p *python3Parser) parseSingle(pyFilename string) (*treeset.Set, map[string]*treeset.Set, *annotations, error) { pyFilenames := treeset.NewWith(godsutils.StringComparator) pyFilenames.Add(pyFilename) return p.parse(pyFilenames) @@ -109,7 +109,7 @@ func (p *python3Parser) parseSingle(pyFilename string) (*treeset.Set, map[string // parse parses multiple Python files and returns the extracted modules from // the import statements as well as the parsed comments. -func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[string]*treeset.Set, error) { +func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[string]*treeset.Set, *annotations, error) { parserMutex.Lock() defer parserMutex.Unlock() @@ -122,28 +122,29 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin } encoder := json.NewEncoder(parserStdin) if err := encoder.Encode(&req); err != nil { - return nil, nil, fmt.Errorf("failed to parse: %w", err) + return nil, nil, nil, fmt.Errorf("failed to parse: %w", err) } reader := bufio.NewReader(parserStdout) data, err := reader.ReadBytes(0) if err != nil { - return nil, nil, fmt.Errorf("failed to parse: %w", err) + return nil, nil, nil, fmt.Errorf("failed to parse: %w", err) } data = data[:len(data)-1] var allRes []parserResponse if err := json.Unmarshal(data, &allRes); err != nil { - return nil, nil, fmt.Errorf("failed to parse: %w", err) + return nil, nil, nil, fmt.Errorf("failed to parse: %w", err) } mainModules := make(map[string]*treeset.Set, len(allRes)) + annotations := &annotations{} for _, res := range allRes { if res.HasMain { mainModules[res.FileName] = treeset.NewWith(moduleComparator) } annotations, err := annotationsFromComments(res.Comments) if err != nil { - return nil, nil, fmt.Errorf("failed to parse annotations: %w", err) + return nil, nil, nil, fmt.Errorf("failed to parse annotations: %w", err) } for _, m := range res.Modules { @@ -166,7 +167,7 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin } } - return modules, mainModules, nil + return modules, mainModules, annotations, nil } // parserResponse represents a response returned by the parser.py for a given From eccf7bd01695cbd4d0d099c2c035590e975a85b1 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 21 Apr 2024 15:52:16 -0700 Subject: [PATCH 06/25] Add target.addResolvedDependencies --- gazelle/python/target.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gazelle/python/target.go b/gazelle/python/target.go index a941a7cc7..c40d6fb3b 100644 --- a/gazelle/python/target.go +++ b/gazelle/python/target.go @@ -99,6 +99,15 @@ func (t *targetBuilder) addResolvedDependency(dep string) *targetBuilder { return t } +// addResolvedDependencies adds multiple dependencies, that have already been +// resolved or generated, to the target. +func (t *targetBuilder) addResolvedDependencies(deps []string) *targetBuilder { + for _, dep := range deps { + t.addResolvedDependency(dep) + } + return t +} + // addVisibility adds visibility labels to the target. func (t *targetBuilder) addVisibility(visibility []string) *targetBuilder { for _, item := range visibility { From 92adfc1fcbf1e875f3a31a80d1c9b227662ec659 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 21 Apr 2024 20:01:21 -0700 Subject: [PATCH 07/25] Actually use the returned annotations --- gazelle/python/generate.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index e2a405b58..c1877aa06 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -233,7 +233,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes collisionErrors := singlylinkedlist.New() appendPyLibrary := func(srcs *treeset.Set, pyLibraryTargetName string) { - allDeps, mainModules, _, err := parser.parse(srcs) + allDeps, mainModules, annotations, err := parser.parse(srcs) if err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -263,6 +263,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addVisibility(visibility). addSrc(filename). addModuleDependencies(mainModules[filename]). + addResolvedDependencies(annotations.includeDep). generateImportsAttribute().build() result.Gen = append(result.Gen, pyBinary) result.Imports = append(result.Imports, pyBinary.PrivateAttr(config.GazelleImportsKey)) @@ -290,6 +291,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addVisibility(visibility). addSrcs(srcs). addModuleDependencies(allDeps). + addResolvedDependencies(annotations.includeDep). generateImportsAttribute(). build() @@ -314,7 +316,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes } if hasPyBinaryEntryPointFile { - deps, _, _, err := parser.parseSingle(pyBinaryEntrypointFilename) + deps, _, annotations, err := parser.parseSingle(pyBinaryEntrypointFilename) if err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -338,6 +340,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addVisibility(visibility). addSrc(pyBinaryEntrypointFilename). addModuleDependencies(deps). + addResolvedDependencies(annotations.includeDep). generateImportsAttribute() pyBinary := pyBinaryTarget.build() @@ -348,7 +351,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes var conftest *rule.Rule if hasConftestFile { - deps, _, _, err := parser.parseSingle(conftestFilename) + deps, _, annotations, err := parser.parseSingle(conftestFilename) if err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -367,6 +370,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes conftestTarget := newTargetBuilder(pyLibraryKind, conftestTargetname, pythonProjectRoot, args.Rel, pyFileNames). addSrc(conftestFilename). addModuleDependencies(deps). + addResolvedDependencies(annotations.includeDep). addVisibility(visibility). setTestonly(). generateImportsAttribute() @@ -379,7 +383,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes var pyTestTargets []*targetBuilder newPyTestTargetBuilder := func(srcs *treeset.Set, pyTestTargetName string) *targetBuilder { - deps, _, _, err := parser.parse(srcs) + deps, _, annotations, err := parser.parse(srcs) if err != nil { log.Fatalf("ERROR: %v\n", err) } @@ -397,6 +401,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes return newTargetBuilder(pyTestKind, pyTestTargetName, pythonProjectRoot, args.Rel, pyFileNames). addSrcs(srcs). addModuleDependencies(deps). + addResolvedDependencies(annotations.includeDep). generateImportsAttribute() } if (hasPyTestEntryPointFile || hasPyTestEntryPointTarget || cfg.CoarseGrainedGeneration()) && !cfg.PerFileGeneration() { From cdf2a4979b934ff4c76ef3950e6b8bcc57adb8d5 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 21 Apr 2024 20:01:32 -0700 Subject: [PATCH 08/25] Correctly return the annotations --- gazelle/python/parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index cc852e63c..c162b7da5 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -137,12 +137,12 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin } mainModules := make(map[string]*treeset.Set, len(allRes)) - annotations := &annotations{} + annotations := new(annotations) for _, res := range allRes { if res.HasMain { mainModules[res.FileName] = treeset.NewWith(moduleComparator) } - annotations, err := annotationsFromComments(res.Comments) + annotations, err = annotationsFromComments(res.Comments) if err != nil { return nil, nil, nil, fmt.Errorf("failed to parse annotations: %w", err) } From 975011b96f8d98eb80569dfab2c532750b61337e Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 11:29:47 -0700 Subject: [PATCH 09/25] git things working --- gazelle/python/parser.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index c162b7da5..d5b6ac128 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -142,7 +142,7 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin if res.HasMain { mainModules[res.FileName] = treeset.NewWith(moduleComparator) } - annotations, err = annotationsFromComments(res.Comments) + thisFileAnnotations, err := annotationsFromComments(res.Comments) if err != nil { return nil, nil, nil, fmt.Errorf("failed to parse annotations: %w", err) } @@ -150,7 +150,7 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin for _, m := range res.Modules { // Check for ignored dependencies set via an annotation to the Python // module. - if annotations.ignores(m.Name) || annotations.ignores(m.From) { + if thisFileAnnotations.ignores(m.Name) || thisFileAnnotations.ignores(m.From) { continue } @@ -165,8 +165,13 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin mainModules[res.FileName].Add(m) } } + + // Collect all annotations from each file into a single annotations struct. + // annotations.ignores = append(annotations.ignores, thisFileAnnotations.ignores) + annotations.includeDep = append(annotations.includeDep, thisFileAnnotations.includeDep...) } + // Remove dupes from the annotations. return modules, mainModules, annotations, nil } From 4cbfecde4d698a724ffd29b60f70f60f4fb6c6c5 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 12:02:26 -0700 Subject: [PATCH 10/25] Finish up the deduping; format stuff --- gazelle/python/parser.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index d5b6ac128..632f2cc50 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -167,11 +167,23 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin } // Collect all annotations from each file into a single annotations struct. - // annotations.ignores = append(annotations.ignores, thisFileAnnotations.ignores) + for k, v := range thisFileAnnotations.ignore { + annotations.ignore[k] = v + } annotations.includeDep = append(annotations.includeDep, thisFileAnnotations.includeDep...) } - // Remove dupes from the annotations. + // Remove dupes. Make a treeset and then "cast" back to []string + depsSet := treeset.NewWith(godsutils.StringComparator) + for _, d := range annotations.includeDep { + depsSet.Add(d) + } + s := make([]string, depsSet.Size()) + for i, v := range depsSet.Values() { + s[i] = fmt.Sprint(v) + } + annotations.includeDep = s + return modules, mainModules, annotations, nil } @@ -217,7 +229,7 @@ const ( // The Gazelle annotation prefix. annotationPrefix string = "gazelle:" // The ignore annotation kind. E.g. '# gazelle:ignore '. - annotationKindIgnore annotationKind = "ignore" + annotationKindIgnore annotationKind = "ignore" annotationKindIncludeDep annotationKind = "include_dep" ) @@ -292,7 +304,7 @@ func annotationsFromComments(comments []comment) (*annotations, error) { } } return &annotations{ - ignore: ignore, + ignore: ignore, includeDep: includeDep, }, nil } From 5ad84718fd53cb11a7d5805cbe93e089ebf98814 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 12:04:05 -0700 Subject: [PATCH 11/25] rename variables --- gazelle/python/parser.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 632f2cc50..2b68eff43 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -137,12 +137,12 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin } mainModules := make(map[string]*treeset.Set, len(allRes)) - annotations := new(annotations) + allAnnotations := new(annotations) for _, res := range allRes { if res.HasMain { mainModules[res.FileName] = treeset.NewWith(moduleComparator) } - thisFileAnnotations, err := annotationsFromComments(res.Comments) + annotations, err := annotationsFromComments(res.Comments) if err != nil { return nil, nil, nil, fmt.Errorf("failed to parse annotations: %w", err) } @@ -150,7 +150,7 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin for _, m := range res.Modules { // Check for ignored dependencies set via an annotation to the Python // module. - if thisFileAnnotations.ignores(m.Name) || thisFileAnnotations.ignores(m.From) { + if annotations.ignores(m.Name) || annotations.ignores(m.From) { continue } @@ -167,24 +167,24 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin } // Collect all annotations from each file into a single annotations struct. - for k, v := range thisFileAnnotations.ignore { - annotations.ignore[k] = v + for k, v := range annotations.ignore { + allAnnotations.ignore[k] = v } - annotations.includeDep = append(annotations.includeDep, thisFileAnnotations.includeDep...) + allAnnotations.includeDep = append(allAnnotations.includeDep, annotations.includeDep...) } // Remove dupes. Make a treeset and then "cast" back to []string depsSet := treeset.NewWith(godsutils.StringComparator) - for _, d := range annotations.includeDep { + for _, d := range allAnnotations.includeDep { depsSet.Add(d) } s := make([]string, depsSet.Size()) for i, v := range depsSet.Values() { s[i] = fmt.Sprint(v) } - annotations.includeDep = s + allAnnotations.includeDep = s - return modules, mainModules, annotations, nil + return modules, mainModules, allAnnotations, nil } // parserResponse represents a response returned by the parser.py for a given From ae8e2008bdffaf2d3385e36847490fd6c9b8bf09 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 12:11:50 -0700 Subject: [PATCH 12/25] factor out a function --- gazelle/python/parser.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 2b68eff43..8cd769197 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -173,20 +173,26 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin allAnnotations.includeDep = append(allAnnotations.includeDep, annotations.includeDep...) } - // Remove dupes. Make a treeset and then "cast" back to []string - depsSet := treeset.NewWith(godsutils.StringComparator) - for _, d := range allAnnotations.includeDep { - depsSet.Add(d) - } - s := make([]string, depsSet.Size()) - for i, v := range depsSet.Values() { - s[i] = fmt.Sprint(v) - } - allAnnotations.includeDep = s + allAnnotations.includeDep = removeDupesFromStringTreeSetSlice(allAnnotations.includeDep) return modules, mainModules, allAnnotations, nil } +// removeDupesFromStringTreeSetSlice takes a []string, makes a set out of the +// elements, and then returns a new []string with all duplicates removed. Order +// is preserved. +func removeDupesFromStringTreeSetSlice(array []string) []string { + s := treeset.NewWith(godsutils.StringComparator) + for _, v := range array { + s.Add(v) + } + dedupe := make([]string, s.Size()) + for i, v := range s.Values() { + dedupe[i] = fmt.Sprint(v) + } + return dedupe +} + // parserResponse represents a response returned by the parser.py for a given // parsed Python module. type parserResponse struct { From b5f3bf9f09ae2900f519d88ce250a38349c87b40 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 12:23:10 -0700 Subject: [PATCH 13/25] update test: check deduping --- gazelle/python/testdata/annotation_include_dep/__main__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gazelle/python/testdata/annotation_include_dep/__main__.py b/gazelle/python/testdata/annotation_include_dep/__main__.py index 7b4569aa4..6d9d8aa24 100644 --- a/gazelle/python/testdata/annotation_include_dep/__main__.py +++ b/gazelle/python/testdata/annotation_include_dep/__main__.py @@ -1,4 +1,6 @@ # gazelle:include_dep //checking/py_binary/from/__main__:works +# Check deduping +# gazelle:include_dep //checking/py_binary/from/__main__:works import module2 From 6f0b7ebba96827af6c8293a463a06a2e4b813342 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 12:41:31 -0700 Subject: [PATCH 14/25] Apply forgotten stash in testdata README --- gazelle/python/testdata/annotation_include_dep/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gazelle/python/testdata/annotation_include_dep/README.md b/gazelle/python/testdata/annotation_include_dep/README.md index a41054acd..4c8afbe5e 100644 --- a/gazelle/python/testdata/annotation_include_dep/README.md +++ b/gazelle/python/testdata/annotation_include_dep/README.md @@ -2,3 +2,9 @@ Test that the Python gazelle annotation `# gazelle:include_dep` correctly adds dependences to the generated target even if those dependencies are not imported by the Python module. + +The root directory tests that all `py_*` targets will correctly include the additional +dependencies. + +The `subpkg` directory tests that all `# gazlle:include_dep` annotations found in all source +files are included in the generated target (such as during `generation_mode package`). From 4d15f7d96ec6451c2777b4b31f948219752734a4 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 13:02:25 -0700 Subject: [PATCH 15/25] Update docs --- gazelle/README.md | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/gazelle/README.md b/gazelle/README.md index 4c1cb2799..f3ff97a9c 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -425,6 +425,100 @@ py_library( [issue-1826]: https://github.com/bazelbuild/rules_python/issues/1826 +### Annotations + +*Annotations* refer to comments found _within Python files_ that configure how +Gazelle acts for that particular file. + +Annotations have the form: + +```python +# gazelle:annotation_name value +``` + +and can reside anywhere within a Python file where comments are valid. For example: + +```python +import foo +# gazelle:annotation_name value + +def bar(): # gazelle:annotation_name value + pass +``` + +The annotations are: + +| **Annotation** | **Default value** | +|---------------------------------------------------------------|-------------------| +| [`# gazelle:ignore imports`](#annotation-ignore) | N/A | +| Tell Gazelle to ignore import statements. `imports` is a comma-separated list of imports to ignore. | | +| [`# gazelle:include_dep targets`](#annotation-include_dep) | N/A | +| Tell Gazelle to include import statements. `targets` is a comma-separated list of target names to include as dependencies. | | + + +#### Annotation: `ignore` + +This annotation accepts a comma-separated string of values. Values are names of Python +imports that Gazelle should _not_ include in target dependencies. + +The annotation can be added multiple times, and all values are combined and +de-duplicated. + +For `python_generation_mode = "package"`, the `ignore` annotations +found across all files included in the generated target are removed from `deps`. + +Example: + +```python +import numpy # a pypi package + +# gazelle:ignore bar.baz.hello,foo +import bar.baz.hello +import foo + +# Ignore this import because _reasons_ +import baz # gazelle:ignore baz +``` + +will cause Gazelle to generate: + +```starlark +deps = ["@pypi//numpy"], +``` + + +#### Annotation: `include_deps` + +This annotation accepts a comma-separated string of values. Values _should_ +be targets names, but no validation is done. + +The annotation can be added multiple times, and all values are combined +and de-duplicated. + +For `python_generation_mode = "package"`, the `include_dep` annotations +found across all files included in the generated target are included in `deps`. + +Example: + +```python +# gazelle:include_dep //foo:bar,:hello_world,//:abc +# gazelle:include_dep //:def,//foo:bar +import numpy # a pypi package +``` + +will cause Gazelle to generate: + +```starlark +deps = [ + ":hello_world", + "//:abc", + "//:def", + "//foo:bar", + "@pypi//numpy", +] +``` + + ### Libraries Python source files are those ending in `.py` but not ending in `_test.py`. From a00b5dab1e6a11727201d4341507482c88e407ba Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 13:03:39 -0700 Subject: [PATCH 16/25] Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4c059ec5..92d8c2af7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,8 @@ A brief description of the categories of changes: downloader. If you see any issues, report in [#1357](https://github.com/bazelbuild/rules_python/issues/1357). The URLs for the whl and sdist files will be written to the lock file. +* (gazelle) Add a new annotation `include_deps`. Also add documentation for + annotations to `gazelle/README.md`. [0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0 [python_default_visibility]: gazelle/README.md#directive-python_default_visibility From 8832835accbf719877542fbbcbc1a88949cc4a0e Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 13:04:33 -0700 Subject: [PATCH 17/25] fix title in docs --- gazelle/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gazelle/README.md b/gazelle/README.md index f3ff97a9c..14686afef 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -487,7 +487,7 @@ deps = ["@pypi//numpy"], ``` -#### Annotation: `include_deps` +#### Annotation: `include_dep` This annotation accepts a comma-separated string of values. Values _should_ be targets names, but no validation is done. From 5a5a5635caab4e1040b6e56d55aa10bf02985746 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 14:15:56 -0700 Subject: [PATCH 18/25] Fix failing test --- gazelle/python/parser.go | 1 + 1 file changed, 1 insertion(+) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 8cd769197..da4023cae 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -138,6 +138,7 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin mainModules := make(map[string]*treeset.Set, len(allRes)) allAnnotations := new(annotations) + allAnnotations.ignore = make(map[string]struct{}) for _, res := range allRes { if res.HasMain { mainModules[res.FileName] = treeset.NewWith(moduleComparator) From ac6151e0aebf5412f517d289daf1ad319e6b2a3e Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 14:23:38 -0700 Subject: [PATCH 19/25] Update README.md --- gazelle/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gazelle/README.md b/gazelle/README.md index 14686afef..cc2e76086 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -489,8 +489,13 @@ deps = ["@pypi//numpy"], #### Annotation: `include_dep` -This annotation accepts a comma-separated string of values. Values _should_ -be targets names, but no validation is done. +This annotation accepts a comma-separated string of values. Values _must_ +be Python targets, but _no validation is done_. If a value is not a Python +target, building will result in an error saying: + +``` + does not have mandatory providers: 'PyInfo' or 'CcInfo' or 'PyInfo'. +``` The annotation can be added multiple times, and all values are combined and de-duplicated. From 98c61fb2db4f97285dc62675adbff47e5c0af5dc Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Tue, 23 Apr 2024 14:33:08 -0700 Subject: [PATCH 20/25] Update readme with link to 'data' feature request --- gazelle/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gazelle/README.md b/gazelle/README.md index cc2e76086..b690767a8 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -497,6 +497,9 @@ target, building will result in an error saying: does not have mandatory providers: 'PyInfo' or 'CcInfo' or 'PyInfo'. ``` +Adding non-Python targets to the generated target is a feature request being +tracked in [Issue #1865](https://github.com/bazelbuild/rules_python/issues/1865). + The annotation can be added multiple times, and all values are combined and de-duplicated. From 8c68bdb0453c2915486a3616643cff419e1ac160 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Wed, 8 May 2024 14:17:03 -0700 Subject: [PATCH 21/25] Update gazelle/README.md Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- gazelle/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gazelle/README.md b/gazelle/README.md index b690767a8..1b418d75a 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -451,7 +451,7 @@ The annotations are: | **Annotation** | **Default value** | |---------------------------------------------------------------|-------------------| | [`# gazelle:ignore imports`](#annotation-ignore) | N/A | -| Tell Gazelle to ignore import statements. `imports` is a comma-separated list of imports to ignore. | | +| Tells Gazelle to ignore import statements. `imports` is a comma-separated list of imports to ignore. | | | [`# gazelle:include_dep targets`](#annotation-include_dep) | N/A | | Tell Gazelle to include import statements. `targets` is a comma-separated list of target names to include as dependencies. | | From cb650d7d6e05f0d33bd01202525abbc85e5e59dd Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Wed, 8 May 2024 14:17:16 -0700 Subject: [PATCH 22/25] Update gazelle/README.md Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- gazelle/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gazelle/README.md b/gazelle/README.md index 1b418d75a..e7b17669a 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -453,7 +453,7 @@ The annotations are: | [`# gazelle:ignore imports`](#annotation-ignore) | N/A | | Tells Gazelle to ignore import statements. `imports` is a comma-separated list of imports to ignore. | | | [`# gazelle:include_dep targets`](#annotation-include_dep) | N/A | -| Tell Gazelle to include import statements. `targets` is a comma-separated list of target names to include as dependencies. | | +| Tells Gazelle to include a set of dependencies, even if they are not imported in a Python module. `targets` is a comma-separated list of target names to include as dependencies. | | #### Annotation: `ignore` From 43f82a810eb0cd32b8839c69bada1b02b7866d0a Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Wed, 8 May 2024 14:20:49 -0700 Subject: [PATCH 23/25] s/includeDep/includeDeps/g --- gazelle/python/generate.go | 10 +++++----- gazelle/python/parser.go | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index c1877aa06..8889438c0 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -263,7 +263,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addVisibility(visibility). addSrc(filename). addModuleDependencies(mainModules[filename]). - addResolvedDependencies(annotations.includeDep). + addResolvedDependencies(annotations.includeDeps). generateImportsAttribute().build() result.Gen = append(result.Gen, pyBinary) result.Imports = append(result.Imports, pyBinary.PrivateAttr(config.GazelleImportsKey)) @@ -291,7 +291,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addVisibility(visibility). addSrcs(srcs). addModuleDependencies(allDeps). - addResolvedDependencies(annotations.includeDep). + addResolvedDependencies(annotations.includeDeps). generateImportsAttribute(). build() @@ -340,7 +340,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addVisibility(visibility). addSrc(pyBinaryEntrypointFilename). addModuleDependencies(deps). - addResolvedDependencies(annotations.includeDep). + addResolvedDependencies(annotations.includeDeps). generateImportsAttribute() pyBinary := pyBinaryTarget.build() @@ -370,7 +370,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes conftestTarget := newTargetBuilder(pyLibraryKind, conftestTargetname, pythonProjectRoot, args.Rel, pyFileNames). addSrc(conftestFilename). addModuleDependencies(deps). - addResolvedDependencies(annotations.includeDep). + addResolvedDependencies(annotations.includeDeps). addVisibility(visibility). setTestonly(). generateImportsAttribute() @@ -401,7 +401,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes return newTargetBuilder(pyTestKind, pyTestTargetName, pythonProjectRoot, args.Rel, pyFileNames). addSrcs(srcs). addModuleDependencies(deps). - addResolvedDependencies(annotations.includeDep). + addResolvedDependencies(annotations.includeDeps). generateImportsAttribute() } if (hasPyTestEntryPointFile || hasPyTestEntryPointTarget || cfg.CoarseGrainedGeneration()) && !cfg.PerFileGeneration() { diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index da4023cae..9218001ec 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -171,10 +171,10 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin for k, v := range annotations.ignore { allAnnotations.ignore[k] = v } - allAnnotations.includeDep = append(allAnnotations.includeDep, annotations.includeDep...) + allAnnotations.includeDeps = append(allAnnotations.includeDeps, annotations.includeDeps...) } - allAnnotations.includeDep = removeDupesFromStringTreeSetSlice(allAnnotations.includeDep) + allAnnotations.includeDeps = removeDupesFromStringTreeSetSlice(allAnnotations.includeDeps) return modules, mainModules, allAnnotations, nil } @@ -274,14 +274,14 @@ type annotations struct { // The parsed modules to be ignored by Gazelle. ignore map[string]struct{} // Labels that Gazelle should include as deps of the generated target. - includeDep []string + includeDeps []string } // annotationsFromComments returns all the annotations parsed out of the // comments of a Python module. func annotationsFromComments(comments []comment) (*annotations, error) { ignore := make(map[string]struct{}) - includeDep := []string{} + includeDeps := []string{} for _, comment := range comments { annotation, err := comment.asAnnotation() if err != nil { @@ -305,14 +305,14 @@ func annotationsFromComments(comments []comment) (*annotations, error) { continue } t = strings.TrimSpace(t) - includeDep = append(includeDep, t) + includeDeps = append(includeDeps, t) } } } } return &annotations{ ignore: ignore, - includeDep: includeDep, + includeDeps: includeDeps, }, nil } From ba2ed39e2345867d4cf8c14711aff9442fc5c8d6 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Wed, 8 May 2024 14:23:10 -0700 Subject: [PATCH 24/25] Correct copyright year --- .../python/testdata/annotation_include_dep/gazelle_python.yaml | 2 +- .../python/testdata/invalid_annotation_include_dep/__init__.py | 2 +- .../python/testdata/invalid_annotation_include_dep/test.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gazelle/python/testdata/annotation_include_dep/gazelle_python.yaml b/gazelle/python/testdata/annotation_include_dep/gazelle_python.yaml index 4b12372b4..7afe81f81 100644 --- a/gazelle/python/testdata/annotation_include_dep/gazelle_python.yaml +++ b/gazelle/python/testdata/annotation_include_dep/gazelle_python.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. +# Copyright 2024 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/gazelle/python/testdata/invalid_annotation_include_dep/__init__.py b/gazelle/python/testdata/invalid_annotation_include_dep/__init__.py index f707fd0ca..61f4c76c3 100644 --- a/gazelle/python/testdata/invalid_annotation_include_dep/__init__.py +++ b/gazelle/python/testdata/invalid_annotation_include_dep/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. +# Copyright 2024 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/gazelle/python/testdata/invalid_annotation_include_dep/test.yaml b/gazelle/python/testdata/invalid_annotation_include_dep/test.yaml index 3325c7c1c..f2159a6cd 100644 --- a/gazelle/python/testdata/invalid_annotation_include_dep/test.yaml +++ b/gazelle/python/testdata/invalid_annotation_include_dep/test.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. +# Copyright 2024 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From b1bfd1ae7127f22d803104b1d450afc1966c7df7 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Wed, 8 May 2024 14:27:53 -0700 Subject: [PATCH 25/25] format --- gazelle/python/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 9218001ec..184fad7c1 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -311,7 +311,7 @@ func annotationsFromComments(comments []comment) (*annotations, error) { } } return &annotations{ - ignore: ignore, + ignore: ignore, includeDeps: includeDeps, }, nil }