Add linting for pipelines, workflows and metadata. Fixes #3103

assemblies/static/src/main/resources/hop

@@ -127,7 +127,9 @@ if [ -d "plugins/engines/beam" ]; then
   CLASSPATH="${CLASSPATH}:plugins/engines/beam/*"
 fi
 
-"${_HOP_JAVA}" ${HOP_OPTIONS} -Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}" org.apache.hop.hop.Hop "$@"
+# The JVM runs from the Hop installation, so user.dir is never where the user typed the command.
+# Pass the directory they were in, for subcommands which take a path from them.
+"${_HOP_JAVA}" ${HOP_OPTIONS} -Dhop.origin.dir="${ORIGINDIR}" -Djava.library.path="${LIBPATH}" -classpath "${CLASSPATH}" org.apache.hop.hop.Hop "$@"
 EXITCODE=$?
 
 cd "${ORIGINDIR}" || exit 1

assemblies/static/src/main/resources/hop.bat

@@ -20,6 +20,9 @@ REM
 setlocal EnableDelayedExpansion
 
 REM switch to script directory
+REM The JVM runs from the Hop installation, so user.dir is never where the user typed the
+REM command. Remember the directory they were in, for subcommands which take a path from them.
+set _ORIGINDIR=%CD%
 cd /D %~dp0
 
 REM Optional user-level env written by `hop setup` (does not override already-set variables)
@@ -107,5 +110,5 @@ set _cmdline=%*
 
 :Run
 
-%_HOP_JAVA% -classpath %CLASSPATH% -Djava.library.path=%LIBSPATH% %_HOP_OPTIONS% org.apache.hop.hop.Hop %_cmdline%%
+%_HOP_JAVA% -classpath %CLASSPATH% -Djava.library.path=%LIBSPATH% -Dhop.origin.dir="%_ORIGINDIR%" %_HOP_OPTIONS% org.apache.hop.hop.Hop %_cmdline%%
 

core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java

@@ -138,6 +138,21 @@ public enum HopExtensionPoint {
   HopGuiPipelineAfterClose("Called after a pipeline is closed in the Hop GUI (PipelineMeta)"),
   HopGuiWorkflowAfterClose("Called after a workflow is closed in the Hop GUI (WorkflowMeta)"),
 
+  /**
+   * Called before Hop GUI commits files to git, so that optional plugins can inspect what is about
+   * to be committed and refuse it. Payload is a GUI extension object (see ui module) carrying the
+   * git directory, the files being committed, and a cancel flag the listener sets to stop the
+   * commit.
+   *
+   * <p>Like git's own {@code pre-commit} hook this is called for commits the user authors, and not
+   * for revert or cherry-pick, which replay content that is already in the history.
+   *
+   * <p>Refusing the commit leaves the working tree and the index as they are, again as git's hook
+   * behaves: files which were already staged stay staged.
+   */
+  HopGuiFileBeforeCommit(
+      "Called before Hop GUI commits files to git (HopGuiFileBeforeCommitExtension)"),
+
   GetFieldsExtension("Get Fields dialog"),
 
   HopEnvironmentAfterInit(

docs/hop-user-manual/modules/ROOT/nav.adoc

@@ -547,6 +547,7 @@ under the License.
 ** xref:hop-server/async-web-service.adoc[Async Web Service]
 * xref:hop-rest/index.adoc[Hop REST Web Application]
 * xref:hop-tools/index.adoc[Hop Tools]
+** xref:hop-tools/hop-lint.adoc[hop lint]
 ** xref:hop-tools/hop-marketplace.adoc[hop marketplace]
 ** xref:hop-tools/hop-setup.adoc[hop setup]
 ** xref:hop-tools/hop-conf/hop-conf.adoc[hop-conf]
@@ -558,6 +559,9 @@ under the License.
 ** xref:hop-tools/hop-import.adoc[hop-import]
 ** xref:hop-server/index.adoc[hop-server]
 * xref:best-practices/index.adoc[Best Practices]
+* xref:linting/index.adoc[Linting]
+** xref:linting/lint-rules.adoc[Lint rules]
+** xref:linting/lint-rule-packs.adoc[Lint rule packs]
 * xref:protips/index.adoc[Pro Tips]
 * xref:hop-usps.adoc[Unique Selling Propositions]
 * xref:how-to-guides/index.adoc[How-to guides]

docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-lint.adoc

@@ -0,0 +1,152 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+[[HopLint]]
+:imagesdir: ../assets/images
+:description: hop lint runs the Hop linter from the command line, for a quick check while you work or as a build step in CI.
+
+= hop lint
+
+`hop lint` runs the Hop linter over a file, a folder or a whole project and reports what it finds.
+
+It performs static analysis only: it reads pipelines, workflows and metadata without running anything and without connecting to a database, so it is safe to run in a build.
+
+The linter is not part of the standard Hop client, so `hop lint` only exists once the plugin is installed. See xref:linting/index.adoc#_installing[Installing].
+
+See xref:linting/index.adoc[Linting] for what the linter does, and xref:linting/lint-rules.adoc[Lint rules] for the rules and how a project configures them.
+
+== Usage
+
+[source,bash]
+----
+hop lint                              # lint the current directory
+hop lint pipelines/load.hpl           # lint one file
+hop lint /path/to/project             # lint a folder
+hop lint --list-rules                 # show the effective rule set
+hop lint --help                       # full option list
+----
+
+== Options
+
+[cols="1,3", options="header"]
+|===
+|Option |Description
+
+|`--format <format>`
+|Report format: `text` (default), `json` or `sarif`
+
+|`--output <file>`
+|Write the report to a file instead of stdout
+
+|`--fail-on <severity>`
+|Exit with 1 when a finding reaches this severity. Default `ERROR`.
+
+|`--max-warnings <n>`
+|Exit with 1 when more than `n` warnings are reported
+
+|`--baseline <file>`
+|Report only findings that are not in this baseline file
+
+|`--write-baseline <file>`
+|Write today's findings to a baseline file and exit
+
+|`--list-rules`
+|Print the effective rule set, with the pack each rule came from, and exit
+
+|`--list-metadata-types`
+|Print the metadata type keys this installation has registered, and exit
+
+|`--list-fields <plugin-id>`
+|Print the field names a rule can use for a transform or action plugin, and exit
+
+|`--install-hook`
+|Write a git pre-commit hook into `.git/hooks/pre-commit` and exit
+
+|`-c`, `--config <file>`
+|Use this `hop-lint.yml` instead of the one in the project root
+
+|`-s`, `--severity <severity>`
+|Report only findings at this severity or above. This filters the report; it does not change the build verdict, which `--fail-on` decides.
+
+|`-v`, `--verbose`
+|More detail in the report
+
+|`-q`, `--quiet`
+|Report findings only, without the summary
+|===
+
+`--pre-commit` and `--staged-file` are used by the generated git hook and are not normally typed by hand.
+
+== Exit codes
+
+[cols="1,3", options="header"]
+|===
+|Code |Meaning
+
+|`0`
+|No findings at or above the `--fail-on` threshold
+
+|`1`
+|Findings at or above the threshold, or more warnings than `--max-warnings`
+|===
+
+== Continuous integration
+
+[source,bash]
+----
+hop lint --format sarif --output lint.sarif .
+hop lint --fail-on WARNING .
+hop lint --max-warnings 20 .
+----
+
+`sarif` is the format GitHub code scanning, Azure DevOps and most review tooling read, so findings show up as annotations on the pull request.
+`json` is there for everything else.
+Both formats write only the report to stdout, so they can be piped.
+
+[[baseline]]
+== Adopting the linter on an existing project
+
+Pointed at a project that has been running for years, the linter will report a great deal at once.
+Nobody triages that, so record it and fail only on what is added from then on:
+
+[source,bash]
+----
+hop lint --write-baseline hop-lint-baseline.json .   # accept today's findings
+hop lint --baseline hop-lint-baseline.json .         # report only what is new
+----
+
+Commit the baseline file.
+From then on the build passes on the existing backlog and fails the moment something new appears, and the backlog is paid down by deleting entries from the file.
+Runs report how many baseline entries no longer occur, so the file can be pruned as things get fixed.
+
+Findings are matched on the rule, the file and the transform or action they point at.
+They are deliberately not matched on the message, which carries values that change for the same underlying problem, nor on a line number, which lint findings do not have.
+Counts are kept, so a second orphaned transform in a file that already had one is still reported.
+
+== Git pre-commit hook
+
+[source,bash]
+----
+hop lint --install-hook
+----
+
+This writes a `pre-commit` hook into the current repository's `.git/hooks`.
+The hook lints the staged Hop files and blocks the commit when they fail.
+
+Hop Gui can do the same for commits made from within Hop Gui. See *Configuration -> Linter -> Block Git Commits*, which is *off by default*: installing and enabling the linter does not on its own change how committing behaves.
+
+As with git's own `pre-commit` hook, only commits you author are checked.
+Reverting a file and cherry-picking a file are not, because both replay content that is already in the history — and reverting is often how you fix a lint failure in the first place.

docs/hop-user-manual/modules/ROOT/pages/linting/index.adoc

@@ -0,0 +1,118 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+[[Linting]]
+:imagesdir: ../assets/images
+:description: The Hop linter checks pipelines, workflows and metadata against a set of rules, in Hop Gui while you work and on the command line in CI.
+
+= Linting
+
+== Introduction
+
+The linter performs static analysis on a Hop project: it reads pipelines (`.hpl`), workflows (`.hwf`) and metadata, and reports where they break a rule.
+It never runs a pipeline and never connects to a database, so it is safe to run on any project at any time, including in a build.
+
+A linter answers questions a review would otherwise have to catch by eye:
+
+* Is a password hardcoded in a connection instead of coming from a variable?
+* Are there transforms on the canvas that nothing is connected to?
+* Are transforms still carrying their auto-generated names?
+
+Apache Hop ships a small set of rules that are enabled by default.
+Everything beyond those — naming conventions, size ceilings, mandatory descriptions — is a matter of house style, so those rules ship disabled or come from a xref:linting/lint-rule-packs.adoc[rule pack] you install or write yourself.
+
+See xref:linting/lint-rules.adoc[Lint rules] for the rules themselves and how to configure them, and xref:hop-tools/hop-lint.adoc[hop lint] for the command line.
+
+== Installing
+
+The linter is not part of the standard Hop client.
+Install it from the xref:hop-tools/hop-marketplace.adoc[marketplace], where it is listed as *Lint* under Miscellaneous, and restart Hop Gui so the plugin registry picks it up.
+
+From the command line:
+
+[source,bash]
+----
+hop marketplace install hop-misc-lint
+----
+
+== Linting in Hop Gui
+
+[cols="1,3", options="header"]
+|===
+|Where |What it does
+
+|*Tools -> Lint -> Lint Project*
+|Lints every pipeline, workflow and metadata file in the current project
+
+|*Tools -> Lint -> Manage Custom Rules*
+|Shows the effective rule set and edits the rules for this project
+
+|*Tools -> Lint -> Show Lint Results*
+|Opens a window with the findings for the whole project
+
+|Explorer perspective, right-click
+|Lints a single file or a folder
+
+|Pipeline and workflow editor, *Problems* tab
+|Lists the findings for the file being edited, next to Hop's own checks
+
+|Pipeline and workflow canvas
+|Draws a severity badge on each transform or action with findings, and a totals overlay in the top left corner
+
+|*Verify*
+|Adds the lint findings to the Problems tab alongside Hop's own checks
+|===
+
+Files are linted again when you open them, when you save them, and shortly after you stop editing.
+
+Findings for a file that is open are shown in that editor's *Problems* tab, which fills as soon as the file has been linted.
+Clicking the totals overlay on the canvas brings that tab to the front.
+The separate window is used for the cases a Problems tab cannot serve: a whole project or folder, and files which are not open.
+
+Clicking a badge on the canvas, or a row in the results, jumps to the transform or action the finding is about.
+
+== Severities
+
+[cols="1,3", options="header"]
+|===
+|Severity |Meaning
+
+|`ERROR`
+|A defect. In CI this fails the build by default.
+
+|`WARNING`
+|Worth fixing, but does not fail the build unless you ask it to.
+
+|`INFO`
+|Informational.
+|===
+
+A rule's severity is part of the rule, and a project can override it.
+
+== Configuration
+
+Linter behaviour is configured under *Configuration -> Linter* in Hop Gui, and stored in `hop-config.json`.
+The linter can be switched off entirely there, and each of its parts — linting while you edit, the problems bar, the canvas overlays, the checks added to *Verify*, and blocking git commits — can be switched off individually.
+
+Blocking git commits is *off by default*. Nothing about committing changes until you switch it on, and see xref:hop-tools/hop-lint.adoc[hop lint] for which git operations it applies to.
+
+Which *rules* apply is a separate matter, and belongs to the project rather than to the installation: see xref:linting/lint-rules.adoc[Lint rules].
+
+== Adopting the linter on an existing project
+
+Pointed at a project that has been running for years, the linter will report a great deal at once.
+Rather than triaging all of it, record today's findings as a baseline and let the build fail only on what gets added from then on.
+See xref:hop-tools/hop-lint.adoc#baseline[the baseline section] of the `hop lint` documentation.