fix: support heredoc

packages/checksum-collector/src/checksum-collector.ts

@@ -59,23 +59,6 @@ export const tokenize = (shellText: string): ShellWordToken[] => {
     return parts;
 };
 
-function* lineSeparator(text: string) {
-    const lines = text.split("\n");
-    let currentStatements: string[] = [];
-    let currentLineNumber = 0;
-    for (const line of lines) {
-        currentStatements.push(line);
-        if (/\\\s*$/.test(line)) {
-            yield { line: line, statement: currentStatements.join("\n"), lineNumber: currentLineNumber };
-            currentLineNumber++;
-            continue;
-        }
-        yield { line: line, statement: currentStatements.join("\n"), lineNumber: currentLineNumber };
-        currentStatements = [];
-        currentLineNumber++;
-    }
-}
-
 const memorize = <CB extends (arg: string) => unknown>(fn: CB): CB => {
     const cacheMap = new Map<unknown, unknown>();
     return ((proxyArg: string) => {
@@ -104,7 +87,10 @@ export const collectExecutableCommands = (content: string): ExecutableCommand[]
     const memorizedTokenize = memorize(tokenize);
     // collect chmod as executable commands
     const chmodList: ExecutableCommand[] = [];
-    for (const { line, lineNumber } of lineSeparator(content)) {
+    const lines = content.split("\n");
+    for (let index = 0; index < lines.length - 1; index++) {
+        const line = lines[index];
+        const lineNumber = index + 1;
         const absoluteIndex = source.positionToIndex({
             line: lineNumber,
             column: 0
@@ -132,12 +118,47 @@ export const collectExecutableCommands = (content: string): ExecutableCommand[]
         }
     }
     // check checksum commands
-    for (const { statement, lineNumber } of lineSeparator(content)) {
-        const absoluteIndex = source.positionToIndex({
+    // Note: command → line → parse for avoiding here document parsing
+    const checksums = new RegExp(CHECKSUM_COMMANDS.join("|"), "g");
+    const matches = content.matchAll(checksums);
+    for (const match of matches) {
+        if (!match.index) {
+            continue;
+        }
+        const position = source.indexToPosition(match.index);
+        const getTokens = (
+            lineNumber: number,
+            text: string
+        ):
+            | {
+                  lineNumber: number;
+                  tokens: ShellWordToken[];
+              }
+            | undefined => {
+            try {
+                const tokens = memorizedTokenize(text);
+                return {
+                    lineNumber,
+                    tokens
+                };
+            } catch {
+                const prevLine = lineNumber - 1;
+                // reach first line
+                if (prevLine === 0) {
+                    return;
+                }
+                return getTokens(prevLine, lines[prevLine - 1] + "\n" + text);
+            }
+        };
+        const result = getTokens(position.line, lines[position.line - 1]);
+        if (!result) {
+            continue;
+        }
+        const { tokens, lineNumber } = result;
+        const absoluteLineStartIndex = source.positionToIndex({
             line: lineNumber,
             column: 0
         });
-        const tokens = memorizedTokenize(statement);
         const checksumToken = tokens.find((token) => {
             return CHECKSUM_COMMANDS.includes(token.value);
         });
@@ -148,10 +169,16 @@ export const collectExecutableCommands = (content: string): ExecutableCommand[]
                     chmod.checked = true;
                     chmod.checkedCommand = {
                         binary: checksumToken.value,
-                        range: [checksumToken.range[0] + absoluteIndex, checksumToken.range[1] + absoluteIndex],
+                        range: [
+                            checksumToken.range[0] + absoluteLineStartIndex,
+                            checksumToken.range[1] + absoluteLineStartIndex
+                        ],
                         targetToken: {
                             ...checkedToken,
-                            range: [checkedToken.range[0] + absoluteIndex, checkedToken.range[1] + absoluteIndex]
+                            range: [
+                                checkedToken.range[0] + absoluteLineStartIndex,
+                                checkedToken.range[1] + absoluteLineStartIndex
+                            ]
                         }
                     };
                 }

packages/checksum-collector/test/snapshots/checksum/output.json

@@ -1,8 +1,8 @@
 [
     {
         "range": [
-            462,
-            483
+            513,
+            534
         ],
         "statement": "chmod 755 jq-linux64 ",
         "binary": "jq-linux64",

packages/checksum-collector/test/snapshots/install-shellscript/output.json

@@ -1,44 +1,44 @@
 [
     {
         "range": [
-            465,
-            494
+            496,
+            525
         ],
         "statement": "chmod +x ${BIN_DIR}/kustomize",
         "binary": "kustomize",
         "checked": false
     },
     {
         "range": [
-            545,
-            572
+            678,
+            705
         ],
         "statement": "chmod +x ${BIN_DIR}/kubectl",
         "binary": "kubectl",
         "checked": false
     },
     {
         "range": [
-            943,
-            971
+            973,
+            1001
         ],
         "statement": "chmod +x ${BIN_DIR}/conftest",
         "binary": "conftest",
         "checked": false
     },
     {
         "range": [
-            1201,
-            1233
+            1202,
+            1234
         ],
         "statement": "chmod 755 /usr/local/bin/kubectl",
         "binary": "kubectl",
         "checked": false
     },
     {
         "range": [
-            1271,
-            1298
+            1372,
+            1399
         ],
         "statement": "chmod +x /usr/local/bin/jq ",
         "binary": "jq",

packages/checksum-collector/test/snapshots/parse/input.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -x
+username="composer"
+
+sudo useradd ${username} -u 2000 -G docker -s /bin/bash
+
+conf_dir="conf"
+
+cat << EOF > /etc/systemd/system/composer.service
+[Unit]
+    Description=Composer Service
+    Requires=docker.service network-online.target
+    After=docker.service network-online.target
+    [Service]
+    User=composer
+    Environment="HOME=/home/composer/${conf_dir}"
+    ExecStart=/usr/bin/docker run --rm -v  /var/run/docker.sock:/var/run/docker.sock -v "/home/composer/.docker:/root/.docker" -v "/home/composer:/home/composer" -w="/home/composer/${conf_dir}" docker/compose up -d
+    ExecStop=/usr/bin/docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v "/home/composer/.docker:/root/.docker" -v "/home/composer:/home/composer" -w="/home/composer/${conf_dir}" docker/compose rm -f
+    Restart=on-failure
+    RestartSec=10
+    [Install]
+    WantedBy=multi-user.target
+EOF
+
+sudo chmod 644 /etc/systemd/system/composer.service
+sudo chown -R ${username}:${username} /home/${username}
+sudo systemctl daemon-reload
+sudo systemctl enable --now --no-block composer.service
+sudo systemctl start composer.service
+
+
+# The application log before datadog-agent is started is not sent to datadog-logging.
+# So send it again.
+sleep 30
+
+ids=(`docker ps -f name=${conf_dir}_ -q |xargs`)
+for id in "${ids[@]}"
+do
+    echo "==========="
+    echo "id: $id"
+    log_file="$id.log"
+    docker logs $id > /tmp/${log_file}
+    docker cp /tmp/${log_file}  $id:/tmp/${log_file}
+    docker exec $id /bin/sh -c "cat /tmp/${log_file} >> /proc/1/fd/1"
+done

packages/checksum-collector/test/snapshots/parse/output.json

@@ -0,0 +1 @@
+[]

packages/secretlint-rule-checksum/src/secretlint-rule-checksum.ts

@@ -53,7 +53,7 @@ export const creator: SecretLintRuleCreator<Options> = {
                         });
                     });
                 } catch (error) {
-                    console.error(error);
+                    console.error("parse error", error, source);
                 }
             }
         };