diff --git a/packages/checksum-collector/src/checksum-collector.ts b/packages/checksum-collector/src/checksum-collector.ts index cf977ad..06c7ca3 100644 --- a/packages/checksum-collector/src/checksum-collector.ts +++ b/packages/checksum-collector/src/checksum-collector.ts @@ -59,23 +59,6 @@ export const tokenize = (shellText: string): ShellWordToken[] => { return parts; }; -function* lineSeparator(text: string) { - const lines = text.split("\n"); - let currentStatements: string[] = []; - let currentLineNumber = 0; - for (const line of lines) { - currentStatements.push(line); - if (/\\\s*$/.test(line)) { - yield { line: line, statement: currentStatements.join("\n"), lineNumber: currentLineNumber }; - currentLineNumber++; - continue; - } - yield { line: line, statement: currentStatements.join("\n"), lineNumber: currentLineNumber }; - currentStatements = []; - currentLineNumber++; - } -} - const memorize = unknown>(fn: CB): CB => { const cacheMap = new Map(); return ((proxyArg: string) => { @@ -104,7 +87,10 @@ export const collectExecutableCommands = (content: string): ExecutableCommand[] const memorizedTokenize = memorize(tokenize); // collect chmod as executable commands const chmodList: ExecutableCommand[] = []; - for (const { line, lineNumber } of lineSeparator(content)) { + const lines = content.split("\n"); + for (let index = 0; index < lines.length - 1; index++) { + const line = lines[index]; + const lineNumber = index + 1; const absoluteIndex = source.positionToIndex({ line: lineNumber, column: 0 @@ -132,12 +118,47 @@ export const collectExecutableCommands = (content: string): ExecutableCommand[] } } // check checksum commands - for (const { statement, lineNumber } of lineSeparator(content)) { - const absoluteIndex = source.positionToIndex({ + // Note: command → line → parse for avoiding here document parsing + const checksums = new RegExp(CHECKSUM_COMMANDS.join("|"), "g"); + const matches = content.matchAll(checksums); + for (const match of matches) { + if (!match.index) { + continue; + } + const position = source.indexToPosition(match.index); + const getTokens = ( + lineNumber: number, + text: string + ): + | { + lineNumber: number; + tokens: ShellWordToken[]; + } + | undefined => { + try { + const tokens = memorizedTokenize(text); + return { + lineNumber, + tokens + }; + } catch { + const prevLine = lineNumber - 1; + // reach first line + if (prevLine === 0) { + return; + } + return getTokens(prevLine, lines[prevLine - 1] + "\n" + text); + } + }; + const result = getTokens(position.line, lines[position.line - 1]); + if (!result) { + continue; + } + const { tokens, lineNumber } = result; + const absoluteLineStartIndex = source.positionToIndex({ line: lineNumber, column: 0 }); - const tokens = memorizedTokenize(statement); const checksumToken = tokens.find((token) => { return CHECKSUM_COMMANDS.includes(token.value); }); @@ -148,10 +169,16 @@ export const collectExecutableCommands = (content: string): ExecutableCommand[] chmod.checked = true; chmod.checkedCommand = { binary: checksumToken.value, - range: [checksumToken.range[0] + absoluteIndex, checksumToken.range[1] + absoluteIndex], + range: [ + checksumToken.range[0] + absoluteLineStartIndex, + checksumToken.range[1] + absoluteLineStartIndex + ], targetToken: { ...checkedToken, - range: [checkedToken.range[0] + absoluteIndex, checkedToken.range[1] + absoluteIndex] + range: [ + checkedToken.range[0] + absoluteLineStartIndex, + checkedToken.range[1] + absoluteLineStartIndex + ] } }; } diff --git a/packages/checksum-collector/test/snapshots/checksum/output.json b/packages/checksum-collector/test/snapshots/checksum/output.json index 5f39c8e..6b06d8a 100644 --- a/packages/checksum-collector/test/snapshots/checksum/output.json +++ b/packages/checksum-collector/test/snapshots/checksum/output.json @@ -1,8 +1,8 @@ [ { "range": [ - 462, - 483 + 513, + 534 ], "statement": "chmod 755 jq-linux64 ", "binary": "jq-linux64", diff --git a/packages/checksum-collector/test/snapshots/install-shellscript/output.json b/packages/checksum-collector/test/snapshots/install-shellscript/output.json index 2c8bb91..13e5ee1 100644 --- a/packages/checksum-collector/test/snapshots/install-shellscript/output.json +++ b/packages/checksum-collector/test/snapshots/install-shellscript/output.json @@ -1,8 +1,8 @@ [ { "range": [ - 465, - 494 + 496, + 525 ], "statement": "chmod +x ${BIN_DIR}/kustomize", "binary": "kustomize", @@ -10,8 +10,8 @@ }, { "range": [ - 545, - 572 + 678, + 705 ], "statement": "chmod +x ${BIN_DIR}/kubectl", "binary": "kubectl", @@ -19,8 +19,8 @@ }, { "range": [ - 943, - 971 + 973, + 1001 ], "statement": "chmod +x ${BIN_DIR}/conftest", "binary": "conftest", @@ -28,8 +28,8 @@ }, { "range": [ - 1201, - 1233 + 1202, + 1234 ], "statement": "chmod 755 /usr/local/bin/kubectl", "binary": "kubectl", @@ -37,8 +37,8 @@ }, { "range": [ - 1271, - 1298 + 1372, + 1399 ], "statement": "chmod +x /usr/local/bin/jq ", "binary": "jq", diff --git a/packages/checksum-collector/test/snapshots/parse/input.sh b/packages/checksum-collector/test/snapshots/parse/input.sh new file mode 100644 index 0000000..91d329c --- /dev/null +++ b/packages/checksum-collector/test/snapshots/parse/input.sh @@ -0,0 +1,44 @@ +#!/bin/bash -x +username="composer" + +sudo useradd ${username} -u 2000 -G docker -s /bin/bash + +conf_dir="conf" + +cat << EOF > /etc/systemd/system/composer.service +[Unit] + Description=Composer Service + Requires=docker.service network-online.target + After=docker.service network-online.target + [Service] + User=composer + Environment="HOME=/home/composer/${conf_dir}" + ExecStart=/usr/bin/docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v "/home/composer/.docker:/root/.docker" -v "/home/composer:/home/composer" -w="/home/composer/${conf_dir}" docker/compose up -d + ExecStop=/usr/bin/docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v "/home/composer/.docker:/root/.docker" -v "/home/composer:/home/composer" -w="/home/composer/${conf_dir}" docker/compose rm -f + Restart=on-failure + RestartSec=10 + [Install] + WantedBy=multi-user.target +EOF + +sudo chmod 644 /etc/systemd/system/composer.service +sudo chown -R ${username}:${username} /home/${username} +sudo systemctl daemon-reload +sudo systemctl enable --now --no-block composer.service +sudo systemctl start composer.service + + +# The application log before datadog-agent is started is not sent to datadog-logging. +# So send it again. +sleep 30 + +ids=(`docker ps -f name=${conf_dir}_ -q |xargs`) +for id in "${ids[@]}" +do + echo "===========" + echo "id: $id" + log_file="$id.log" + docker logs $id > /tmp/${log_file} + docker cp /tmp/${log_file} $id:/tmp/${log_file} + docker exec $id /bin/sh -c "cat /tmp/${log_file} >> /proc/1/fd/1" +done diff --git a/packages/checksum-collector/test/snapshots/parse/output.json b/packages/checksum-collector/test/snapshots/parse/output.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/packages/checksum-collector/test/snapshots/parse/output.json @@ -0,0 +1 @@ +[] diff --git a/packages/secretlint-rule-checksum/src/secretlint-rule-checksum.ts b/packages/secretlint-rule-checksum/src/secretlint-rule-checksum.ts index cc4d674..6ad5d3e 100644 --- a/packages/secretlint-rule-checksum/src/secretlint-rule-checksum.ts +++ b/packages/secretlint-rule-checksum/src/secretlint-rule-checksum.ts @@ -53,7 +53,7 @@ export const creator: SecretLintRuleCreator = { }); }); } catch (error) { - console.error(error); + console.error("parse error", error, source); } } };