Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/linters/.trivyignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Ignore the dataplexAdmin role issue
AVD-GCP-0007
File renamed without changes.
1 change: 1 addition & 0 deletions .github/linters/trivy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ignorefile: ".github/linters/.trivyignore"
4 changes: 4 additions & 0 deletions .github/linters/zizmor.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
rules:
unpinned-uses:
ignore:
- ci.yaml
4 changes: 2 additions & 2 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ jobs:
uses: actions/checkout@v5
with:
fetch-depth: 0
persist-credentials: false

- name: Lint Code Base
uses: super-linter/super-linter/slim@v8.0.0
uses: super-linter/super-linter/slim@v8.1.0
env:
DEFAULT_BRANCH: main
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LINTER_RULES_PATH: .
VALIDATE_JSCPD: false
VALIDATE_JAVASCRIPT_PRETTIER: false
VALIDATE_MARKDOWN_PRETTIER: false
Expand Down
12 changes: 12 additions & 0 deletions infra/bigquery-export/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ FROM node:22-slim
# Set the working directory
WORKDIR /app

# Create a non-root user
RUN groupadd -r appuser && useradd -r -g appuser appuser

# Copy package files first for better layer caching
COPY package*.json ./

Expand All @@ -15,4 +18,13 @@ ENV EXPORT_CONFIG=""
# Copy source code
COPY . .

# Change ownership of the app directory to the non-root user
RUN chown -R appuser:appuser /app

# Switch to non-root user
USER appuser

# No healthcheck needed for one-time job containers
HEALTHCHECK NONE

CMD ["node", "index.js"]
13 changes: 13 additions & 0 deletions infra/dataform-service/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ FROM node:22-slim
# Set the working directory
WORKDIR /app

# Create a non-root user
RUN groupadd -r appuser && useradd -r -g appuser appuser

# Copy package files first for better layer caching
COPY package*.json ./

Expand All @@ -12,11 +15,21 @@ RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --f
# Copy source code
COPY . .

# Change ownership of the app directory to the non-root user
RUN chown -R appuser:appuser /app

# Switch to non-root user
USER appuser

# Set default port (Cloud Run will override this)
ENV PORT=8080

# Expose port for Cloud Run
EXPOSE 8080

# Add healthcheck
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
CMD node -e "require('http').get('http://localhost:$PORT/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => { process.exit(1) })" || exit 1

# Start the function
CMD ["npm", "start"]
10 changes: 8 additions & 2 deletions infra/dataform-service/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -223,14 +223,20 @@ async function mainHandler (req, res) {

console.info(`Received request for path: ${path}`)

if (path === '/trigger' || path.startsWith('/trigger/')) {
if (path === '/health') {
// Health check endpoint
res.status(200).json({
status: 'healthy',
timestamp: new Date().toISOString()
})
} else if (path === '/trigger' || path.startsWith('/trigger/')) {
await handleTrigger(req, res)
} else if (path === '/') {
await handleExport(req, res)
} else {
res.status(404).json({
error: 'Not Found',
message: 'Available endpoints: /, /export'
message: 'Available endpoints: /, /trigger, /health'
})
}
}
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
"name": "crawl-data",
"author": "@max-ostapenko",
"scripts": {
"format": "npx eslint --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",
"lint": "npx eslint .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile",
"format": "npx eslint -c .github/linters/eslint.config.mjs --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",
"lint": "npx eslint -c .github/linters/eslint.config.mjs .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile",
"superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest"
},
"dependencies": {
Expand Down