Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support custom report dir for dependency-submission #189

Merged
merged 3 commits into from
Apr 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Test dependency graph
name: Test dependency submission failures

on:
workflow_call:
Expand Down
78 changes: 77 additions & 1 deletion .github/workflows/integ-test-dependency-submission.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Test dependency graph
name: Test dependency submission

on:
workflow_call:
Expand Down Expand Up @@ -245,3 +245,79 @@ jobs:
uses: ./dependency-submission
with:
build-root-directory: .github/workflow-samples/groovy-dsl

custom-report-dir-submit:
strategy:
fail-fast: false
matrix:
os: ${{fromJSON(inputs.runner-os)}}
runs-on: ${{ matrix.os }}
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Initialize integ-test
uses: ./.github/actions/init-integ-test

- name: Generate dependency graph
id: dependency-graph
uses: ./dependency-submission
with:
dependency-graph: generate-and-submit
build-root-directory: .github/workflow-samples/groovy-dsl
env:
DEPENDENCY_GRAPH_REPORT_DIR: '${{ github.workspace }}/custom/report-dir'
- name: Check generated dependency graphs
shell: bash
run: |
echo "report file: ${{ steps.dependency-graph.outputs.dependency-graph-file }}"

if [ ! -e "${{ steps.dependency-graph.outputs.dependency-graph-file }}" ]; then
echo "Did not find dependency graph file"
exit 1
fi

if [ -z "$(ls -A "${{ github.workspace }}/custom/report-dir")" ]; then
echo "No dependency graph files found in custom directory"
exit 1
fi

custom-report-dir-upload:
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Initialize integ-test
uses: ./.github/actions/init-integ-test

- name: Generate and upload dependency graph
id: dependency-graph
uses: ./dependency-submission
with:
dependency-graph: generate-and-upload
build-root-directory: .github/workflow-samples/groovy-dsl
env:
DEPENDENCY_GRAPH_REPORT_DIR: '${{ github.workspace }}/custom/report-dir'

custom-report-dir-download-and-submit:
needs: custom-report-dir-upload
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v4
- name: Initialize integ-test
uses: ./.github/actions/init-integ-test

- name: Download and submit dependency graph
uses: ./dependency-submission
with:
dependency-graph: download-and-submit
build-root-directory: .github/workflow-samples/groovy-dsl
env:
DEPENDENCY_GRAPH_REPORT_DIR: '${{ github.workspace }}/custom/report-dir'
- name: Check downloaded dependency graph
shell: bash
run: |
if [ -z "$(ls -A "${{ github.workspace }}/custom/report-dir")" ]; then
echo "No dependency graph files found in custom directory"
exit 1
fi
4 changes: 4 additions & 0 deletions sources/src/configuration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ export class DependencyGraphConfig {
return DependencyGraphConfig.constructJobCorrelator(github.context.workflow, github.context.job, getJobMatrix())
}

getReportDirectory(): string {
return path.resolve(getWorkspaceDirectory(), 'dependency-graph-reports')
}

static constructJobCorrelator(workflow: string, jobId: string, matrixJson: string): string {
const matrixString = this.describeMatrix(matrixJson)
const label = matrixString ? `${workflow}-${jobId}-${matrixString}` : `${workflow}-${jobId}`
Expand Down
41 changes: 22 additions & 19 deletions sources/src/dependency-graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,7 @@ export async function setup(config: DependencyGraphConfig): Promise<void> {
maybeExportVariable('GITHUB_DEPENDENCY_GRAPH_REF', github.context.ref)
maybeExportVariable('GITHUB_DEPENDENCY_GRAPH_SHA', getShaFromContext())
maybeExportVariable('GITHUB_DEPENDENCY_GRAPH_WORKSPACE', getWorkspaceDirectory())
maybeExportVariable(
'DEPENDENCY_GRAPH_REPORT_DIR',
path.resolve(getWorkspaceDirectory(), 'dependency-graph-reports')
)
maybeExportVariable('DEPENDENCY_GRAPH_REPORT_DIR', config.getReportDirectory())

// To clear the dependency graph, we generate an empty graph by excluding all projects and configurations
if (option === DependencyGraphOption.Clear) {
Expand All @@ -62,22 +59,22 @@ export async function complete(config: DependencyGraphConfig): Promise<void> {
return
case DependencyGraphOption.GenerateAndSubmit:
case DependencyGraphOption.Clear: // Submit the empty dependency graph
await submitDependencyGraphs(await findGeneratedDependencyGraphFiles())
await submitDependencyGraphs(await findDependencyGraphFiles())
return
case DependencyGraphOption.GenerateAndUpload:
await uploadDependencyGraphs(await findGeneratedDependencyGraphFiles(), config)
await uploadDependencyGraphs(await findDependencyGraphFiles(), config)
}
} catch (e) {
warnOrFail(config, option, e)
}
}

async function findGeneratedDependencyGraphFiles(): Promise<string[]> {
const workspaceDirectory = getWorkspaceDirectory()
return await findDependencyGraphFiles(workspaceDirectory)
}

async function uploadDependencyGraphs(dependencyGraphFiles: string[], config: DependencyGraphConfig): Promise<void> {
if (dependencyGraphFiles.length === 0) {
core.info('No dependency graph files found to upload.')
return
}

if (isRunningInActEnvironment()) {
core.info('Dependency graph upload not supported in the ACT environment.')
core.info(`Would upload: ${dependencyGraphFiles.join(', ')}`)
Expand Down Expand Up @@ -111,6 +108,11 @@ async function downloadAndSubmitDependencyGraphs(config: DependencyGraphConfig):
}

async function submitDependencyGraphs(dependencyGraphFiles: string[]): Promise<void> {
if (dependencyGraphFiles.length === 0) {
core.info('No dependency graph files found to submit.')
return
}

if (isRunningInActEnvironment()) {
core.info('Dependency graph submit not supported in the ACT environment.')
core.info(`Would submit: ${dependencyGraphFiles.join(', ')}`)
Expand Down Expand Up @@ -156,8 +158,6 @@ async function submitDependencyGraphFile(jsonFile: string): Promise<void> {
}

async function downloadDependencyGraphs(): Promise<string[]> {
const workspaceDirectory = getWorkspaceDirectory()

const findBy = github.context.payload.workflow_run
? {
token: getGithubToken(),
Expand All @@ -168,34 +168,37 @@ async function downloadDependencyGraphs(): Promise<string[]> {
: undefined

const artifactClient = new DefaultArtifactClient()
const downloadPath = path.resolve(workspaceDirectory, 'dependency-graph')

const dependencyGraphArtifacts = (
await artifactClient.listArtifacts({
latest: true,
findBy
})
).artifacts.filter(candidate => candidate.name.startsWith(DEPENDENCY_GRAPH_PREFIX))
).artifacts.filter(artifact => artifact.name.startsWith(DEPENDENCY_GRAPH_PREFIX))

for (const artifact of dependencyGraphArtifacts) {
const downloadedArtifact = await artifactClient.downloadArtifact(artifact.id, {
path: downloadPath,
findBy
})
core.info(`Downloading dependency-graph artifact ${artifact.name} to ${downloadedArtifact.downloadPath}`)
}

return findDependencyGraphFiles(downloadPath)
return findDependencyGraphFiles()
}

async function findDependencyGraphFiles(dir: string): Promise<string[]> {
const globber = await glob.create(`${dir}/dependency-graph-reports/*.json`)
async function findDependencyGraphFiles(): Promise<string[]> {
const globber = await glob.create(`${getReportDirectory()}/**/*.json`)
const allFiles = await globber.glob()
const unprocessedFiles = allFiles.filter(file => !isProcessed(file))
unprocessedFiles.forEach(markProcessed)
core.info(`Found dependency graph files: ${unprocessedFiles.join(', ')}`)
return unprocessedFiles
}

function getReportDirectory(): string {
return process.env.DEPENDENCY_GRAPH_REPORT_DIR!
}

function isProcessed(dependencyGraphFile: string): boolean {
const markerFile = `${dependencyGraphFile}.processed`
return fs.existsSync(markerFile)
Expand Down
Loading