Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add fetchJobs option to parallelize submodule updates #323

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Add fetchJobs option to parallelize submodule updates #323

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ad5dc19
Add fetchJobs option to parallelize submodule updates
trylle Aug 9, 2020
a5c1ce9
Support fetch-jobs: 0 to use as many jobs as there are processors
trylle Oct 10, 2020
cd648cf
Merge remote-tracking branch 'actions/main'
trylle Mar 22, 2022
d46e61f
Only process fetch-jobs argument if submodules are enabled
trylle Dec 31, 2022
c4fe551
Merge remote-tracking branch 'actions/main'
trylle Dec 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Merge remote-tracking branch 'actions/main'
  • Loading branch information
@trylle
trylle committed Dec 31, 2022
commit c4fe5518253fa506597ab13d068d5dadeff7f136
243 changes: 29 additions & 214 deletions dist/index.js
View file
Original file line number Diff line number Diff line change
@@ -7851,219 +7851,34 @@ class GitOutput {
Object.defineProperty(exports, "__esModule", {
value: true
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.cleanup = exports.getSource = void 0;
const core = __importStar(__webpack_require__(470));
const fsHelper = __importStar(__webpack_require__(618));
const gitAuthHelper = __importStar(__webpack_require__(287));
const gitCommandManager = __importStar(__webpack_require__(289));
const gitDirectoryHelper = __importStar(__webpack_require__(438));
const githubApiHelper = __importStar(__webpack_require__(464));
const io = __importStar(__webpack_require__(1));
const path = __importStar(__webpack_require__(622));
const refHelper = __importStar(__webpack_require__(227));
const stateHelper = __importStar(__webpack_require__(153));
const urlHelper = __importStar(__webpack_require__(81));
function getSource(settings) {
return __awaiter(this, void 0, void 0, function* () {
// Repository URL
core.info(`Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`);
const repositoryUrl = urlHelper.getFetchUrl(settings);
// Remove conflicting file path
if (fsHelper.fileExistsSync(settings.repositoryPath)) {
yield io.rmRF(settings.repositoryPath);
}
// Create directory
let isExisting = true;
if (!fsHelper.directoryExistsSync(settings.repositoryPath)) {
isExisting = false;
yield io.mkdirP(settings.repositoryPath);
}
// Git command manager
core.startGroup('Getting Git version info');
const git = yield getGitCommandManager(settings);
core.endGroup();
// Prepare existing directory, otherwise recreate
if (isExisting) {
yield gitDirectoryHelper.prepareExistingDirectory(git, settings.repositoryPath, repositoryUrl, settings.clean, settings.ref);
}
if (!git) {
// Downloading using REST API
core.info(`The repository will be downloaded using the GitHub REST API`);
core.info(`To create a local Git repository instead, add Git ${gitCommandManager.MinimumGitVersion} or higher to the PATH`);
if (settings.submodules) {
throw new Error(`Input 'submodules' not supported when falling back to download using the GitHub REST API. To create a local Git repository instead, add Git ${gitCommandManager.MinimumGitVersion} or higher to the PATH.`);
}
else if (settings.sshKey) {
throw new Error(`Input 'ssh-key' not supported when falling back to download using the GitHub REST API. To create a local Git repository instead, add Git ${gitCommandManager.MinimumGitVersion} or higher to the PATH.`);
}
yield githubApiHelper.downloadRepository(settings.authToken, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit, settings.repositoryPath);
return;
}
// Save state for POST action
stateHelper.setRepositoryPath(settings.repositoryPath);
// Initialize the repository
if (!fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git'))) {
core.startGroup('Initializing the repository');
yield git.init();
yield git.remoteAdd('origin', repositoryUrl);
core.endGroup();
}
// Disable automatic garbage collection
core.startGroup('Disabling automatic garbage collection');
if (!(yield git.tryDisableAutomaticGarbageCollection())) {
core.warning(`Unable to turn off git automatic garbage collection. The git fetch operation may trigger garbage collection and cause a delay.`);
}
core.endGroup();
const authHelper = gitAuthHelper.createAuthHelper(git, settings);
try {
// Configure auth
core.startGroup('Setting up auth');
yield authHelper.configureAuth();
core.endGroup();
// Determine the default branch
if (!settings.ref && !settings.commit) {
core.startGroup('Determining the default branch');
if (settings.sshKey) {
settings.ref = yield git.getDefaultBranch(repositoryUrl);
}
else {
settings.ref = yield githubApiHelper.getDefaultBranch(settings.authToken, settings.repositoryOwner, settings.repositoryName);
}
core.endGroup();
}
// LFS install
if (settings.lfs) {
yield git.lfsInstall();
}
// Fetch
core.startGroup('Fetching the repository');
if (settings.fetchDepth <= 0) {
// Fetch all branches and tags
let refSpec = refHelper.getRefSpecForAllHistory(settings.ref, settings.commit);
yield git.fetch(refSpec);
// When all history is fetched, the ref we're interested in may have moved to a different
// commit (push or force push). If so, fetch again with a targeted refspec.
if (!(yield refHelper.testRef(git, settings.ref, settings.commit))) {
refSpec = refHelper.getRefSpec(settings.ref, settings.commit);
yield git.fetch(refSpec);
}
}
else {
const refSpec = refHelper.getRefSpec(settings.ref, settings.commit);
yield git.fetch(refSpec, settings.fetchDepth);
}
core.endGroup();
// Checkout info
core.startGroup('Determining the checkout info');
const checkoutInfo = yield refHelper.getCheckoutInfo(git, settings.ref, settings.commit);
core.endGroup();
// LFS fetch
// Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time).
// Explicit lfs fetch will fetch lfs objects in parallel.
if (settings.lfs) {
core.startGroup('Fetching LFS objects');
yield git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref);
core.endGroup();
}
// Checkout
core.startGroup('Checking out the ref');
yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint);
core.endGroup();
// Submodules
if (settings.submodules) {
try {
// Temporarily override global config
core.startGroup('Setting up auth for fetching submodules');
yield authHelper.configureGlobalAuth();
core.endGroup();
// Checkout submodules
core.startGroup('Fetching submodules');
yield git.submoduleSync(settings.nestedSubmodules);
yield git.submoduleUpdate(settings.fetchDepth, settings.nestedSubmodules, settings.fetchJobs);
yield git.submoduleForeach('git config --local gc.auto 0', settings.nestedSubmodules);
core.endGroup();
// Persist credentials
if (settings.persistCredentials) {
core.startGroup('Persisting credentials for submodules');
yield authHelper.configureSubmoduleAuth();
core.endGroup();
}
}
finally {
// Remove temporary global config override
yield authHelper.removeGlobalAuth();
}
}
// Get commit information
const commitInfo = yield git.log1();
// Log commit sha
yield git.log1("--format='%H'");
// Check for incorrect pull request merge commit
yield refHelper.checkCommitInfo(settings.authToken, commitInfo, settings.repositoryOwner, settings.repositoryName, settings.ref, settings.commit);
}
finally {
// Remove auth
if (!settings.persistCredentials) {
core.startGroup('Removing auth');
yield authHelper.removeAuth();
core.endGroup();
}
}
});
}
exports.getSource = getSource;
function cleanup(repositoryPath) {
return __awaiter(this, void 0, void 0, function* () {
// Repo exists?
if (!repositoryPath ||
!fsHelper.fileExistsSync(path.join(repositoryPath, '.git', 'config'))) {
return;
}
let git;
try {
git = yield gitCommandManager.createCommandManager(repositoryPath, false);
}
catch (_a) {
return;
}
// Remove auth
const authHelper = gitAuthHelper.createAuthHelper(git);
yield authHelper.removeAuth();
});
}
exports.cleanup = cleanup;
function getGitCommandManager(settings) {
return __awaiter(this, void 0, void 0, function* () {
core.info(`Working directory is '${settings.repositoryPath}'`);
try {
return yield gitCommandManager.createCommandManager(settings.repositoryPath, settings.lfs);
}
catch (err) {
// Git is required for LFS
if (settings.lfs) {
throw err;
}
// Otherwise fallback to REST API
return undefined;
}
});
exports.default = void 0;

var _rng = _interopRequireDefault(__webpack_require__(506));

var _stringify = _interopRequireDefault(__webpack_require__(960));

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function v4(options, buf, offset) {
options = options || {};

const rnds = options.random || (options.rng || _rng.default)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved`


rnds[6] = rnds[6] & 0x0f | 0x40;
rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided

if (buf) {
offset = offset || 0;

for (let i = 0; i < 16; ++i) {
buf[offset + i] = rnds[i];
}

return buf;
}

return (0, _stringify.default)(rnds);
}

var _default = v4;
@@ -32206,7 +32021,7 @@ function getSource(settings) {
// Checkout submodules
core.startGroup('Fetching submodules');
yield git.submoduleSync(settings.nestedSubmodules);
yield git.submoduleUpdate(settings.fetchDepth, settings.nestedSubmodules);
yield git.submoduleUpdate(settings.fetchDepth, settings.nestedSubmodules, settings.fetchJobs);
yield git.submoduleForeach('git config --local gc.auto 0', settings.nestedSubmodules);
core.endGroup();
// Persist credentials
26 changes: 13 additions & 13 deletions src/git-source-provider.ts
View file
Original file line number Diff line number Diff line change
@@ -203,19 +203,19 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
await authHelper.configureGlobalAuth()
core.endGroup()

// Checkout submodules
core.startGroup('Fetching submodules')
await git.submoduleSync(settings.nestedSubmodules)
await git.submoduleUpdate(
settings.fetchDepth,
settings.nestedSubmodules,
settings.fetchJobs
)
await git.submoduleForeach(
'git config --local gc.auto 0',
settings.nestedSubmodules
)
core.endGroup()
// Checkout submodules
core.startGroup('Fetching submodules')
await git.submoduleSync(settings.nestedSubmodules)
await git.submoduleUpdate(
settings.fetchDepth,
settings.nestedSubmodules,
settings.fetchJobs
)
await git.submoduleForeach(
'git config --local gc.auto 0',
settings.nestedSubmodules
)
core.endGroup()

// Persist credentials
if (settings.persistCredentials) {
You are viewing a condensed version of this merge commit. You can view the full changes here.