diff --git a/.licenses/npm/@actions/cache.dep.yml b/.licenses/npm/@actions/cache.dep.yml index b716d1d..2cf3e93 100644 Binary files a/.licenses/npm/@actions/cache.dep.yml and b/.licenses/npm/@actions/cache.dep.yml differ diff --git a/README.md b/README.md index d2aa0fa..6f3eff7 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,11 @@ See ["Caching dependencies to speed up workflows"](https://help.github.com/githu * Updated the minimum runner version support from node 12 -> node 16. * Fixed avoiding empty cache save when no files are available for caching. * Fixed tar creation error while trying to create tar with path as `~/` home folder on `ubuntu-latest`. -* Fixed zstd failing on amazon linux 2.0 runners -* Fixed cache not working with github workspace directory or current directory +* Fixed zstd failing on amazon linux 2.0 runners. +* Fixed cache not working with github workspace directory or current directory. * Fixed the download stuck problem by introducing a timeout of 1 hour for cache downloads. +* Fix zstd not working for windows on gnu tar in issues. +* Allowing users to provide a custom timeout as input for aborting download of a cache segment using an environment variable `SEGMENT_DOWNLOAD_TIMEOUT_MIN`. Default is 60 minutes. Refer [here](https://github.com/actions/cache/blob/v2/README.md) for previous versions @@ -35,6 +37,9 @@ If you are using this inside a container, a POSIX-compliant `tar` needs to be in * `restore-keys` - An ordered list of keys to use for restoring stale cache if no cache hit occurred for key. Note `cache-hit` returns false in this case. +#### Environment Variables +* `SEGMENT_DOWNLOAD_TIMEOUT_MIN` - Segment download timeout (in minutes, default `60`) to abort download of the segment if not completed in the defined number of minutes. [Read more](#cache-segment-restore-timeout) + ### Outputs * `cache-hit` - A boolean value to indicate an exact match was found for the key @@ -218,6 +223,11 @@ jobs: if: steps.cache-primes.outputs.cache-hit != 'true' run: ./generate-primes -d prime-numbers ``` +## Cache segment restore timeout + +A cache gets downloaded in multiple segments of fixed sizes (`1GB` for a `32-bit` runner and `2GB` for a `64-bit` runner). Sometimes, a segment download gets stuck which causes the workflow job to be stuck forever and fail. Version `v3.0.8` of `actions/cache` introduces a segment download timeout. The segment download timeout will allow the segment download to get aborted and hence allow the job to proceed with a cache miss. + +Default value of this timeout is 60 minutes and can be customized by specifying an [environment variable](https://docs.github.com/en/actions/learn-github-actions/environment-variables) named `SEGMENT_DOWNLOAD_TIMEOUT_MINS` with timeout value in minutes. ## Contributing We would love for you to contribute to `actions/cache`, pull requests are welcome! Please see the [CONTRIBUTING.md](CONTRIBUTING.md) for more information. diff --git a/RELEASES.md b/RELEASES.md index 59f2c3b..cf12dab 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -24,4 +24,8 @@ - Fixed [#833](https://github.com/actions/cache/issues/833) - cache doesn't work with github workspace directory ### 3.0.7 -- Fixed [#810](https://github.com/actions/cache/issues/810) - download stuck issue. A new timeout is introduced in the download process to abort the download if it gets stuck and doesn't finish within an hour. \ No newline at end of file +- Fixed [#810](https://github.com/actions/cache/issues/810) - download stuck issue. A new timeout is introduced in the download process to abort the download if it gets stuck and doesn't finish within an hour. + +### 3.0.8 +- Fix zstd not working for windows on gnu tar in issues [#888](https://github.com/actions/cache/issues/888) and [#891](https://github.com/actions/cache/issues/891). +- Allowing users to provide a custom timeout as input for aborting download of a cache segment using an environment variable `SEGMENT_DOWNLOAD_TIMEOUT_MIN`. Default is 60 minutes. \ No newline at end of file diff --git a/dist/restore/index.js b/dist/restore/index.js index 119ca14..b4b9aa8 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -37240,6 +37240,7 @@ const fs_1 = __webpack_require__(747); const path = __importStar(__webpack_require__(622)); const utils = __importStar(__webpack_require__(15)); const constants_1 = __webpack_require__(931); +const IS_WINDOWS = process.platform === 'win32'; function getTarPath(args, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { switch (process.platform) { @@ -37287,26 +37288,43 @@ function getWorkingDirectory() { var _a; return (_a = process.env['GITHUB_WORKSPACE']) !== null && _a !== void 0 ? _a : process.cwd(); } +// Common function for extractTar and listTar to get the compression method +function getCompressionProgram(compressionMethod) { + // -d: Decompress. + // unzstd is equivalent to 'zstd -d' + // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. + // Using 30 here because we also support 32-bit self-hosted runners. + switch (compressionMethod) { + case constants_1.CompressionMethod.Zstd: + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + ]; + case constants_1.CompressionMethod.ZstdWithoutLong: + return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']; + default: + return ['-z']; + } +} +function listTar(archivePath, compressionMethod) { + return __awaiter(this, void 0, void 0, function* () { + const args = [ + ...getCompressionProgram(compressionMethod), + '-tf', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ]; + yield execTar(args, compressionMethod); + }); +} +exports.listTar = listTar; function extractTar(archivePath, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into const workingDirectory = getWorkingDirectory(); yield io.mkdirP(workingDirectory); - // --d: Decompress. - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - function getCompressionProgram() { - switch (compressionMethod) { - case constants_1.CompressionMethod.Zstd: - return ['--use-compress-program', 'unzstd --long=30']; - case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', 'unzstd']; - default: - return ['-z']; - } - } const args = [ - ...getCompressionProgram(), + ...getCompressionProgram(compressionMethod), '-xf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -37325,15 +37343,19 @@ function createTar(archiveFolder, sourceDirectories, compressionMethod) { fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n')); const workingDirectory = getWorkingDirectory(); // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. + // zstdmt is equivalent to 'zstd -T0' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. function getCompressionProgram() { switch (compressionMethod) { case constants_1.CompressionMethod.Zstd: - return ['--use-compress-program', 'zstdmt --long=30']; + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ]; case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', 'zstdmt']; + return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']; default: return ['-z']; } @@ -37355,32 +37377,6 @@ function createTar(archiveFolder, sourceDirectories, compressionMethod) { }); } exports.createTar = createTar; -function listTar(archivePath, compressionMethod) { - return __awaiter(this, void 0, void 0, function* () { - // --d: Decompress. - // --long=#: Enables long distance matching with # bits. - // Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - function getCompressionProgram() { - switch (compressionMethod) { - case constants_1.CompressionMethod.Zstd: - return ['--use-compress-program', 'unzstd --long=30']; - case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', 'unzstd']; - default: - return ['-z']; - } - } - const args = [ - ...getCompressionProgram(), - '-tf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ]; - yield execTar(args, compressionMethod); - }); -} -exports.listTar = listTar; //# sourceMappingURL=tar.js.map /***/ }), @@ -40832,9 +40828,16 @@ function getDownloadOptions(copy) { result.segmentTimeoutInMs = copy.segmentTimeoutInMs; } } + const segmentDownloadTimeoutMins = process.env['SEGMENT_DOWNLOAD_TIMEOUT_MINS']; + if (segmentDownloadTimeoutMins && + !isNaN(Number(segmentDownloadTimeoutMins)) && + isFinite(Number(segmentDownloadTimeoutMins))) { + result.segmentTimeoutInMs = Number(segmentDownloadTimeoutMins) * 60 * 1000; + } core.debug(`Use Azure SDK: ${result.useAzureSdk}`); core.debug(`Download concurrency: ${result.downloadConcurrency}`); core.debug(`Request timeout (ms): ${result.timeoutInMs}`); + core.debug(`Cache segment download timeout mins env var: ${process.env['SEGMENT_DOWNLOAD_TIMEOUT_MINS']}`); core.debug(`Segment download timeout (ms): ${result.segmentTimeoutInMs}`); return result; } diff --git a/dist/save/index.js b/dist/save/index.js index 18825d3..80b8752 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -37240,6 +37240,7 @@ const fs_1 = __webpack_require__(747); const path = __importStar(__webpack_require__(622)); const utils = __importStar(__webpack_require__(15)); const constants_1 = __webpack_require__(931); +const IS_WINDOWS = process.platform === 'win32'; function getTarPath(args, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { switch (process.platform) { @@ -37287,26 +37288,43 @@ function getWorkingDirectory() { var _a; return (_a = process.env['GITHUB_WORKSPACE']) !== null && _a !== void 0 ? _a : process.cwd(); } +// Common function for extractTar and listTar to get the compression method +function getCompressionProgram(compressionMethod) { + // -d: Decompress. + // unzstd is equivalent to 'zstd -d' + // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. + // Using 30 here because we also support 32-bit self-hosted runners. + switch (compressionMethod) { + case constants_1.CompressionMethod.Zstd: + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' + ]; + case constants_1.CompressionMethod.ZstdWithoutLong: + return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']; + default: + return ['-z']; + } +} +function listTar(archivePath, compressionMethod) { + return __awaiter(this, void 0, void 0, function* () { + const args = [ + ...getCompressionProgram(compressionMethod), + '-tf', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ]; + yield execTar(args, compressionMethod); + }); +} +exports.listTar = listTar; function extractTar(archivePath, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into const workingDirectory = getWorkingDirectory(); yield io.mkdirP(workingDirectory); - // --d: Decompress. - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - function getCompressionProgram() { - switch (compressionMethod) { - case constants_1.CompressionMethod.Zstd: - return ['--use-compress-program', 'unzstd --long=30']; - case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', 'unzstd']; - default: - return ['-z']; - } - } const args = [ - ...getCompressionProgram(), + ...getCompressionProgram(compressionMethod), '-xf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -37325,15 +37343,19 @@ function createTar(archiveFolder, sourceDirectories, compressionMethod) { fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n')); const workingDirectory = getWorkingDirectory(); // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. + // zstdmt is equivalent to 'zstd -T0' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. function getCompressionProgram() { switch (compressionMethod) { case constants_1.CompressionMethod.Zstd: - return ['--use-compress-program', 'zstdmt --long=30']; + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ]; case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', 'zstdmt']; + return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']; default: return ['-z']; } @@ -37355,32 +37377,6 @@ function createTar(archiveFolder, sourceDirectories, compressionMethod) { }); } exports.createTar = createTar; -function listTar(archivePath, compressionMethod) { - return __awaiter(this, void 0, void 0, function* () { - // --d: Decompress. - // --long=#: Enables long distance matching with # bits. - // Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - function getCompressionProgram() { - switch (compressionMethod) { - case constants_1.CompressionMethod.Zstd: - return ['--use-compress-program', 'unzstd --long=30']; - case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', 'unzstd']; - default: - return ['-z']; - } - } - const args = [ - ...getCompressionProgram(), - '-tf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ]; - yield execTar(args, compressionMethod); - }); -} -exports.listTar = listTar; //# sourceMappingURL=tar.js.map /***/ }), @@ -40832,9 +40828,16 @@ function getDownloadOptions(copy) { result.segmentTimeoutInMs = copy.segmentTimeoutInMs; } } + const segmentDownloadTimeoutMins = process.env['SEGMENT_DOWNLOAD_TIMEOUT_MINS']; + if (segmentDownloadTimeoutMins && + !isNaN(Number(segmentDownloadTimeoutMins)) && + isFinite(Number(segmentDownloadTimeoutMins))) { + result.segmentTimeoutInMs = Number(segmentDownloadTimeoutMins) * 60 * 1000; + } core.debug(`Use Azure SDK: ${result.useAzureSdk}`); core.debug(`Download concurrency: ${result.downloadConcurrency}`); core.debug(`Request timeout (ms): ${result.timeoutInMs}`); + core.debug(`Cache segment download timeout mins env var: ${process.env['SEGMENT_DOWNLOAD_TIMEOUT_MINS']}`); core.debug(`Segment download timeout (ms): ${result.segmentTimeoutInMs}`); return result; } diff --git a/package-lock.json b/package-lock.json index e60f053..698c822 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "cache", - "version": "3.0.7", + "version": "3.0.8", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cache", - "version": "3.0.7", + "version": "3.0.8", "license": "MIT", "dependencies": { - "@actions/cache": "^3.0.3", + "@actions/cache": "^3.0.4", "@actions/core": "^1.7.0", "@actions/exec": "^1.1.1", "@actions/io": "^1.1.2" @@ -36,9 +36,9 @@ } }, "node_modules/@actions/cache": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.0.3.tgz", - "integrity": "sha512-kn0pZRQNFRg1IQnW/N7uTNbbLqYalvQW2bmrznn3C34LMY/rSuEmH6Uo69HDh335Q0vKs9kg/jsIarzUBKzEXg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.0.4.tgz", + "integrity": "sha512-9RwVL8/ISJoYWFNH1wR/C26E+M3HDkGPWmbFJMMCKwTkjbNZJreMT4XaR/EB1bheIvN4PREQxEQQVJ18IPnf/Q==", "dependencies": { "@actions/core": "^1.2.6", "@actions/exec": "^1.0.1", @@ -9533,9 +9533,9 @@ }, "dependencies": { "@actions/cache": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.0.3.tgz", - "integrity": "sha512-kn0pZRQNFRg1IQnW/N7uTNbbLqYalvQW2bmrznn3C34LMY/rSuEmH6Uo69HDh335Q0vKs9kg/jsIarzUBKzEXg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.0.4.tgz", + "integrity": "sha512-9RwVL8/ISJoYWFNH1wR/C26E+M3HDkGPWmbFJMMCKwTkjbNZJreMT4XaR/EB1bheIvN4PREQxEQQVJ18IPnf/Q==", "requires": { "@actions/core": "^1.2.6", "@actions/exec": "^1.0.1", diff --git a/package.json b/package.json index aaa1370..20786f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cache", - "version": "3.0.7", + "version": "3.0.8", "private": true, "description": "Cache dependencies and build outputs", "main": "dist/restore/index.js", @@ -23,7 +23,7 @@ "author": "GitHub", "license": "MIT", "dependencies": { - "@actions/cache": "^3.0.3", + "@actions/cache": "^3.0.4", "@actions/core": "^1.7.0", "@actions/exec": "^1.1.1", "@actions/io": "^1.1.2" @@ -48,4 +48,4 @@ "ts-jest": "^28.0.2", "typescript": "^4.6.4" } -} +} \ No newline at end of file