From 84e606dfac347c57a599fe0acefc86daab454178 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan Date: Thu, 9 Apr 2020 10:29:33 -0400 Subject: [PATCH 01/12] Fallback to GNU tar if BSD tar is unavailable --- __tests__/tar.test.ts | 55 +++++++++++++++++++++++---- dist/restore/index.js | 87 ++++++++++++++++++++++++++++++++++++++++--- dist/save/index.js | 87 ++++++++++++++++++++++++++++++++++++++++--- src/tar.ts | 48 +++++++++++++++++++----- 4 files changed, 249 insertions(+), 28 deletions(-) diff --git a/__tests__/tar.test.ts b/__tests__/tar.test.ts index 55ff4c7..cc94a4d 100644 --- a/__tests__/tar.test.ts +++ b/__tests__/tar.test.ts @@ -1,5 +1,7 @@ import * as exec from "@actions/exec"; import * as io from "@actions/io"; +import * as fs from "fs"; +import * as path from "path"; import * as tar from "../src/tar"; jest.mock("@actions/exec"); @@ -11,17 +13,19 @@ beforeAll(() => { }); }); -test("extract tar", async () => { +test("extract BSD tar", async () => { const mkdirMock = jest.spyOn(io, "mkdirP"); const execMock = jest.spyOn(exec, "exec"); - const archivePath = "cache.tar"; + const IS_WINDOWS = process.platform === "win32"; + const archivePath = IS_WINDOWS + ? `${process.env["windir"]}\\fakepath\\cache.tar` + : "cache.tar"; const targetDirectory = "~/.npm/cache"; await tar.extractTar(archivePath, targetDirectory); expect(mkdirMock).toHaveBeenCalledWith(targetDirectory); - const IS_WINDOWS = process.platform === "win32"; const tarPath = IS_WINDOWS ? `${process.env["windir"]}\\System32\\tar.exe` : "tar"; @@ -29,13 +33,48 @@ test("extract tar", async () => { expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [ "-xz", "-f", - archivePath, + archivePath?.replace(/\\/g, "/"), "-C", - targetDirectory + targetDirectory?.replace(/\\/g, "/"), ]); }); -test("create tar", async () => { +test("extract GNU tar", async () => { + const IS_WINDOWS = process.platform === "win32"; + if (IS_WINDOWS) { + jest.mock("fs"); + + const execMock = jest.spyOn(exec, "exec"); + const existsSyncMock = jest + .spyOn(fs, "existsSync") + .mockReturnValue(false); + const isGnuTarMock = jest + .spyOn(tar, "isGnuTar") + .mockReturnValue(Promise.resolve(true)); + const archivePath = `${process.env["windir"]}\\fakepath\\cache.tar`; + const targetDirectory = "~/.npm/cache"; + + await tar.extractTar(archivePath, targetDirectory); + + expect(existsSyncMock).toHaveBeenCalledTimes(1); + expect(isGnuTarMock).toHaveBeenCalledTimes(1); + expect(execMock).toHaveBeenCalledTimes(2); + expect(execMock).toHaveBeenLastCalledWith( + "tar", + [ + "-xz", + "-f", + archivePath?.replace(/\\/g, "/"), + "-C", + targetDirectory?.replace(/\\/g, "/"), + "--force-local" + ], + { cwd: undefined } + ); + } +}); + +test("create BSD tar", async () => { const execMock = jest.spyOn(exec, "exec"); const archivePath = "cache.tar"; @@ -50,9 +89,9 @@ test("create tar", async () => { expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [ "-cz", "-f", - archivePath, + archivePath?.replace(/\\/g, "/"), "-C", - sourceDirectory, + sourceDirectory?.replace(/\\/g, "/"), "." ]); }); diff --git a/dist/restore/index.js b/dist/restore/index.js index a3ea855..8ccadfb 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -2928,10 +2928,34 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", { value: true }); +const core = __importStar(__webpack_require__(470)); const exec_1 = __webpack_require__(986); const io = __importStar(__webpack_require__(1)); const fs_1 = __webpack_require__(747); +<<<<<<< HEAD function getTarPath() { +======= +const path = __importStar(__webpack_require__(622)); +const constants_1 = __webpack_require__(694); +function isGnuTar() { + return __awaiter(this, void 0, void 0, function* () { + core.debug("Checking tar --version"); + let versionOutput = ""; + yield exec_1.exec("tar --version", [], { + ignoreReturnCode: true, + silent: true, + listeners: { + stdout: (data) => (versionOutput += data.toString()), + stderr: (data) => (versionOutput += data.toString()) + } + }); + core.debug(versionOutput.trim()); + return versionOutput.toUpperCase().includes("GNU TAR"); + }); +} +exports.isGnuTar = isGnuTar; +function getTarPath(args) { +>>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable return __awaiter(this, void 0, void 0, function* () { // Explicitly use BSD Tar on Windows const IS_WINDOWS = process.platform === "win32"; @@ -2940,38 +2964,91 @@ function getTarPath() { if (fs_1.existsSync(systemTar)) { return systemTar; } + else if (isGnuTar()) { + args.push("--force-local"); + } } return yield io.which("tar", true); }); } +<<<<<<< HEAD function execTar(args) { var _a, _b; return __awaiter(this, void 0, void 0, function* () { try { yield exec_1.exec(`"${yield getTarPath()}"`, args); +======= +function execTar(args, cwd) { + var _a; + return __awaiter(this, void 0, void 0, function* () { + try { + yield exec_1.exec(`"${yield getTarPath(args)}"`, args, { cwd: cwd }); +>>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable } catch (error) { - const IS_WINDOWS = process.platform === "win32"; - if (IS_WINDOWS) { - throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`); - } - throw new Error(`Tar failed with error: ${(_b = error) === null || _b === void 0 ? void 0 : _b.message}`); + throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}`); } }); } +<<<<<<< HEAD function extractTar(archivePath, targetDirectory) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into yield io.mkdirP(targetDirectory); const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; +======= +function getWorkingDirectory() { + var _a; + return _a = process.env["GITHUB_WORKSPACE"], (_a !== null && _a !== void 0 ? _a : process.cwd()); +} +function extractTar(archivePath) { + var _a, _b; + return __awaiter(this, void 0, void 0, function* () { + // Create directory to extract tar into + const workingDirectory = getWorkingDirectory(); + yield io.mkdirP(workingDirectory); + const args = [ + "-xz", + "-f", + (_a = archivePath) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), + "-P", + "-C", + (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/") + ]; +>>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable yield execTar(args); }); } exports.extractTar = extractTar; +<<<<<<< HEAD function createTar(archivePath, sourceDirectory) { return __awaiter(this, void 0, void 0, function* () { const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; yield execTar(args); +======= +function createTar(archiveFolder, sourceDirectories) { + var _a, _b; + return __awaiter(this, void 0, void 0, function* () { + // Write source directories to manifest.txt to avoid command length limits + const manifestFilename = "manifest.txt"; + fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join("\n")); + const workingDirectory = getWorkingDirectory(); + const args = [ + "-cz", + "-f", +<<<<<<< HEAD + constants_1.CacheFilename, + "-P", +======= + (_a = constants_1.CacheFilename) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), +>>>>>>> Fallback to GNU tar if BSD tar is unavailable + "-C", + (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/"), + "--files-from", + manifestFilename + ]; + yield execTar(args, archiveFolder); +>>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable }); } exports.createTar = createTar; diff --git a/dist/save/index.js b/dist/save/index.js index e7e0eae..33d75c6 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2909,10 +2909,34 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", { value: true }); +const core = __importStar(__webpack_require__(470)); const exec_1 = __webpack_require__(986); const io = __importStar(__webpack_require__(1)); const fs_1 = __webpack_require__(747); +<<<<<<< HEAD function getTarPath() { +======= +const path = __importStar(__webpack_require__(622)); +const constants_1 = __webpack_require__(694); +function isGnuTar() { + return __awaiter(this, void 0, void 0, function* () { + core.debug("Checking tar --version"); + let versionOutput = ""; + yield exec_1.exec("tar --version", [], { + ignoreReturnCode: true, + silent: true, + listeners: { + stdout: (data) => (versionOutput += data.toString()), + stderr: (data) => (versionOutput += data.toString()) + } + }); + core.debug(versionOutput.trim()); + return versionOutput.toUpperCase().includes("GNU TAR"); + }); +} +exports.isGnuTar = isGnuTar; +function getTarPath(args) { +>>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable return __awaiter(this, void 0, void 0, function* () { // Explicitly use BSD Tar on Windows const IS_WINDOWS = process.platform === "win32"; @@ -2921,38 +2945,91 @@ function getTarPath() { if (fs_1.existsSync(systemTar)) { return systemTar; } + else if (isGnuTar()) { + args.push("--force-local"); + } } return yield io.which("tar", true); }); } +<<<<<<< HEAD function execTar(args) { var _a, _b; return __awaiter(this, void 0, void 0, function* () { try { yield exec_1.exec(`"${yield getTarPath()}"`, args); +======= +function execTar(args, cwd) { + var _a; + return __awaiter(this, void 0, void 0, function* () { + try { + yield exec_1.exec(`"${yield getTarPath(args)}"`, args, { cwd: cwd }); +>>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable } catch (error) { - const IS_WINDOWS = process.platform === "win32"; - if (IS_WINDOWS) { - throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`); - } - throw new Error(`Tar failed with error: ${(_b = error) === null || _b === void 0 ? void 0 : _b.message}`); + throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}`); } }); } +<<<<<<< HEAD function extractTar(archivePath, targetDirectory) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into yield io.mkdirP(targetDirectory); const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; +======= +function getWorkingDirectory() { + var _a; + return _a = process.env["GITHUB_WORKSPACE"], (_a !== null && _a !== void 0 ? _a : process.cwd()); +} +function extractTar(archivePath) { + var _a, _b; + return __awaiter(this, void 0, void 0, function* () { + // Create directory to extract tar into + const workingDirectory = getWorkingDirectory(); + yield io.mkdirP(workingDirectory); + const args = [ + "-xz", + "-f", + (_a = archivePath) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), + "-P", + "-C", + (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/") + ]; +>>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable yield execTar(args); }); } exports.extractTar = extractTar; +<<<<<<< HEAD function createTar(archivePath, sourceDirectory) { return __awaiter(this, void 0, void 0, function* () { const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; yield execTar(args); +======= +function createTar(archiveFolder, sourceDirectories) { + var _a, _b; + return __awaiter(this, void 0, void 0, function* () { + // Write source directories to manifest.txt to avoid command length limits + const manifestFilename = "manifest.txt"; + fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join("\n")); + const workingDirectory = getWorkingDirectory(); + const args = [ + "-cz", + "-f", +<<<<<<< HEAD + constants_1.CacheFilename, + "-P", +======= + (_a = constants_1.CacheFilename) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), +>>>>>>> Fallback to GNU tar if BSD tar is unavailable + "-C", + (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/"), + "--files-from", + manifestFilename + ]; + yield execTar(args, archiveFolder); +>>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable }); } exports.createTar = createTar; diff --git a/src/tar.ts b/src/tar.ts index 1f572d1..c20c15d 100644 --- a/src/tar.ts +++ b/src/tar.ts @@ -1,14 +1,35 @@ +import * as core from "@actions/core"; import { exec } from "@actions/exec"; import * as io from "@actions/io"; import { existsSync } from "fs"; +import * as path from "path"; -async function getTarPath(): Promise { +export async function isGnuTar(): Promise { + core.debug("Checking tar --version"); + let versionOutput = ""; + await exec("tar --version", [], { + ignoreReturnCode: true, + silent: true, + listeners: { + stdout: (data: Buffer): string => + (versionOutput += data.toString()), + stderr: (data: Buffer): string => (versionOutput += data.toString()) + } + }); + + core.debug(versionOutput.trim()); + return versionOutput.toUpperCase().includes("GNU TAR"); +} + +async function getTarPath(args: string[]): Promise { // Explicitly use BSD Tar on Windows const IS_WINDOWS = process.platform === "win32"; if (IS_WINDOWS) { const systemTar = `${process.env["windir"]}\\System32\\tar.exe`; if (existsSync(systemTar)) { return systemTar; + } else if (isGnuTar()) { + args.push("--force-local"); } } return await io.which("tar", true); @@ -16,14 +37,8 @@ async function getTarPath(): Promise { async function execTar(args: string[]): Promise { try { - await exec(`"${await getTarPath()}"`, args); + await exec(`"${await getTarPath(args)}"`, args); } catch (error) { - const IS_WINDOWS = process.platform === "win32"; - if (IS_WINDOWS) { - throw new Error( - `Tar failed with error: ${error?.message}. Ensure BSD tar is installed and on the PATH.` - ); - } throw new Error(`Tar failed with error: ${error?.message}`); } } @@ -34,7 +49,13 @@ export async function extractTar( ): Promise { // Create directory to extract tar into await io.mkdirP(targetDirectory); - const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; + const args = [ + "-xz", + "-f", + archivePath?.replace(/\\/g, "/"), + "-C", + targetDirectory?.replace(/\\/g, "/") + ]; await execTar(args); } @@ -42,6 +63,13 @@ export async function createTar( archivePath: string, sourceDirectory: string ): Promise { - const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; + const args = [ + "-cz", + "-f", + archivePath?.replace(/\\/g, "/"), + "-C", + sourceDirectory?.replace(/\\/g, "/"), + "." + ]; await execTar(args); } From 96e5a46c57ebbeb88f20116716d147b285ba34d0 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan Date: Fri, 10 Apr 2020 15:26:15 -0400 Subject: [PATCH 02/12] Fix test --- __tests__/tar.test.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/__tests__/tar.test.ts b/__tests__/tar.test.ts index cc94a4d..8e9dd5e 100644 --- a/__tests__/tar.test.ts +++ b/__tests__/tar.test.ts @@ -1,9 +1,10 @@ import * as exec from "@actions/exec"; import * as io from "@actions/io"; -import * as fs from "fs"; import * as path from "path"; import * as tar from "../src/tar"; +import fs = require("fs"); + jest.mock("@actions/exec"); jest.mock("@actions/io"); @@ -42,25 +43,18 @@ test("extract BSD tar", async () => { test("extract GNU tar", async () => { const IS_WINDOWS = process.platform === "win32"; if (IS_WINDOWS) { - jest.mock("fs"); + jest.spyOn(fs, "existsSync").mockReturnValueOnce(false); + jest.spyOn(tar, "isGnuTar").mockReturnValue(Promise.resolve(true)); const execMock = jest.spyOn(exec, "exec"); - const existsSyncMock = jest - .spyOn(fs, "existsSync") - .mockReturnValue(false); - const isGnuTarMock = jest - .spyOn(tar, "isGnuTar") - .mockReturnValue(Promise.resolve(true)); const archivePath = `${process.env["windir"]}\\fakepath\\cache.tar`; const targetDirectory = "~/.npm/cache"; await tar.extractTar(archivePath, targetDirectory); - expect(existsSyncMock).toHaveBeenCalledTimes(1); - expect(isGnuTarMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenCalledTimes(2); expect(execMock).toHaveBeenLastCalledWith( - "tar", + `"tar"`, [ "-xz", "-f", From 7c7d003bbbad33ad70cbf095976259f7c974d91b Mon Sep 17 00:00:00 2001 From: Aiqiao Yan Date: Fri, 10 Apr 2020 15:34:34 -0400 Subject: [PATCH 03/12] Rebase and rebuild --- dist/restore/index.js | 6 +----- dist/save/index.js | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/dist/restore/index.js b/dist/restore/index.js index 8ccadfb..eeb2634 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -3036,12 +3036,8 @@ function createTar(archiveFolder, sourceDirectories) { const args = [ "-cz", "-f", -<<<<<<< HEAD - constants_1.CacheFilename, - "-P", -======= (_a = constants_1.CacheFilename) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), ->>>>>>> Fallback to GNU tar if BSD tar is unavailable + "-P", "-C", (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/"), "--files-from", diff --git a/dist/save/index.js b/dist/save/index.js index 33d75c6..251e559 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -3017,12 +3017,8 @@ function createTar(archiveFolder, sourceDirectories) { const args = [ "-cz", "-f", -<<<<<<< HEAD - constants_1.CacheFilename, - "-P", -======= (_a = constants_1.CacheFilename) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), ->>>>>>> Fallback to GNU tar if BSD tar is unavailable + "-P", "-C", (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/"), "--files-from", From 9fe7ad8b07cdf5e7e198f006d6e5156a9012ef87 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan Date: Mon, 13 Apr 2020 12:20:27 -0400 Subject: [PATCH 04/12] Use path.sep in path replace --- __tests__/tar.test.ts | 29 +++++++---------- dist/restore/index.js | 54 -------------------------------- dist/save/index.js | 73 ------------------------------------------- src/tar.ts | 10 +++--- 4 files changed, 17 insertions(+), 149 deletions(-) diff --git a/__tests__/tar.test.ts b/__tests__/tar.test.ts index 8e9dd5e..d5d9b62 100644 --- a/__tests__/tar.test.ts +++ b/__tests__/tar.test.ts @@ -1,6 +1,5 @@ import * as exec from "@actions/exec"; import * as io from "@actions/io"; -import * as path from "path"; import * as tar from "../src/tar"; import fs = require("fs"); @@ -34,9 +33,9 @@ test("extract BSD tar", async () => { expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [ "-xz", "-f", - archivePath?.replace(/\\/g, "/"), + IS_WINDOWS ? archivePath.replace(/\\/g, "/") : archivePath, "-C", - targetDirectory?.replace(/\\/g, "/"), + IS_WINDOWS ? targetDirectory?.replace(/\\/g, "/") : targetDirectory ]); }); @@ -53,18 +52,14 @@ test("extract GNU tar", async () => { await tar.extractTar(archivePath, targetDirectory); expect(execMock).toHaveBeenCalledTimes(2); - expect(execMock).toHaveBeenLastCalledWith( - `"tar"`, - [ - "-xz", - "-f", - archivePath?.replace(/\\/g, "/"), - "-C", - targetDirectory?.replace(/\\/g, "/"), - "--force-local" - ], - { cwd: undefined } - ); + expect(execMock).toHaveBeenLastCalledWith(`"tar"`, [ + "-xz", + "-f", + archivePath.replace(/\\/g, "/"), + "-C", + targetDirectory?.replace(/\\/g, "/"), + "--force-local" + ]); } }); @@ -83,9 +78,9 @@ test("create BSD tar", async () => { expect(execMock).toHaveBeenCalledWith(`"${tarPath}"`, [ "-cz", "-f", - archivePath?.replace(/\\/g, "/"), + IS_WINDOWS ? archivePath.replace(/\\/g, "/") : archivePath, "-C", - sourceDirectory?.replace(/\\/g, "/"), + IS_WINDOWS ? sourceDirectory?.replace(/\\/g, "/") : sourceDirectory, "." ]); }); diff --git a/dist/restore/index.js b/dist/restore/index.js index eeb2634..732f7fc 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -2932,9 +2932,6 @@ const core = __importStar(__webpack_require__(470)); const exec_1 = __webpack_require__(986); const io = __importStar(__webpack_require__(1)); const fs_1 = __webpack_require__(747); -<<<<<<< HEAD -function getTarPath() { -======= const path = __importStar(__webpack_require__(622)); const constants_1 = __webpack_require__(694); function isGnuTar() { @@ -2955,7 +2952,6 @@ function isGnuTar() { } exports.isGnuTar = isGnuTar; function getTarPath(args) { ->>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable return __awaiter(this, void 0, void 0, function* () { // Explicitly use BSD Tar on Windows const IS_WINDOWS = process.platform === "win32"; @@ -2971,80 +2967,30 @@ function getTarPath(args) { return yield io.which("tar", true); }); } -<<<<<<< HEAD function execTar(args) { var _a, _b; return __awaiter(this, void 0, void 0, function* () { try { yield exec_1.exec(`"${yield getTarPath()}"`, args); -======= -function execTar(args, cwd) { - var _a; - return __awaiter(this, void 0, void 0, function* () { - try { - yield exec_1.exec(`"${yield getTarPath(args)}"`, args, { cwd: cwd }); ->>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable } catch (error) { throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}`); } }); } -<<<<<<< HEAD function extractTar(archivePath, targetDirectory) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into yield io.mkdirP(targetDirectory); const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; -======= -function getWorkingDirectory() { - var _a; - return _a = process.env["GITHUB_WORKSPACE"], (_a !== null && _a !== void 0 ? _a : process.cwd()); -} -function extractTar(archivePath) { - var _a, _b; - return __awaiter(this, void 0, void 0, function* () { - // Create directory to extract tar into - const workingDirectory = getWorkingDirectory(); - yield io.mkdirP(workingDirectory); - const args = [ - "-xz", - "-f", - (_a = archivePath) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), - "-P", - "-C", - (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/") - ]; ->>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable yield execTar(args); }); } exports.extractTar = extractTar; -<<<<<<< HEAD function createTar(archivePath, sourceDirectory) { return __awaiter(this, void 0, void 0, function* () { const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; yield execTar(args); -======= -function createTar(archiveFolder, sourceDirectories) { - var _a, _b; - return __awaiter(this, void 0, void 0, function* () { - // Write source directories to manifest.txt to avoid command length limits - const manifestFilename = "manifest.txt"; - fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join("\n")); - const workingDirectory = getWorkingDirectory(); - const args = [ - "-cz", - "-f", - (_a = constants_1.CacheFilename) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), - "-P", - "-C", - (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/"), - "--files-from", - manifestFilename - ]; - yield execTar(args, archiveFolder); ->>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable }); } exports.createTar = createTar; diff --git a/dist/save/index.js b/dist/save/index.js index 251e559..b83d036 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2913,30 +2913,7 @@ const core = __importStar(__webpack_require__(470)); const exec_1 = __webpack_require__(986); const io = __importStar(__webpack_require__(1)); const fs_1 = __webpack_require__(747); -<<<<<<< HEAD function getTarPath() { -======= -const path = __importStar(__webpack_require__(622)); -const constants_1 = __webpack_require__(694); -function isGnuTar() { - return __awaiter(this, void 0, void 0, function* () { - core.debug("Checking tar --version"); - let versionOutput = ""; - yield exec_1.exec("tar --version", [], { - ignoreReturnCode: true, - silent: true, - listeners: { - stdout: (data) => (versionOutput += data.toString()), - stderr: (data) => (versionOutput += data.toString()) - } - }); - core.debug(versionOutput.trim()); - return versionOutput.toUpperCase().includes("GNU TAR"); - }); -} -exports.isGnuTar = isGnuTar; -function getTarPath(args) { ->>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable return __awaiter(this, void 0, void 0, function* () { // Explicitly use BSD Tar on Windows const IS_WINDOWS = process.platform === "win32"; @@ -2952,80 +2929,30 @@ function getTarPath(args) { return yield io.which("tar", true); }); } -<<<<<<< HEAD function execTar(args) { var _a, _b; return __awaiter(this, void 0, void 0, function* () { try { yield exec_1.exec(`"${yield getTarPath()}"`, args); -======= -function execTar(args, cwd) { - var _a; - return __awaiter(this, void 0, void 0, function* () { - try { - yield exec_1.exec(`"${yield getTarPath(args)}"`, args, { cwd: cwd }); ->>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable } catch (error) { throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}`); } }); } -<<<<<<< HEAD function extractTar(archivePath, targetDirectory) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into yield io.mkdirP(targetDirectory); const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; -======= -function getWorkingDirectory() { - var _a; - return _a = process.env["GITHUB_WORKSPACE"], (_a !== null && _a !== void 0 ? _a : process.cwd()); -} -function extractTar(archivePath) { - var _a, _b; - return __awaiter(this, void 0, void 0, function* () { - // Create directory to extract tar into - const workingDirectory = getWorkingDirectory(); - yield io.mkdirP(workingDirectory); - const args = [ - "-xz", - "-f", - (_a = archivePath) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), - "-P", - "-C", - (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/") - ]; ->>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable yield execTar(args); }); } exports.extractTar = extractTar; -<<<<<<< HEAD function createTar(archivePath, sourceDirectory) { return __awaiter(this, void 0, void 0, function* () { const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; yield execTar(args); -======= -function createTar(archiveFolder, sourceDirectories) { - var _a, _b; - return __awaiter(this, void 0, void 0, function* () { - // Write source directories to manifest.txt to avoid command length limits - const manifestFilename = "manifest.txt"; - fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join("\n")); - const workingDirectory = getWorkingDirectory(); - const args = [ - "-cz", - "-f", - (_a = constants_1.CacheFilename) === null || _a === void 0 ? void 0 : _a.replace(/\\/g, "/"), - "-P", - "-C", - (_b = workingDirectory) === null || _b === void 0 ? void 0 : _b.replace(/\\/g, "/"), - "--files-from", - manifestFilename - ]; - yield execTar(args, archiveFolder); ->>>>>>> 4fa017f... Fallback to GNU tar if BSD tar is unavailable }); } exports.createTar = createTar; diff --git a/src/tar.ts b/src/tar.ts index c20c15d..dde9b61 100644 --- a/src/tar.ts +++ b/src/tar.ts @@ -28,7 +28,7 @@ async function getTarPath(args: string[]): Promise { const systemTar = `${process.env["windir"]}\\System32\\tar.exe`; if (existsSync(systemTar)) { return systemTar; - } else if (isGnuTar()) { + } else if (await isGnuTar()) { args.push("--force-local"); } } @@ -52,9 +52,9 @@ export async function extractTar( const args = [ "-xz", "-f", - archivePath?.replace(/\\/g, "/"), + archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"), "-C", - targetDirectory?.replace(/\\/g, "/") + targetDirectory.replace(new RegExp("\\" + path.sep, "g"), "/") ]; await execTar(args); } @@ -66,9 +66,9 @@ export async function createTar( const args = [ "-cz", "-f", - archivePath?.replace(/\\/g, "/"), + archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"), "-C", - sourceDirectory?.replace(/\\/g, "/"), + sourceDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"), "." ]; await execTar(args); From 5a0add1806bb8f47699878d071e546b093793bd2 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Wed, 22 Apr 2020 18:23:41 -0400 Subject: [PATCH 05/12] Adds socket timeout and validate file size --- src/cacheHttpClient.ts | 28 ++++++++++++++++++++++++++++ src/constants.ts | 2 ++ 2 files changed, 30 insertions(+) diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 62ae2c1..4cc9c92 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -7,6 +7,8 @@ import { IRequestOptions, ITypedResponse } from "@actions/http-client/interfaces"; + +import { SocketTimeout } from "./constants"; import { ArtifactCacheEntry, CommitCacheRequest, @@ -123,7 +125,33 @@ export async function downloadCache( const stream = fs.createWriteStream(archivePath); const httpClient = new HttpClient("actions/cache"); const downloadResponse = await httpClient.get(archiveLocation); + + // Abort download if no traffic received over the socket. + downloadResponse.message.socket.setTimeout(SocketTimeout, () => { + downloadResponse.message.destroy(); + core.debug( + `Aborting download, socket timed out after ${SocketTimeout} ms` + ); + }); + await pipeResponseToStream(downloadResponse, stream); + + // Validate download size. + var contentLengthHeader = + downloadResponse.message.headers["content-length"]; + + if (contentLengthHeader) { + const expectedLength = parseInt(contentLengthHeader); + const actualLength = utils.getArchiveFileSize(archivePath); + + if (actualLength != expectedLength) { + throw new Error( + `Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}` + ); + } + } else { + core.debug("Unable to validate download, no Content-Length header"); + } } // Reserve Cache diff --git a/src/constants.ts b/src/constants.ts index 5f26e8c..a39e5e0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -18,3 +18,5 @@ export enum Events { Push = "push", PullRequest = "pull_request" } + +export const SocketTimeout = 5000; From cbbb8b4d4f487d1b7589fff9ee89e26d12b54961 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Wed, 22 Apr 2020 18:35:16 -0400 Subject: [PATCH 06/12] Fix lint issue, build .js files --- dist/restore/index.js | 30 ++++++++++++++++++++++++++++++ dist/save/index.js | 29 +++++++++++++++++++++++++++++ src/cacheHttpClient.ts | 2 +- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/dist/restore/index.js b/dist/restore/index.js index 732f7fc..77148ca 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -1256,6 +1256,7 @@ const fs = __importStar(__webpack_require__(747)); const auth_1 = __webpack_require__(226); const http_client_1 = __webpack_require__(539); const utils = __importStar(__webpack_require__(443)); +const constants_1 = __webpack_require__(694); function isSuccessStatusCode(statusCode) { if (!statusCode) { return false; @@ -1339,7 +1340,24 @@ function downloadCache(archiveLocation, archivePath) { const stream = fs.createWriteStream(archivePath); const httpClient = new http_client_1.HttpClient("actions/cache"); const downloadResponse = yield httpClient.get(archiveLocation); + // Abort download if no traffic received over the socket. + downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => { + downloadResponse.message.destroy(); + core.debug(`Aborting download, socket timed out after ${constants_1.SocketTimeout} ms`); + }); yield pipeResponseToStream(downloadResponse, stream); + // Validate download size. + const contentLengthHeader = downloadResponse.message.headers["content-length"]; + if (contentLengthHeader) { + const expectedLength = parseInt(contentLengthHeader); + const actualLength = utils.getArchiveFileSize(archivePath); + if (actualLength != expectedLength) { + throw new Error(`Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}`); + } + } + else { + core.debug("Unable to validate download, no Content-Length header"); + } }); } exports.downloadCache = downloadCache; @@ -1647,6 +1665,10 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); +<<<<<<< HEAD +======= +const glob = __importStar(__webpack_require__(281)); +>>>>>>> 9bb13c7... Fix lint issue, build .js files const io = __importStar(__webpack_require__(1)); const fs = __importStar(__webpack_require__(747)); const os = __importStar(__webpack_require__(87)); @@ -2022,6 +2044,12 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); + this.message.on('aborted', () => { + reject("Request was aborted or closed prematurely"); + }); + this.message.on('timeout', (socket) => { + reject("Request timed out"); + }); this.message.on('end', () => { resolve(output.toString()); }); @@ -2143,6 +2171,7 @@ class HttpClient { let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); + // Check if it's an authentication challenge if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; @@ -2721,6 +2750,7 @@ var Events; Events["Push"] = "push"; Events["PullRequest"] = "pull_request"; })(Events = exports.Events || (exports.Events = {})); +exports.SocketTimeout = 5000; /***/ }), diff --git a/dist/save/index.js b/dist/save/index.js index b83d036..bdc5c09 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -1339,7 +1339,24 @@ function downloadCache(archiveLocation, archivePath) { const stream = fs.createWriteStream(archivePath); const httpClient = new http_client_1.HttpClient("actions/cache"); const downloadResponse = yield httpClient.get(archiveLocation); + // Abort download if no traffic received over the socket. + downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => { + downloadResponse.message.destroy(); + core.debug(`Aborting download, socket timed out after ${constants_1.SocketTimeout} ms`); + }); yield pipeResponseToStream(downloadResponse, stream); + // Validate download size. + const contentLengthHeader = downloadResponse.message.headers["content-length"]; + if (contentLengthHeader) { + const expectedLength = parseInt(contentLengthHeader); + const actualLength = utils.getArchiveFileSize(archivePath); + if (actualLength != expectedLength) { + throw new Error(`Incomplete download. Expected file size: ${expectedLength}, actual file size: ${actualLength}`); + } + } + else { + core.debug("Unable to validate download, no Content-Length header"); + } }); } exports.downloadCache = downloadCache; @@ -1647,6 +1664,10 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); +<<<<<<< HEAD +======= +const glob = __importStar(__webpack_require__(281)); +>>>>>>> 9bb13c7... Fix lint issue, build .js files const io = __importStar(__webpack_require__(1)); const fs = __importStar(__webpack_require__(747)); const os = __importStar(__webpack_require__(87)); @@ -2022,6 +2043,12 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); + this.message.on('aborted', () => { + reject("Request was aborted or closed prematurely"); + }); + this.message.on('timeout', (socket) => { + reject("Request timed out"); + }); this.message.on('end', () => { resolve(output.toString()); }); @@ -2143,6 +2170,7 @@ class HttpClient { let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); + // Check if it's an authentication challenge if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; @@ -2802,6 +2830,7 @@ var Events; Events["Push"] = "push"; Events["PullRequest"] = "pull_request"; })(Events = exports.Events || (exports.Events = {})); +exports.SocketTimeout = 5000; /***/ }), diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 4cc9c92..702a01a 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -137,7 +137,7 @@ export async function downloadCache( await pipeResponseToStream(downloadResponse, stream); // Validate download size. - var contentLengthHeader = + const contentLengthHeader = downloadResponse.message.headers["content-length"]; if (contentLengthHeader) { From 2a973a0f4ea7187ed295bd4e479051fb057deb32 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Tue, 28 Apr 2020 21:31:41 -0400 Subject: [PATCH 07/12] Add comment for SocketTimeout --- dist/restore/index.js | 3 +++ dist/save/index.js | 3 +++ src/constants.ts | 3 +++ 3 files changed, 9 insertions(+) diff --git a/dist/restore/index.js b/dist/restore/index.js index 77148ca..e175183 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -2750,6 +2750,9 @@ var Events; Events["Push"] = "push"; Events["PullRequest"] = "pull_request"; })(Events = exports.Events || (exports.Events = {})); +// Socket timeout in milliseconds during download. If no traffic is received +// over the socket during this period, the socket is destroyed and the download +// is aborted. exports.SocketTimeout = 5000; diff --git a/dist/save/index.js b/dist/save/index.js index bdc5c09..602b1cf 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2830,6 +2830,9 @@ var Events; Events["Push"] = "push"; Events["PullRequest"] = "pull_request"; })(Events = exports.Events || (exports.Events = {})); +// Socket timeout in milliseconds during download. If no traffic is received +// over the socket during this period, the socket is destroyed and the download +// is aborted. exports.SocketTimeout = 5000; diff --git a/src/constants.ts b/src/constants.ts index a39e5e0..2e60e34 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -19,4 +19,7 @@ export enum Events { PullRequest = "pull_request" } +// Socket timeout in milliseconds during download. If no traffic is received +// over the socket during this period, the socket is destroyed and the download +// is aborted. export const SocketTimeout = 5000; From ec7f7ebd08bf74c5f5ae1a027e3cd2cbb62a2164 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Wed, 29 Apr 2020 09:31:53 -0400 Subject: [PATCH 08/12] Use promisify of stream.pipeline for downloading --- dist/restore/index.js | 23 +++++++++++++---------- dist/save/index.js | 23 +++++++++++++---------- src/cacheHttpClient.ts | 15 +++++++-------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/dist/restore/index.js b/dist/restore/index.js index e175183..1fd26f2 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -1255,6 +1255,9 @@ const core = __importStar(__webpack_require__(470)); const fs = __importStar(__webpack_require__(747)); const auth_1 = __webpack_require__(226); const http_client_1 = __webpack_require__(539); +const stream = __importStar(__webpack_require__(794)); +const util = __importStar(__webpack_require__(669)); +const constants_1 = __webpack_require__(694); const utils = __importStar(__webpack_require__(443)); const constants_1 = __webpack_require__(694); function isSuccessStatusCode(statusCode) { @@ -1326,13 +1329,10 @@ function getCacheEntry(keys) { }); } exports.getCacheEntry = getCacheEntry; -function pipeResponseToStream(response, stream) { +function pipeResponseToStream(response, output) { return __awaiter(this, void 0, void 0, function* () { - return new Promise(resolve => { - response.message.pipe(stream).on("close", () => { - resolve(); - }); - }); + const pipeline = util.promisify(stream.pipeline); + yield pipeline(response.message, output); }); } function downloadCache(archiveLocation, archivePath) { @@ -1665,10 +1665,6 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); -<<<<<<< HEAD -======= -const glob = __importStar(__webpack_require__(281)); ->>>>>>> 9bb13c7... Fix lint issue, build .js files const io = __importStar(__webpack_require__(1)); const fs = __importStar(__webpack_require__(747)); const os = __importStar(__webpack_require__(87)); @@ -2894,6 +2890,13 @@ run(); exports.default = run; +/***/ }), + +/***/ 794: +/***/ (function(module) { + +module.exports = require("stream"); + /***/ }), /***/ 826: diff --git a/dist/save/index.js b/dist/save/index.js index 602b1cf..a595665 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -1255,6 +1255,9 @@ const core = __importStar(__webpack_require__(470)); const fs = __importStar(__webpack_require__(747)); const auth_1 = __webpack_require__(226); const http_client_1 = __webpack_require__(539); +const stream = __importStar(__webpack_require__(794)); +const util = __importStar(__webpack_require__(669)); +const constants_1 = __webpack_require__(694); const utils = __importStar(__webpack_require__(443)); function isSuccessStatusCode(statusCode) { if (!statusCode) { @@ -1325,13 +1328,10 @@ function getCacheEntry(keys) { }); } exports.getCacheEntry = getCacheEntry; -function pipeResponseToStream(response, stream) { +function pipeResponseToStream(response, output) { return __awaiter(this, void 0, void 0, function* () { - return new Promise(resolve => { - response.message.pipe(stream).on("close", () => { - resolve(); - }); - }); + const pipeline = util.promisify(stream.pipeline); + yield pipeline(response.message, output); }); } function downloadCache(archiveLocation, archivePath) { @@ -1664,10 +1664,6 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); -<<<<<<< HEAD -======= -const glob = __importStar(__webpack_require__(281)); ->>>>>>> 9bb13c7... Fix lint issue, build .js files const io = __importStar(__webpack_require__(1)); const fs = __importStar(__webpack_require__(747)); const os = __importStar(__webpack_require__(87)); @@ -2876,6 +2872,13 @@ module.exports = require("fs"); /***/ }), +/***/ 794: +/***/ (function(module) { + +module.exports = require("stream"); + +/***/ }), + /***/ 826: /***/ (function(module, __unusedexports, __webpack_require__) { diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 702a01a..41078b3 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -1,12 +1,14 @@ import * as core from "@actions/core"; -import * as fs from "fs"; -import { BearerCredentialHandler } from "@actions/http-client/auth"; import { HttpClient, HttpCodes } from "@actions/http-client"; +import { BearerCredentialHandler } from "@actions/http-client/auth"; import { IHttpClientResponse, IRequestOptions, ITypedResponse } from "@actions/http-client/interfaces"; +import * as fs from "fs"; +import * as stream from "stream"; +import * as util from "util"; import { SocketTimeout } from "./constants"; import { @@ -109,13 +111,10 @@ export async function getCacheEntry( async function pipeResponseToStream( response: IHttpClientResponse, - stream: NodeJS.WritableStream + output: NodeJS.WritableStream ): Promise { - return new Promise(resolve => { - response.message.pipe(stream).on("close", () => { - resolve(); - }); - }); + const pipeline = util.promisify(stream.pipeline); + await pipeline(response.message, output); } export async function downloadCache( From da9f90cb83eb605958d3e7e1c09c21678745e735 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Mon, 11 May 2020 10:49:48 -0400 Subject: [PATCH 09/12] Fix upload chunk retries --- dist/restore/index.js | 110 +++++++++++++++++++++++++++-------------- dist/save/index.js | 110 +++++++++++++++++++++++++++-------------- src/cacheHttpClient.ts | 18 +++---- 3 files changed, 153 insertions(+), 85 deletions(-) diff --git a/dist/restore/index.js b/dist/restore/index.js index 1fd26f2..d2b9af7 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -1382,7 +1382,7 @@ function getContentRange(start, end) { // Content-Range: bytes 0-199/* return `bytes ${start}-${end}/*`; } -function uploadChunk(httpClient, resourceUrl, data, start, end) { +function uploadChunk(httpClient, resourceUrl, openStream, start, end) { return __awaiter(this, void 0, void 0, function* () { core.debug(`Uploading chunk of size ${end - start + @@ -1392,7 +1392,7 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) { "Content-Range": getContentRange(start, end) }; const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { - return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders); + return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders); }); const response = yield uploadChunkRequest(); if (isSuccessStatusCode(response.message.statusCode)) { @@ -1435,13 +1435,12 @@ function uploadFile(httpClient, cacheId, archivePath) { const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { fd, start, end, autoClose: false - }); - yield uploadChunk(httpClient, resourceUrl, chunk, start, end); + }), start, end); } }))); } @@ -1496,7 +1495,9 @@ class BasicCredentialHandler { this.password = password; } prepareRequest(options) { - options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64'); + options.headers['Authorization'] = + 'Basic ' + + Buffer.from(this.username + ':' + this.password).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -1532,7 +1533,8 @@ class PersonalAccessTokenCredentialHandler { // currently implements pre-authorization // TODO: support preAuth = false where it hooks on 401 prepareRequest(options) { - options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); + options.headers['Authorization'] = + 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -2001,6 +2003,7 @@ var HttpCodes; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; + HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; @@ -2025,8 +2028,18 @@ function getProxyUrl(serverUrl) { return proxyUrl ? proxyUrl.href : ''; } exports.getProxyUrl = getProxyUrl; -const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect]; -const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout]; +const HttpRedirectCodes = [ + HttpCodes.MovedPermanently, + HttpCodes.ResourceMoved, + HttpCodes.SeeOther, + HttpCodes.TemporaryRedirect, + HttpCodes.PermanentRedirect +]; +const HttpResponseRetryCodes = [ + HttpCodes.BadGateway, + HttpCodes.ServiceUnavailable, + HttpCodes.GatewayTimeout +]; const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const ExponentialBackoffCeiling = 10; const ExponentialBackoffTimeSlice = 5; @@ -2157,19 +2170,23 @@ class HttpClient { */ async request(verb, requestUrl, data, headers) { if (this._disposed) { - throw new Error("Client has already been disposed."); + throw new Error('Client has already been disposed.'); } let parsedUrl = url.parse(requestUrl); let info = this._prepareRequest(verb, parsedUrl, headers); // Only perform retries on reads since writes may not be idempotent. - let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1; + let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1 + ? this._maxRetries + 1 + : 1; let numTries = 0; let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); // Check if it's an authentication challenge - if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { + if (response && + response.message && + response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; for (let i = 0; i < this.handlers.length; i++) { if (this.handlers[i].canHandleAuthentication(response)) { @@ -2187,21 +2204,32 @@ class HttpClient { } } let redirectsRemaining = this._maxRedirects; - while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 - && this._allowRedirects - && redirectsRemaining > 0) { - const redirectUrl = response.message.headers["location"]; + while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 && + this._allowRedirects && + redirectsRemaining > 0) { + const redirectUrl = response.message.headers['location']; if (!redirectUrl) { // if there's no location to redirect to, we won't break; } let parsedRedirectUrl = url.parse(redirectUrl); - if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) { - throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true."); + if (parsedUrl.protocol == 'https:' && + parsedUrl.protocol != parsedRedirectUrl.protocol && + !this._allowRedirectDowngrade) { + throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.'); } // we need to finish reading the response before reassigning response // which will leak the open socket. await response.readBody(); + // strip authorization header if redirected to a different hostname + if (parsedRedirectUrl.hostname !== parsedUrl.hostname) { + for (let header in headers) { + // header names are case insensitive + if (header.toLowerCase() === 'authorization') { + delete headers[header]; + } + } + } // let's make the request with the new redirectUrl info = this._prepareRequest(verb, parsedRedirectUrl, headers); response = await this.requestRaw(info, data); @@ -2252,8 +2280,8 @@ class HttpClient { */ requestRawWithCallback(info, data, onResult) { let socket; - if (typeof (data) === 'string') { - info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8'); + if (typeof data === 'string') { + info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8'); } let callbackCalled = false; let handleResult = (err, res) => { @@ -2266,7 +2294,7 @@ class HttpClient { let res = new HttpClientResponse(msg); handleResult(null, res); }); - req.on('socket', (sock) => { + req.on('socket', sock => { socket = sock; }); // If we ever get disconnected, we want the socket to timeout eventually @@ -2281,10 +2309,10 @@ class HttpClient { // res should have headers handleResult(err, null); }); - if (data && typeof (data) === 'string') { + if (data && typeof data === 'string') { req.write(data, 'utf8'); } - if (data && typeof (data) !== 'string') { + if (data && typeof data !== 'string') { data.on('close', function () { req.end(); }); @@ -2311,31 +2339,34 @@ class HttpClient { const defaultPort = usingSsl ? 443 : 80; info.options = {}; info.options.host = info.parsedUrl.hostname; - info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort; - info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); + info.options.port = info.parsedUrl.port + ? parseInt(info.parsedUrl.port) + : defaultPort; + info.options.path = + (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); info.options.method = method; info.options.headers = this._mergeHeaders(headers); if (this.userAgent != null) { - info.options.headers["user-agent"] = this.userAgent; + info.options.headers['user-agent'] = this.userAgent; } info.options.agent = this._getAgent(info.parsedUrl); // gives handlers an opportunity to participate if (this.handlers) { - this.handlers.forEach((handler) => { + this.handlers.forEach(handler => { handler.prepareRequest(info.options); }); } return info; } _mergeHeaders(headers) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); if (this.requestOptions && this.requestOptions.headers) { return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers)); } return lowercaseKeys(headers || {}); } _getExistingOrDefaultHeader(additionalHeaders, header, _default) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); let clientHeader; if (this.requestOptions && this.requestOptions.headers) { clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; @@ -2373,7 +2404,7 @@ class HttpClient { proxyAuth: proxyUrl.auth, host: proxyUrl.hostname, port: proxyUrl.port - }, + } }; let tunnelAgent; const overHttps = proxyUrl.protocol === 'https:'; @@ -2400,7 +2431,9 @@ class HttpClient { // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // we have to cast it to any and change it directly - agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }); + agent.options = Object.assign(agent.options || {}, { + rejectUnauthorized: false + }); } return agent; } @@ -2461,7 +2494,7 @@ class HttpClient { msg = contents; } else { - msg = "Failed request: (" + statusCode + ")"; + msg = 'Failed request: (' + statusCode + ')'; } let err = new Error(msg); // attach statusCode and body obj (if available) to the error object @@ -3049,12 +3082,10 @@ function getProxyUrl(reqUrl) { } let proxyVar; if (usingSsl) { - proxyVar = process.env["https_proxy"] || - process.env["HTTPS_PROXY"]; + proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY']; } else { - proxyVar = process.env["http_proxy"] || - process.env["HTTP_PROXY"]; + proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY']; } if (proxyVar) { proxyUrl = url.parse(proxyVar); @@ -3066,7 +3097,7 @@ function checkBypass(reqUrl) { if (!reqUrl.hostname) { return false; } - let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || ''; + let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; if (!noProxy) { return false; } @@ -3087,7 +3118,10 @@ function checkBypass(reqUrl) { upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); } // Compare request host against noproxy - for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) { + for (let upperNoProxyItem of noProxy + .split(',') + .map(x => x.trim().toUpperCase()) + .filter(x => x)) { if (upperReqHosts.some(x => x === upperNoProxyItem)) { return true; } diff --git a/dist/save/index.js b/dist/save/index.js index a595665..acb0391 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -1381,7 +1381,7 @@ function getContentRange(start, end) { // Content-Range: bytes 0-199/* return `bytes ${start}-${end}/*`; } -function uploadChunk(httpClient, resourceUrl, data, start, end) { +function uploadChunk(httpClient, resourceUrl, openStream, start, end) { return __awaiter(this, void 0, void 0, function* () { core.debug(`Uploading chunk of size ${end - start + @@ -1391,7 +1391,7 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) { "Content-Range": getContentRange(start, end) }; const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { - return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders); + return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders); }); const response = yield uploadChunkRequest(); if (isSuccessStatusCode(response.message.statusCode)) { @@ -1434,13 +1434,12 @@ function uploadFile(httpClient, cacheId, archivePath) { const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { + yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { fd, start, end, autoClose: false - }); - yield uploadChunk(httpClient, resourceUrl, chunk, start, end); + }), start, end); } }))); } @@ -1495,7 +1494,9 @@ class BasicCredentialHandler { this.password = password; } prepareRequest(options) { - options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64'); + options.headers['Authorization'] = + 'Basic ' + + Buffer.from(this.username + ':' + this.password).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -1531,7 +1532,8 @@ class PersonalAccessTokenCredentialHandler { // currently implements pre-authorization // TODO: support preAuth = false where it hooks on 401 prepareRequest(options) { - options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); + options.headers['Authorization'] = + 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -2000,6 +2002,7 @@ var HttpCodes; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; + HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; @@ -2024,8 +2027,18 @@ function getProxyUrl(serverUrl) { return proxyUrl ? proxyUrl.href : ''; } exports.getProxyUrl = getProxyUrl; -const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect]; -const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout]; +const HttpRedirectCodes = [ + HttpCodes.MovedPermanently, + HttpCodes.ResourceMoved, + HttpCodes.SeeOther, + HttpCodes.TemporaryRedirect, + HttpCodes.PermanentRedirect +]; +const HttpResponseRetryCodes = [ + HttpCodes.BadGateway, + HttpCodes.ServiceUnavailable, + HttpCodes.GatewayTimeout +]; const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const ExponentialBackoffCeiling = 10; const ExponentialBackoffTimeSlice = 5; @@ -2156,19 +2169,23 @@ class HttpClient { */ async request(verb, requestUrl, data, headers) { if (this._disposed) { - throw new Error("Client has already been disposed."); + throw new Error('Client has already been disposed.'); } let parsedUrl = url.parse(requestUrl); let info = this._prepareRequest(verb, parsedUrl, headers); // Only perform retries on reads since writes may not be idempotent. - let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1; + let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1 + ? this._maxRetries + 1 + : 1; let numTries = 0; let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); // Check if it's an authentication challenge - if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { + if (response && + response.message && + response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; for (let i = 0; i < this.handlers.length; i++) { if (this.handlers[i].canHandleAuthentication(response)) { @@ -2186,21 +2203,32 @@ class HttpClient { } } let redirectsRemaining = this._maxRedirects; - while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 - && this._allowRedirects - && redirectsRemaining > 0) { - const redirectUrl = response.message.headers["location"]; + while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 && + this._allowRedirects && + redirectsRemaining > 0) { + const redirectUrl = response.message.headers['location']; if (!redirectUrl) { // if there's no location to redirect to, we won't break; } let parsedRedirectUrl = url.parse(redirectUrl); - if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) { - throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true."); + if (parsedUrl.protocol == 'https:' && + parsedUrl.protocol != parsedRedirectUrl.protocol && + !this._allowRedirectDowngrade) { + throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.'); } // we need to finish reading the response before reassigning response // which will leak the open socket. await response.readBody(); + // strip authorization header if redirected to a different hostname + if (parsedRedirectUrl.hostname !== parsedUrl.hostname) { + for (let header in headers) { + // header names are case insensitive + if (header.toLowerCase() === 'authorization') { + delete headers[header]; + } + } + } // let's make the request with the new redirectUrl info = this._prepareRequest(verb, parsedRedirectUrl, headers); response = await this.requestRaw(info, data); @@ -2251,8 +2279,8 @@ class HttpClient { */ requestRawWithCallback(info, data, onResult) { let socket; - if (typeof (data) === 'string') { - info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8'); + if (typeof data === 'string') { + info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8'); } let callbackCalled = false; let handleResult = (err, res) => { @@ -2265,7 +2293,7 @@ class HttpClient { let res = new HttpClientResponse(msg); handleResult(null, res); }); - req.on('socket', (sock) => { + req.on('socket', sock => { socket = sock; }); // If we ever get disconnected, we want the socket to timeout eventually @@ -2280,10 +2308,10 @@ class HttpClient { // res should have headers handleResult(err, null); }); - if (data && typeof (data) === 'string') { + if (data && typeof data === 'string') { req.write(data, 'utf8'); } - if (data && typeof (data) !== 'string') { + if (data && typeof data !== 'string') { data.on('close', function () { req.end(); }); @@ -2310,31 +2338,34 @@ class HttpClient { const defaultPort = usingSsl ? 443 : 80; info.options = {}; info.options.host = info.parsedUrl.hostname; - info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort; - info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); + info.options.port = info.parsedUrl.port + ? parseInt(info.parsedUrl.port) + : defaultPort; + info.options.path = + (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); info.options.method = method; info.options.headers = this._mergeHeaders(headers); if (this.userAgent != null) { - info.options.headers["user-agent"] = this.userAgent; + info.options.headers['user-agent'] = this.userAgent; } info.options.agent = this._getAgent(info.parsedUrl); // gives handlers an opportunity to participate if (this.handlers) { - this.handlers.forEach((handler) => { + this.handlers.forEach(handler => { handler.prepareRequest(info.options); }); } return info; } _mergeHeaders(headers) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); if (this.requestOptions && this.requestOptions.headers) { return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers)); } return lowercaseKeys(headers || {}); } _getExistingOrDefaultHeader(additionalHeaders, header, _default) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); let clientHeader; if (this.requestOptions && this.requestOptions.headers) { clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; @@ -2372,7 +2403,7 @@ class HttpClient { proxyAuth: proxyUrl.auth, host: proxyUrl.hostname, port: proxyUrl.port - }, + } }; let tunnelAgent; const overHttps = proxyUrl.protocol === 'https:'; @@ -2399,7 +2430,9 @@ class HttpClient { // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // we have to cast it to any and change it directly - agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }); + agent.options = Object.assign(agent.options || {}, { + rejectUnauthorized: false + }); } return agent; } @@ -2460,7 +2493,7 @@ class HttpClient { msg = contents; } else { - msg = "Failed request: (" + statusCode + ")"; + msg = 'Failed request: (' + statusCode + ')'; } let err = new Error(msg); // attach statusCode and body obj (if available) to the error object @@ -3010,12 +3043,10 @@ function getProxyUrl(reqUrl) { } let proxyVar; if (usingSsl) { - proxyVar = process.env["https_proxy"] || - process.env["HTTPS_PROXY"]; + proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY']; } else { - proxyVar = process.env["http_proxy"] || - process.env["HTTP_PROXY"]; + proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY']; } if (proxyVar) { proxyUrl = url.parse(proxyVar); @@ -3027,7 +3058,7 @@ function checkBypass(reqUrl) { if (!reqUrl.hostname) { return false; } - let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || ''; + let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; if (!noProxy) { return false; } @@ -3048,7 +3079,10 @@ function checkBypass(reqUrl) { upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); } // Compare request host against noproxy - for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) { + for (let upperNoProxyItem of noProxy + .split(',') + .map(x => x.trim().toUpperCase()) + .filter(x => x)) { if (upperReqHosts.some(x => x === upperNoProxyItem)) { return true; } diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 41078b3..1b34a58 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -179,7 +179,7 @@ function getContentRange(start: number, end: number): string { async function uploadChunk( httpClient: HttpClient, resourceUrl: string, - data: NodeJS.ReadableStream, + openStream: () => NodeJS.ReadableStream, start: number, end: number ): Promise { @@ -200,7 +200,7 @@ async function uploadChunk( return await httpClient.sendStream( "PATCH", resourceUrl, - data, + openStream(), additionalHeaders ); }; @@ -263,17 +263,17 @@ async function uploadFile( const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - const chunk = fs.createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }); await uploadChunk( httpClient, resourceUrl, - chunk, + () => + fs.createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }), start, end ); From ee7a57c6158120f107592e03bf5b612fc582ff88 Mon Sep 17 00:00:00 2001 From: Aiqiao Yan Date: Fri, 8 May 2020 14:27:52 -0400 Subject: [PATCH 10/12] error handling for stream --- dist/restore/index.js | 6 +++++- dist/save/index.js | 6 +++++- src/cacheHttpClient.ts | 18 ++++++++++++------ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/dist/restore/index.js b/dist/restore/index.js index d2b9af7..6d88c89 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -1435,11 +1435,15 @@ function uploadFile(httpClient, cacheId, archivePath) { const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { + yield uploadChunk(httpClient, resourceUrl, () => fs + .createReadStream(archivePath, { fd, start, end, autoClose: false + }) + .on("error", error => { + throw new Error(`Cache upload failed because file read failed with ${error.Message}`); }), start, end); } }))); diff --git a/dist/save/index.js b/dist/save/index.js index acb0391..e91ab93 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -1434,11 +1434,15 @@ function uploadFile(httpClient, cacheId, archivePath) { const start = offset; const end = offset + chunkSize - 1; offset += MAX_CHUNK_SIZE; - yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, { + yield uploadChunk(httpClient, resourceUrl, () => fs + .createReadStream(archivePath, { fd, start, end, autoClose: false + }) + .on("error", error => { + throw new Error(`Cache upload failed because file read failed with ${error.Message}`); }), start, end); } }))); diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index 1b34a58..cecdaae 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -268,12 +268,18 @@ async function uploadFile( httpClient, resourceUrl, () => - fs.createReadStream(archivePath, { - fd, - start, - end, - autoClose: false - }), + fs + .createReadStream(archivePath, { + fd, + start, + end, + autoClose: false + }) + .on("error", error => { + throw new Error( + `Cache upload failed because file read failed with ${error.Message}` + ); + }), start, end ); From 0232e3178d8d3a56b774396ff2cc1136a0b1bbf2 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Mon, 11 May 2020 11:11:25 -0400 Subject: [PATCH 11/12] Add retries to all API calls --- __tests__/cacheHttpsClient.test.ts | 144 ++++++++++++++++++ dist/restore/index.js | 211 +++++++++++++------------- dist/save/index.js | 229 ++++++++++++++++------------- src/cacheHttpClient.ts | 136 ++++++++++++----- 4 files changed, 480 insertions(+), 240 deletions(-) create mode 100644 __tests__/cacheHttpsClient.test.ts diff --git a/__tests__/cacheHttpsClient.test.ts b/__tests__/cacheHttpsClient.test.ts new file mode 100644 index 0000000..c9f4fac --- /dev/null +++ b/__tests__/cacheHttpsClient.test.ts @@ -0,0 +1,144 @@ +import { retry } from "../src/cacheHttpClient"; +import * as testUtils from "../src/utils/testUtils"; + +afterEach(() => { + testUtils.clearInputs(); +}); + +interface TestResponse { + statusCode: number; + result: string | null; +} + +function handleResponse( + response: TestResponse | undefined +): Promise { + if (!response) { + fail("Retry method called too many times"); + } + + if (response.statusCode === 999) { + throw Error("Test Error"); + } else { + return Promise.resolve(response); + } +} + +async function testRetryExpectingResult( + responses: Array, + expectedResult: string | null +): Promise { + responses = responses.reverse(); // Reverse responses since we pop from end + + const actualResult = await retry( + "test", + () => handleResponse(responses.pop()), + (response: TestResponse) => response.statusCode + ); + + expect(actualResult.result).toEqual(expectedResult); +} + +async function testRetryExpectingError( + responses: Array +): Promise { + responses = responses.reverse(); // Reverse responses since we pop from end + + expect( + retry( + "test", + () => handleResponse(responses.pop()), + (response: TestResponse) => response.statusCode + ) + ).rejects.toBeInstanceOf(Error); +} + +test("retry works on successful response", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); + +test("retry works after retryable status code", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 503, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); + +test("retry fails after exhausting retries", async () => { + await testRetryExpectingError([ + { + statusCode: 503, + result: null + }, + { + statusCode: 503, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ]); +}); + +test("retry fails after non-retryable status code", async () => { + await testRetryExpectingError([ + { + statusCode: 500, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ]); +}); + +test("retry works after error", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 999, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ], + "Ok" + ); +}); + +test("retry returns after client error", async () => { + await testRetryExpectingResult( + [ + { + statusCode: 400, + result: null + }, + { + statusCode: 200, + result: "Ok" + } + ], + null + ); +}); diff --git a/dist/restore/index.js b/dist/restore/index.js index 6d88c89..c90f729 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -1252,20 +1252,25 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); -const fs = __importStar(__webpack_require__(747)); -const auth_1 = __webpack_require__(226); const http_client_1 = __webpack_require__(539); +const auth_1 = __webpack_require__(226); +const fs = __importStar(__webpack_require__(747)); const stream = __importStar(__webpack_require__(794)); const util = __importStar(__webpack_require__(669)); const constants_1 = __webpack_require__(694); const utils = __importStar(__webpack_require__(443)); -const constants_1 = __webpack_require__(694); function isSuccessStatusCode(statusCode) { if (!statusCode) { return false; } return statusCode >= 200 && statusCode < 300; } +function isServerErrorStatusCode(statusCode) { + if (!statusCode) { + return true; + } + return statusCode >= 500; +} function isRetryableStatusCode(statusCode) { if (!statusCode) { return false; @@ -1305,12 +1310,56 @@ function createHttpClient() { const bearerCredentialHandler = new auth_1.BearerCredentialHandler(token); return new http_client_1.HttpClient("actions/cache", [bearerCredentialHandler], getRequestOptions()); } +function retry(name, method, getStatusCode, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + let response = undefined; + let statusCode = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + while (attempt <= maxAttempts) { + try { + response = yield method(); + statusCode = getStatusCode(response); + if (!isServerErrorStatusCode(statusCode)) { + return response; + } + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } + catch (error) { + isRetryable = true; + errorMessage = error.message; + } + core.debug(`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`); + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + attempt++; + } + throw Error(`${name} failed: ${errorMessage}`); + }); +} +exports.retry = retry; +function retryTypedResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.statusCode, maxAttempts); + }); +} +exports.retryTypedResponse = retryTypedResponse; +function retryHttpClientResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.message.statusCode, maxAttempts); + }); +} +exports.retryHttpClientResponse = retryHttpClientResponse; function getCacheEntry(keys) { var _a; return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`; - const response = yield httpClient.getJson(getCacheApiUrl(resource)); + const response = yield retryTypedResponse("getCacheEntry", () => httpClient.getJson(getCacheApiUrl(resource))); if (response.statusCode === 204) { return null; } @@ -1339,7 +1388,7 @@ function downloadCache(archiveLocation, archivePath) { return __awaiter(this, void 0, void 0, function* () { const stream = fs.createWriteStream(archivePath); const httpClient = new http_client_1.HttpClient("actions/cache"); - const downloadResponse = yield httpClient.get(archiveLocation); + const downloadResponse = yield retryHttpClientResponse("downloadCache", () => httpClient.get(archiveLocation)); // Abort download if no traffic received over the socket. downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => { downloadResponse.message.destroy(); @@ -1369,7 +1418,7 @@ function reserveCache(key) { const reserveCacheRequest = { key }; - const response = yield httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest); + const response = yield retryTypedResponse("reserveCache", () => httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest)); return _c = (_b = (_a = response) === null || _a === void 0 ? void 0 : _a.result) === null || _b === void 0 ? void 0 : _b.cacheId, (_c !== null && _c !== void 0 ? _c : -1); }); } @@ -1391,21 +1440,7 @@ function uploadChunk(httpClient, resourceUrl, openStream, start, end) { "Content-Type": "application/octet-stream", "Content-Range": getContentRange(start, end) }; - const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { - return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders); - }); - const response = yield uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug(`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`); - const retryResponse = yield uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - throw new Error(`Cache service responded with ${response.message.statusCode} during chunk upload.`); + yield retryHttpClientResponse(`uploadChunk (start: ${start}, end: ${end})`, () => httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders)); }); } function parseEnvNumber(key) { @@ -1457,7 +1492,7 @@ function uploadFile(httpClient, cacheId, archivePath) { function commitCache(httpClient, cacheId, filesize) { return __awaiter(this, void 0, void 0, function* () { const commitCacheRequest = { size: filesize }; - return yield httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest); + return yield retryTypedResponse("commitCache", () => httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest)); }); } function saveCache(cacheId, archivePath) { @@ -1499,9 +1534,7 @@ class BasicCredentialHandler { this.password = password; } prepareRequest(options) { - options.headers['Authorization'] = - 'Basic ' + - Buffer.from(this.username + ':' + this.password).toString('base64'); + options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -1537,8 +1570,7 @@ class PersonalAccessTokenCredentialHandler { // currently implements pre-authorization // TODO: support preAuth = false where it hooks on 401 prepareRequest(options) { - options.headers['Authorization'] = - 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); + options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -2007,7 +2039,6 @@ var HttpCodes; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; - HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; @@ -2032,18 +2063,8 @@ function getProxyUrl(serverUrl) { return proxyUrl ? proxyUrl.href : ''; } exports.getProxyUrl = getProxyUrl; -const HttpRedirectCodes = [ - HttpCodes.MovedPermanently, - HttpCodes.ResourceMoved, - HttpCodes.SeeOther, - HttpCodes.TemporaryRedirect, - HttpCodes.PermanentRedirect -]; -const HttpResponseRetryCodes = [ - HttpCodes.BadGateway, - HttpCodes.ServiceUnavailable, - HttpCodes.GatewayTimeout -]; +const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect]; +const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout]; const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const ExponentialBackoffCeiling = 10; const ExponentialBackoffTimeSlice = 5; @@ -2057,12 +2078,6 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); - this.message.on('aborted', () => { - reject("Request was aborted or closed prematurely"); - }); - this.message.on('timeout', (socket) => { - reject("Request timed out"); - }); this.message.on('end', () => { resolve(output.toString()); }); @@ -2174,23 +2189,18 @@ class HttpClient { */ async request(verb, requestUrl, data, headers) { if (this._disposed) { - throw new Error('Client has already been disposed.'); + throw new Error("Client has already been disposed."); } let parsedUrl = url.parse(requestUrl); let info = this._prepareRequest(verb, parsedUrl, headers); // Only perform retries on reads since writes may not be idempotent. - let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1 - ? this._maxRetries + 1 - : 1; + let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1; let numTries = 0; let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); - // Check if it's an authentication challenge - if (response && - response.message && - response.message.statusCode === HttpCodes.Unauthorized) { + if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; for (let i = 0; i < this.handlers.length; i++) { if (this.handlers[i].canHandleAuthentication(response)) { @@ -2208,32 +2218,21 @@ class HttpClient { } } let redirectsRemaining = this._maxRedirects; - while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 && - this._allowRedirects && - redirectsRemaining > 0) { - const redirectUrl = response.message.headers['location']; + while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 + && this._allowRedirects + && redirectsRemaining > 0) { + const redirectUrl = response.message.headers["location"]; if (!redirectUrl) { // if there's no location to redirect to, we won't break; } let parsedRedirectUrl = url.parse(redirectUrl); - if (parsedUrl.protocol == 'https:' && - parsedUrl.protocol != parsedRedirectUrl.protocol && - !this._allowRedirectDowngrade) { - throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.'); + if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) { + throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true."); } // we need to finish reading the response before reassigning response // which will leak the open socket. await response.readBody(); - // strip authorization header if redirected to a different hostname - if (parsedRedirectUrl.hostname !== parsedUrl.hostname) { - for (let header in headers) { - // header names are case insensitive - if (header.toLowerCase() === 'authorization') { - delete headers[header]; - } - } - } // let's make the request with the new redirectUrl info = this._prepareRequest(verb, parsedRedirectUrl, headers); response = await this.requestRaw(info, data); @@ -2284,8 +2283,8 @@ class HttpClient { */ requestRawWithCallback(info, data, onResult) { let socket; - if (typeof data === 'string') { - info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8'); + if (typeof (data) === 'string') { + info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8'); } let callbackCalled = false; let handleResult = (err, res) => { @@ -2298,7 +2297,7 @@ class HttpClient { let res = new HttpClientResponse(msg); handleResult(null, res); }); - req.on('socket', sock => { + req.on('socket', (sock) => { socket = sock; }); // If we ever get disconnected, we want the socket to timeout eventually @@ -2313,10 +2312,10 @@ class HttpClient { // res should have headers handleResult(err, null); }); - if (data && typeof data === 'string') { + if (data && typeof (data) === 'string') { req.write(data, 'utf8'); } - if (data && typeof data !== 'string') { + if (data && typeof (data) !== 'string') { data.on('close', function () { req.end(); }); @@ -2343,34 +2342,31 @@ class HttpClient { const defaultPort = usingSsl ? 443 : 80; info.options = {}; info.options.host = info.parsedUrl.hostname; - info.options.port = info.parsedUrl.port - ? parseInt(info.parsedUrl.port) - : defaultPort; - info.options.path = - (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); + info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort; + info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); info.options.method = method; info.options.headers = this._mergeHeaders(headers); if (this.userAgent != null) { - info.options.headers['user-agent'] = this.userAgent; + info.options.headers["user-agent"] = this.userAgent; } info.options.agent = this._getAgent(info.parsedUrl); // gives handlers an opportunity to participate if (this.handlers) { - this.handlers.forEach(handler => { + this.handlers.forEach((handler) => { handler.prepareRequest(info.options); }); } return info; } _mergeHeaders(headers) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); if (this.requestOptions && this.requestOptions.headers) { return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers)); } return lowercaseKeys(headers || {}); } _getExistingOrDefaultHeader(additionalHeaders, header, _default) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); let clientHeader; if (this.requestOptions && this.requestOptions.headers) { clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; @@ -2408,7 +2404,7 @@ class HttpClient { proxyAuth: proxyUrl.auth, host: proxyUrl.hostname, port: proxyUrl.port - } + }, }; let tunnelAgent; const overHttps = proxyUrl.protocol === 'https:'; @@ -2435,9 +2431,7 @@ class HttpClient { // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // we have to cast it to any and change it directly - agent.options = Object.assign(agent.options || {}, { - rejectUnauthorized: false - }); + agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }); } return agent; } @@ -2498,7 +2492,7 @@ class HttpClient { msg = contents; } else { - msg = 'Failed request: (' + statusCode + ')'; + msg = "Failed request: (" + statusCode + ")"; } let err = new Error(msg); // attach statusCode and body obj (if available) to the error object @@ -3006,7 +3000,6 @@ const exec_1 = __webpack_require__(986); const io = __importStar(__webpack_require__(1)); const fs_1 = __webpack_require__(747); const path = __importStar(__webpack_require__(622)); -const constants_1 = __webpack_require__(694); function isGnuTar() { return __awaiter(this, void 0, void 0, function* () { core.debug("Checking tar --version"); @@ -3033,7 +3026,7 @@ function getTarPath(args) { if (fs_1.existsSync(systemTar)) { return systemTar; } - else if (isGnuTar()) { + else if (yield isGnuTar()) { args.push("--force-local"); } } @@ -3041,10 +3034,10 @@ function getTarPath(args) { }); } function execTar(args) { - var _a, _b; + var _a; return __awaiter(this, void 0, void 0, function* () { try { - yield exec_1.exec(`"${yield getTarPath()}"`, args); + yield exec_1.exec(`"${yield getTarPath(args)}"`, args); } catch (error) { throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}`); @@ -3055,14 +3048,27 @@ function extractTar(archivePath, targetDirectory) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into yield io.mkdirP(targetDirectory); - const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; + const args = [ + "-xz", + "-f", + archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"), + "-C", + targetDirectory.replace(new RegExp("\\" + path.sep, "g"), "/") + ]; yield execTar(args); }); } exports.extractTar = extractTar; function createTar(archivePath, sourceDirectory) { return __awaiter(this, void 0, void 0, function* () { - const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; + const args = [ + "-cz", + "-f", + archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"), + "-C", + sourceDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"), + "." + ]; yield execTar(args); }); } @@ -3086,10 +3092,12 @@ function getProxyUrl(reqUrl) { } let proxyVar; if (usingSsl) { - proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY']; + proxyVar = process.env["https_proxy"] || + process.env["HTTPS_PROXY"]; } else { - proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY']; + proxyVar = process.env["http_proxy"] || + process.env["HTTP_PROXY"]; } if (proxyVar) { proxyUrl = url.parse(proxyVar); @@ -3101,7 +3109,7 @@ function checkBypass(reqUrl) { if (!reqUrl.hostname) { return false; } - let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; + let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || ''; if (!noProxy) { return false; } @@ -3122,10 +3130,7 @@ function checkBypass(reqUrl) { upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); } // Compare request host against noproxy - for (let upperNoProxyItem of noProxy - .split(',') - .map(x => x.trim().toUpperCase()) - .filter(x => x)) { + for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) { if (upperReqHosts.some(x => x === upperNoProxyItem)) { return true; } diff --git a/dist/save/index.js b/dist/save/index.js index e91ab93..76c820c 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -1252,9 +1252,9 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); -const fs = __importStar(__webpack_require__(747)); -const auth_1 = __webpack_require__(226); const http_client_1 = __webpack_require__(539); +const auth_1 = __webpack_require__(226); +const fs = __importStar(__webpack_require__(747)); const stream = __importStar(__webpack_require__(794)); const util = __importStar(__webpack_require__(669)); const constants_1 = __webpack_require__(694); @@ -1265,6 +1265,12 @@ function isSuccessStatusCode(statusCode) { } return statusCode >= 200 && statusCode < 300; } +function isServerErrorStatusCode(statusCode) { + if (!statusCode) { + return true; + } + return statusCode >= 500; +} function isRetryableStatusCode(statusCode) { if (!statusCode) { return false; @@ -1304,12 +1310,56 @@ function createHttpClient() { const bearerCredentialHandler = new auth_1.BearerCredentialHandler(token); return new http_client_1.HttpClient("actions/cache", [bearerCredentialHandler], getRequestOptions()); } +function retry(name, method, getStatusCode, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + let response = undefined; + let statusCode = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + while (attempt <= maxAttempts) { + try { + response = yield method(); + statusCode = getStatusCode(response); + if (!isServerErrorStatusCode(statusCode)) { + return response; + } + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } + catch (error) { + isRetryable = true; + errorMessage = error.message; + } + core.debug(`${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}`); + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + attempt++; + } + throw Error(`${name} failed: ${errorMessage}`); + }); +} +exports.retry = retry; +function retryTypedResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.statusCode, maxAttempts); + }); +} +exports.retryTypedResponse = retryTypedResponse; +function retryHttpClientResponse(name, method, maxAttempts = 2) { + return __awaiter(this, void 0, void 0, function* () { + return yield retry(name, method, (response) => response.message.statusCode, maxAttempts); + }); +} +exports.retryHttpClientResponse = retryHttpClientResponse; function getCacheEntry(keys) { var _a; return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`; - const response = yield httpClient.getJson(getCacheApiUrl(resource)); + const response = yield retryTypedResponse("getCacheEntry", () => httpClient.getJson(getCacheApiUrl(resource))); if (response.statusCode === 204) { return null; } @@ -1338,7 +1388,7 @@ function downloadCache(archiveLocation, archivePath) { return __awaiter(this, void 0, void 0, function* () { const stream = fs.createWriteStream(archivePath); const httpClient = new http_client_1.HttpClient("actions/cache"); - const downloadResponse = yield httpClient.get(archiveLocation); + const downloadResponse = yield retryHttpClientResponse("downloadCache", () => httpClient.get(archiveLocation)); // Abort download if no traffic received over the socket. downloadResponse.message.socket.setTimeout(constants_1.SocketTimeout, () => { downloadResponse.message.destroy(); @@ -1368,7 +1418,7 @@ function reserveCache(key) { const reserveCacheRequest = { key }; - const response = yield httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest); + const response = yield retryTypedResponse("reserveCache", () => httpClient.postJson(getCacheApiUrl("caches"), reserveCacheRequest)); return _c = (_b = (_a = response) === null || _a === void 0 ? void 0 : _a.result) === null || _b === void 0 ? void 0 : _b.cacheId, (_c !== null && _c !== void 0 ? _c : -1); }); } @@ -1390,21 +1440,7 @@ function uploadChunk(httpClient, resourceUrl, openStream, start, end) { "Content-Type": "application/octet-stream", "Content-Range": getContentRange(start, end) }; - const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { - return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders); - }); - const response = yield uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug(`Received ${response.message.statusCode}, retrying chunk at offset ${start}.`); - const retryResponse = yield uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - throw new Error(`Cache service responded with ${response.message.statusCode} during chunk upload.`); + yield retryHttpClientResponse(`uploadChunk (start: ${start}, end: ${end})`, () => httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders)); }); } function parseEnvNumber(key) { @@ -1456,7 +1492,7 @@ function uploadFile(httpClient, cacheId, archivePath) { function commitCache(httpClient, cacheId, filesize) { return __awaiter(this, void 0, void 0, function* () { const commitCacheRequest = { size: filesize }; - return yield httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest); + return yield retryTypedResponse("commitCache", () => httpClient.postJson(getCacheApiUrl(`caches/${cacheId.toString()}`), commitCacheRequest)); }); } function saveCache(cacheId, archivePath) { @@ -1498,9 +1534,7 @@ class BasicCredentialHandler { this.password = password; } prepareRequest(options) { - options.headers['Authorization'] = - 'Basic ' + - Buffer.from(this.username + ':' + this.password).toString('base64'); + options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -1536,8 +1570,7 @@ class PersonalAccessTokenCredentialHandler { // currently implements pre-authorization // TODO: support preAuth = false where it hooks on 401 prepareRequest(options) { - options.headers['Authorization'] = - 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); + options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); } // This handler cannot handle 401 canHandleAuthentication(response) { @@ -2006,7 +2039,6 @@ var HttpCodes; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; - HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; @@ -2031,18 +2063,8 @@ function getProxyUrl(serverUrl) { return proxyUrl ? proxyUrl.href : ''; } exports.getProxyUrl = getProxyUrl; -const HttpRedirectCodes = [ - HttpCodes.MovedPermanently, - HttpCodes.ResourceMoved, - HttpCodes.SeeOther, - HttpCodes.TemporaryRedirect, - HttpCodes.PermanentRedirect -]; -const HttpResponseRetryCodes = [ - HttpCodes.BadGateway, - HttpCodes.ServiceUnavailable, - HttpCodes.GatewayTimeout -]; +const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect]; +const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout]; const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const ExponentialBackoffCeiling = 10; const ExponentialBackoffTimeSlice = 5; @@ -2056,12 +2078,6 @@ class HttpClientResponse { this.message.on('data', (chunk) => { output = Buffer.concat([output, chunk]); }); - this.message.on('aborted', () => { - reject("Request was aborted or closed prematurely"); - }); - this.message.on('timeout', (socket) => { - reject("Request timed out"); - }); this.message.on('end', () => { resolve(output.toString()); }); @@ -2173,23 +2189,18 @@ class HttpClient { */ async request(verb, requestUrl, data, headers) { if (this._disposed) { - throw new Error('Client has already been disposed.'); + throw new Error("Client has already been disposed."); } let parsedUrl = url.parse(requestUrl); let info = this._prepareRequest(verb, parsedUrl, headers); // Only perform retries on reads since writes may not be idempotent. - let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1 - ? this._maxRetries + 1 - : 1; + let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1; let numTries = 0; let response; while (numTries < maxTries) { response = await this.requestRaw(info, data); - // Check if it's an authentication challenge - if (response && - response.message && - response.message.statusCode === HttpCodes.Unauthorized) { + if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { let authenticationHandler; for (let i = 0; i < this.handlers.length; i++) { if (this.handlers[i].canHandleAuthentication(response)) { @@ -2207,32 +2218,21 @@ class HttpClient { } } let redirectsRemaining = this._maxRedirects; - while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 && - this._allowRedirects && - redirectsRemaining > 0) { - const redirectUrl = response.message.headers['location']; + while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 + && this._allowRedirects + && redirectsRemaining > 0) { + const redirectUrl = response.message.headers["location"]; if (!redirectUrl) { // if there's no location to redirect to, we won't break; } let parsedRedirectUrl = url.parse(redirectUrl); - if (parsedUrl.protocol == 'https:' && - parsedUrl.protocol != parsedRedirectUrl.protocol && - !this._allowRedirectDowngrade) { - throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.'); + if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) { + throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true."); } // we need to finish reading the response before reassigning response // which will leak the open socket. await response.readBody(); - // strip authorization header if redirected to a different hostname - if (parsedRedirectUrl.hostname !== parsedUrl.hostname) { - for (let header in headers) { - // header names are case insensitive - if (header.toLowerCase() === 'authorization') { - delete headers[header]; - } - } - } // let's make the request with the new redirectUrl info = this._prepareRequest(verb, parsedRedirectUrl, headers); response = await this.requestRaw(info, data); @@ -2283,8 +2283,8 @@ class HttpClient { */ requestRawWithCallback(info, data, onResult) { let socket; - if (typeof data === 'string') { - info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8'); + if (typeof (data) === 'string') { + info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8'); } let callbackCalled = false; let handleResult = (err, res) => { @@ -2297,7 +2297,7 @@ class HttpClient { let res = new HttpClientResponse(msg); handleResult(null, res); }); - req.on('socket', sock => { + req.on('socket', (sock) => { socket = sock; }); // If we ever get disconnected, we want the socket to timeout eventually @@ -2312,10 +2312,10 @@ class HttpClient { // res should have headers handleResult(err, null); }); - if (data && typeof data === 'string') { + if (data && typeof (data) === 'string') { req.write(data, 'utf8'); } - if (data && typeof data !== 'string') { + if (data && typeof (data) !== 'string') { data.on('close', function () { req.end(); }); @@ -2342,34 +2342,31 @@ class HttpClient { const defaultPort = usingSsl ? 443 : 80; info.options = {}; info.options.host = info.parsedUrl.hostname; - info.options.port = info.parsedUrl.port - ? parseInt(info.parsedUrl.port) - : defaultPort; - info.options.path = - (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); + info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort; + info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); info.options.method = method; info.options.headers = this._mergeHeaders(headers); if (this.userAgent != null) { - info.options.headers['user-agent'] = this.userAgent; + info.options.headers["user-agent"] = this.userAgent; } info.options.agent = this._getAgent(info.parsedUrl); // gives handlers an opportunity to participate if (this.handlers) { - this.handlers.forEach(handler => { + this.handlers.forEach((handler) => { handler.prepareRequest(info.options); }); } return info; } _mergeHeaders(headers) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); if (this.requestOptions && this.requestOptions.headers) { return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers)); } return lowercaseKeys(headers || {}); } _getExistingOrDefaultHeader(additionalHeaders, header, _default) { - const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); + const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); let clientHeader; if (this.requestOptions && this.requestOptions.headers) { clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; @@ -2407,7 +2404,7 @@ class HttpClient { proxyAuth: proxyUrl.auth, host: proxyUrl.hostname, port: proxyUrl.port - } + }, }; let tunnelAgent; const overHttps = proxyUrl.protocol === 'https:'; @@ -2434,9 +2431,7 @@ class HttpClient { // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // we have to cast it to any and change it directly - agent.options = Object.assign(agent.options || {}, { - rejectUnauthorized: false - }); + agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }); } return agent; } @@ -2497,7 +2492,7 @@ class HttpClient { msg = contents; } else { - msg = 'Failed request: (' + statusCode + ')'; + msg = "Failed request: (" + statusCode + ")"; } let err = new Error(msg); // attach statusCode and body obj (if available) to the error object @@ -2985,7 +2980,25 @@ const core = __importStar(__webpack_require__(470)); const exec_1 = __webpack_require__(986); const io = __importStar(__webpack_require__(1)); const fs_1 = __webpack_require__(747); -function getTarPath() { +const path = __importStar(__webpack_require__(622)); +function isGnuTar() { + return __awaiter(this, void 0, void 0, function* () { + core.debug("Checking tar --version"); + let versionOutput = ""; + yield exec_1.exec("tar --version", [], { + ignoreReturnCode: true, + silent: true, + listeners: { + stdout: (data) => (versionOutput += data.toString()), + stderr: (data) => (versionOutput += data.toString()) + } + }); + core.debug(versionOutput.trim()); + return versionOutput.toUpperCase().includes("GNU TAR"); + }); +} +exports.isGnuTar = isGnuTar; +function getTarPath(args) { return __awaiter(this, void 0, void 0, function* () { // Explicitly use BSD Tar on Windows const IS_WINDOWS = process.platform === "win32"; @@ -2994,7 +3007,7 @@ function getTarPath() { if (fs_1.existsSync(systemTar)) { return systemTar; } - else if (isGnuTar()) { + else if (yield isGnuTar()) { args.push("--force-local"); } } @@ -3002,10 +3015,10 @@ function getTarPath() { }); } function execTar(args) { - var _a, _b; + var _a; return __awaiter(this, void 0, void 0, function* () { try { - yield exec_1.exec(`"${yield getTarPath()}"`, args); + yield exec_1.exec(`"${yield getTarPath(args)}"`, args); } catch (error) { throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}`); @@ -3016,14 +3029,27 @@ function extractTar(archivePath, targetDirectory) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into yield io.mkdirP(targetDirectory); - const args = ["-xz", "-f", archivePath, "-C", targetDirectory]; + const args = [ + "-xz", + "-f", + archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"), + "-C", + targetDirectory.replace(new RegExp("\\" + path.sep, "g"), "/") + ]; yield execTar(args); }); } exports.extractTar = extractTar; function createTar(archivePath, sourceDirectory) { return __awaiter(this, void 0, void 0, function* () { - const args = ["-cz", "-f", archivePath, "-C", sourceDirectory, "."]; + const args = [ + "-cz", + "-f", + archivePath.replace(new RegExp("\\" + path.sep, "g"), "/"), + "-C", + sourceDirectory.replace(new RegExp("\\" + path.sep, "g"), "/"), + "." + ]; yield execTar(args); }); } @@ -3047,10 +3073,12 @@ function getProxyUrl(reqUrl) { } let proxyVar; if (usingSsl) { - proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY']; + proxyVar = process.env["https_proxy"] || + process.env["HTTPS_PROXY"]; } else { - proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY']; + proxyVar = process.env["http_proxy"] || + process.env["HTTP_PROXY"]; } if (proxyVar) { proxyUrl = url.parse(proxyVar); @@ -3062,7 +3090,7 @@ function checkBypass(reqUrl) { if (!reqUrl.hostname) { return false; } - let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; + let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || ''; if (!noProxy) { return false; } @@ -3083,10 +3111,7 @@ function checkBypass(reqUrl) { upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); } // Compare request host against noproxy - for (let upperNoProxyItem of noProxy - .split(',') - .map(x => x.trim().toUpperCase()) - .filter(x => x)) { + for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) { if (upperReqHosts.some(x => x === upperNoProxyItem)) { return true; } diff --git a/src/cacheHttpClient.ts b/src/cacheHttpClient.ts index cecdaae..19124e9 100644 --- a/src/cacheHttpClient.ts +++ b/src/cacheHttpClient.ts @@ -26,6 +26,13 @@ function isSuccessStatusCode(statusCode?: number): boolean { return statusCode >= 200 && statusCode < 300; } +function isServerErrorStatusCode(statusCode?: number): boolean { + if (!statusCode) { + return true; + } + return statusCode >= 500; +} + function isRetryableStatusCode(statusCode?: number): boolean { if (!statusCode) { return false; @@ -81,14 +88,83 @@ function createHttpClient(): HttpClient { ); } +export async function retry( + name: string, + method: () => Promise, + getStatusCode: (T) => number | undefined, + maxAttempts = 2 +): Promise { + let response: T | undefined = undefined; + let statusCode: number | undefined = undefined; + let isRetryable = false; + let errorMessage = ""; + let attempt = 1; + + while (attempt <= maxAttempts) { + try { + response = await method(); + statusCode = getStatusCode(response); + + if (!isServerErrorStatusCode(statusCode)) { + return response; + } + + isRetryable = isRetryableStatusCode(statusCode); + errorMessage = `Cache service responded with ${statusCode}`; + } catch (error) { + isRetryable = true; + errorMessage = error.message; + } + + core.debug( + `${name} - Attempt ${attempt} of ${maxAttempts} failed with error: ${errorMessage}` + ); + + if (!isRetryable) { + core.debug(`${name} - Error is not retryable`); + break; + } + + attempt++; + } + + throw Error(`${name} failed: ${errorMessage}`); +} + +export async function retryTypedResponse( + name: string, + method: () => Promise>, + maxAttempts = 2 +): Promise> { + return await retry( + name, + method, + (response: ITypedResponse) => response.statusCode, + maxAttempts + ); +} + +export async function retryHttpClientResponse( + name: string, + method: () => Promise, + maxAttempts = 2 +): Promise { + return await retry( + name, + method, + (response: IHttpClientResponse) => response.message.statusCode, + maxAttempts + ); +} + export async function getCacheEntry( keys: string[] ): Promise { const httpClient = createHttpClient(); const resource = `cache?keys=${encodeURIComponent(keys.join(","))}`; - const response = await httpClient.getJson( - getCacheApiUrl(resource) + const response = await retryTypedResponse("getCacheEntry", () => + httpClient.getJson(getCacheApiUrl(resource)) ); if (response.statusCode === 204) { return null; @@ -123,7 +199,10 @@ export async function downloadCache( ): Promise { const stream = fs.createWriteStream(archivePath); const httpClient = new HttpClient("actions/cache"); - const downloadResponse = await httpClient.get(archiveLocation); + const downloadResponse = await retryHttpClientResponse( + "downloadCache", + () => httpClient.get(archiveLocation) + ); // Abort download if no traffic received over the socket. downloadResponse.message.socket.setTimeout(SocketTimeout, () => { @@ -160,9 +239,11 @@ export async function reserveCache(key: string): Promise { const reserveCacheRequest: ReserveCacheRequest = { key }; - const response = await httpClient.postJson( - getCacheApiUrl("caches"), - reserveCacheRequest + const response = await retryTypedResponse("reserveCache", () => + httpClient.postJson( + getCacheApiUrl("caches"), + reserveCacheRequest + ) ); return response?.result?.cacheId ?? -1; } @@ -196,32 +277,15 @@ async function uploadChunk( "Content-Range": getContentRange(start, end) }; - const uploadChunkRequest = async (): Promise => { - return await httpClient.sendStream( - "PATCH", - resourceUrl, - openStream(), - additionalHeaders - ); - }; - - const response = await uploadChunkRequest(); - if (isSuccessStatusCode(response.message.statusCode)) { - return; - } - - if (isRetryableStatusCode(response.message.statusCode)) { - core.debug( - `Received ${response.message.statusCode}, retrying chunk at offset ${start}.` - ); - const retryResponse = await uploadChunkRequest(); - if (isSuccessStatusCode(retryResponse.message.statusCode)) { - return; - } - } - - throw new Error( - `Cache service responded with ${response.message.statusCode} during chunk upload.` + await retryHttpClientResponse( + `uploadChunk (start: ${start}, end: ${end})`, + () => + httpClient.sendStream( + "PATCH", + resourceUrl, + openStream(), + additionalHeaders + ) ); } @@ -298,9 +362,11 @@ async function commitCache( filesize: number ): Promise> { const commitCacheRequest: CommitCacheRequest = { size: filesize }; - return await httpClient.postJson( - getCacheApiUrl(`caches/${cacheId.toString()}`), - commitCacheRequest + return await retryTypedResponse("commitCache", () => + httpClient.postJson( + getCacheApiUrl(`caches/${cacheId.toString()}`), + commitCacheRequest + ) ); } From 3f662ca624fc3e0ca4791e54930fa9939fd9936b Mon Sep 17 00:00:00 2001 From: Aiqiao Yan Date: Tue, 12 May 2020 16:36:56 -0400 Subject: [PATCH 12/12] Add Eric's e2e test change to get more coverage --- .github/workflows/workflow.yml | 111 +++++++++++++++++++++++++++----- __tests__/create-cache-files.sh | 11 ++++ __tests__/tar.test.ts | 2 +- __tests__/verify-cache-files.sh | 30 +++++++++ dist/restore/index.js | 3 +- dist/save/index.js | 3 +- src/tar.ts | 3 +- 7 files changed, 143 insertions(+), 20 deletions(-) create mode 100755 __tests__/create-cache-files.sh create mode 100755 __tests__/verify-cache-files.sh diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index f6c5448..629953d 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -4,51 +4,130 @@ on: pull_request: branches: - master + - releases/** paths-ignore: - '**.md' push: branches: - master + - releases/** paths-ignore: - '**.md' jobs: - test: - name: Test on ${{ matrix.os }} - + # Build and unit test + build: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v1 - - - uses: actions/setup-node@v1 + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Node.js + uses: actions/setup-node@v1 with: node-version: '12.x' - - - name: Get npm cache directory + - name: Determine npm cache directory id: npm-cache run: | echo "::set-output name=dir::$(npm config get cache)" - - - uses: actions/cache@v1 + - name: Restore npm cache + uses: actions/cache@v1 with: path: ${{ steps.npm-cache.outputs.dir }} key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} restore-keys: | ${{ runner.os }}-node- - - run: npm ci - - name: Prettier Format Check run: npm run format-check - - name: ESLint Check run: npm run lint - - name: Build & Test run: npm run test + + # End to end save and restore + test-save: + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + fail-fast: false + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Generate files + shell: bash + run: __tests__/create-cache-files.sh ${{ runner.os }} + - name: Save cache + uses: ./ + with: + key: test-${{ runner.os }}-${{ github.run_id }} + path: test-cache + test-restore: + needs: test-save + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + fail-fast: false + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Restore cache + uses: ./ + with: + key: test-${{ runner.os }}-${{ github.run_id }} + path: test-cache + - name: Verify cache + shell: bash + run: __tests__/verify-cache-files.sh ${{ runner.os }} + + # End to end with proxy + test-proxy-save: + runs-on: ubuntu-latest + container: + image: ubuntu:latest + options: --dns 127.0.0.1 + services: + squid-proxy: + image: datadog/squid:latest + ports: + - 3128:3128 + env: + https_proxy: http://squid-proxy:3128 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Generate files + run: __tests__/create-cache-files.sh proxy + - name: Save cache + uses: ./ + with: + key: test-proxy-${{ github.run_id }} + path: test-cache + test-proxy-restore: + needs: test-proxy-save + runs-on: ubuntu-latest + container: + image: ubuntu:latest + options: --dns 127.0.0.1 + services: + squid-proxy: + image: datadog/squid:latest + ports: + - 3128:3128 + env: + https_proxy: http://squid-proxy:3128 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Restore cache + uses: ./ + with: + key: test-proxy-${{ github.run_id }} + path: test-cache + - name: Verify cache + run: __tests__/verify-cache-files.sh proxy \ No newline at end of file diff --git a/__tests__/create-cache-files.sh b/__tests__/create-cache-files.sh new file mode 100755 index 0000000..885a5f2 --- /dev/null +++ b/__tests__/create-cache-files.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Validate args +prefix="$1" +if [ -z "$prefix" ]; then + echo "Must supply prefix argument" + exit 1 +fi + +mkdir test-cache +echo "$prefix $GITHUB_RUN_ID" > test-cache/test-file.txt \ No newline at end of file diff --git a/__tests__/tar.test.ts b/__tests__/tar.test.ts index d5d9b62..6de03c3 100644 --- a/__tests__/tar.test.ts +++ b/__tests__/tar.test.ts @@ -51,7 +51,7 @@ test("extract GNU tar", async () => { await tar.extractTar(archivePath, targetDirectory); - expect(execMock).toHaveBeenCalledTimes(2); + expect(execMock).toHaveBeenCalledTimes(1); expect(execMock).toHaveBeenLastCalledWith(`"tar"`, [ "-xz", "-f", diff --git a/__tests__/verify-cache-files.sh b/__tests__/verify-cache-files.sh new file mode 100755 index 0000000..c7b75ae --- /dev/null +++ b/__tests__/verify-cache-files.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# Validate args +prefix="$1" +if [ -z "$prefix" ]; then + echo "Must supply prefix argument" + exit 1 +fi + +# Sanity check GITHUB_RUN_ID defined +if [ -z "$GITHUB_RUN_ID" ]; then + echo "GITHUB_RUN_ID not defined" + exit 1 +fi + +# Verify file exists +file="test-cache/test-file.txt" +echo "Checking for $file" +if [ ! -e $file ]; then + echo "File does not exist" + exit 1 +fi + +# Verify file content +content="$(cat $file)" +echo "File content:\n$content" +if [ -z "$(echo $content | grep --fixed-strings "$prefix $GITHUB_RUN_ID")" ]; then + echo "Unexpected file content" + exit 1 +fi \ No newline at end of file diff --git a/dist/restore/index.js b/dist/restore/index.js index c90f729..b5e894b 100644 --- a/dist/restore/index.js +++ b/dist/restore/index.js @@ -3000,6 +3000,7 @@ const exec_1 = __webpack_require__(986); const io = __importStar(__webpack_require__(1)); const fs_1 = __webpack_require__(747); const path = __importStar(__webpack_require__(622)); +const tar = __importStar(__webpack_require__(943)); function isGnuTar() { return __awaiter(this, void 0, void 0, function* () { core.debug("Checking tar --version"); @@ -3026,7 +3027,7 @@ function getTarPath(args) { if (fs_1.existsSync(systemTar)) { return systemTar; } - else if (yield isGnuTar()) { + else if (yield tar.isGnuTar()) { args.push("--force-local"); } } diff --git a/dist/save/index.js b/dist/save/index.js index 76c820c..a90a6e4 100644 --- a/dist/save/index.js +++ b/dist/save/index.js @@ -2981,6 +2981,7 @@ const exec_1 = __webpack_require__(986); const io = __importStar(__webpack_require__(1)); const fs_1 = __webpack_require__(747); const path = __importStar(__webpack_require__(622)); +const tar = __importStar(__webpack_require__(943)); function isGnuTar() { return __awaiter(this, void 0, void 0, function* () { core.debug("Checking tar --version"); @@ -3007,7 +3008,7 @@ function getTarPath(args) { if (fs_1.existsSync(systemTar)) { return systemTar; } - else if (yield isGnuTar()) { + else if (yield tar.isGnuTar()) { args.push("--force-local"); } } diff --git a/src/tar.ts b/src/tar.ts index dde9b61..00bed5a 100644 --- a/src/tar.ts +++ b/src/tar.ts @@ -3,6 +3,7 @@ import { exec } from "@actions/exec"; import * as io from "@actions/io"; import { existsSync } from "fs"; import * as path from "path"; +import * as tar from "./tar"; export async function isGnuTar(): Promise { core.debug("Checking tar --version"); @@ -28,7 +29,7 @@ async function getTarPath(args: string[]): Promise { const systemTar = `${process.env["windir"]}\\System32\\tar.exe`; if (existsSync(systemTar)) { return systemTar; - } else if (await isGnuTar()) { + } else if (await tar.isGnuTar()) { args.push("--force-local"); } }