diff --git a/.changeset/wicked-aliens-vanish.md b/.changeset/wicked-aliens-vanish.md new file mode 100644 index 00000000..7bb48b08 --- /dev/null +++ b/.changeset/wicked-aliens-vanish.md @@ -0,0 +1,5 @@ +--- +"@nodesecure/scanner": minor +--- + +Implement cache lookup for from and workingDir APIs diff --git a/README.md b/README.md index 7b15ed73..1b5ac2d9 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,22 @@ type WorkingDirOptions = Options & { * It is optionally used to fetch registry authentication tokens */ npmRcConfig?: Config; + /** + * Optional cache lookup called after reading the local package.json. + */ + cacheLookup?: ( + packageJSON: PackageJSON + ) => Promise; }; -type FromOptions = Omit; +type FromOptions = Omit & { + /** + * Optional cache lookup called after fetching the remote manifest. + */ + cacheLookup?: ( + manifest: pacote.AbbreviatedManifest & pacote.ManifestResult + ) => Promise; +}; interface Options { /** diff --git a/workspaces/scanner/docs/from.md b/workspaces/scanner/docs/from.md index e09cc15e..49edae3a 100644 --- a/workspaces/scanner/docs/from.md +++ b/workspaces/scanner/docs/from.md @@ -56,6 +56,38 @@ It is important here to dig and learn some vocabulary related to npm: To simplify it, the first step is to check the package's existence on the remote registry and to get a structure similar to the `package.json`. +## Steps 1.5: Cache Lookup (optional) + +After fetching the manifest, if a `cacheLookup` function was provided in the options, it is called with the resolved manifest. If it returns a non-null `Payload`, that value is returned immediately and the dependency walker is **never executed**. This is useful to avoid redundant network I/O when a fresh result is already available. + +```ts +const payload = await scanner.from("fastify", { + cacheLookup: async(manifest) => { + const cached = await myCache.get(`${manifest.name}@${manifest.version}`); + return cached ?? null; + } +}); +``` + +The same `cacheLookup` mechanism is available on `workingDir`. In that case, the callback receives the parsed `package.json` object instead of the npm manifest: + +```ts +const payload = await scanner.workingDir(process.cwd(), { + cacheLookup: async(packageJSON) => { + const cached = await myCache.get(`${packageJSON.name}@${packageJSON.version}`); + return cached ?? null; + } +}); +``` + +```mermaid +graph LR; + A[From API]-->|Spec|B[Fetching Manifest]; + B-->C{cacheLookup?}; + C-->|Payload returned|D[Return cached result]; + C-->|null returned|E[Dependency Walker]; +``` + ## Steps 2: Dependency Walker This step aims to identify and walk through the package dependencies (that's why we call this the dependency walker). To do this, we retrieve the dependencies from the root of the Manifest in step 1 and start a recursive mechanism. diff --git a/workspaces/scanner/src/index.ts b/workspaces/scanner/src/index.ts index b3176944..bd8ec22a 100644 --- a/workspaces/scanner/src/index.ts +++ b/workspaces/scanner/src/index.ts @@ -12,11 +12,21 @@ import type Config from "@npmcli/config"; // Import Internal Dependencies import { depWalker } from "./depWalker.ts"; -import { NPM_TOKEN, urlToString, readNpmRc } from "./utils/index.ts"; -import { Logger, ScannerLoggerEvents } from "./class/logger.class.ts"; +import { + NPM_TOKEN, + urlToString, + readNpmRc +} from "./utils/index.ts"; +import { + Logger, + ScannerLoggerEvents +} from "./class/logger.class.ts"; import { TempDirectory } from "./class/TempDirectory.class.ts"; import { comparePayloads } from "./comparePayloads.ts"; -import type { Options } from "./types.ts"; +import type { + Options, + Payload +} from "./types.ts"; // CONSTANTS const kDefaultWorkingDirOptions = { @@ -33,13 +43,16 @@ export type WorkingDirOptions = Options & { * It is optionally used to fetch registry authentication tokens */ npmRcConfig?: Config; + cacheLookup?: ( + packageJSON: PackageJSON + ) => Promise; }; export async function workingDir( location = process.cwd(), options: WorkingDirOptions = {}, logger = new Logger() -) { +): Promise { const registry = options.registry ? urlToString(options.registry) : getLocalRegistryURL(); @@ -66,20 +79,30 @@ export async function workingDir( const str = await fs.readFile(packagePath, "utf-8"); logger.end(ScannerLoggerEvents.manifest.read); + const packageJSON = JSON.parse(str) as PackageJSON; + const cachedPayload = await options.cacheLookup?.(packageJSON); + if (cachedPayload) { + return cachedPayload; + } + return depWalker( - JSON.parse(str) as PackageJSON, + packageJSON, finalizedOptions, logger ); } -export type FromOptions = Omit; +export type FromOptions = Omit & { + cacheLookup?: ( + manifest: pacote.AbbreviatedManifest & pacote.ManifestResult + ) => Promise; +}; export async function from( packageName: string, options: FromOptions = {}, logger = new Logger() -) { +): Promise { const registry = options.registry ? urlToString(options.registry) : getLocalRegistryURL(); @@ -91,6 +114,11 @@ export async function from( }); logger.end(ScannerLoggerEvents.manifest.fetch); + const cachedPayload = await options.cacheLookup?.(manifest); + if (cachedPayload) { + return cachedPayload; + } + return depWalker( // FIX: find a way to merge pacote & registry interfaces manifest as pacote.AbbreviatedManifest, diff --git a/workspaces/scanner/test/depWalker.spec.ts b/workspaces/scanner/test/depWalker.spec.ts index f1d32e94..90a61214 100644 --- a/workspaces/scanner/test/depWalker.spec.ts +++ b/workspaces/scanner/test/depWalker.spec.ts @@ -1,7 +1,7 @@ // Import Node.js Dependencies import path from "node:path"; import { readFileSync } from "node:fs"; -import { test, describe } from "node:test"; +import { describe, it } from "node:test"; import assert from "node:assert"; // Import Third-party Dependencies @@ -12,11 +12,8 @@ import { getLocalRegistryURL } from "@nodesecure/npm-registry-sdk"; import { depWalker } from "../src/depWalker.ts"; import { Logger, - from, - workingDir, type Payload, - type DependencyVersion, - type Identifier + type DependencyVersion } from "../src/index.ts"; // VARS @@ -76,411 +73,248 @@ function cleanupPayload(payload: Payload) { } } -test("execute depWalker on @slimio/is", { skip }, async(test) => { - Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); - const { logger, errorCount } = buildLogger(); - test.after(() => logger.removeAllListeners()); - - const result = await depWalker( - is, - structuredClone(kDefaultWalkerOptions), - logger - ); - const resultAsJSON = JSON.parse(JSON.stringify(result.dependencies, null, 2)); - cleanupPayload(resultAsJSON); - - const expectedResult = JSON.parse(readFileSync(path.join(kFixturePath, "slimio.is-result.json"), "utf-8")); - assert.deepEqual(resultAsJSON, expectedResult); - assert.strictEqual(errorCount(), 0); -}); - -test("execute depWalker on @slimio/config", { skip }, async(test) => { - Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); - const { logger, errorCount } = buildLogger(); - test.after(() => logger.removeAllListeners()); - - const result = await depWalker( - config, - structuredClone(kDefaultWalkerOptions), - logger - ); - const resultAsJSON = JSON.parse(JSON.stringify(result.dependencies, null, 2)); - - const packages = Object.keys(resultAsJSON).sort(); - assert.deepEqual(packages, [ - "lodash.clonedeep", - "zen-observable", - "lodash.set", - "lodash.get", - "node-watch", - "fast-deep-equal", - "fast-json-stable-stringify", - "json-schema-traverse", - "punycode", - "uri-js", - "ajv", - "@slimio/is", - "@iarna/toml", - "@slimio/config" - ].sort()); - - const ajvDescriptor = resultAsJSON.ajv.versions["6.14.0"]; - const ajvUsedBy = Object.keys(ajvDescriptor.usedBy); - assert.deepEqual(ajvUsedBy, [ - "@slimio/config" - ]); - assert.strictEqual(errorCount(), 0); -}); - -test("execute depWalker on pkg.gitdeps", { skip }, async(test) => { - Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); - const { logger, errors, statsCount } = buildLogger(); - test.after(() => logger.removeAllListeners()); - - const result = await depWalker( - pkgGitdeps, - { - ...structuredClone(kDefaultWalkerOptions), - isVerbose: true - }, - logger - ); - const resultAsJSON = JSON.parse(JSON.stringify(result.dependencies, null, 2)); - - const packages = Object.keys(resultAsJSON).sort(); - assert.deepEqual(packages, [ - "@nodesecure/npm-registry-sdk", - "@nodesecure/npm-types", - "@openally/httpie", - "@openally/result", - "lru-cache", - "nanodelay", - "nanoevents", - "nanoid", - "pkg.gitdeps", - "undici", - "zen-observable" - ].sort()); - - const walkErrors = errors(); - - assert.deepStrictEqual(walkErrors, [ - { - name: "pacote.manifest pkg.gitdeps@0.1.0", - error: "404 Not Found - GET https://registry.npmjs.org/pkg.gitdeps - Not found", - phase: "tree-walk" - }, - { - name: "pacote.extract pkg.gitdeps@0.1.0", - error: "404 Not Found - GET https://registry.npmjs.org/pkg.gitdeps - Not found", - phase: "tarball-scan" - } - ]); - const { metadata } = result; - assert.strictEqual(typeof metadata.startedAt, "number"); - assert.strictEqual(typeof metadata.executionTime, "number"); - assert.strictEqual(Array.isArray(metadata.apiCalls), true); - assert.strictEqual(metadata.apiCallsCount, 42); - assert.strictEqual(metadata.errorCount, 2); - assert.strictEqual(metadata.errors.length, 2); - assert.strictEqual(statsCount(), 40); -}); - -test("execute depWalker on typo-squatting (with location)", { skip }, async(test) => { - Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); - const { logger, errors, statsCount } = buildLogger(); - test.after(() => logger.removeAllListeners()); - - const result = await depWalker( - pkgTypoSquatting, - { - ...structuredClone(kDefaultWalkerOptions), - location: "", - isVerbose: true - }, - logger - ); - - assert.ok(result.warnings.length > 0); - const warning = result.warnings[0]; - - assert.equal(warning.type, "typo-squatting"); - assert.match( - result.warnings[0].message, - /.*'mecha'.*fecha, mocha/ - ); - - const walkErrors = errors(); - assert.deepStrictEqual(walkErrors, [ - { - name: "pacote.manifest mecha@1.0.0", - error: "No matching version found for mecha@1.0.0.", - phase: "tree-walk" - }, - { - name: "pacote.extract mecha@1.0.0", - error: "No matching version found for mecha@1.0.0.", - phase: "tarball-scan" - } - ]); - assert.strictEqual(statsCount(), 0); -}); - -test("execute depWalker on typo-squatting (with no location)", { skip }, async(test) => { - Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); - const { logger, errors } = buildLogger(); - test.after(() => logger.removeAllListeners()); - - const result = await depWalker( - pkgTypoSquatting, - { - ...structuredClone(kDefaultWalkerOptions), - isVerbose: true - }, - logger - ); - - assert.ok(result.warnings.length === 0); - const walkErrors = errors(); - assert.deepStrictEqual(walkErrors, [ - { - name: "pacote.manifest mecha@1.0.0", - error: "No matching version found for mecha@1.0.0.", - phase: "tree-walk" - }, - { - name: "pacote.extract mecha@1.0.0", - error: "No matching version found for mecha@1.0.0.", - phase: "tarball-scan" - } - ]); -}); - -test("should highlight the given packages", { skip }, async() => { - const { logger } = buildLogger(); - test.after(() => logger.removeAllListeners()); - - const hightlightPackages = { - "zen-observable": "0.8.14 || 0.8.15", - nanoid: "*" - }; - - const result = await depWalker( - pkgHighlightedPackages, - structuredClone({ - ...kDefaultWalkerOptions, - highlight: { - packages: hightlightPackages, - contacts: [] - } - }), - logger - ); - - assert.deepStrictEqual( - result.highlighted.packages.sort(), - [ - "nanoid@5.1.6", - "zen-observable@0.8.15" - ] - ); -}); +describe("depWalker", () => { + it("should resolve and match the full dependency tree of @slimio/is", { skip }, async(t) => { + Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); + const { logger, errorCount } = buildLogger(); + t.after(() => logger.removeAllListeners()); -test("should support multiple formats for packages highlighted", { skip }, async() => { - const { logger } = buildLogger(); - test.after(() => logger.removeAllListeners()); - - const hightlightPackages = ["zen-observable@0.8.14 || 0.8.15", "nanoid"]; - - const result = await depWalker( - pkgHighlightedPackages, - structuredClone({ - ...kDefaultWalkerOptions, - highlight: { - packages: hightlightPackages, - contacts: [] - } - }), - logger - ); - - assert.deepStrictEqual( - result.highlighted.packages.sort(), - [ - "nanoid@5.1.6", - "zen-observable@0.8.15" - ] - ); -}); - -test("fetch payload of pacote on the npm registry", { skip }, async() => { - const result = await from( - "pacote", - { - maxDepth: 10, - vulnerabilityStrategy: Vulnera.strategies.GITHUB_ADVISORY - } - ); - - assert.deepEqual(Object.keys(result), [ - "id", - "rootDependency", - "scannerVersion", - "vulnerabilityStrategy", - "warnings", - "highlighted", - "dependencies", - "metadata" - ]); - assert.strictEqual(typeof result.rootDependency.integrity, "string"); -}); + const result = await depWalker( + is, + structuredClone(kDefaultWalkerOptions), + logger + ); + const resultAsJSON = JSON.parse(JSON.stringify(result.dependencies, null, 2)); + cleanupPayload(resultAsJSON); -test.skip("fetch payload of pacote on the gitlab registry", { skip }, async() => { - const result = await from("pacote", { - registry: "https://gitlab.com/api/v4/packages/npm/", - maxDepth: 10, - vulnerabilityStrategy: Vulnera.strategies.GITHUB_ADVISORY + const expectedResult = JSON.parse(readFileSync(path.join(kFixturePath, "slimio.is-result.json"), "utf-8")); + assert.deepEqual(resultAsJSON, expectedResult); + assert.strictEqual(errorCount(), 0); }); - assert.deepEqual(Object.keys(result), [ - "id", - "rootDependency", - "scannerVersion", - "vulnerabilityStrategy", - "warnings", - "highlighted", - "dependencies", - "metadata" - ]); - assert.strictEqual(typeof result.rootDependency.integrity, "string"); -}); + it("should resolve all packages and usedBy relations for @slimio/config", { skip }, async(t) => { + Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); + const { logger, errorCount } = buildLogger(); + t.after(() => logger.removeAllListeners()); -test("highlight contacts from a remote package", { skip }, async() => { - const spec = "@adonisjs/logger"; - const result = await from(spec, { - highlight: { - contacts: [ - { - name: "/.*virk.*/i" - } - ] - } + const result = await depWalker( + config, + structuredClone(kDefaultWalkerOptions), + logger + ); + const resultAsJSON = JSON.parse(JSON.stringify(result.dependencies, null, 2)); + + const packages = Object.keys(resultAsJSON).sort(); + assert.deepEqual(packages, [ + "lodash.clonedeep", + "zen-observable", + "lodash.set", + "lodash.get", + "node-watch", + "fast-deep-equal", + "fast-json-stable-stringify", + "json-schema-traverse", + "punycode", + "uri-js", + "ajv", + "@slimio/is", + "@iarna/toml", + "@slimio/config" + ].sort()); + + const ajvDescriptor = resultAsJSON.ajv.versions["6.14.0"]; + const ajvUsedBy = Object.keys(ajvDescriptor.usedBy); + assert.deepEqual(ajvUsedBy, [ + "@slimio/config" + ]); + assert.strictEqual(errorCount(), 0); }); - assert.ok(result.highlighted.contacts.length > 0); - const maintainer = result.highlighted.contacts[0]!; - assert.ok( - maintainer.dependencies.includes(spec) - ); -}); - -describe("scanner.cwd()", () => { - test("should parse author, homepage and links for a local package who doesn't exist on the remote registry", async() => { - const file = path.join(kFixturePath, "non-npm-package"); - const result = await workingDir(file, { - highlight: { - identifiers: ["foobar@gmail.com", "https://foobar.com/something", "foobar.com", "127.0.0.1"] - }, - scanRootNode: true - }); - - const dep = result.dependencies["non-npm-package"]; - const v1 = dep.versions["1.0.0"]; - - assert.deepEqual(v1.author, { - name: "NodeSecure" - }); - assert.deepStrictEqual(v1.links, { - npm: null, - homepage: "https://nodesecure.com", - repository: "https://github.com/NodeSecure/non-npm-package" - }); - assert.deepStrictEqual(v1.repository, { - type: "git", - url: "https://github.com/NodeSecure/non-npm-package.git" - }); + it("should collect walk errors and metadata stats for unresolvable git dependencies", { skip }, async(t) => { + Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); + const { logger, errors, statsCount } = buildLogger(); + t.after(() => logger.removeAllListeners()); - assert.deepStrictEqual(dep.metadata.author, { - name: "NodeSecure" - }); - assert.strictEqual(dep.metadata.homepage, "https://nodesecure.com"); - assert.strictEqual(typeof result.rootDependency.integrity, "string"); - const spec = "non-npm-package@1.0.0"; - assert.partialDeepStrictEqual(sortIdentifiers(result.highlighted.identifiers), sortIdentifiers([ - { - value: "foobar@gmail.com", - spec, - location: { - file - } - }, + const result = await depWalker( + pkgGitdeps, { - value: "foobar@gmail.com", - spec, - location: { - file: path.join(file, "email") - } + ...structuredClone(kDefaultWalkerOptions), + isVerbose: true }, + logger + ); + const resultAsJSON = JSON.parse(JSON.stringify(result.dependencies, null, 2)); + + const packages = Object.keys(resultAsJSON).sort(); + assert.deepEqual(packages, [ + "@nodesecure/npm-registry-sdk", + "@nodesecure/npm-types", + "@openally/httpie", + "@openally/result", + "lru-cache", + "nanodelay", + "nanoevents", + "nanoid", + "pkg.gitdeps", + "undici", + "zen-observable" + ].sort()); + + const walkErrors = errors(); + + assert.deepStrictEqual(walkErrors, [ { - value: "https://foobar.com/something", - spec, - location: { - file - } + name: "pacote.manifest pkg.gitdeps@0.1.0", + error: "404 Not Found - GET https://registry.npmjs.org/pkg.gitdeps - Not found", + phase: "tree-walk" }, { - value: "foobar.com", - spec, - location: { - file - } - }, - { - value: "127.0.0.1", - spec, - location: { - file - } + name: "pacote.extract pkg.gitdeps@0.1.0", + error: "404 Not Found - GET https://registry.npmjs.org/pkg.gitdeps - Not found", + phase: "tarball-scan" } - ])); + ]); + const { metadata } = result; + assert.strictEqual(typeof metadata.startedAt, "number"); + assert.strictEqual(typeof metadata.executionTime, "number"); + assert.strictEqual(Array.isArray(metadata.apiCalls), true); + assert.strictEqual(metadata.apiCallsCount, 42); + assert.strictEqual(metadata.errorCount, 2); + assert.strictEqual(metadata.errors.length, 2); + assert.strictEqual(statsCount(), 40); }); - test("should parse local manifest author field without throwing when attempting to highlight contacts", async() => { - const { dependencies } = await workingDir( - path.join(kFixturePath, "non-valid-authors") - ); - const pkg = dependencies["random-package"]; + describe("typo-squatting", () => { + it("should emit a global warning when a location is provided", { skip }, async(t) => { + Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); + const { logger, errors, statsCount } = buildLogger(); + t.after(() => logger.removeAllListeners()); - assert.deepEqual(pkg.metadata.author, { - email: "john.doe@gmail.com", - name: "John Doe" + const result = await depWalker( + pkgTypoSquatting, + { + ...structuredClone(kDefaultWalkerOptions), + location: "", + isVerbose: true + }, + logger + ); + + assert.ok(result.warnings.length > 0); + const warning = result.warnings[0]; + + assert.equal(warning.type, "typo-squatting"); + assert.match( + result.warnings[0].message, + /.*'mecha'.*fecha, mocha/ + ); + + const walkErrors = errors(); + assert.deepStrictEqual(walkErrors, [ + { + name: "pacote.manifest mecha@1.0.0", + error: "No matching version found for mecha@1.0.0.", + phase: "tree-walk" + }, + { + name: "pacote.extract mecha@1.0.0", + error: "No matching version found for mecha@1.0.0.", + phase: "tarball-scan" + } + ]); + assert.strictEqual(statsCount(), 0); }); - }); - test("should scan a workspace package.json and assign 'workspace' as the package name", async() => { - const result = await workingDir( - path.join(kFixturePath, "workspace-no-name-version") - ); + it("should not emit a global warning when no location is provided", { skip }, async(t) => { + Vulnera.setStrategy(Vulnera.strategies.GITHUB_ADVISORY); + const { logger, errors } = buildLogger(); + t.after(() => logger.removeAllListeners()); - assert.deepStrictEqual(result.rootDependency, { - name: "workspace", - version: "0.0.0", - integrity: null + const result = await depWalker( + pkgTypoSquatting, + { + ...structuredClone(kDefaultWalkerOptions), + isVerbose: true + }, + logger + ); + + assert.ok(result.warnings.length === 0); + const walkErrors = errors(); + assert.deepStrictEqual(walkErrors, [ + { + name: "pacote.manifest mecha@1.0.0", + error: "No matching version found for mecha@1.0.0.", + phase: "tree-walk" + }, + { + name: "pacote.extract mecha@1.0.0", + error: "No matching version found for mecha@1.0.0.", + phase: "tarball-scan" + } + ]); }); }); -}); - -type PartialIdentifer = Omit & { location: { file: string | null; }; }; -function sortIdentifiers( - identifiers: PartialIdentifer[] -) { - return identifiers.toSorted((a, b) => uniqueIdenfier(a).localeCompare(uniqueIdenfier(b))); -} + describe("highlight", () => { + it("should highlight packages matching a semver range map", { skip }, async(t) => { + const { logger } = buildLogger(); + t.after(() => logger.removeAllListeners()); + + const hightlightPackages = { + "zen-observable": "0.8.14 || 0.8.15", + nanoid: "*" + }; + + const result = await depWalker( + pkgHighlightedPackages, + structuredClone({ + ...kDefaultWalkerOptions, + highlight: { + packages: hightlightPackages, + contacts: [] + } + }), + logger + ); + + assert.deepStrictEqual( + result.highlighted.packages.sort(), + [ + "nanoid@5.1.6", + "zen-observable@0.8.15" + ] + ); + }); -function uniqueIdenfier(identifer: PartialIdentifer) { - return `${identifer.value} ${identifer.location.file}`; -} + it("should highlight packages from an array of specs", { skip }, async(t) => { + const { logger } = buildLogger(); + t.after(() => logger.removeAllListeners()); + + const hightlightPackages = ["zen-observable@0.8.14 || 0.8.15", "nanoid"]; + + const result = await depWalker( + pkgHighlightedPackages, + structuredClone({ + ...kDefaultWalkerOptions, + highlight: { + packages: hightlightPackages, + contacts: [] + } + }), + logger + ); + + assert.deepStrictEqual( + result.highlighted.packages.sort(), + [ + "nanoid@5.1.6", + "zen-observable@0.8.15" + ] + ); + }); + }); +}); function buildLogger() { const errors: ({ diff --git a/workspaces/scanner/test/from.spec.ts b/workspaces/scanner/test/from.spec.ts new file mode 100644 index 00000000..946e5599 --- /dev/null +++ b/workspaces/scanner/test/from.spec.ts @@ -0,0 +1,136 @@ +// Import Node.js Dependencies +import assert from "node:assert"; +import { describe, it } from "node:test"; + +// Import Third-party Dependencies +import * as Vulnera from "@nodesecure/vulnera"; +import { getLocalRegistryURL } from "@nodesecure/npm-registry-sdk"; +import type pacote from "pacote"; + +// Import Internal Dependencies +import { + from, + type Payload +} from "../src/index.ts"; + +function buildFakePayload(): Payload { + return { + id: "cached-payload-id", + rootDependency: { name: "fake", version: "1.0.0", integrity: null }, + warnings: [], + highlighted: { contacts: [], packages: [], identifiers: [] }, + dependencies: {}, + scannerVersion: "0.0.0", + vulnerabilityStrategy: Vulnera.strategies.NONE, + metadata: { + startedAt: 0, + executionTime: 0, + apiCallsCount: 0, + apiCalls: [], + errorCount: 0, + errors: [] + } + }; +} + +describe("scanner.from()", () => { + it("should fetch the payload of pacote on the npm registry", async() => { + const result = await from( + "pacote", + { + maxDepth: 10, + vulnerabilityStrategy: Vulnera.strategies.GITHUB_ADVISORY + } + ); + + assert.deepEqual(Object.keys(result), [ + "id", + "rootDependency", + "scannerVersion", + "vulnerabilityStrategy", + "warnings", + "highlighted", + "dependencies", + "metadata" + ]); + assert.strictEqual(typeof result.rootDependency.integrity, "string"); + }); + + it.skip("should fetch the payload of pacote on the gitlab registry", async() => { + const result = await from("pacote", { + registry: "https://gitlab.com/api/v4/packages/npm/", + maxDepth: 10, + vulnerabilityStrategy: Vulnera.strategies.GITHUB_ADVISORY + }); + + assert.deepEqual(Object.keys(result), [ + "id", + "rootDependency", + "scannerVersion", + "vulnerabilityStrategy", + "warnings", + "highlighted", + "dependencies", + "metadata" + ]); + assert.strictEqual(typeof result.rootDependency.integrity, "string"); + }); + + it("should highlight contacts from a remote package", async() => { + const spec = "@adonisjs/logger"; + const result = await from(spec, { + highlight: { + contacts: [ + { + name: "/.*virk.*/i" + } + ] + } + }); + + assert.ok(result.highlighted.contacts.length > 0); + const maintainer = result.highlighted.contacts[0]!; + assert.ok( + maintainer.dependencies.includes(spec) + ); + }); + + describe("cacheLookup", () => { + it("should return the cached payload without running the dependency walker", async() => { + const fakePayload = buildFakePayload(); + + const capturedManifests: (pacote.AbbreviatedManifest & pacote.ManifestResult)[] = []; + const result = await from("@slimio/is", { + registry: getLocalRegistryURL(), + cacheLookup: async(manifest) => { + capturedManifests.push(manifest); + + return fakePayload; + } + }); + + assert.strictEqual(result, fakePayload, "should return the exact cached payload instance"); + assert.strictEqual(capturedManifests.length, 1); + assert.strictEqual(capturedManifests[0].name, "@slimio/is"); + }); + + it("should proceed with a full scan when null is returned", async() => { + let callCount = 0; + const result = await from("@slimio/is", { + registry: getLocalRegistryURL(), + maxDepth: 1, + cacheLookup: async() => { + callCount++; + + return null; + } + }); + + assert.strictEqual(callCount, 1, "cacheLookup should have been called once"); + assert.ok( + result.dependencies["@slimio/is"] !== undefined, + "should have scanned the package normally" + ); + }); + }); +}); diff --git a/workspaces/scanner/test/verify.spec.ts b/workspaces/scanner/test/verify.spec.ts index 4cbdcbbe..69098bea 100644 --- a/workspaces/scanner/test/verify.spec.ts +++ b/workspaces/scanner/test/verify.spec.ts @@ -2,7 +2,7 @@ import path from "node:path"; import fs from "node:fs"; import assert from "node:assert"; -import { test } from "node:test"; +import { describe, it } from "node:test"; // Import Internal Dependencies import { verify } from "../src/index.ts"; @@ -10,71 +10,73 @@ import { verify } from "../src/index.ts"; // CONSTANTS const kFixturePath = path.join(import.meta.dirname, "fixtures", "verify"); -test("verify 'express' package", async() => { - const data = await verify("express@4.17.0"); - data.files.extensions.sort(); +describe("scanner.verify()", () => { + it("should scan the files, licenses and AST warnings of the express package", async() => { + const data = await verify("express@4.17.0"); + data.files.extensions.sort(); - assert.deepEqual(data.files, { - list: [ - "History.md", - "LICENSE", - "Readme.md", - "index.js", - "lib\\application.js", - "lib\\express.js", - "lib\\middleware\\init.js", - "lib\\middleware\\query.js", - "lib\\request.js", - "lib\\response.js", - "lib\\router\\index.js", - "lib\\router\\layer.js", - "lib\\router\\route.js", - "lib\\utils.js", - "lib\\view.js", - "package.json" - ].map((location) => location.replaceAll("\\", path.sep)), - extensions: ["", ".md", ".js", ".json"].sort(), - minified: [] - }); - assert.ok(data.directorySize > 0); + assert.deepEqual(data.files, { + list: [ + "History.md", + "LICENSE", + "Readme.md", + "index.js", + "lib\\application.js", + "lib\\express.js", + "lib\\middleware\\init.js", + "lib\\middleware\\query.js", + "lib\\request.js", + "lib\\response.js", + "lib\\router\\index.js", + "lib\\router\\layer.js", + "lib\\router\\route.js", + "lib\\utils.js", + "lib\\view.js", + "package.json" + ].map((location) => location.replaceAll("\\", path.sep)), + extensions: ["", ".md", ".js", ".json"].sort(), + minified: [] + }); + assert.ok(data.directorySize > 0); - // licenses - assert.deepEqual(data.uniqueLicenseIds, ["MIT"]); - assert.deepEqual(data.licenses, [ - { - licenses: { - MIT: "https://spdx.org/licenses/MIT.html#licenseText" - }, - spdx: { - osi: true, - fsf: true, - fsfAndOsi: true, - includesDeprecated: false - }, - fileName: "package.json" - }, - { - licenses: { - MIT: "https://spdx.org/licenses/MIT.html#licenseText" + // licenses + assert.deepEqual(data.uniqueLicenseIds, ["MIT"]); + assert.deepEqual(data.licenses, [ + { + licenses: { + MIT: "https://spdx.org/licenses/MIT.html#licenseText" + }, + spdx: { + osi: true, + fsf: true, + fsfAndOsi: true, + includesDeprecated: false + }, + fileName: "package.json" }, - spdx: { - osi: true, - fsf: true, - fsfAndOsi: true, - includesDeprecated: false - }, - fileName: "LICENSE" - } - ]); + { + licenses: { + MIT: "https://spdx.org/licenses/MIT.html#licenseText" + }, + spdx: { + osi: true, + fsf: true, + fsfAndOsi: true, + includesDeprecated: false + }, + fileName: "LICENSE" + } + ]); - assert.ok(data.ast.warnings.length === 1); - const warningName = data.ast.warnings.map((row) => row.kind); - assert.deepEqual(warningName, ["unsafe-import"]); + assert.ok(data.ast.warnings.length === 1); + const warningName = data.ast.warnings.map((row) => row.kind); + assert.deepEqual(warningName, ["unsafe-import"]); - const expectedResult = JSON.parse( - fs.readFileSync(path.join(kFixturePath, "express-result.json"), "utf-8") - .replaceAll("\\", path.sep) - .replaceAll("//", "/") - ); - assert.deepEqual(data.ast.dependencies, expectedResult); + const expectedResult = JSON.parse( + fs.readFileSync(path.join(kFixturePath, "express-result.json"), "utf-8") + .replaceAll("\\", path.sep) + .replaceAll("//", "/") + ); + assert.deepEqual(data.ast.dependencies, expectedResult); + }); }); diff --git a/workspaces/scanner/test/workingDir.spec.ts b/workspaces/scanner/test/workingDir.spec.ts new file mode 100644 index 00000000..fbc19786 --- /dev/null +++ b/workspaces/scanner/test/workingDir.spec.ts @@ -0,0 +1,177 @@ +// Import Node.js Dependencies +import path from "node:path"; +import assert from "node:assert"; +import { describe, it } from "node:test"; + +// Import Third-party Dependencies +import * as Vulnera from "@nodesecure/vulnera"; +import type { PackageJSON } from "@nodesecure/npm-types"; + +// Import Internal Dependencies +import { + workingDir, + type Payload, + type Identifier +} from "../src/index.ts"; + +// CONSTANTS +const kFixturePath = path.join(import.meta.dirname, "fixtures", "depWalker"); + +function buildFakePayload(): Payload { + return { + id: "cached-payload-id", + rootDependency: { name: "fake", version: "1.0.0", integrity: null }, + warnings: [], + highlighted: { contacts: [], packages: [], identifiers: [] }, + dependencies: {}, + scannerVersion: "0.0.0", + vulnerabilityStrategy: Vulnera.strategies.NONE, + metadata: { + startedAt: 0, + executionTime: 0, + apiCallsCount: 0, + apiCalls: [], + errorCount: 0, + errors: [] + } + }; +} + +describe("scanner.workingDir()", () => { + it("should parse author, homepage and links for a local package who doesn't exist on the remote registry", async() => { + const file = path.join(kFixturePath, "non-npm-package"); + const result = await workingDir(file, { + highlight: { + identifiers: ["foobar@gmail.com", "https://foobar.com/something", "foobar.com", "127.0.0.1"] + }, + scanRootNode: true + }); + + const dep = result.dependencies["non-npm-package"]; + const v1 = dep.versions["1.0.0"]; + + assert.deepEqual(v1.author, { + name: "NodeSecure" + }); + assert.deepStrictEqual(v1.links, { + npm: null, + homepage: "https://nodesecure.com", + repository: "https://github.com/NodeSecure/non-npm-package" + }); + assert.deepStrictEqual(v1.repository, { + type: "git", + url: "https://github.com/NodeSecure/non-npm-package.git" + }); + + assert.deepStrictEqual(dep.metadata.author, { + name: "NodeSecure" + }); + assert.strictEqual(dep.metadata.homepage, "https://nodesecure.com"); + assert.strictEqual(typeof result.rootDependency.integrity, "string"); + + const spec = "non-npm-package@1.0.0"; + assert.partialDeepStrictEqual(sortIdentifiers(result.highlighted.identifiers), sortIdentifiers([ + { + value: "foobar@gmail.com", + spec, + location: { file } + }, + { + value: "foobar@gmail.com", + spec, + location: { file: path.join(file, "email") } + }, + { + value: "https://foobar.com/something", + spec, + location: { file } + }, + { + value: "foobar.com", + spec, + location: { file } + }, + { + value: "127.0.0.1", + spec, + location: { file } + } + ])); + }); + + it("should parse local manifest author field without throwing when attempting to highlight contacts", async() => { + const { dependencies } = await workingDir( + path.join(kFixturePath, "non-valid-authors") + ); + const pkg = dependencies["random-package"]; + + assert.deepEqual(pkg.metadata.author, { + email: "john.doe@gmail.com", + name: "John Doe" + }); + }); + + it("should scan a workspace package.json and assign 'workspace' as the package name", async() => { + const result = await workingDir( + path.join(kFixturePath, "workspace-no-name-version") + ); + + assert.deepStrictEqual(result.rootDependency, { + name: "workspace", + version: "0.0.0", + integrity: null + }); + }); + + describe("cacheLookup", () => { + it("should return the cached payload without running the dependency walker", async() => { + const fakePayload = buildFakePayload(); + const file = path.join(kFixturePath, "non-npm-package"); + + const capturedPackageJSONs: PackageJSON[] = []; + const result = await workingDir(file, { + cacheLookup: async(packageJSON) => { + capturedPackageJSONs.push(packageJSON); + + return fakePayload; + } + }); + + assert.strictEqual(result, fakePayload, "should return the exact cached payload instance"); + assert.strictEqual(capturedPackageJSONs.length, 1); + assert.strictEqual(capturedPackageJSONs[0].name, "non-npm-package"); + assert.strictEqual(capturedPackageJSONs[0].version, "1.0.0"); + }); + + it("should proceed with a full scan when null is returned", async() => { + const file = path.join(kFixturePath, "non-npm-package"); + + let callCount = 0; + const result = await workingDir(file, { + cacheLookup: async() => { + callCount++; + + return null; + } + }); + + assert.strictEqual(callCount, 1, "cacheLookup should have been called once"); + assert.ok( + result.dependencies["non-npm-package"] !== undefined, + "should have scanned the package normally" + ); + }); + }); +}); + +type PartialIdentifer = Omit & { location: { file: string | null; }; }; + +function sortIdentifiers( + identifiers: PartialIdentifer[] +) { + return identifiers.toSorted((a, b) => uniqueIdentifier(a).localeCompare(uniqueIdentifier(b))); +} + +function uniqueIdentifier(identifier: PartialIdentifer) { + return `${identifier.value} ${identifier.location.file}`; +}