${info.flags} ${info.name}
${info.version}`; + searchBarPackagesContainer.insertAdjacentHTML( + "beforeend", + `` + ); + } + window.searchbar = new SearchBar(nsn, secureDataSet.linker); + } +} + +function initPackagesNavigation(data) { + const fragment = document.createDocumentFragment(); + const packages = data.lru; + + const hasAtLeast2Packages = packages.length > 1; + const hasExactly2Packages = packages.length === 2; + const container = createDOMElement("div", { + classList: ["packages"] + }); + + for (const pkg of packages) { + const { name, version } = parseNpmSpec(pkg); + + const pkgElement = createDOMElement("div", { + classList: ["package"], + childs: [ + createDOMElement("p", { text: name }), + createDOMElement("b", { text: `v${version}` }) + ] + }); + pkgElement.dataset.name = pkg; + if (pkg === data.current) { + window.activePackage = pkg; + pkgElement.classList.add("active"); + } + pkgElement.addEventListener("click", () => { + if (window.activePackage !== pkg) { + window.socket.send(JSON.stringify({ pkg, action: "SEARCH" })); + } + }); + + if (hasAtLeast2Packages && pkg !== data.root) { + pkgElement.appendChild( + renderPackageRemoveButton(pkgElement.dataset.name, { hasExactly2Packages }) + ); + } + + container.appendChild(pkgElement); + } + + const plusButtonElement = createDOMElement("button", { + classList: ["add"], + childs: [ + createDOMElement("p", { text: "+" }) + ] + }); + plusButtonElement.addEventListener("click", () => { + window.navigation.setNavByName("search--view"); + }); + + fragment.append(container, plusButtonElement); + + return fragment; +} + +function renderPackageRemoveButton(packageName, options) { + const { + hasExactly2Packages + } = options; + + // we allow to remove a package when at least 2 packages are present + const removeButton = createDOMElement("button", { + classList: ["remove"], + text: "x" + }); + + removeButton.addEventListener("click", (event) => { + event.stopPropagation(); + window.socket.send(JSON.stringify({ action: "REMOVE", pkg: packageName })); + + if (hasExactly2Packages) { + document + .getElementById("search-nav") + .querySelectorAll(".package") + .forEach((element) => element.querySelector(".remove")?.remove()); + } + }, { once: true }); + + return removeButton; +} diff --git a/public/main.css b/public/main.css index 7e90e505..138021b3 100644 --- a/public/main.css +++ b/public/main.css @@ -16,6 +16,7 @@ @import url("./components/searchbar/searchbar.css"); @import url("./components/views/home/home.css"); @import url("./components/views/network/network.css"); +@import url("./components/views/search/search.css"); @import url("./components/views/settings/settings.css"); @import url("./components/wiki/wiki.css"); @import url("./components/gauge/gauge.css"); diff --git a/public/main.js b/public/main.js index 1e65c0cf..774cda3c 100644 --- a/public/main.js +++ b/public/main.js @@ -5,7 +5,6 @@ import { NodeSecureDataSet, NodeSecureNetwork } from "@nodesecure/vis-network"; import { PackageInfo } from "./components/package/package.js"; import { ViewNavigation } from "./components/navigation/navigation.js"; import { Wiki } from "./components/wiki/wiki.js"; -import { SearchBar } from "./components/searchbar/searchbar.js"; import { Popup } from "./components/popup/popup.js"; import { Locker } from "./components/locker/locker.js"; import { Legend } from "./components/legend/legend.js"; @@ -13,23 +12,90 @@ import { Legend } from "./components/legend/legend.js"; // Import Views Components import { Settings } from "./components/views/settings/settings.js"; import { HomeView } from "./components/views/home/home.js"; +import { SearchView } from "./components/views/search/search.js"; // Import Core Components import { NetworkNavigation } from "./core/network-navigation.js"; import { i18n } from "./core/i18n.js"; +import { initSearchNav } from "./core/search-nav.js"; // Import Utils import * as utils from "./common/utils.js"; +let secureDataSet; +let nsn; +let searchview; +let packageInfoOpened = false; + document.addEventListener("DOMContentLoaded", async() => { + window.scannedPackageCache = []; window.locker = null; window.popup = new Popup(); window.settings = await new Settings().fetchUserConfig(); window.i18n = await new i18n().fetch(); window.navigation = new ViewNavigation(); window.wiki = new Wiki(); - let packageInfoOpened = false; - const secureDataSet = new NodeSecureDataSet({ + + await init(); + onSettingsSaved(); + + window.socket = new WebSocket(`ws://${window.location.hostname}:1338`); + window.socket.addEventListener("message", async(event) => { + const data = JSON.parse(event.data); + console.log(`[WEBSOCKET] data status = '${data.status || "NONE"}'`); + + if (data.rootDependencyName) { + window.activePackage = data.rootDependencyName; + await init({ navigateToNetworkView: true }); + initSearchNav(data, { + initFromZero: false, + searchOptions: { + nsn, secureDataSet + } + }); + } + else if (data.status === "INIT" || data.status === "RELOAD") { + window.scannedPackageCache = data.older; + console.log( + "[INFO] Older packages are loaded!", + window.scannedPackageCache + ); + + initSearchNav(data, { + searchOptions: { + nsn, secureDataSet + } + }); + searchview.reset(); + const nsnActivePackage = secureDataSet.linker.get(0); + const nsnRootPackage = nsnActivePackage ? `${nsnActivePackage.name}@${nsnActivePackage.version}` : null; + if (data.status === "RELOAD" && nsnRootPackage !== null && nsnRootPackage !== window.activePackage) { + // it means we removed the previous active package, which is still active in network, so we need to re-init + await init(); + + // FIXME: initSearchNav is called twice, we need to fix this + initSearchNav(data, { + searchOptions: { + nsn, secureDataSet + } + }); + } + } + else if (data.status === "SCAN") { + searchview.onScan(data.pkg); + } + }); + + window.onbeforeunload = () => { + window.socket.onclose = () => void 0; + window.socket.close(); + }; +}); + +async function init(options = {}) { + const { navigateToNetworkView = false } = options; + + secureDataSet = new NodeSecureDataSet({ flagsToIgnore: window.settings.config.ignore.flags, warningsToIgnore: window.settings.config.ignore.warnings }); @@ -39,39 +105,90 @@ document.addEventListener("DOMContentLoaded", async() => { // Initialize vis Network NodeSecureNetwork.networkElementId = "dependency-graph"; - const nsn = new NodeSecureNetwork(secureDataSet, { i18n: window.i18n[utils.currentLang()] }); + nsn = new NodeSecureNetwork(secureDataSet, { i18n: window.i18n[utils.currentLang()] }); window.locker = new Locker(nsn); - const legend = new Legend({ show: window.settings.config.showFriendlyDependencies }); + window.legend = new Legend({ show: window.settings.config.showFriendlyDependencies }); new HomeView(secureDataSet, nsn); + searchview ??= new SearchView(secureDataSet, nsn); window.addEventListener("package-info-closed", () => { - networkNavigation.currentNodeParams = null; + window.networkNav.currentNodeParams = null; packageInfoOpened = false; }); nsn.network.on("click", updateShowInfoMenu); - function getNodeLevel(node) { - const rootNode = secureDataSet.linker.get(0); - if (node.id === rootNode.id) { - return 0; - } + const networkNavigation = new NetworkNavigation(secureDataSet, nsn); + window.networkNav = networkNavigation; - let level = 1; - let currentNode = node; - while (currentNode.usedBy[rootNode.name] === undefined) { - currentNode = secureDataSet.linker.get( - [...secureDataSet.linker].find(([_, { name }]) => Object.keys(currentNode.usedBy)[0] === name)[0] - ); - level++; + if (navigateToNetworkView) { + window.navigation.setNavByName("network--view"); + } + + // update search nav + const pkgs = document.querySelectorAll("#search-nav .packages > .package"); + for (const pkg of pkgs) { + if (pkg.dataset.name.startsWith(window.activePackage)) { + pkg.classList.add("active"); } + else { + pkg.classList.remove("active"); + } + } - return level; + PackageInfo.close(); + + console.log("[INFO] Node-Secure is ready!"); +} + +function getNodeLevel(node) { + const rootNode = secureDataSet.linker.get(0); + if (node.id === rootNode.id) { + return 0; } - const networkNavigation = new NetworkNavigation(secureDataSet, nsn); - window.networkNav = networkNavigation; + let level = 1; + let currentNode = node; + while (currentNode.usedBy[rootNode.name] === undefined) { + currentNode = secureDataSet.linker.get( + [...secureDataSet.linker].find(([_, { name }]) => Object.keys(currentNode.usedBy)[0] === name)[0] + ); + level++; + } + + return level; +} +async function updateShowInfoMenu(params) { + if (params.nodes.length === 0) { + window.networkNav.currentNodeParams = null; + + return PackageInfo.close(); + } + + if (window.networkNav.currentNodeParams?.nodes[0] === params.nodes[0] && packageInfoOpened === true) { + return void 0; + } + + packageInfoOpened = true; + window.networkNav.currentNodeParams = params; + const currentNode = window.networkNav.currentNodeParams.nodes[0]; + const selectedNode = secureDataSet.linker.get( + Number(currentNode) + ); + const selectedNodeLevel = getNodeLevel(selectedNode); + + window.networkNav.setLevel(selectedNodeLevel); + if (window.networkNav.dependenciesMapByLevel.get(selectedNodeLevel) === undefined) { + window.networkNav.dependenciesMapByLevel.set(selectedNodeLevel, params); + } + + new PackageInfo(selectedNode, currentNode, secureDataSet.data.dependencies[selectedNode.name], nsn); + + return void 0; +} + +function onSettingsSaved() { window.addEventListener("settings-saved", async(event) => { const warningsToIgnore = new Set(event.detail.ignore.warnings); const flagsToIgnore = new Set(event.detail.ignore.flags); @@ -86,57 +203,20 @@ document.addEventListener("DOMContentLoaded", async() => { ); const { nodes } = secureDataSet.build(); nsn.nodes.update(nodes.get()); + const rootNode = secureDataSet.linker.get(0); + window.activePackage = rootNode.name; - if (networkNavigation.currentNodeParams !== null) { + if (window.networkNav.currentNodeParams !== null) { window.navigation.setNavByName("network--view"); - nsn.neighbourHighlight(networkNavigation.currentNodeParams, window.i18n[utils.currentLang()]); - updateShowInfoMenu(networkNavigation.currentNodeParams); + nsn.neighbourHighlight(window.networkNav.currentNodeParams, window.i18n[utils.currentLang()]); + updateShowInfoMenu(window.networkNav.currentNodeParams); } if (event.detail.showFriendlyDependencies) { - legend.show(); + window.legend.show(); } else { - legend.hide(); + window.legend.hide(); } }); - - // Initialize searchbar - { - const dataListElement = document.getElementById("package-list"); - for (const info of secureDataSet.packages) { - const content = `${info.flags} ${info.name}
${info.version}`; - dataListElement.insertAdjacentHTML("beforeend", ``); - } - } - window.searchbar = new SearchBar(nsn, secureDataSet.linker); - - async function updateShowInfoMenu(params) { - if (params.nodes.length === 0) { - networkNavigation.currentNodeParams = null; - - return PackageInfo.close(); - } - - if (networkNavigation.currentNodeParams?.nodes[0] === params.nodes[0] && packageInfoOpened === true) { - return void 0; - } - - packageInfoOpened = true; - networkNavigation.currentNodeParams = params; - const currentNode = networkNavigation.currentNodeParams.nodes[0]; - const selectedNode = secureDataSet.linker.get( - Number(currentNode) - ); - const selectedNodeLevel = getNodeLevel(selectedNode); - - networkNavigation.setLevel(selectedNodeLevel); - if (networkNavigation.dependenciesMapByLevel.get(selectedNodeLevel) === undefined) { - networkNavigation.dependenciesMapByLevel.set(selectedNodeLevel, params); - } - - new PackageInfo(selectedNode, currentNode, secureDataSet.data.dependencies[selectedNode.name], nsn); - - return void 0; - } -}); +} diff --git a/scripts/clear-cache.js b/scripts/clear-cache.js new file mode 100644 index 00000000..06577b47 --- /dev/null +++ b/scripts/clear-cache.js @@ -0,0 +1,9 @@ +// Import Third-party Dependencies +import cacache from "cacache"; + +// Import Internal Dependencies +import { CACHE_PATH } from "../src/http-server/cache.js"; + +await cacache.rm.all(CACHE_PATH); + +console.log("Cache cleared successfully!"); diff --git a/src/commands/scorecard.js b/src/commands/scorecard.js index dafc2ff8..47b4d348 100644 --- a/src/commands/scorecard.js +++ b/src/commands/scorecard.js @@ -41,7 +41,7 @@ export async function main(repo, opts) { try { const [repo, vcs] = result.unwrap(); repository = repo; - platform = vcs.slice(-4) === ".com" ? vsc : `${vcs}.com`; + platform = vcs.slice(-4) === ".com" ? vcs : `${vcs}.com`; } catch (error) { console.log(white().bold(result.err)); diff --git a/src/http-server/cache.js b/src/http-server/cache.js new file mode 100644 index 00000000..c2b9a753 --- /dev/null +++ b/src/http-server/cache.js @@ -0,0 +1,132 @@ +// Import Node.js Dependencies +import os from "node:os"; +import path from "node:path"; +import fs from "node:fs"; + +// Import Third-party Dependencies +import cacache from "cacache"; + +// Import Internal Dependencies +import { logger } from "./logger.js"; + +// CONSTANTS +const kConfigCache = "___config"; +const kPayloadsCache = "___payloads"; +const kPayloadsPath = path.join(os.homedir(), ".nsecure", "payloads"); +const kMaxPayloads = 3; + +export const CACHE_PATH = path.join(os.tmpdir(), "nsecure-cli"); +export const DEFAULT_PAYLOAD_PATH = path.join(process.cwd(), "nsecure-result.json"); + +class _AppCache { + constructor() { + fs.mkdirSync(kPayloadsPath, { recursive: true }); + } + + async updateConfig(newValue) { + await cacache.put(CACHE_PATH, kConfigCache, JSON.stringify(newValue)); + } + + async getConfig() { + const { data } = await cacache.get(CACHE_PATH, kConfigCache); + + return JSON.parse(data.toString()); + } + + updatePayload(pkg, payload) { + fs.writeFileSync(path.join(kPayloadsPath, pkg), JSON.stringify(payload)); + } + + async getPayload(pkg) { + try { + return JSON.parse(fs.readFileSync(path.join(kPayloadsPath, pkg.replaceAll("/", "-")), "utf-8")); + } + catch (err) { + logger.error(`[cache|get](pkg: ${pkg}|cache: not found)`); + + throw err; + } + } + + async getPayloadOrNull(pkg) { + try { + return await this.getPayload(pkg); + } + catch { + return null; + } + } + + async updatePayloadsList(payloadsList) { + await cacache.put(CACHE_PATH, kPayloadsCache, JSON.stringify(payloadsList)); + } + + async payloadsList() { + try { + const { data } = await cacache.get(CACHE_PATH, kPayloadsCache); + + return JSON.parse(data.toString()); + } + catch (err) { + logger.error(`[cache|get](cache: not found)`); + + throw err; + } + } + + async #initDefaultPayloadsList() { + const payload = JSON.parse(fs.readFileSync(DEFAULT_PAYLOAD_PATH, "utf-8")); + const version = Object.keys(payload.dependencies[payload.rootDependencyName].versions)[0]; + const formatted = `${payload.rootDependencyName}@${version}`; + const payloadsList = { + lru: [formatted], + current: formatted, + older: [], + lastUsed: { + [formatted]: Date.now() + }, + root: formatted + }; + + logger.info(`[cache|init](dep: ${formatted}|version: ${version}|rootDependencyName: ${payload.rootDependencyName})`); + await cacache.put(CACHE_PATH, kPayloadsCache, JSON.stringify(payloadsList)); + this.updatePayload(formatted.replaceAll("/", "-"), payload); + } + + async initPayloadsList() { + const packagesInFolder = fs.readdirSync(kPayloadsPath); + if (packagesInFolder.length === 0) { + this.#initDefaultPayloadsList(); + + return; + } + + const list = packagesInFolder.map(({ name }) => name); + logger.info(`[cache|init](list: ${list})`); + + await cacache.put(CACHE_PATH, kPayloadsCache, JSON.stringify({ list, current: list[0] })); + } + + removePayload(pkg) { + fs.rmSync(path.join(kPayloadsPath, pkg)); + } + + async removeLastLRU() { + const { lru, lastUsed, older, root } = await this.payloadsList(); + if (lru.length < kMaxPayloads) { + return { lru, older, lastUsed, root }; + } + const packageToBeRemoved = Object.keys(lastUsed) + .filter((key) => lru.includes(key)) + .sort((a, b) => lastUsed[a] - lastUsed[b])[0]; + + return { + lru: lru.filter((pkg) => pkg !== packageToBeRemoved), + older: [...older, packageToBeRemoved], + lastUsed, + root + }; + } +} + +export const appCache = new _AppCache(); diff --git a/src/http-server/config.js b/src/http-server/config.js index 02aa02a5..4f0998d5 100644 --- a/src/http-server/config.js +++ b/src/http-server/config.js @@ -1,32 +1,49 @@ -// Import Node.js Dependencies -import path from "node:path"; -import os from "node:os"; - -// Import Third-party Depedencies -import cacache from "cacache"; +// Import Internal Dependencies +import { appCache } from "./cache.js"; +import { logger } from "./logger.js"; // CONSTANTS -const kCachePath = path.join(os.tmpdir(), "nsecure-cli"); -const kConfigKey = "cli-config"; +const kDefaultConfig = { + defaultPackageMenu: "info", + ignore: { flags: [], warnings: [] } +}; export async function get() { try { - const { data } = await cacache.get(kCachePath, kConfigKey); + const config = await appCache.getConfig(); + + const { + defaultPackageMenu, + ignore: { + flags, + warnings + } = {} + } = config; + logger.info(`[config|get](defaultPackageMenu: ${defaultPackageMenu}|ignore-flag: ${flags}|ignore-warnings: ${warnings})`); - return JSON.parse(data.toString()); + return config; } - catch { - const defaultValue = { - defaultPackageMenu: "info", - ignore: { flags: [], warnings: [] } - }; + catch (err) { + logger.error(`[config|get](error: ${err.message})`); - await cacache.put(kCachePath, kConfigKey, JSON.stringify(defaultValue)); + await appCache.updateConfig(kDefaultConfig); - return defaultValue; + logger.info(`[config|get](fallback to default: ${JSON.stringify(kDefaultConfig)})`); + + return kDefaultConfig; } } export async function set(newValue) { - await cacache.put(kCachePath, kConfigKey, JSON.stringify(newValue)); + logger.info(`[config|set](config: ${newValue})`); + try { + await appCache.updateConfig(newValue); + + logger.info(`[config|set](sucess)`); + } + catch (err) { + logger.error(`[config|set](error: ${err.message})`); + + throw err; + } } diff --git a/src/http-server/endpoints/data.js b/src/http-server/endpoints/data.js index 6a5e34d8..02194a86 100644 --- a/src/http-server/endpoints/data.js +++ b/src/http-server/endpoints/data.js @@ -1,18 +1,47 @@ // Import Node.js Dependencies import fs from "node:fs"; -import { pipeline } from "node:stream"; +import path from "node:path"; + +// Import Third-party Dependencies +import send from "@polka/send-type"; // Import Internal Dependencies -import { context } from "../context.js"; +import { appCache } from "../cache.js"; +import { logger } from "../logger.js"; + +// CONSTANTS +const kDefaultPayloadPath = path.join(process.cwd(), "nsecure-result.json"); export async function get(req, res) { - const { dataFilePath } = context.getStore(); + try { + const { current, lru } = await appCache.payloadsList(); + logger.info(`[data|get](current: ${current})`); + logger.debug(`[data|get](lru: ${lru})`); + + const formatted = current.replaceAll("/", "-"); + send(res, 200, await appCache.getPayload(formatted)); + } + catch { + logger.error(`[data|get](No cache yet. Creating one...)`); + + const payload = JSON.parse(fs.readFileSync(kDefaultPayloadPath, "utf-8")); + const version = Object.keys(payload.dependencies[payload.rootDependencyName].versions)[0]; + const formatted = `${payload.rootDependencyName}@${version}`; + const payloadsList = { + lru: [formatted], + current: formatted, + older: [], + lastUsed: { + [formatted]: Date.now() + }, + root: formatted + }; + logger.info(`[data|get](dep: ${formatted}|version: ${version}|rootDependencyName: ${payload.rootDependencyName})`); - res.writeHead(200, { "Content-Type": "application/json" }); + await appCache.updatePayloadsList(payloadsList); + appCache.updatePayload(formatted.replaceAll("/", "-"), payload); + logger.info(`[data|get](cache: created|payloadsList: ${payloadsList.lru})`); - pipeline(fs.createReadStream(dataFilePath), res, (err) => { - if (err) { - console.error(err); - } - }); + send(res, 200, payload); + } } diff --git a/src/http-server/endpoints/search.js b/src/http-server/endpoints/search.js new file mode 100644 index 00000000..2f221981 --- /dev/null +++ b/src/http-server/endpoints/search.js @@ -0,0 +1,41 @@ +// Import Third-party Dependencies +import send from "@polka/send-type"; +import * as npm from "@nodesecure/npm-registry-sdk"; + +// Import Internal Dependencies +import { logger } from "../logger.js"; + +export async function get(req, res) { + const { packageName } = req.params; + logger.info(`[search|get](packageName: ${packageName}|formatted: ${decodeURIComponent(packageName)})`); + + const { objects, total } = await npm.search({ + text: decodeURIComponent(packageName) + }); + logger.debug(`[search|get](npmSearchResult: ${JSON.stringify(objects.map((pkg) => pkg.package.name))})`); + + send(res, 200, { + count: total, + result: objects.map((pkg) => { + return { + name: pkg.package.name, + version: pkg.package.version, + description: pkg.package.description + }; + }) + }); +} + +export async function versions(req, res) { + const { packageName } = req.params; + + logger.info(`[search|versions](packageName: ${packageName}|formatted: ${decodeURIComponent(packageName)})`); + + const packument = await npm.packument(decodeURIComponent(packageName)); + const versions = Object.keys(packument.versions); + + logger.info(`[search|versions](packageName: ${packageName}|versions: ${versions})`); + logger.debug(`[search|versions](packument: ${packument})`); + + send(res, 200, versions); +} diff --git a/src/http-server/index.js b/src/http-server/index.js index 93cfa9e2..bab1b82d 100644 --- a/src/http-server/index.js +++ b/src/http-server/index.js @@ -6,22 +6,27 @@ import kleur from "kleur"; import polka from "polka"; import open from "open"; import * as i18n from "@nodesecure/i18n"; +import { WebSocketServer } from "ws"; // Import Internal Dependencies import * as root from "./endpoints/root.js"; import * as data from "./endpoints/data.js"; import * as flags from "./endpoints/flags.js"; import * as config from "./endpoints/config.js"; +import * as search from "./endpoints/search.js"; import * as bundle from "./endpoints/bundle.js"; import * as npmDownloads from "./endpoints/npm-downloads.js"; import * as scorecard from "./endpoints/ossf-scorecard.js"; import * as locali18n from "./endpoints/i18n.js"; import * as report from "./endpoints/report.js"; import * as middleware from "./middleware.js"; +import * as wsHandlers from "./websocket/index.js"; +import { logger } from "./logger.js"; export function buildServer(dataFilePath, options = {}) { const httpConfigPort = typeof options.port === "number" ? options.port : 0; const openLink = typeof options.openLink === "boolean" ? options.openLink : true; + const enableWS = options.enableWS ?? process.env.NODE_ENV !== "test"; fs.accessSync(dataFilePath, fs.constants.R_OK | fs.constants.W_OK); @@ -30,12 +35,15 @@ export function buildServer(dataFilePath, options = {}) { httpServer.use(middleware.buildContextMiddleware(dataFilePath)); httpServer.use(middleware.addStaticFiles); httpServer.get("/", root.get); - httpServer.get("/data", data.get); + httpServer.get("/data", data.get); httpServer.get("/config", config.get); httpServer.put("/config", config.save); httpServer.get("/i18n", locali18n.get); + httpServer.get("/search/:packageName", search.get); + httpServer.get("/search-versions/:packageName", search.versions); + httpServer.get("/flags", flags.getAll); httpServer.get("/flags/description/:title", flags.get); httpServer.get("/bundle/:pkgName", bundle.get); @@ -54,5 +62,29 @@ export function buildServer(dataFilePath, options = {}) { } }); + if (enableWS) { + const websocket = new WebSocketServer({ port: 1338 }); + websocket.on("connection", async(socket) => { + socket.on("message", async(rawMessage) => { + const message = JSON.parse(rawMessage); + logger.info(`[ws](message: ${JSON.stringify(message)})`); + + if (message.action === "SEARCH") { + wsHandlers.search(socket, message.pkg); + } + else if (message.action === "REMOVE") { + wsHandlers.remove(socket, message.pkg); + } + }); + + wsHandlers.init(socket); + }); + } + return httpServer; } + +process.on("SIGINT", () => { + console.log(kleur.red().bold("SIGINT signal received.")); + process.exit(0); +}); diff --git a/src/http-server/logger.js b/src/http-server/logger.js new file mode 100644 index 00000000..e313792e --- /dev/null +++ b/src/http-server/logger.js @@ -0,0 +1,14 @@ +// Import Third-party Dependencies +import pino from "pino"; + +// CONSTANTS +const kDefaultLogLevel = "info"; + +const logger = pino({ + level: process.env.LOG_LEVEL ?? kDefaultLogLevel, + transport: { + target: "pino-pretty" + } +}); + +export { logger }; diff --git a/src/http-server/websocket/index.js b/src/http-server/websocket/index.js new file mode 100644 index 00000000..9c19c095 --- /dev/null +++ b/src/http-server/websocket/index.js @@ -0,0 +1,3 @@ +export * from "./search.js"; +export * from "./remove.js"; +export * from "./init.js"; diff --git a/src/http-server/websocket/init.js b/src/http-server/websocket/init.js new file mode 100644 index 00000000..fe3a84ac --- /dev/null +++ b/src/http-server/websocket/init.js @@ -0,0 +1,33 @@ +// Import Internal Dependencies +import { appCache } from "../cache.js"; +import { logger } from "../logger.js"; + +export async function init(socket, lock = false) { + try { + const { current, lru, older, root } = await appCache.payloadsList(); + logger.info(`[ws|init](lru: ${lru}|older: ${older}|current: ${current}|root: ${root})`); + + if (lru === void 0 || current === void 0) { + throw new Error("Payloads list not found in cache."); + } + + socket.send(JSON.stringify({ + status: "INIT", + current, + lru, + older, + root + })); + } + catch { + logger.error(`[ws|init](No cache yet. Creating one...)`); + + if (lock) { + return; + } + + await appCache.initPayloadsList(); + + init(socket, true); + } +} diff --git a/src/http-server/websocket/remove.js b/src/http-server/websocket/remove.js new file mode 100644 index 00000000..005529fe --- /dev/null +++ b/src/http-server/websocket/remove.js @@ -0,0 +1,85 @@ +// Import Internal Dependencies +import { appCache } from "../cache.js"; +import { logger } from "../logger.js"; + +export async function remove(ws, pkg) { + const formattedPkg = pkg.replace("/", "-"); + logger.info(`[ws|remove](pkg: ${pkg}|formatted: ${formattedPkg})`); + + try { + const { lru, older, current, lastUsed, root } = await appCache.payloadsList(); + logger.debug(`[ws|remove](lru: ${lru}|current: ${current})`); + + if (lru.length === 1 && older.length === 0) { + throw new Error("Cannot remove the last package."); + } + + const lruIndex = lru.findIndex((pkgName) => pkgName === pkg); + const olderIndex = older.findIndex((pkgName) => pkgName === pkg); + + if (lruIndex === -1 && olderIndex === -1) { + throw new Error("Package not found in cache."); + } + + if (lruIndex > -1) { + logger.info(`[ws|remove](remove from lru)`); + const updatedLru = lru.filter((pkgName) => pkgName !== pkg); + if (older.length > 0) { + // We need to move the first older package to the lru list + const olderPkg = older.sort((a, b) => { + const aDate = lastUsed[a]; + const bDate = lastUsed[b]; + + return aDate - bDate; + }); + updatedLru.push(olderPkg[0]); + older.splice(older.indexOf(olderPkg[0]), 1); + } + + const updatedList = { + lru: updatedLru, + older, + lastUsed: { + ...lastUsed, + [pkg]: void 0 + }, + current: current === pkg ? updatedLru[0] : current, + root + }; + await appCache.updatePayloadsList(updatedList); + + ws.send(JSON.stringify({ + status: "RELOAD", + ...updatedList + })); + } + else { + logger.info(`[ws|remove](remove from older)`); + const updatedOlder = older.filter((pkgName) => pkgName !== pkg); + const updatedList = { + lru, + older: updatedOlder, + lastUsed: { + ...lastUsed, + [pkg]: void 0 + }, + current, + root + }; + await appCache.updatePayloadsList(updatedList); + + ws.send(JSON.stringify({ + status: "RELOAD", + ...updatedList + })); + } + + appCache.removePayload(formattedPkg.replaceAll("/", "-")); + } + catch (error) { + logger.error(`[ws|remove](error: ${error.message})`); + logger.debug(error); + + throw error; + } +} diff --git a/src/http-server/websocket/search.js b/src/http-server/websocket/search.js new file mode 100644 index 00000000..8837b045 --- /dev/null +++ b/src/http-server/websocket/search.js @@ -0,0 +1,80 @@ +// Import Third-party Dependencies +import * as Scanner from "@nodesecure/scanner"; + +// Import Internal Dependencies +import { logger } from "../logger.js"; +import { appCache } from "../cache.js"; + +export async function search(ws, pkg) { + logger.info(`[ws|search](pkg: ${pkg})`); + + const cache = await appCache.getPayloadOrNull(pkg); + if (cache) { + logger.info(`[ws|search](payload: ${pkg} found in cache)`); + const cacheList = await appCache.payloadsList(); + if (cacheList.lru.includes(pkg)) { + logger.info(`[ws|search](payload: ${pkg} is already in the LRU)`); + const updatedList = { + ...cacheList, + current: pkg, + lastUsed: { ...cacheList.lastUsed, [pkg]: Date.now() } + }; + await appCache.updatePayloadsList(updatedList); + ws.send(JSON.stringify(cache)); + + return; + } + const { lru, older, lastUsed, root } = await appCache.removeLastLRU(); + const updatedList = { + lru: [...new Set([...lru, pkg])], + current: pkg, + older: older.filter((pckg) => pckg !== pkg), + lastUsed: { ...lastUsed, [pkg]: Date.now() }, + root + }; + await appCache.updatePayloadsList(updatedList); + + ws.send(JSON.stringify(cache)); + ws.send(JSON.stringify({ + status: "RELOAD", + ...updatedList + })); + + return; + } + + // at this point we don't have the payload in cache so we have to scan it. + logger.info(`[ws|search](scan ${pkg} in progress)`); + ws.send(JSON.stringify({ status: "SCAN", pkg })); + + const payload = await Scanner.from(pkg, { maxDepth: 4 }); + const name = payload.rootDependencyName; + const version = Object.keys(payload.dependencies[name].versions)[0]; + + { + // save the payload in cache + const pkg = `${name}@${version}`; + logger.info(`[ws|search](scan ${pkg} done|cache: updated)`); + + // update the payloads list + const { lru, older, lastUsed, root } = await appCache.removeLastLRU(); + lru.push(pkg); + appCache.updatePayload(pkg.replaceAll("/", "-"), payload); + const updatedList = { + lru: [...new Set(lru)], + older, + lastUsed: { ...lastUsed, [pkg]: Date.now() }, + current: pkg, + root + }; + await appCache.updatePayloadsList(updatedList); + + ws.send(JSON.stringify(payload)); + ws.send(JSON.stringify({ + status: "RELOAD", + ...updatedList + })); + + logger.info(`[ws|search](data sent to client|cache: updated)`); + } +} diff --git a/test/config.test.js b/test/config.test.js index d2db3b1f..37c79d11 100644 --- a/test/config.test.js +++ b/test/config.test.js @@ -1,6 +1,4 @@ // Import Node.js Dependencies -import path from "node:path"; -import os from "node:os"; import { describe, it, before, after } from "node:test"; import assert from "node:assert"; @@ -9,12 +7,12 @@ import cacache from "cacache"; // Import Internal Dependencies import { get, set } from "../src/http-server/config.js"; +import { CACHE_PATH } from "../src/http-server/cache.js"; // CONSTANTS -const kCachePath = path.join(os.tmpdir(), "nsecure-cli"); -const kConfigKey = "cli-config"; +const kConfigKey = "___config"; -describe("config", () => { +describe("config", { concurrency: 1 }, () => { let actualConfig; before(async() => { @@ -26,7 +24,7 @@ describe("config", () => { }); it("should get default config from empty cache", async() => { - await cacache.rm(kCachePath, kConfigKey); + await cacache.rm(CACHE_PATH, kConfigKey); const value = await get(); assert.deepStrictEqual(value, { @@ -36,7 +34,7 @@ describe("config", () => { }); it("should get config from cache", async() => { - await cacache.put(kCachePath, kConfigKey, JSON.stringify({ foo: "bar" })); + await cacache.put(CACHE_PATH, kConfigKey, JSON.stringify({ foo: "bar" })); const value = await get(); assert.deepStrictEqual(value, { foo: "bar" }); diff --git a/test/httpServer.test.js b/test/httpServer.test.js index c2ae7178..f59500d4 100644 --- a/test/httpServer.test.js +++ b/test/httpServer.test.js @@ -1,10 +1,9 @@ // Import Node.js Dependencies -import { readFileSync } from "node:fs"; +import fs from "node:fs"; import { fileURLToPath } from "node:url"; import { after, before, describe, test } from "node:test"; import { once } from "node:events"; import path from "node:path"; -import os from "node:os"; import assert from "node:assert"; // Import Third-party Dependencies @@ -18,6 +17,7 @@ import cacache from "cacache"; // Require Internal Dependencies import { buildServer } from "../src/http-server/index.js"; +import { CACHE_PATH } from "../src/http-server/cache.js"; // CONSTANTS const HTTP_PORT = 17049; @@ -25,33 +25,40 @@ const HTTP_URL = new URL(`http://localhost:${HTTP_PORT}`); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const JSON_PATH = path.join(__dirname, "fixtures", "httpServer.json"); -const INDEX_HTML = readFileSync(path.join(__dirname, "..", "views", "index.html"), "utf-8"); +const INDEX_HTML = fs.readFileSync(path.join(__dirname, "..", "views", "index.html"), "utf-8"); -const kCachePath = path.join(os.tmpdir(), "nsecure-cli"); -const kConfigKey = "cli-config"; +const kConfigKey = "___config"; const kGlobalDispatcher = getGlobalDispatcher(); const kMockAgent = new MockAgent(); const kBundlephobiaPool = kMockAgent.get("https://bundlephobia.com"); +const kDefaultPayloadPath = path.join(process.cwd(), "nsecure-result.json"); -describe("httpServer", () => { +describe("httpServer", { concurrency: 1 }, () => { let httpServer; before(async() => { setGlobalDispatcher(kMockAgent); + await i18n.extendFromSystemPath( + path.join(__dirname, "..", "i18n") + ); httpServer = buildServer(JSON_PATH, { port: HTTP_PORT, - openLink: false + openLink: false, + enableWS: false }); await once(httpServer.server, "listening"); - await i18n.extendFromSystemPath( - path.join(__dirname, "..", "i18n") - ); enableDestroy(httpServer.server); + + if (fs.existsSync(kDefaultPayloadPath) === false) { + // When running tests on CI, we need to create the nsecure-result.json file + const payload = fs.readFileSync(JSON_PATH, "utf-8"); + fs.writeFileSync(kDefaultPayloadPath, payload); + } }, { timeout: 5000 }); - after(() => { + after(async() => { httpServer.server.destroy(); kBundlephobiaPool.close(); setGlobalDispatcher(kGlobalDispatcher); @@ -123,7 +130,6 @@ describe("httpServer", () => { createReadStream: () => "foo" } }); - const consoleError = console.error; const logs = []; console.error = (data) => logs.push(data); @@ -135,31 +141,7 @@ describe("httpServer", () => { const result = await get(new URL("/data", HTTP_URL)); assert.equal(result.statusCode, 200); - assert.equal(result.headers["content-type"], "application/json"); - }); - - test("'/data' should fail", async() => { - const module = await esmock("../src/http-server/endpoints/data.js", { - "../src/http-server/context.js": { - context: { - getStore: () => { - return { dataFilePath: "foo" }; - } - } - }, - stream: { - pipeline: (stream, res, err) => err("fake error") - }, - fs: { - createReadStream: () => "foo" - } - }); - const consoleError = console.error; - const logs = []; - console.error = (data) => logs.push(data); - - await module.get({}, ({ writeHead: () => true })); - assert.deepEqual(logs, ["fake error"]); + assert.equal(result.headers["content-type"], "application/json;charset=utf-8"); }); test("'/bundle/:name/:version' should return the bundle size", async() => { @@ -230,7 +212,7 @@ describe("httpServer", () => { test("GET '/config' should return the config", async() => { const { data: actualConfig } = await get(new URL("/config", HTTP_URL)); - await cacache.put(kCachePath, kConfigKey, JSON.stringify({ foo: "bar" })); + await cacache.put(CACHE_PATH, kConfigKey, JSON.stringify({ foo: "bar" })); const result = await get(new URL("/config", HTTP_URL)); assert.deepEqual(result.data, { foo: "bar" }); @@ -254,7 +236,7 @@ describe("httpServer", () => { assert.equal(status, 204); - const inCache = await cacache.get(kCachePath, kConfigKey); + const inCache = await cacache.get(CACHE_PATH, kConfigKey); assert.deepEqual(JSON.parse(inCache.data.toString()), { fooz: "baz" }); await fetch(new URL("/config", HTTP_URL), { @@ -327,11 +309,22 @@ describe("httpServer", () => { const json = JSON.parse(result.data); assert.strictEqual(json.data.type, "Buffer"); }); + + test("'/search' should return the package list", async() => { + const result = await get(new URL("/search/nodesecure", HTTP_URL)); + + assert.equal(result.statusCode, 200); + assert.ok(result.data); + assert.ok(Array.isArray(result.data.result)); + assert.ok(result.data.count); + }); }); describe("httpServer without options", () => { let httpServer; let opened = false; + // We want to disable WS + process.env.NODE_ENV = "test"; before(async() => { const module = await esmock("../src/http-server/index.js", { @@ -343,7 +336,7 @@ describe("httpServer without options", () => { enableDestroy(httpServer.server); }); - after(() => { + after(async() => { httpServer.server.destroy(); }); diff --git a/views/index.html b/views/index.html index 3f11668b..c43afa67 100644 --- a/views/index.html +++ b/views/index.html @@ -9,18 +9,19 @@ +