diff --git a/Dockerfile b/Dockerfile index 7b01cd5..44ab7b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY . . FROM --platform=$TARGETPLATFORM node:lts-alpine as runner -RUN apk add --no-cache tini +RUN apk add --no-cache tini git WORKDIR /app diff --git a/README.md b/README.md index 133e8e9..48f8645 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The application requires a configuration file `./config.json` to be able to run. | informationLevel | The level of severity of the information outputted to the log. The severity can be 0 (everything), 1 (warnings and errors), 2 (only errors). Defaults to 2. You can override this per runtime by setting the `INFORMATION_LEVEL` environment variable. | | port | The port the server application will listen to. Defaults to 80. | | secureToken | The secret token used to validate the GitHub webhook post to /update. See [GitHub Developer - Securing your webhooks](https://developer.github.com/webhooks/securing/) for more information. | -| token | Personal access token used to gain access to GitHub API. The token scope is requires only access to public repositories. See [GitHub Help - Creating a personal access token for the command line](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) for more information. | +| token | Personal access token used to gain access to GitHub API (unused in git-only mode). The token scope is requires only access to public repositories. See [GitHub Help - Creating a personal access token for the command line](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) for more information. | | cleanInterval | How often the server should check if it is time to clean. Defaults to every 2 hours (Note that the cleanup job| | cleanThreshold | The amount of downloaded branches that will trigger the clean up job. Defaults to 1000 | | tmpLifetime | How many hours since last request to keep a branch when cleaning up | @@ -48,6 +48,16 @@ INFORMATION_LEVEL=0 npm start Environment variables take precedence over values in `config.json`. +### Git-only runtime cache +This server now uses a local git clone under `tmp/git-cache/repo` for all source retrieval. +The GitHub API is not used in production. + +Environment variables: +- `GIT_SYNC_INTERVAL_MS`: Interval for `git fetch` + `git checkout -B master origin/master` (default 30 minutes). +- `GIT_REF_CLEAN_INTERVAL_MS`: Interval for cleaning inactive refs in `tmp/` (default 30 minutes). + +The cleanup process keeps active refs based on the `info.json` timestamp in each `tmp/{ref}` directory. + ## Run the application Open a CLI and run the command: `npm start` diff --git a/app/dashboards.js b/app/dashboards.js index e04940d..367203d 100644 --- a/app/dashboards.js +++ b/app/dashboards.js @@ -4,7 +4,7 @@ const { join } = require('node:path') const { stat, opendir, readFile, writeFile, symlink } = require('node:fs/promises') const { existsSync } = require('node:fs') const { downloadSourceFolder, getBranchInfo, getCommitInfo } = require('./download') -const { compileTypeScript, log } = require('./utilities') +const { compileTypeScript, log, updateBranchAccess } = require('./utilities') const { build } = require('@highcharts/highcharts-assembler/src/build') const { JobQueue } = require('./JobQueue') @@ -41,7 +41,7 @@ async function assembleDashboards (pathCacheDirectory, commit) { try { build({ // TODO: Remove trailing slash when assembler has fixed path concatenation - base: jsMastersDirectory + '/', + base: `${jsMastersDirectory}/`, output: pathOutputFolder, pretty: false, version: commit, @@ -70,7 +70,7 @@ async function assembleDashboards (pathCacheDirectory, commit) { await modifyFiles(join(dirPath, dirent.name)) } else if (dirent.name.endsWith('.src.js')) { const contents = await readFile(join(dirPath, dirent.name), 'utf-8') - const toReplace = 'code.highcharts.com.*' + commit + '\/' // eslint-disable-line + const toReplace = `code.highcharts.com.*${commit}\/` // eslint-disable-line if (contents) { await writeFile( join(dirPath, dirent.name), @@ -142,7 +142,10 @@ async function dashboardsHandler (req, res, next) { } const pathCacheDirectory = join(PATH_TMP_DIRECTORY, commit) - const downloadURL = 'https://raw.githubusercontent.com/highcharts/highcharts/' + + await updateBranchAccess(pathCacheDirectory).catch(error => { + log(1, `Failed to update access for ${commit}: ${error.message}`) + }) try { await stat(join(pathCacheDirectory, 'js')) @@ -155,7 +158,7 @@ async function dashboardsHandler (req, res, next) { func: downloadSourceFolder, args: [ pathCacheDirectory, - downloadURL, + null, commit ] } @@ -199,7 +202,7 @@ async function dashboardsHandler (req, res, next) { async () => { await Promise.resolve().then(() => queue.addJob( 'compile', - commit + filepath, + `${commit}/${filepath}`, { func: compileTypeScript, args: [ @@ -210,19 +213,19 @@ async function dashboardsHandler (req, res, next) { ] } )).catch(error => { - log(2, `Failed to enqueue TypeScript compile for ${commit}${filepath}: ${error.message}`) + log(2, `Failed to enqueue TypeScript compile for ${commit}/${filepath}: ${error.message}`) return handleQueueError(error) }) await Promise.resolve().then(() => queue.addJob( 'compile', - commit + filepath, + `${commit}/${filepath}`, { func: assembleDashboards, args: [pathCacheDirectory, commit] } )).catch(error => { - log(2, `Failed to enqueue dashboard assembly for ${commit}${filepath}: ${error.message}`) + log(2, `Failed to enqueue dashboard assembly for ${commit}/${filepath}: ${error.message}`) return handleQueueError(error) }) diff --git a/app/download.js b/app/download.js index df65f90..ea59688 100644 --- a/app/download.js +++ b/app/download.js @@ -3,25 +3,47 @@ * @author Jon Arild Nygard * @todo Add license */ -'use strict' - // Import dependencies, sorted by path. -const { token, repo } = require('../config.json') -const { writeFile } = require('./filesystem.js') +const { repo } = require('../config.json') +const { createDirectory, exists, removeDirectory, writeFile } = require('./filesystem.js') const { log } = require('./utilities.js') -const { get: httpsGet } = require('https') -const { join } = require('path') -const authToken = token ? { Authorization: `token ${token}` } : {} - -const degit = require('tiged') +const { execFile } = require('node:child_process') +const { unlink } = require('node:fs/promises') +const { join } = require('node:path') +const { promisify } = require('node:util') const DEFAULT_CACHE_TTL = Number(process.env.GITHUB_LOOKUP_CACHE_TTL || 60_000) const NEGATIVE_CACHE_TTL = Number(process.env.GITHUB_LOOKUP_NEGATIVE_CACHE_TTL || 10_000) +const DEFAULT_BRANCH = process.env.DEFAULT_BRANCH || 'master' +const GIT_AUTH_TOKEN = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || process.env.GIT_TOKEN const branchInfoCache = new Map() const commitInfoCache = new Map() +let defaultBranchCache let githubRequest +const execFileAsync = promisify(execFile) +const PATH_TMP_DIRECTORY = join(__dirname, '../tmp') +const GIT_CACHE_DIR = join(PATH_TMP_DIRECTORY, 'git-cache') +const GIT_REPO_DIR = join(GIT_CACHE_DIR, 'repo') +const REQUIRED_GIT_PATHS = ['css', 'js', 'ts', 'tools/webpacks', 'tools/libs'] +const GIT_MAX_BUFFER = 50 * 1024 * 1024 + +function getRepoUrl () { + if (!GIT_AUTH_TOKEN) { + return `https://github.com/${repo}.git` + } + + const token = encodeURIComponent(GIT_AUTH_TOKEN) + return `https://x-access-token:${token}@github.com/${repo}.git` +} + +const disabledGitHubRequest = () => Promise.reject( + new Error('GitHub API requests are disabled') +) + +githubRequest = disabledGitHubRequest + /** * Global rate limit state tracking. * When rate limit is exhausted, we store the reset time to avoid @@ -32,20 +54,6 @@ const rateLimitState = { reset: undefined } -/** - * Update the global rate limit state from response headers. - * @param {number|undefined} remaining - * @param {number|undefined} reset - */ -function updateRateLimitState (remaining, reset) { - if (remaining !== undefined) { - rateLimitState.remaining = remaining - } - if (reset !== undefined) { - rateLimitState.reset = reset - } -} - /** * Check if we are currently rate limited. * Returns an object with `limited` boolean and `retryAfter` seconds if limited. @@ -84,48 +92,366 @@ function getRateLimitState () { } } -/** - * Extracts rate limit information from a headers object. - * @param {import('http').IncomingHttpHeaders|undefined} headers - */ -function getRateLimitInfo (headers) { - if (!headers) { - return { remaining: undefined, reset: undefined } +async function execGit (args, options = {}) { + const command = `git ${Array.isArray(args) ? args.join(' ') : String(args)}` + + try { + const { stdout, stderr } = await execFileAsync('git', args, { + maxBuffer: GIT_MAX_BUFFER, + ...options + }) + + if (stderr?.trim()) { + log(1, `git stderr for "${command}": ${stderr.trim()}`) + } + + return stdout.trim() + } catch (error) { + const stderr = error?.stderr ? String(error.stderr).trim() : '' + const messageParts = [`git command failed: ${command}`] + + if (options?.cwd) { + messageParts.push(`cwd: ${options.cwd}`) + } + + if (stderr) { + messageParts.push(`stderr: ${stderr}`) + } + + const message = messageParts.join(' | ') + + if (error instanceof Error) { + const originalMessage = error.message ? ` | original error: ${error.message}` : '' + error.message = `${message}${originalMessage}` + error.command = command + if (options?.cwd) { + error.cwd = options.cwd + } + throw error + } + + const wrappedError = new Error(message) + wrappedError.cause = error + throw wrappedError } +} - const remainingHeader = headers['x-ratelimit-remaining'] ?? headers['X-RateLimit-Remaining'] - const resetHeader = headers['x-ratelimit-reset'] ?? headers['X-RateLimit-Reset'] +async function ensureGitRepo () { + const gitDir = join(GIT_REPO_DIR, '.git') + if (exists(gitDir)) { + return + } - const remaining = Number.parseInt(remainingHeader, 10) - const reset = Number.parseInt(resetHeader, 10) + if (exists(GIT_REPO_DIR)) { + await removeDirectory(GIT_REPO_DIR) + } - return { - remaining: Number.isNaN(remaining) ? undefined : remaining, - reset: Number.isNaN(reset) ? undefined : reset + await createDirectory(GIT_CACHE_DIR) + + // Clone with filter to avoid downloading blobs upfront (partial clone) + await execGit([ + 'clone', + '--no-checkout', + '--filter=blob:none', + getRepoUrl(), + GIT_REPO_DIR + ], { + cwd: GIT_CACHE_DIR + }) + + // Enable sparse checkout for the paths we need + await execGit(['sparse-checkout', 'init', '--cone'], { + cwd: GIT_REPO_DIR + }) + + await execGit(['sparse-checkout', 'set', ...REQUIRED_GIT_PATHS], { + cwd: GIT_REPO_DIR + }) +} + +async function getDefaultBranchName () { + if (defaultBranchCache) { + return defaultBranchCache + } + + await ensureGitRepo() + + const headRef = await execGit(['symbolic-ref', 'refs/remotes/origin/HEAD'], { + cwd: GIT_REPO_DIR + }).catch(() => '') + + const match = headRef.match(/refs\/remotes\/origin\/(.+)$/) + let branch = match ? match[1] : '' + + if (!branch) { + const candidates = [DEFAULT_BRANCH, 'main', 'master'] + .filter((candidate, index, list) => list.indexOf(candidate) === index) + + for (const candidate of candidates) { + const exists = await execGit( + ['show-ref', '--verify', `refs/remotes/origin/${candidate}`], + { cwd: GIT_REPO_DIR } + ).then(() => true).catch(() => false) + + if (exists) { + branch = candidate + break + } + } } + + defaultBranchCache = branch || DEFAULT_BRANCH + return defaultBranchCache } -/** - * Logs a warning when rate limit remaining is depleted. - * Also updates the global rate limit state. - * @param {import('http').IncomingHttpHeaders|undefined} headers - * @param {string} context - * @returns {{ remaining: number|undefined, reset: number|undefined }} - */ -function logRateLimitIfDepleted (headers, context) { - const { remaining, reset } = getRateLimitInfo(headers) +async function syncMaster () { + await ensureGitRepo() + await execGit(['fetch', 'origin', '--prune', '--tags'], { + cwd: GIT_REPO_DIR + }) + const defaultBranch = await getDefaultBranchName() + await execGit(['checkout', '-B', defaultBranch, `origin/${defaultBranch}`], { + cwd: GIT_REPO_DIR + }) +} + +function isCommitHash (ref) { + return /^[0-9a-f]{7,40}$/i.test(ref) +} - // Update global state - updateRateLimitState(remaining, reset) +function getUnqualifiedRef (ref) { + return ref + .replace(/^origin\//, '') + .replace(/^refs\/heads\//, '') + .replace(/^refs\/remotes\/origin\//, '') + .replace(/^refs\/tags\//, '') +} - if (remaining === 0) { - const resetTime = reset - ? new Date(reset * 1000).toISOString() - : 'unknown' - log(2, `GitHub API rate limit exhausted while ${context}. Next reset: ${resetTime}`) +function getRemoteRefCandidates (ref) { + if (ref.startsWith('refs/')) { + return [ref] } - return { remaining, reset } + const name = getUnqualifiedRef(ref) + + if (ref.startsWith('origin/')) { + return [`refs/remotes/origin/${name}`] + } + + return [ + `refs/remotes/origin/${name}`, + `refs/tags/${name}` + ] +} + +async function remoteRefExists (ref) { + const candidates = getRemoteRefCandidates(ref) + + for (const candidate of candidates) { + const exists = await execGit(['show-ref', '--verify', candidate], { + cwd: GIT_REPO_DIR + }).then(() => true).catch(() => false) + + if (exists) { + return true + } + } + + return false +} + +async function fetchRef (ref) { + await ensureGitRepo() + + if (!ref) { + return false + } + + const unqualifiedRef = getUnqualifiedRef(ref) + const isTagRef = ref.startsWith('refs/tags/') + const isRemoteRef = ref.startsWith('origin/') || ref.startsWith('refs/remotes/') + + if (!isCommitHash(ref) && !isTagRef && !isRemoteRef) { + const refspec = `+refs/heads/${unqualifiedRef}:refs/remotes/origin/${unqualifiedRef}` + const fetched = await execGit(['fetch', 'origin', '--prune', '--tags', refspec], { + cwd: GIT_REPO_DIR + }).then(() => true).catch(() => false) + + if (fetched && await remoteRefExists(unqualifiedRef)) { + return true + } + } + + if (!isCommitHash(ref)) { + const tagRefspec = `+refs/tags/${unqualifiedRef}:refs/tags/${unqualifiedRef}` + const tagFetched = await execGit(['fetch', 'origin', '--prune', tagRefspec], { + cwd: GIT_REPO_DIR + }).then(() => true).catch(() => false) + + if (tagFetched && await remoteRefExists(`refs/tags/${unqualifiedRef}`)) { + return true + } + } + + const fetched = await execGit(['fetch', 'origin', '--prune', '--tags', ref], { + cwd: GIT_REPO_DIR + }).then(() => true).catch(() => false) + + if (!fetched) { + return false + } + + if (isCommitHash(ref)) { + return true + } + + return remoteRefExists(ref) +} + +async function resolveGitRef (ref) { + if (!ref) { + return null + } + + await ensureGitRepo() + + const candidates = new Set([ref]) + if (ref === 'master' || ref === 'main') { + const defaultBranch = await getDefaultBranchName() + if (defaultBranch) { + candidates.add(defaultBranch) + } + if (ref === 'master') { + candidates.add('main') + } + if (ref === 'main') { + candidates.add('master') + } + } + + const resolveCandidates = [] + const resolveCandidateSet = new Set() + const addResolveCandidate = (candidate) => { + if (!resolveCandidateSet.has(candidate)) { + resolveCandidateSet.add(candidate) + resolveCandidates.push(candidate) + } + } + + for (const candidate of candidates) { + addResolveCandidate(candidate) + + if (candidate.startsWith('origin/')) { + addResolveCandidate(`refs/remotes/${candidate}`) + continue + } + + if (!isCommitHash(candidate)) { + addResolveCandidate(`origin/${candidate}`) + addResolveCandidate(`refs/remotes/origin/${candidate}`) + addResolveCandidate(`refs/tags/${candidate}`) + } + } + + for (const candidate of resolveCandidates) { + try { + return await execGit(['rev-parse', '--verify', `${candidate}^{commit}`], { + cwd: GIT_REPO_DIR + }) + } catch (error) { + } + } + + if (await fetchRef(ref)) { + for (const candidate of resolveCandidates) { + try { + return await execGit(['rev-parse', '--verify', `${candidate}^{commit}`], { + cwd: GIT_REPO_DIR + }) + } catch (error) { + } + } + } + + log(1, `Failed resolving ref ${ref}`) + return null +} + +async function pathExistsInRepo (ref, path) { + const resolvedRef = await resolveGitRef(ref) + if (!resolvedRef) { + return false + } + + try { + await execGit(['cat-file', '-e', `${resolvedRef}:${path}`], { + cwd: GIT_REPO_DIR + }) + return true + } catch (error) { + return false + } +} + +async function exportGitArchive (ref, outputDir) { + const resolvedRef = await resolveGitRef(ref) + if (!resolvedRef) { + throw new Error(`Unable to resolve git ref ${ref}`) + } + + const existingPaths = [] + for (const path of REQUIRED_GIT_PATHS) { + if (await pathExistsInRepo(resolvedRef, path)) { + existingPaths.push(path) + } + } + + if (!existingPaths.length) { + throw new Error(`No exportable paths found for ${resolvedRef}`) + } + + await createDirectory(outputDir) + + const archivePath = join(GIT_CACHE_DIR, `${resolvedRef}-${Date.now()}.tar`) + await execGit([ + 'archive', + '--format=tar', + '--output', + archivePath, + resolvedRef, + ...existingPaths + ], { + cwd: GIT_REPO_DIR + }) + + await execFileAsync('tar', ['-xf', archivePath, '-C', outputDir]) + await unlink(archivePath).catch(error => { + log(2, `Failed to remove git archive ${archivePath}: ${error.message}`) + }) + + return { + ref: resolvedRef, + paths: existingPaths + } +} + +async function exportGitFile (ref, filePath, outputPath) { + const resolvedRef = await resolveGitRef(ref) + if (!resolvedRef) { + return { statusCode: 404, success: false } + } + + const existsInRepo = await pathExistsInRepo(resolvedRef, filePath) + if (!existsInRepo) { + return { statusCode: 404, success: false } + } + + const contents = await execGit(['show', `${resolvedRef}:${filePath}`], { + cwd: GIT_REPO_DIR + }) + await writeFile(outputPath, contents) + + return { statusCode: 200, success: true } } /** @@ -166,6 +492,23 @@ function getWithCache (cache, key, fetcher) { return promise } +function parseGitRawUrl (url) { + try { + const parsed = new URL(url) + const parts = parsed.pathname.split('/').filter(Boolean) + if (parts.length < 4) { + return null + } + + const ref = parts[2] + const filePath = parts.slice(3).join('/') + + return { ref, filePath } + } catch (error) { + return null + } +} + /** * Downloads the content of a url and writes it to the given output path. * The Promise is resolved with an object containing information of the result @@ -175,28 +518,25 @@ function getWithCache (cache, key, fetcher) { * @param {string} outputPath The path to output the content at the URL. */ async function downloadFile (url, outputPath) { - const { body, statusCode, headers } = await get(url) - const result = { - outputPath, - statusCode, - success: false, - url + if (!url || !outputPath) { + throw new Error('Invalid download request') } - if (statusCode === 200) { - log(0, `Downloading ${url}`) - await writeFile(outputPath, body) - result.success = true - } - const { remaining, reset } = logRateLimitIfDepleted(headers, `downloading ${url}`) - if (typeof remaining === 'number') { - result.rateLimitRemaining = remaining - } - if (typeof reset === 'number') { - result.rateLimitReset = reset + const parsed = parseGitRawUrl(url) + if (!parsed) { + throw new Error('Invalid download request') } - return result + const { ref, filePath } = parsed + log(0, `Downloading ${filePath} from ${ref} using git`) + + const result = await exportGitFile(ref, filePath, outputPath) + return { + outputPath, + statusCode: result.statusCode, + success: result.success, + url + } } /** @@ -239,80 +579,29 @@ async function downloadFiles (baseURL, subpaths, outputDir) { * @param {string} branch The name of the branch the files are located in. */ async function downloadSourceFolder (outputDir, repositoryURL, branch) { - log(0, `Downloading source for commit ${branch} using GH api`) - const url = `${repositoryURL}${branch}` - const files = await getDownloadFiles(branch) - const responses = await downloadFiles(url, files, outputDir) - const errors = responses - .filter(({ statusCode }) => statusCode !== 200) - .map(({ url, statusCode }) => `${statusCode}: ${url}`) + log(0, `Downloading source for commit ${branch} using git`) + + const hasSources = exists(join(outputDir, 'ts')) || exists(join(outputDir, 'js')) + if (hasSources) { + return { statusCode: 200, success: true } + } - // Log possible errors - if (errors.length) { - log(2, `Some files did not download in branch "${branch}"\n${errors.join('\n') - }`) + const result = await exportGitArchive(branch, outputDir) + return { + statusCode: 200, + success: true, + ref: result.ref } } /** - * Download the source folder using git (via https://github.com/tiged/tiged) + * Download the source folder using git. * @param {string} outputDir * @param {string} branch * @returns Promise<[{}]> */ -async function downloadSourceFolderGit (outputDir, branch, mode = 'tar') { - log(0, `Downloading source for commit ${branch} using git`) - - const responses = [] - const promises = ['css', 'js', 'ts'].map(folder => { - const outputPath = join(outputDir, folder) - const uri = `${repo}/${folder}#${branch}` - return new Promise((resolve, reject) => { - const result = { - success: false, - statusCode: 400, - url: uri - } - try { - const emitter = degit(uri, { - cache: false, - force: true, - verbose: false, - mode - }) - emitter.clone(outputPath).then(() => { - result.success = true - result.statusCode = 200 - }).catch((error) => { - // Error here is mostly degit not finding the branch - log(0, error.message) - }).finally(() => { - return resolve(result) - }) - } catch (error) { - log(0, error) - return resolve(result) - } - }) - }) - - /* eslint-disable */ - for await (const promise of promises) { - responses.push(promise) - } - /* eslint-disable */ - - const errors = responses - .filter(({ statusCode }) => statusCode !== 200) - .map(({ url, statusCode }) => `${statusCode}: ${url}`) - - // Log possible errors - if (errors.length) { - log(2, `Some files did not download in branch "${branch}"\n${errors.join('\n') - }`) - } - - return responses +async function downloadSourceFolderGit (outputDir, branch) { + return downloadSourceFolder(outputDir, null, branch) } /** @@ -323,27 +612,10 @@ async function downloadSourceFolderGit (outputDir, branch, mode = 'tar') { * @param {object|string} options Can either be an https request options object, * or an url string. */ -function get(options) { - return new Promise((resolve, reject) => { - const request = httpsGet(options, response => { - const body = [] - response.setEncoding('utf8') - response.on('data', (data) => { body.push(data) }) - response.on('end', () => - resolve({ - statusCode: response.statusCode, - body: body.join(''), - headers: response.headers - }) - ) - }) - request.on('error', reject) - request.end() - }) +function get (options) { + return githubRequest(options) } -githubRequest = get - /** * Gives a list of all the source files in the given branch in the repository. * The Promise resolves with a list of objects containing information on each @@ -351,23 +623,17 @@ githubRequest = get * * @param {string} branch The name of the branch the files are located in. */ -async function getDownloadFiles(branch) { - const promises = [ - 'css', - 'ts', - 'js', - 'tools/webpacks', - 'tools/libs' - ].map(folder => getFilesInFolder(folder, branch)) +async function getDownloadFiles (branch) { + const promises = REQUIRED_GIT_PATHS.map(folder => getFilesInFolder(folder, branch)) - const folders = await Promise.all(promises) - const files = [].concat.apply([], folders) + const folders = await Promise.all(promises) + const files = [].concat.apply([], folders) - const extensions = ['ts', 'js', 'css', 'scss', 'json', 'mjs'] + const extensions = ['ts', 'js', 'css', 'scss', 'json', 'mjs'] - const isValidFile = ({ path, size }) => - (extensions.some(ext => path.endsWith('.' + ext))) && size > 0 - return files.filter(isValidFile).map(({ path }) => path) + const isValidFile = ({ path, size }) => + (extensions.some(ext => path.endsWith(`.${ext}`))) && size > 0 + return files.filter(isValidFile).map(({ path }) => path) } /** @@ -379,39 +645,30 @@ async function getDownloadFiles(branch) { * @param {string} path The path to the directory. * @param {string} branch The name of the branch the files are located in. */ -async function getFilesInFolder(path, branch) { - const { body, statusCode, headers } = await get({ - hostname: 'api.github.com', - path: `/repos/${repo}/contents/${path}?ref=${branch}`, - headers: { - 'user-agent': 'github.highcharts.com', - ...authToken - } - }) - logRateLimitIfDepleted(headers, `listing files in ${path} for ${branch}`) +async function getFilesInFolder (path, branch) { + const resolvedRef = await resolveGitRef(branch) + if (!resolvedRef) { + return [] + } - if (statusCode !== 200) { - console.warn(`Could not get files in folder ${path}. This is only an issue if the requested path exists in the branch ${branch}. (HTTP ${statusCode})`) - } + const output = await execGit(['ls-tree', '-r', '-l', resolvedRef, path], { + cwd: GIT_REPO_DIR + }).catch(() => '') - let promises = [] - if (statusCode === 200) { - promises = JSON.parse(body).map(obj => { - const name = path + '/' + obj.name - return ( - (obj.type === 'dir') - ? getFilesInFolder(name, branch) - : [{ - download: obj.download_url, - path: name, - size: obj.size, - type: obj.type - }] - ) - }) + if (!output) { + return [] + } + + return output.split('\n').filter(Boolean).map(line => { + const [meta, filePath] = line.split('\t') + const parts = meta.split(' ') + const size = Number(parts[3]) + return { + path: filePath, + size: Number.isFinite(size) ? size : 0, + type: parts[1] } - const arr = await Promise.all(promises) - return arr.reduce((arr1, arr2) => arr1.concat(arr2), []) + }) } /** @@ -421,17 +678,17 @@ async function getFilesInFolder(path, branch) { * * @param {string} url The URL to check if exists. */ -async function urlExists(url) { - try { - const response = await get(url) - return response.statusCode === 200 - } catch (e) { - return false - } +async function urlExists (url) { + const parsed = parseGitRawUrl(url) + if (!parsed) { + return false + } + + return pathExistsInRepo(parsed.ref, parsed.filePath) } /** - * Gets branch info from the github api. + * Gets branch info from git. * @param {string} branch * The branch name * @@ -440,25 +697,16 @@ async function urlExists(url) { */ async function getBranchInfo (branch) { return getWithCache(branchInfoCache, branch, async () => { - const { body, statusCode, headers } = await githubRequest({ - hostname: 'api.github.com', - path: `/repos/${repo}/branches/${branch}`, - headers: { - 'user-agent': 'github.highcharts.com', - ...authToken - } - }) - logRateLimitIfDepleted(headers, `fetching branch info for ${branch}`) - if (statusCode === 200) { - return JSON.parse(body) + const sha = await resolveGitRef(branch) + if (!sha) { + return false } - return false + return { commit: { sha } } }) } - /** - * Gets commit info from the github api. + * Gets commit info from git. * @param {string} commit * The commit sha, long or short * @@ -467,29 +715,22 @@ async function getBranchInfo (branch) { */ async function getCommitInfo (commit) { return getWithCache(commitInfoCache, commit, async () => { - const { body, statusCode, headers } = await githubRequest({ - hostname: 'api.github.com', - path: `/repos/${repo}/commits/${commit}`, - headers: { - 'user-agent': 'github.highcharts.com', - ...authToken - } - }) - logRateLimitIfDepleted(headers, `fetching commit info for ${commit}`) - if (statusCode === 200) { - return JSON.parse(body) + const sha = await resolveGitRef(commit) + if (!sha) { + return false } - return false + return { sha } }) } function setGitHubRequest (fn) { - githubRequest = typeof fn === 'function' ? fn : get + githubRequest = typeof fn === 'function' ? fn : disabledGitHubRequest } function clearGitHubCache () { branchInfoCache.clear() commitInfoCache.clear() + defaultBranchCache = undefined } /** @@ -512,19 +753,23 @@ function setRateLimitState (remaining, reset) { // Export download functions module.exports = { - downloadFile, - downloadFiles, - downloadSourceFolder, - downloadSourceFolderGit, - getDownloadFiles, - httpsGetPromise: get, - urlExists, - getBranchInfo, - getCommitInfo, - isRateLimited, - getRateLimitState, - __setGitHubRequest: setGitHubRequest, - __clearGitHubCache: clearGitHubCache, - __clearRateLimitState: clearRateLimitState, - __setRateLimitState: setRateLimitState + downloadFile, + downloadFiles, + downloadSourceFolder, + downloadSourceFolderGit, + ensureGitRepo, + syncMaster, + resolveGitRef, + pathExistsInRepo, + getDownloadFiles, + httpsGetPromise: get, + urlExists, + getBranchInfo, + getCommitInfo, + isRateLimited, + getRateLimitState, + __setGitHubRequest: setGitHubRequest, + __clearGitHubCache: clearGitHubCache, + __clearRateLimitState: clearRateLimitState, + __setRateLimitState: setRateLimitState } diff --git a/app/filesystem.js b/app/filesystem.js index 4cd89da..b6cc13d 100644 --- a/app/filesystem.js +++ b/app/filesystem.js @@ -15,7 +15,8 @@ const { rmdir, stat, unlink, - writeFile + writeFile, + readFile } } = require('fs') const { dirname, join, normalize, sep } = require('path') @@ -141,7 +142,8 @@ const { cleanThreshold, tmpLifetime } = require('../config.json') */ async function shouldClean () { const files = await readdir(join(__dirname, '../tmp/')).catch(() => []) - if (files.length > (cleanThreshold || 100)) return true + const filteredFiles = files.filter(file => file !== 'git-cache') + if (filteredFiles.length > (cleanThreshold || 100)) return true return false } @@ -156,6 +158,10 @@ async function cleanUp (force = false) { const cleaned = [] for (const file of files) { + if (file === 'git-cache') { + continue + } + const folderpath = join(path, file) const fileInfo = await fsStat(folderpath) @@ -165,7 +171,24 @@ async function cleanUp (force = false) { if (fileInfo && fileInfo.isDirectory()) { try { - const accessTime = fileInfo.atime || fileInfo.mtime || fileInfo.ctime + let accessTime = fileInfo.atime || fileInfo.mtime || fileInfo.ctime + const infoPath = join(folderpath, 'info.json') + const infoStat = await fsStat(infoPath) + + if (infoStat && infoStat.isFile()) { + const contents = await readFile(infoPath, 'utf-8').catch(() => null) + if (contents) { + try { + const parsed = JSON.parse(contents) + if (parsed?.last_access) { + accessTime = new Date(parsed.last_access) + } + } catch (parseError) { + log(1, `Failed to parse ${infoPath}: ${parseError.message}`) + } + } + } + const diff = new Date() - accessTime if ((diff > timeToKill) || force) { diff --git a/app/handlers.js b/app/handlers.js index c819920..2e2215a 100644 --- a/app/handlers.js +++ b/app/handlers.js @@ -8,9 +8,15 @@ 'use strict' // Import dependencies, sorted by path name. -const { secureToken, repo } = require('../config.json') -const { downloadFile, downloadSourceFolder, urlExists, getBranchInfo, getCommitInfo, isRateLimited } = require('./download.js') -const { log } = require('./utilities') +const { secureToken } = require('../config.json') +const { + downloadSourceFolder, + getBranchInfo, + getCommitInfo, + isRateLimited, + pathExistsInRepo +} = require('./download.js') +const { log, updateBranchAccess } = require('./utilities') const { exists, @@ -43,7 +49,6 @@ const { compileWithEsbuild } = require('./esbuild.js') // Constants const PATH_TMP_DIRECTORY = join(__dirname, '../tmp') -const URL_DOWNLOAD = `https://raw.githubusercontent.com/${repo}/` const queue = new JobQueue() const RATE_LIMIT_HEADER_REMAINING = 'X-GitHub-RateLimit-Remaining' @@ -99,19 +104,16 @@ function applyRateLimitMeta (target, meta) { /** * Tries to look for a remote tsconfig file if the branch/ref is of newer date typically 2019+. * If one exists it will return true - * @param {String} repoURL to repository on GitHub * @param {String} branch or ref for commit * @return {Promise} true if a tsconfig.json file exists for the branch/ref */ -async function shouldDownloadTypeScriptFolders (repoURL, branch) { - const urlPath = `${repoURL}${branch}/ts/tsconfig.json` +async function shouldDownloadTypeScriptFolders (branch) { const tsConfigPath = join(PATH_TMP_DIRECTORY, branch, 'ts', 'tsconfig.json') if (exists(tsConfigPath)) { return true } - const tsConfigResponse = await downloadFile(urlPath, tsConfigPath) - return (tsConfigResponse.statusCode >= 200 && tsConfigResponse.statusCode < 300) + return pathExistsInRepo(branch, 'ts/tsconfig.json') } /** @@ -150,32 +152,24 @@ async function handlerDefault (req, res) { let branch = await getBranch(req.path) let url = req.url - let useGitDownloader = branch === 'master' || /^\/v[0-9]/.test(req.path) // version tags // If we can get it, save by commit sha - // This also means we can use the degit downloader - // (only works on latest commit in a branch) // Only `v8.0.0` gets saved by their proper names - const { commit } = await getBranchInfo(branch) - if (commit) { - branch = commit.sha - useGitDownloader = true - } - - // If this is still not true, the request may be for a short SHA - // Get the long form sha - // Todo: find a way to check if it is the latest commit in the branch - if (!useGitDownloader) { - const { sha } = await getCommitInfo(branch) - if (sha) { - url = url.replace(branch, sha) - branch = sha + const branchInfo = await getBranchInfo(branch) + if (branchInfo?.commit?.sha) { + branch = branchInfo.commit.sha + } else { + // If the request is for a short SHA, resolve to full SHA + const commitInfo = await getCommitInfo(branch) + if (commitInfo?.sha) { + url = url.replace(branch, commitInfo.sha) + branch = commitInfo.sha } } // Handle esbuild mode if (useEsbuild) { - const result = await serveEsbuildFile(branch, url, useGitDownloader) + const result = await serveEsbuildFile(branch, url) if (!res.headersSent) { res.header('ETag', branch) res.header('X-Built-With', 'esbuild') @@ -190,11 +184,13 @@ async function handlerDefault (req, res) { if (result?.status !== 200) { // Try to build the file - const buildResult = await serveBuildFile(branch, url, useGitDownloader) + const buildResult = await serveBuildFile(branch, url) result = applyRateLimitMeta(buildResult, rateLimitMeta) } - // await updateBranchAccess(join(PATH_TMP_DIRECTORY, branch)) + await updateBranchAccess(join(PATH_TMP_DIRECTORY, branch)).catch(error => { + log(1, `Failed to update access for ${branch}: ${error.message}`) + }) if (!result) { result = applyRateLimitMeta({ @@ -358,7 +354,7 @@ async function respondToClient (result, response, request) { * @param {string} branch * @param {string} requestURL The url which the request was sent to. */ -async function serveBuildFile (branch, requestURL, useGitDownloader = true) { +async function serveBuildFile (branch, requestURL) { const type = getType(branch, requestURL) const file = getFile(branch, type, requestURL) @@ -394,7 +390,7 @@ async function serveBuildFile (branch, requestURL, useGitDownloader = true) { { func: downloadSourceFolder, args: [ - pathCacheDirectory, URL_DOWNLOAD, branch + pathCacheDirectory, null, branch ] } ).catch(error => { @@ -535,9 +531,9 @@ ${error.message}`) const masterFiles = await getFileNamesInDirectory(join(pathCacheDirectory, 'ts', 'masters'), true) || [] return masterFiles.some(product => file.replace(/^masters\//, '').replace('.js', '.ts').startsWith(product)) } - // Otherwise check github - const remoteURL = URL_DOWNLOAD + branch + '/ts/masters/' + file.replace('.js', '.ts') - return urlExists(remoteURL) + // Otherwise check git + const repoPath = `ts/masters/${file.replace('.js', '.ts')}` + return pathExistsInRepo(branch, repoPath) } /** @@ -573,56 +569,51 @@ ${error.message}`) */ async function serveStaticFile (branch, requestURL) { const file = getFile(branch, 'classic', requestURL) - let rateLimitMeta = null // Respond with not found if the interpreter can not find a filename. if (file === false) { return response.missingFile } + const pathCacheDirectory = join(PATH_TMP_DIRECTORY, branch) + if (file.endsWith('.css')) { - // TODO: add fs check before download - const fileLocation = join(PATH_TMP_DIRECTORY, branch, file) + const fileLocation = join(pathCacheDirectory, file) if (!existsSync(fileLocation)) { - const urlFile = `${URL_DOWNLOAD}${branch}/${file}` - const download = await downloadFile(urlFile, fileLocation) - const downloadMeta = extractRateLimitMeta(download) - if (downloadMeta) { - rateLimitMeta = downloadMeta - } - if (download.success) { - return applyRateLimitMeta({ status: 200, file: fileLocation }, rateLimitMeta) - } - } else { - return applyRateLimitMeta({ status: 200, file: fileLocation }, rateLimitMeta) + await downloadSourceFolder(pathCacheDirectory, null, branch) + } + + if (existsSync(fileLocation)) { + return { status: 200, file: fileLocation } } } - const pathFile = join(PATH_TMP_DIRECTORY, branch, 'output', file) + const outputFile = join(pathCacheDirectory, 'output', file) + const jsFile = join(pathCacheDirectory, 'js', file) - // Download the file if it is not already available in cache. - if (!exists(pathFile)) { - const urlFile = `${URL_DOWNLOAD}${branch}/js/${file}` - const download = await downloadFile(urlFile, pathFile) - const downloadMeta = extractRateLimitMeta(download) - if (downloadMeta) { - rateLimitMeta = downloadMeta - } - if (download.statusCode !== 200) { - // we don't always know if it is a static file before we have tried to download it. - // check if this branch contains TypeScript config (we then need to compile it). - if (file.split('/').length <= 1 || await shouldDownloadTypeScriptFolders(URL_DOWNLOAD, branch)) { - const buildResult = await serveBuildFile(branch, requestURL) - return applyRateLimitMeta(buildResult, rateLimitMeta) - } - return applyRateLimitMeta({ ...response.missingFile }, rateLimitMeta) - } + if (exists(outputFile)) { + return { status: 200, file: outputFile } + } + + if (exists(jsFile)) { + return { status: 200, file: jsFile } + } + + await downloadSourceFolder(pathCacheDirectory, null, branch) + + if (exists(jsFile)) { + return { status: 200, file: jsFile } + } + + if (exists(outputFile)) { + return { status: 200, file: outputFile } + } - return applyRateLimitMeta({ status: 200, file: pathFile }, rateLimitMeta) + if (file.split('/').length <= 1 || await shouldDownloadTypeScriptFolders(branch)) { + return serveBuildFile(branch, requestURL) } - // Return path to file location in the cache. - return applyRateLimitMeta({ status: 200, file: pathFile }, rateLimitMeta) + return { ...response.missingFile } } /** @@ -632,9 +623,8 @@ async function serveStaticFile (branch, requestURL) { * * @param {string} branch The branch/commit SHA * @param {string} requestURL The url which the request was sent to. - * @param {boolean} useGitDownloader Whether to use the git downloader */ -async function serveEsbuildFile (branch, requestURL, useGitDownloader = true) { +async function serveEsbuildFile (branch, requestURL) { const type = getType(branch, requestURL) const { filename: file, minify } = getFileForEsbuild(branch, type, requestURL) @@ -656,7 +646,7 @@ async function serveEsbuildFile (branch, requestURL, useGitDownloader = true) { { func: downloadSourceFolder, args: [ - pathCacheDirectory, URL_DOWNLOAD, branch + pathCacheDirectory, null, branch ] } ).catch(error => { diff --git a/app/interpreter.js b/app/interpreter.js index fb3a228..4180138 100644 --- a/app/interpreter.js +++ b/app/interpreter.js @@ -4,13 +4,11 @@ * @author Jon Arild Nygard * @todo Add license */ -'use strict' - // Import dependencies, sorted by path name. const { getOrderedDependencies } = require('@highcharts/highcharts-assembler/src/dependencies.js') -const { join, sep, relative } = require('path') +const { join, sep, relative } = require('node:path') // Constants const BRANCH_TYPES = [ @@ -24,18 +22,29 @@ const BRANCH_TYPES = [ ] const PRODUCTS = ['stock', 'maps', 'gantt'] +const DEFAULT_BRANCH = process.env.DEFAULT_BRANCH || 'master' const replaceAll = (str, search, replace) => str.split(search).join(replace) const stripQuery = (url) => (typeof url === 'string' ? url.split('?')[0] : '') +const stripBranchPrefix = (url) => { + let normalizedUrl = stripQuery(url) + normalizedUrl = normalizedUrl.replace(/^\/(master|main)(?=\/|$)/, '') + normalizedUrl = normalizedUrl.replace(/^\/v[0-9]+(\.[0-9]+)*(-[a-zA-Z0-9.]+)?\//, '/') + const regex = new RegExp(`^\\/(${BRANCH_TYPES.join('|')})\\/([A-Za-z]|[0-9]|-)+\\/`) + if (regex.test(normalizedUrl)) { + normalizedUrl = normalizedUrl.replace(regex, '/') + } + return normalizedUrl +} /** * Finds which branch, tag, or commit that is requested by the client. Defaults - * to master. Returns a string with the resulting reference. + * to the configured branch name. Returns a string with the resulting reference. * * @param {string} url The request URL. */ async function getBranch (url) { - url = stripQuery(url) + const normalizedUrl = stripQuery(url) const folders = ['adapters', 'indicators', 'modules', 'parts-3d', 'parts-map', 'parts-more', 'parts', 'themes'] const isValidBranchName = (str) => ( @@ -45,13 +54,13 @@ async function getBranch (url) { !(str.endsWith('.js') || str.endsWith('.css')) // Not a file ) - let branch = 'master' - const sections = url.substring(1).split('/') + let branch = DEFAULT_BRANCH + const sections = normalizedUrl.substring(1).split('/') // We have more than one section if (sections.length > 1 && BRANCH_TYPES.includes(sections[0])) { branch = ( (sections.length > 2 && isValidBranchName(sections[1])) - ? sections[0] + '/' + sections[1] + ? `${sections[0]}/${sections[1]}` : sections[0] ) /** @@ -61,6 +70,12 @@ async function getBranch (url) { } else if (isValidBranchName(sections[0])) { branch = sections[0] } + + // Treat legacy master URLs as alias for the configured default branch. + if (branch === 'master' && DEFAULT_BRANCH !== 'master') { + branch = DEFAULT_BRANCH + } + return branch } @@ -73,14 +88,8 @@ async function getBranch (url) { * @param {string} url The request URL. */ function getFile (branch, type, url) { - url = stripQuery(url) // Replace branches in url, since we save by commit sha - url = url.replace(/^\/master/, '') - url = url.replace(/^\/v[0-9]+\//, '/') - const regex = new RegExp(`^\\/(${BRANCH_TYPES.join('|')})\\/([A-Za-z]|[0-9]|-)+\\/`) - if (regex.test(url)) { - url = url.replace(regex, '/') - } + const normalizedUrl = stripBranchPrefix(url) const sections = [ x => x === branch.split('/')[0], // Remove first section of branch name x => x === branch.split('/')[1], // Remove second section of branch name @@ -91,7 +100,7 @@ function getFile (branch, type, url) { sections.splice(0, 1) } return sections - }, url.substring(1).split('/')) + }, normalizedUrl.substring(1).split('/')) let filename = sections.join('/') @@ -118,14 +127,8 @@ function getFile (branch, type, url) { * @param {string} url The request URL. */ function getFileForEsbuild (branch, type, url) { - url = stripQuery(url) // Replace branches in url, since we save by commit sha - url = url.replace(/^\/master/, '') - url = url.replace(/^\/v[0-9]+\//, '/') - const regex = new RegExp(`^\\/(${BRANCH_TYPES.join('|')})\\/([A-Za-z]|[0-9]|-)+\\/`) - if (regex.test(url)) { - url = url.replace(regex, '/') - } + const normalizedUrl = stripBranchPrefix(url) const sections = [ x => x === branch.split('/')[0], // Remove first section of branch name x => x === branch.split('/')[1], // Remove second section of branch name @@ -136,7 +139,7 @@ function getFileForEsbuild (branch, type, url) { sections.splice(0, 1) } return sections - }, url.substring(1).split('/')) + }, normalizedUrl.substring(1).split('/')) let filename = sections.join('/') @@ -226,7 +229,7 @@ function getFileOptions (files, pathJS) { * @param {string} url The request URL. */ const getType = (branch, url) => { - url = stripQuery(url) + const normalizedUrl = stripQuery(url) const sections = [ x => x === branch.split('/')[0], // Remove first section of branch name x => x === branch.split('/')[1], // Remove second section of branch name @@ -236,7 +239,7 @@ const getType = (branch, url) => { sections.splice(0, 1) } return sections - }, url.substring(1).split('/')) + }, normalizedUrl.substring(1).split('/')) return sections[0] === 'js' ? 'css' : 'classic' } diff --git a/app/server.js b/app/server.js index cd72219..0bc44a1 100644 --- a/app/server.js +++ b/app/server.js @@ -20,6 +20,7 @@ const { JobQueue } = require('./JobQueue') const router = require('./router.js') const { formatDate, log, compileTypeScript, compileTypeScriptProject } = require('./utilities.js') const { shouldUseWebpack } = require('./utils.js') +const { syncMaster } = require('./download.js') const express = require('express') const { cleanUp, shouldClean } = require('./filesystem') @@ -30,6 +31,9 @@ const { readFileSync } = require('node:fs') const APP = express() const PORT = process.env.PORT || config.port || 80 const DATE = formatDate(new Date()) +const DEFAULT_GIT_INTERVAL_MS = 30 * 60 * 1000 +const GIT_SYNC_INTERVAL_MS = Number(process.env.GIT_SYNC_INTERVAL_MS || DEFAULT_GIT_INTERVAL_MS) +const REF_CLEAN_INTERVAL_MS = Number(process.env.GIT_REF_CLEAN_INTERVAL_MS || config.cleanInterval || DEFAULT_GIT_INTERVAL_MS) const state = { typescriptJobs: {}, @@ -187,6 +191,15 @@ APP.use(ignoreSocketErrors) */ APP.listen(PORT) +async function syncGitMaster () { + await syncMaster().catch(error => { + log(2, `Git sync failed: ${error.message}`) + }) +} + +syncGitMaster() +setInterval(syncGitMaster, GIT_SYNC_INTERVAL_MS) + // Clean up the tmp folder every now and then setInterval(async () => { if (hasActiveJobs()) { @@ -202,7 +215,7 @@ setInterval(async () => { await cleanUp().catch(error => { log(2, `Cleanup failed: ${error.message}`) }) -}, config.cleanInterval || 2 * 60 * 1000) +}, REF_CLEAN_INTERVAL_MS) // Do a cleanup when restarting the server cleanUp().catch(error => { diff --git a/app/utilities.js b/app/utilities.js index 26fca09..10157f9 100644 --- a/app/utilities.js +++ b/app/utilities.js @@ -12,8 +12,7 @@ const { promises: { readFile, stat, - writeFile, - readdir + writeFile } } = require('fs') const tscPath = require.resolve('typescript/lib/tsc.js') @@ -211,26 +210,15 @@ async function compileTypeScriptProject (branch) { async function updateBranchAccess (branchPath) { const filePath = join(branchPath, 'info.json') + const jsonString = JSON.stringify({ last_access: new Date().toISOString() }) - const jsonString = JSON.stringify({ last_access: new Date() }) - - if (await stat(branchPath)) { - // create file if not existant - const isFile = (await readdir(branchPath)).includes('info.json') - if (!isFile) { - return writeFile(filePath, jsonString) - } - - // Only update if the date has changed - const data = require(filePath) - const splitDate = (date) => date.toISOString().split('T')[0] - - if (splitDate(new Date(data.last_access)) !== splitDate(new Date())) { - return writeFile(filePath, jsonString) - } + try { + await stat(branchPath) + } catch (error) { + return Promise.resolve() } - return Promise.resolve() + return writeFile(filePath, jsonString) } async function getGlobalsLocation (filePath) { diff --git a/test/download.js b/test/download.js index 65e1fd3..f680596 100644 --- a/test/download.js +++ b/test/download.js @@ -1,8 +1,7 @@ const defaults = require('../app/download.js') const { expect } = require('chai') -const fs = require('fs') +const fs = require('node:fs') const { after, afterEach, before, describe, it } = require('mocha') -const sinon = require('sinon') describe('download.js', () => { describe('exported properties', () => { @@ -11,6 +10,10 @@ describe('download.js', () => { 'downloadFiles', 'downloadSourceFolder', 'downloadSourceFolderGit', + 'ensureGitRepo', + 'syncMaster', + 'resolveGitRef', + 'pathExistsInRepo', 'getDownloadFiles', 'httpsGetPromise', 'urlExists', @@ -24,10 +27,10 @@ describe('download.js', () => { '__setRateLimitState' ] it('should have a default export', () => { - functions.forEach((name) => { + for (const name of functions) { expect(defaults).to.have.property(name) .that.is.a('function') - }) + } }) it('should not have unexpected properties', () => { const exportedProperties = Object.keys(defaults) @@ -37,62 +40,60 @@ describe('download.js', () => { describe('httpsGetPromise', () => { const { httpsGetPromise } = defaults - const downloadURL = 'https://raw.githubusercontent.com/highcharts/highcharts/' - it('should error if no options is provided', function () { + it('should reject when GitHub API is disabled', function () { this.timeout(5000) - return httpsGetPromise() + return httpsGetPromise('https://raw.githubusercontent.com/highcharts/highcharts/') .then(() => { throw new Error('Promise resolved unexpectedly.') }) .catch(e => { - expect(e.message).to.not.equal('Promise resolved unexpectedly.') - }) - }) - it('should return a response when a url is provided', () => { - return httpsGetPromise(downloadURL) - .then(({ body, statusCode }) => { - expect(statusCode).to.equal(400) - expect(body).to.equal('400: Invalid request') + expect(e.message).to.equal('GitHub API requests are disabled') }) }) }) - describe('downloadFile', () => { + describe('downloadFile', function () { + this.timeout(60000) + const { downloadFile } = defaults const downloadURL = 'https://raw.githubusercontent.com/highcharts/highcharts/' const cleanFiles = () => { - [ + const paths = [ 'tmp/test/downloaded-file1.js', 'tmp/test/downloaded-file2.js', 'tmp/test' - ].forEach(p => { + ] + + for (const path of paths) { try { - const stat = fs.lstatSync(p) + const stat = fs.lstatSync(path) if (stat.isFile()) { - fs.unlinkSync(p) + fs.unlinkSync(path) } else if (stat.isDirectory()) { - fs.rmdirSync(p) + fs.rmSync(path, { recursive: true, force: true }) } } catch (err) {} - }) + } } after(cleanFiles) before(() => { fs.mkdirSync('tmp/test', { recursive: true }) }) it('should resolve with an informational object, and a newly created file.', () => { + const fileUrl = `${downloadURL}master/ts/masters/highcharts.src.ts` return downloadFile( - downloadURL + 'master/ts/masters/highcharts.src.ts', + fileUrl, './tmp/test/downloaded-file1.js' ).then(({ outputPath, statusCode, success, url }) => { expect(outputPath).to.equal('./tmp/test/downloaded-file1.js') expect(statusCode).to.equal(200) expect(success).to.equal(true) - expect(url).to.equal(downloadURL + 'master/ts/masters/highcharts.src.ts') + expect(url).to.equal(fileUrl) expect(fs.lstatSync('./tmp/test/downloaded-file1.js').size).to.be.greaterThan(0) }) }) it('should only create a file if response status is 200', () => { + const fileUrl = `${downloadURL}master/i-do-not-exist.js` return downloadFile( - downloadURL + 'master/i-do-not-exist.js', + fileUrl, './tmp/test/downloaded-file2.js' ).then(({ outputPath, statusCode, success, url }) => { let exists = true @@ -129,53 +130,34 @@ describe('download.js', () => { it('is missings tests') }) - describe('GitHub metadata caching', () => { + describe('Git metadata lookups', function () { + this.timeout(60000) + const { - __setGitHubRequest, - __clearGitHubCache, getBranchInfo, getCommitInfo } = defaults - afterEach(() => { - __setGitHubRequest() - __clearGitHubCache() - }) - - it('reuses the same request for parallel branch lookups', async () => { - const stub = sinon.stub().resolves({ - statusCode: 200, - body: JSON.stringify({ commit: { sha: 'abc123' } }) - }) - - __clearGitHubCache() - __setGitHubRequest(stub) - - const [first, second] = await Promise.all([ - getBranchInfo('feature/test'), - getBranchInfo('feature/test') - ]) - - expect(stub.callCount).to.equal(1) - expect(first?.commit?.sha).to.equal('abc123') - expect(second?.commit?.sha).to.equal('abc123') - }) - - it('returns cached commit info on subsequent calls', async () => { - const stub = sinon.stub().resolves({ - statusCode: 200, - body: JSON.stringify({ sha: 'deadbeef' }) - }) - - __clearGitHubCache() - __setGitHubRequest(stub) - - const first = await getCommitInfo('deadbeef') - const second = await getCommitInfo('deadbeef') - - expect(stub.callCount).to.equal(1) - expect(first?.sha).to.equal('deadbeef') - expect(second?.sha).to.equal('deadbeef') + it('resolves branch info from git', async function () { + const info = await getBranchInfo('master') + if (!info?.commit?.sha) { + this.skip() + } + expect(info).to.have.property('commit') + expect(info.commit.sha).to.be.a('string') + expect(info.commit.sha).to.match(/^[0-9a-f]{40}$/) + }) + + it('resolves commit info from git', async function () { + const branchInfo = await getBranchInfo('master') + if (!branchInfo?.commit?.sha) { + this.skip() + } + expect(branchInfo.commit.sha).to.be.a('string') + expect(branchInfo.commit.sha).to.match(/^[0-9a-f]{40}$/) + + const commitInfo = await getCommitInfo(branchInfo.commit.sha) + expect(commitInfo?.sha).to.equal(branchInfo.commit.sha) }) }) diff --git a/test/filesystem.js b/test/filesystem.js index e6ac498..743035d 100644 --- a/test/filesystem.js +++ b/test/filesystem.js @@ -1,8 +1,8 @@ const mocha = require('mocha') -const fs = require('fs') +const fs = require('node:fs') const expect = require('chai').expect const defaults = require('../app/filesystem.js') -const { join } = require('path') +const { join } = require('node:path') const describe = mocha.describe const it = mocha.it const before = mocha.before @@ -10,7 +10,7 @@ const after = mocha.after const throwErr = (err) => { if (err) throw err } const cleanFiles = () => { - [ + const paths = [ 'tmp/test-empty', 'tmp/test-files/file.txt', 'tmp/test-files/subfolder/file.txt', @@ -19,17 +19,21 @@ const cleanFiles = () => { 'tmp/fakebranchhash/info.json', 'tmp/fakebranchhash', 'tmp' - ].forEach(p => { - let stat = false + ] + + for (const path of paths) { + let stat try { - stat = fs.lstatSync(p) - } catch (err) {} - if (stat && stat.isFile()) { - fs.unlinkSync(p) - } else if (stat && stat.isDirectory()) { - fs.rmdirSync(p) + stat = fs.lstatSync(path) + } catch (err) { + continue } - }) + if (stat.isFile()) { + fs.unlinkSync(path) + } else if (stat.isDirectory()) { + fs.rmSync(path, { recursive: true, force: true }) + } + } } describe('filesystem.js', () => { @@ -41,10 +45,10 @@ describe('filesystem.js', () => { 'removeDirectory', 'writeFile' ] - functions.forEach((name) => { + for (const name of functions) { expect(defaults).to.have.property(name) .that.is.a('function') - }) + } }) describe('getFileNamesInDirectory', () => {