diff --git a/QualityControl/lib/api.js b/QualityControl/lib/api.js index 47e3c9a9c..e609a1b6b 100644 --- a/QualityControl/lib/api.js +++ b/QualityControl/lib/api.js @@ -72,7 +72,9 @@ export const setup = async (http, ws, eventEmitter) => { http.get('/layouts', layoutController.getLayoutsHandler.bind(layoutController)); http.get('/layout/:id', layoutController.getLayoutHandler.bind(layoutController)); http.get('/layout', layoutController.getLayoutByNameHandler.bind(layoutController)); + http.get('/download', layoutController.getDownloadHandler.bind(layoutController)); http.post('/layout', layoutController.postLayoutHandler.bind(layoutController)); + http.post('/download', layoutController.postDownloadHandler.bind(layoutController)); http.put( '/layout/:id', layoutServiceMiddleware(jsonFileService), diff --git a/QualityControl/lib/controllers/LayoutController.js b/QualityControl/lib/controllers/LayoutController.js index c014da497..4378d4eee 100644 --- a/QualityControl/lib/controllers/LayoutController.js +++ b/QualityControl/lib/controllers/LayoutController.js @@ -25,12 +25,16 @@ import { updateAndSendExpressResponseFromNativeError, } from '@aliceo2/web-ui'; +import { parseRequestToConfig, parseRequestToLayout } from '../utils/download/configurator.js'; +import { MapStorage } from '../utils/download/classes/domain/MapStorage.js'; +import { download, saveDownloadData } from '../utils/download/downloadEngine.js'; /** * @typedef {import('../repositories/LayoutRepository.js').LayoutRepository} LayoutRepository */ const logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? 'qcg'}/layout-ctrl`); +const mapStorage = new MapStorage(); /** * Gateway for all HTTP requests with regards to QCG Layouts @@ -219,6 +223,49 @@ export class LayoutController { } } + /** + * Store layout data for later download request. + * @param {Request}req - request + * @param {Response} res - response + */ + async postDownloadHandler(req, res) { + try { + const downloadConfigDomain = parseRequestToConfig(req); + const downloadLayoutDomain = parseRequestToLayout(req); + // Note: if userId becomes 0 it will throw when creating the storagelayout. + const userId = Number(req.query.user_id ?? 0); + const key = saveDownloadData(mapStorage, downloadLayoutDomain, downloadConfigDomain, userId); + logger.infoMessage(`Saved layout key: ${key}`); + res.status(201).send(key); + } catch { + res.status(400).send('Could not save download data'); + } + }; + + /** + * Download objects using key from post download request. + * @param {Request}req - request + * @param {Response} res - response + */ + async getDownloadHandler(req, res) { + const { key } = req.query; + if (key == '') { + res.status(400).send('Key not defined correctly'); + } + try { + const downloadLayoutDomain = mapStorage.readRequest(key)?.[0].toSuper(); + const downloadConfigDomain = mapStorage.readRequest(key)?.[1]; + if (downloadLayoutDomain == undefined || downloadConfigDomain == undefined) { + throw new Error('Layout could not be found with key'); + } + // start the download engine + await download(downloadLayoutDomain, downloadConfigDomain, 1234567, res); + } catch (error) { + logger.errorMessage(error?.message ?? error); + res.status(400).send('Could not download objects'); + } + } + /** * Patch a layout entity with information as per LayoutPatchDto.js * @param {Request} req - HTTP request object with "params" and "body" information on layout diff --git a/QualityControl/lib/utils/download/classes/DownloadConfigMapper.js b/QualityControl/lib/utils/download/classes/DownloadConfigMapper.js new file mode 100644 index 000000000..35493f355 --- /dev/null +++ b/QualityControl/lib/utils/download/classes/DownloadConfigMapper.js @@ -0,0 +1,68 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +import { DownloadConfigDomain } from '../classes/domain/DownloadConfigDomain.js'; +import { NameTemplateOption } from '../enum/NameTemplateOption.js'; +import { DownloadMode } from '../enum/DownloadMode.js'; + +/** + * map download config to domain model + * @param {string} downloadConfigData - DownloadconfigData to map + * @returns {DownloadConfigDomain} - mapped DownloadConfigDomain + */ +export function mapDownloadConfigToDomain(downloadConfigData) { + const archiveNameTemplateOptions = downloadConfigData.archiveNameTemplateOptions.map(mapNameTemplateOption); + const objectNameTemplateOptions = downloadConfigData.objectNameTemplateOptions.map(mapNameTemplateOption); + const downloadMode = mapDownloadMode(downloadConfigData.downloadMode); + // eslint-disable-next-line @stylistic/js/max-len + return new DownloadConfigDomain(downloadConfigData.tabIds, downloadConfigData.objectIds, archiveNameTemplateOptions, objectNameTemplateOptions, downloadMode, downloadConfigData.pathNameStructure); +} + +/** + * map string to name template option + * @param {string} nameTemplateOption - string representation + * @returns {number} - name template option + */ +function mapNameTemplateOption(nameTemplateOption) { + if (typeof nameTemplateOption === 'string') { + nameTemplateOption = nameTemplateOption.trim(); + const mappedNameTemplateOption = NameTemplateOption[nameTemplateOption]; + if (mappedNameTemplateOption === undefined) { + throw new Error('Failed to map NameTemplateOption, perhaps an invalid option was passed?'); + } else { + return mappedNameTemplateOption; + } + } else { + throw new Error('Failed to map NameTemplateOption, it should be a string'); + } +} + +/** + * map number to download mode. + * @param {string} downloadMode - download mode (tab/object/layout) + * @returns {number} - mapped downloadmode + */ +function mapDownloadMode(downloadMode) { + if (typeof downloadMode === 'string') { + downloadMode = downloadMode.trim(); + downloadMode = downloadMode.toLowerCase(); + const mappedDownloadMode = DownloadMode[downloadMode]; + if (mappedDownloadMode === undefined) { + throw new Error('Failed to map DownloadMode, perhaps an invalid option was passed?'); + } else { + return mappedDownloadMode; + } + } else { + throw new Error('Failed to map DownloadMode, it should be a string'); + } +} diff --git a/QualityControl/lib/utils/download/classes/data/DownloadConfigData.js b/QualityControl/lib/utils/download/classes/data/DownloadConfigData.js new file mode 100644 index 000000000..e199b671e --- /dev/null +++ b/QualityControl/lib/utils/download/classes/data/DownloadConfigData.js @@ -0,0 +1,71 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +/* eslint-disable jsdoc/reject-any-type */ +import { DownloadMode } from '../../enum/DownloadMode.js'; + +export class DownloadConfigData { + /** + * constructor + * @param {string[]} tabIds - tabIds to download + * @param {string[]} objectIds - objectIds to download + * @param {string[]} archiveNameTemplateOptions - name options for the archive (*.tar.gz) + * @param {string[]} objectNameTemplateOptions - name options for the individual object files + * @param {string} downloadMode - download mode (layout/tab/object) + * @param {boolean} pathNameStructure - enable full pathname structure + */ + constructor( + tabIds, objectIds, archiveNameTemplateOptions, + objectNameTemplateOptions, downloadMode, pathNameStructure, + ) { + this.tabIds = tabIds, + this.objectIds = objectIds, + this.archiveNameTemplateOptions = archiveNameTemplateOptions, + this.objectNameTemplateOptions = objectNameTemplateOptions, + this.downloadMode = downloadMode, + this.pathNameStructure = pathNameStructure; + } + + tabIds; + + objectIds; + + archiveNameTemplateOptions; + + objectNameTemplateOptions; + + downloadMode; + + pathNameStructure; + + /** + * mapper from plain object to instance of DownloadConfigData. + * @static + * @param {any} downloadConfigPlain - plain object download config. + * @returns {DownloadConfigData} - mapped DownloadConfigData. + */ + static mapFromPlain(downloadConfigPlain) { + if (!downloadConfigPlain || typeof downloadConfigPlain !== 'object') { + throw new Error('invalid DownloadConfig'); + } + return new DownloadConfigData(Array.isArray(downloadConfigPlain.tabIds) ? + downloadConfigPlain.tabIds : downloadConfigPlain.tabIds?.split(',') ?? + [], Array.isArray(downloadConfigPlain.objectIds) ? downloadConfigPlain.objectIds : + downloadConfigPlain.objectIds?.split(',') ?? [], Array.isArray(downloadConfigPlain.archiveNameTemplateOptions) ? + downloadConfigPlain.archiveNameTemplateOptions : downloadConfigPlain.archiveNameTemplateOptions?.split(',') + ?? [], Array.isArray(downloadConfigPlain.objectNameTemplateOptions) ? + downloadConfigPlain.objectNameTemplateOptions : downloadConfigPlain.objectNameTemplateOptions?.split(',') + ?? [], downloadConfigPlain?.downloadMode ?? + DownloadMode.object, downloadConfigPlain?.pathNameStructure == 'true' ? true : false); + } +} diff --git a/QualityControl/lib/utils/download/classes/data/LayoutData.js b/QualityControl/lib/utils/download/classes/data/LayoutData.js new file mode 100644 index 000000000..195ec6f29 --- /dev/null +++ b/QualityControl/lib/utils/download/classes/data/LayoutData.js @@ -0,0 +1,82 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +/* eslint-disable jsdoc/reject-any-type */ +/* eslint-disable jsdoc/require-param-description */ +import { LayoutDomain } from '../../classes/domain/LayoutDomain.js'; +import { TabData } from './TabData.js'; +export class LayoutData { + /** + * constructor + * @param {string} id + * @param {string} name + * @param {number} owner_id + * @param {string} owner_name + * @param {TabData[]} tabs + * @param {any[]} collaborators + * @param {boolean} displayTimestamp + * @param {number} autoTabChange + * @param {boolean} isOfficial + */ + constructor(id, name, owner_id, owner_name, tabs, collaborators, displayTimestamp, autoTabChange, isOfficial) { + this.id = id; + this.name = name, + this.owner_id = owner_id, + this.owner_name = owner_name, + this.tabs = tabs, + this.collaborators = collaborators, + this.displayTimestamp = displayTimestamp, + this.autoTabChange = autoTabChange, + this.isOfficial = isOfficial; + } + + id; + + name; + + owner_id; + + owner_name; + + tabs; + + collaborators; + + displayTimestamp; + + autoTabChange; + + isOfficial; + + /** + * map to an instance of LayoutData from a plain object. + * @static + * @param {any} layoutPlain + * @returns {LayoutData} - mapped layoutData + */ + static mapFromPlain(layoutPlain) { + if (!layoutPlain || typeof layoutPlain !== 'object' || layoutPlain.id == undefined) { + throw new Error('invalid layout'); + } + // eslint-disable-next-line @stylistic/js/max-len + return new LayoutData(layoutPlain.id, layoutPlain.name, Number(layoutPlain.owner_id), layoutPlain.owner_name, Array.isArray(layoutPlain.tabs) ? layoutPlain.tabs.map(TabData.mapFromPlain) : [], Array.isArray(layoutPlain.collaborators) ? layoutPlain.collaborators : [], Boolean(layoutPlain.displayTimestamp), Number(layoutPlain.autoTabChange), Boolean(layoutPlain.isOfficial)); + } + + /** + * mapper to Domain model + * @returns {LayoutDomain} Resulting LayoutDomain. + */ + mapToDomain() { + return new LayoutDomain(this.id, this.name, this.tabs.map((tab) => tab.mapToDomain())); + } +} diff --git a/QualityControl/lib/utils/download/classes/data/ObjectData.js b/QualityControl/lib/utils/download/classes/data/ObjectData.js new file mode 100644 index 000000000..e76de3866 --- /dev/null +++ b/QualityControl/lib/utils/download/classes/data/ObjectData.js @@ -0,0 +1,84 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +/* eslint-disable jsdoc/require-param-description */ +/* eslint-disable jsdoc/reject-any-type */ +import { ObjectDomain } from '../../classes/domain/ObjectDomain.js'; +export class ObjectData { + /** + * constructor + * @param {string} id + * @param {number} x + * @param {number} y + * @param {number} h + * @param {number} w + * @param {string} name + * @param {string[]} options + * @param {boolean} autoSize + * @param {boolean} ignoreDefaults + */ + constructor(id, x, y, h, w, name, options, autoSize, ignoreDefaults) { + this.id = id, + this.x = x, + this.y = y, + this.h = h, + this.w = w, + this.name = name; + this.options = options, + this.autoSize = autoSize, + this.ignoreDefaults = ignoreDefaults; + } + + id; + + x; + + y; + + h; + + w; + + name; + + options; + + autoSize; + + ignoreDefaults; + + /** + * mapper to map from plain object to instance of ObjectData. + * @static + * @param {any} objectPlain - plain js object + * @returns {ObjectData} - mapped object data + */ + static mapFromPlain(objectPlain) { + if (!objectPlain || typeof objectPlain !== 'object') { + throw new Error('invalid object'); + } + return new ObjectData(objectPlain.id, Number(objectPlain.x ?? + 0), Number(objectPlain.y ?? 0), Number(objectPlain.h ?? + 0), Number(objectPlain.w ?? + 0), objectPlain.name, Array.isArray(objectPlain.options) ? objectPlain.options : + [], Boolean(objectPlain.autoSize), Boolean(objectPlain.ignoreDefaults)); + } + + /** + * mapper to domain model. + * @returns {ObjectDomain} - mapped objectDomain model. + */ + mapToDomain() { + return new ObjectDomain(this.id, this.name); + } +} diff --git a/QualityControl/lib/utils/download/classes/data/TabData.js b/QualityControl/lib/utils/download/classes/data/TabData.js new file mode 100644 index 000000000..3f991a521 --- /dev/null +++ b/QualityControl/lib/utils/download/classes/data/TabData.js @@ -0,0 +1,62 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +/* eslint-disable jsdoc/reject-any-type */ +import { TabDomain } from '../domain/TabDomain.js'; +import { ObjectData } from './ObjectData.js'; + +export class TabData { + /** + * constructor + * @param {string} id - id + * @param {string} name - name of tab + * @param {ObjectData[]} objects - objects within tab + * @param {number} columns - columnds + */ + constructor(id, name, objects, columns) { + this.id = id, + this.name = name, + this.objects = objects, + this.columns = columns; + } + + id; + + name; + + objects; + + columns; + + /** + * mapFromPlain, map to an instance of TabData from a plain object. + * @static + * @param {any} tabPlain - plain object of tab. + * @returns {TabData} - mapped TabData. + */ + static mapFromPlain(tabPlain) { + if (!tabPlain || typeof tabPlain !== 'object') { + throw new Error('invalid tab'); + } + return new TabData(tabPlain.id, tabPlain.name, Array.isArray(tabPlain.objects) ? + tabPlain.objects.map(ObjectData.mapFromPlain) : [], Number(tabPlain.columns)); + } + + /** + * mapper to Domain model. + * @returns {TabDomain} - mapped TabDomain. + */ + mapToDomain() { + return new TabDomain(this.id, this.name, this.objects.map((object) => object.mapToDomain())); + } +} diff --git a/QualityControl/lib/utils/download/classes/domain/DownloadConfigDomain.js b/QualityControl/lib/utils/download/classes/domain/DownloadConfigDomain.js new file mode 100644 index 000000000..3cc5ebd92 --- /dev/null +++ b/QualityControl/lib/utils/download/classes/domain/DownloadConfigDomain.js @@ -0,0 +1,48 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +export class DownloadConfigDomain { + /** + * constructor + * @param {string[]} tabIds - tabIds to download + * @param {string[]} objectIds - objectIds to download + * @param {NameTemplateOption[]} archiveNameTemplateOptions - name options for the archive (*.tar.gz) + * @param {NameTemplateOption[]} objectNameTemplateOptions - name options for the individual object files + * @param {DownloadMode} downloadMode - download mode (layout/tab/object) + * @param {boolean} pathNameStructure - enable full pathname structure + */ + constructor( + tabIds, objectIds, archiveNameTemplateOptions, + objectNameTemplateOptions, downloadMode, pathNameStructure, + ) { + this.tabIds = tabIds, + this.objectIds = objectIds, + this.archiveNameTemplateOptions = archiveNameTemplateOptions, + this.objectNameTemplateOptions = objectNameTemplateOptions, + this.downloadMode = downloadMode, + this.pathNameStructure = pathNameStructure; + } + + tabIds; + + objectIds; + + archiveNameTemplateOptions; + + objectNameTemplateOptions; + + downloadMode; + + pathNameStructure; +} diff --git a/QualityControl/lib/utils/download/classes/domain/LayoutDomain.js b/QualityControl/lib/utils/download/classes/domain/LayoutDomain.js new file mode 100644 index 000000000..da4b6e4dd --- /dev/null +++ b/QualityControl/lib/utils/download/classes/domain/LayoutDomain.js @@ -0,0 +1,41 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +/** @import { TabDomain } from './TabDomain.js'; */ + +export class LayoutDomain { + /** + * constructor + * @param {string} id - id + * @param {string} name - name + * @param {TabDomain[]} tabs - tabs + */ + constructor(id, name, tabs) { + if ( + id != undefined && id != '' && name != undefined && name != '' + && tabs.length != 0 + ) { + this.id = id, + this.name = name, + this.tabs = tabs; + } else { + throw new Error('Failed to instanciate new LayoutDomain'); + } + } + + id; + + name; + + tabs; +} diff --git a/QualityControl/lib/utils/download/classes/domain/LayoutDomainStorage.js b/QualityControl/lib/utils/download/classes/domain/LayoutDomainStorage.js new file mode 100644 index 000000000..95f682d10 --- /dev/null +++ b/QualityControl/lib/utils/download/classes/domain/LayoutDomainStorage.js @@ -0,0 +1,49 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +import { LayoutDomain } from './LayoutDomain.js'; + +/** @import { TabDomain } from './TabDomain.js'; */ + +/** + * @augments LayoutDomain + */ +export class LayoutDomainStorage extends LayoutDomain { + /** + * constructor + * @param {string} id - id + * @param {string} name - name + * @param {TabDomain[]} tabs - tabs + * @param {number} downloadUserId - userid of the user who requested this download. + */ + constructor(id, name, tabs, downloadUserId) { + if ( + downloadUserId != 0 + ) { + super(id, name, tabs); + this.downloadUserId = downloadUserId; + } else { + throw new Error('Failed to instanciate LayoutDomainStorage'); + } + } + + downloadUserId; + + /** + * return a representation of the parent. + * @returns {LayoutDomain} return + */ + toSuper() { + return new LayoutDomain(this.id, this.name, this.tabs); + } +} diff --git a/QualityControl/lib/utils/download/classes/domain/MapStorage.js b/QualityControl/lib/utils/download/classes/domain/MapStorage.js new file mode 100644 index 000000000..a7e872e70 --- /dev/null +++ b/QualityControl/lib/utils/download/classes/domain/MapStorage.js @@ -0,0 +1,61 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +export class MapStorage { + constructor() { + this.layoutStorage = new Map(); + } + + layoutStorage; + + /** + * read func + * @param {string} key - key + * @returns {[LayoutDomainStorage, DownloadConfigDomain] | undefined} found Download request + */ + readRequest(key) { + return this.layoutStorage.get(key); + } + + /** + * write func + * @param {LayoutDomainStorage} layout - layout + * @param {DownloadConfigDomain} config - config + * @returns {string} - key + */ + writeRequest(layout, config) { + const mapKey = crypto.randomUUID(); + this.layoutStorage.set(mapKey, [layout, config]); + return mapKey; + } + + /** + * delete func + * @param {string} key - key + * @returns {boolean} - true if deleted + */ + deleteRequest(key) { + return this.layoutStorage.delete(key); + } + + /** + * delete cached download data by user id + * @param {number} userId - userid + */ + deleteByUserId(userId) { + const found = this.layoutStorage.entries().filter((entry) => entry[1][0].downloadUserId == userId); + found.forEach((entry) => { + this.layoutStorage.delete(entry[0]); + }); + } +} diff --git a/QualityControl/lib/utils/download/classes/domain/ObjectDomain.js b/QualityControl/lib/utils/download/classes/domain/ObjectDomain.js new file mode 100644 index 000000000..47743eda4 --- /dev/null +++ b/QualityControl/lib/utils/download/classes/domain/ObjectDomain.js @@ -0,0 +1,32 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +export class ObjectDomain { + /** + * constructor + * @param {string} id - id + * @param {string} name - name + */ + constructor(id, name) { + if (id != undefined && id != '' && name != undefined && name != '') { + this.id = id, + this.name = name; + } else { + throw new Error('Failed to instanciate new ObjectDomain'); + } + } + + id; + + name; +} diff --git a/QualityControl/lib/utils/download/classes/domain/TabDomain.js b/QualityControl/lib/utils/download/classes/domain/TabDomain.js new file mode 100644 index 000000000..a9741852e --- /dev/null +++ b/QualityControl/lib/utils/download/classes/domain/TabDomain.js @@ -0,0 +1,41 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +/** @import { ObjectDomain } from './ObjectDomain.js'; */ + +export class TabDomain { + /** + * constructor + * @param {string} id - id + * @param {string} name - name + * @param {ObjectDomain[]} objects - objects + */ + constructor(id, name, objects) { + if ( + id != undefined && id != '' && name != undefined && name != '' + && objects.length != 0 + ) { + this.id = id, + this.name = name, + this.objects = objects; + } else { + throw new Error('Failed to instanciate new TabDomain'); + } + } + + id; + + name; + + objects; +} diff --git a/QualityControl/lib/utils/download/configurator.js b/QualityControl/lib/utils/download/configurator.js new file mode 100644 index 000000000..950a91622 --- /dev/null +++ b/QualityControl/lib/utils/download/configurator.js @@ -0,0 +1,68 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +import { LayoutData } from './classes/data/LayoutData.js'; +import { DownloadConfigData } from './classes/data/DownloadConfigData.js'; +import { mapDownloadConfigToDomain } from './classes/DownloadConfigMapper.js'; + +/** @import { DownloadConfigDomain } from './classes/domain/DownloadConfigDomain.js'; */ +/** @import { LayoutDomain } from './classes/domain/LayoutDomain.js'; */ +/** @import { Request, Response, NextFunction } from 'express' */ + +/** + * @typedef {object} Query + * @property {string} tabIds - tabIds to download + * @property {string} objectIds - objectIds to download + * @property {string|string[]} archiveNameTemplateOptions - archiveNameTemplateOptions + * @property {string|string[]} objectNameTemplateOptions - objectNameTemplateOptions + * @property {string} key - key received by earlier post if any. + */ + +/** + * parse request to download configuration + * @param {Request} req - request + * @returns {DownloadConfigDomain} - Parsed DownloadConfigDomain model + */ +export function parseRequestToConfig(req) { + const plainConfigReq = req.query; + // Create config + // Data + const configData = DownloadConfigData.mapFromPlain(plainConfigReq); + // Domain + const configDomain = mapDownloadConfigToDomain(configData); + return configDomain; +} + +/** + * parse request to download layout + * @param {Request} req - request + * @returns {LayoutDomain} - parsed LayoutDomainModel + */ +export function parseRequestToLayout(req) { + // Create Layout object + const jsonBody = req.body; + if (Object.keys(jsonBody).length === 0) { + throw new Error('Json cannot be empty'); + } + // Data + const layout = LayoutData.mapFromPlain(jsonBody); + // Domain + if (layout.id == 0) { + throw new Error('Layout cannot have an empty id'); + } + const layoutDomain = layout.mapToDomain(); + if (layout == undefined) { + throw new Error('Layout not found.'); + } + return layoutDomain; +} diff --git a/QualityControl/lib/utils/download/downloadEngine.js b/QualityControl/lib/utils/download/downloadEngine.js new file mode 100644 index 000000000..fb3ebbd76 --- /dev/null +++ b/QualityControl/lib/utils/download/downloadEngine.js @@ -0,0 +1,280 @@ +/* eslint-disable curly */ +/* eslint-disable prefer-destructuring */ +/* eslint-disable no-case-declarations */ +/* eslint-disable @stylistic/js/max-len */ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +import { pipeline } from 'node:stream'; +import { createGzip } from 'node:zlib'; +import { promisify } from 'node:util'; + +import { LayoutDomainStorage } from './classes/domain/LayoutDomainStorage.js'; +import { DownloadMode } from './enum/DownloadMode.js'; +import { NameTemplateOption } from './enum/NameTemplateOption.js'; +import { createTar } from './tar/tar.js'; + +const CONTENT_LENGTH_HEADER = 'Content-Length'; +const CONTENT_TYPE_HEADER = 'Content-Type'; +const CONTENT_TYPE_DEFAULT = 'application/root'; +const _pipelineAsync = promisify(pipeline); + +/** @import { LayoutDomain } from './classes/domain/LayoutDomain.js'; */ +/** @import { MapStorage } from './classes/domain/MapStorage.js'; */ +/** @import { DownloadConfigDomain } from './classes/domain/DownloadConfigDomain.js' */ +/** @import { ObjectDomain } from './classes/domain/ObjectDomain.js'; */ + +/** + * main download function + * @param {LayoutDomain} downloadLayout + * @param {DownloadConfigDomain} downloadConfiguration + * @param {number} runNumber + * @param {Response} res + * @returns {Promise} + */ +export async function download(downloadLayout, downloadConfiguration, runNumber, res) { + switch (downloadConfiguration.downloadMode) { + case DownloadMode.object: + let file = new File([], ''); + const objects = downloadLayout.tabs.flatMap((tab) => tab.objects.filter((object) => downloadConfiguration.objectIds.includes(object.id))); + if (objects.length > 1) { + // Multiple objects requested + const files = await requestObjects(objects, downloadConfiguration, undefined, runNumber); + file = await createTar(files, processFileNameTemplate(downloadConfiguration.archiveNameTemplateOptions, undefined, undefined, runNumber)); + } else { + // One object requested + // Dangerous index access, fix later + const object = downloadLayout.tabs.flatMap((tab) => tab.objects.find((object) => downloadConfiguration.objectIds[0] == object.id))[0]; + if (object == undefined) + throw new Error('ObjectId does not exist in given layout data'); + file = await requestObject(object, downloadConfiguration, undefined, runNumber); + } + setHeaders(res, file, false); + await _streamFileToResponse(file, res); + break; + case DownloadMode.tab: + // const objects = downloadLayout.tabs.flatMap((tab) => tab.objects.filter((object) => downloadConfiguration.objectIds.includes(object.id))) + let tabFiles = []; + const tabs = downloadLayout.tabs.filter((tab) => downloadConfiguration.tabIds.some((id) => id == tab.id)); + if (tabs == undefined) + throw new Error('TabId does not exist in given layout data'); + if (objectRequestLimiter(tabs.flatMap((tab) => tab.objects).length)) { + throw new Error('Too many objects requested at once'); + } + if (tabs.length > 1) { + // multiple tabs + const tabPromises = []; + tabs.forEach((tab) => { + tabPromises.push(requestTab(tab, downloadConfiguration, runNumber)); + }); + const tabsFiles = await Promise.all(tabPromises); + console.log(tabsFiles); + tabFiles = tabsFiles.flat(); + console.log(tabFiles); + } else { + // single tab + tabFiles = await requestTab(tabs[0], downloadConfiguration, runNumber); + } + const tarFile = await createTar(tabFiles, processFileNameTemplate(downloadConfiguration.archiveNameTemplateOptions, undefined, undefined, runNumber)); + setHeaders(res, tarFile, true); + await streamArchiveToResponse(tarFile, res); + break; + case DownloadMode.layout: + break; + default: + break; + } +} + +/** + * save download data to cache + * @param {MapStorage} mapStorage - map storage used to store data from post request + * @param {LayoutDomain} layoutDomain - layoutDomain data to store. + * @param {DownloadConfigDomain} configDomain - configDomain data to store. + * @param {number} userId - userId of user wanting to download. + * @returns {`${string}-${string}-${string}-${string}-${string}`} - UUID key of Map entry + */ +export function saveDownloadData(mapStorage, layoutDomain, configDomain, userId) { + // Delete existing download Layout data. + mapStorage.deleteByUserId(userId); + const layoutDomainStorage = new LayoutDomainStorage(layoutDomain.id, layoutDomain.name, layoutDomain.tabs, userId); + const insertedLayoutKey = mapStorage.writeRequest(layoutDomainStorage, configDomain); + return insertedLayoutKey; +} + +/** + * load saved data from cache + * @param {MapStorage} mapStorage - map storage used to retrieve data from earlier post request + * @param {string} key - UUID key of Map entry to retrieve layout by + * @returns {[LayoutDomainStorage, DownloadConfigDomain] | undefined} - found download request if any + */ +export function loadSavedData(mapStorage, key) { + return mapStorage.readRequest(key); +} + +/** + * Limit the number of requested objectIds + * @param {number} idsCount - number of requested id's + * @returns {boolean} - true if too many ids have been requested + */ +function objectRequestLimiter(idsCount) { + return idsCount > 40; +} + +/** + * set the right headers on the response depending on + * if the response will be an archive or not. + * @param {Response} res + * @param {File} file + * @param {boolean} isArchive + * @returns {void} + */ +function setHeaders(res, file, isArchive) { + res.setHeader('content-disposition', `inline;filename="${file.name}"`); + // Filesize is wrong and thus breaks stream... We cannot tell the size if the file is going to be gzipped and streamed at once.... + isArchive ? undefined : res.setHeader('content-type', file.size); + res.setHeader('content-type', isArchive ? 'application/gzip' : 'application/root'); + return; +} + +/** + * Stream the archive file into the response to the user. + * @param {File} tarFile + * @param {Response} res + * @returns {Promise} + */ +async function streamArchiveToResponse(tarFile, res) { + const gzip = createGzip(); + const read = tarFile.stream(); + try { + await _pipelineAsync(read, gzip, res); + } catch (error) { + console.log(error?.message ?? error); + throw error; + } +} + +/** + * Stream the ROOT file into our reponse back to the user. + * @param {File} file - Root file + * @param {Response} res - Outgoing response object we'll write our data into + * @returns {Promise} + */ +async function _streamFileToResponse(file, res) { + // We will stream the data from QCDB's answer directly back to the user. + await _pipelineAsync(file.stream(), res); +} + +/** + * Tab becomes our tab.tar.gz or MFT.tar.gz with our objects within + * @param {TabDomain} tab + * @param {DownloadConfigDomain} downloadConfiguration + * @param {number} runNumber + * @returns {Promise} + */ +async function requestTab(tab, downloadConfiguration, runNumber) { + return await requestObjects(tab.objects, downloadConfiguration, tab.name, runNumber); +} + +/** + * Request objects from QCDB + * @param {ObjectDomain[]} objects + * @param {DownloadConfigDomain} downloadConfiguration + * @param {string} [tabName] + * @param {number} [runNumber] + * @returns {Promise} + */ +async function requestObjects(objects, downloadConfiguration, tabName, runNumber) { + const requestPromises = []; + objects.forEach((object) => { + requestPromises.push(requestObject(object, downloadConfiguration, tabName, runNumber)); + }); + const objectFiles = await Promise.all(requestPromises); + return objectFiles; +} + +/** + * Request QCDB ROOT object. + * @param {ObjectDomain} object + * @param {DownloadConfigDomain} downloadConfiguration + * @param {string} [tabName=''] + * @param {number} [runNumber=0] + * @returns {Promise} + */ +async function requestObject(object, downloadConfiguration, tabName = '', runNumber = 0) { + try { + // const response = await fetch(`http://localhost:8083/download/${object.id}`); + const response = await fetch(`http://ali-qcdb-gpn.cern.ch:8083/download/${object.id}`); + if (!response.ok) { + console.log(`QCDB returned ${response.status} ${response.statusText}`); + throw new Error(`Cannot get ROOT file from QCDB object id: ${object.id}`); + } + const contentLength = response.headers.get(CONTENT_LENGTH_HEADER); + console.log(`ROOT size (bytes): ${contentLength}`); + return _getFileFromResponse(response, processFileNameTemplate(downloadConfiguration.objectNameTemplateOptions, object, tabName, runNumber, downloadConfiguration.pathNameStructure)); + } catch (error) { + console.log(error?.message ?? error); + throw error; + } +} + +/** + * process the nameTemplateOptions for this file. + * @param {NameTemplateOption[]} nameTemplateOption + * @param {ObjectDomain | undefined} [object=undefined] + * @param {string} [tabName=''] + * @param {number} runNumber + * @param {boolean} [pathNameStructure=false] + * @returns {string} + */ +function processFileNameTemplate(nameTemplateOption, object = undefined, tabName = '', runNumber, pathNameStructure = false) { + // Make sure that if the tabname is set it will be the root folder of its objects... + let rv = tabName === '' ? tabName : `${tabName}/`; + const nameStartingLength = tabName.length + 1; + // Prevent tar from creating a folder structure if wished for. + const index = object?.name.lastIndexOf('/') ?? 0; + const name = pathNameStructure ? object?.name : object?.name.slice(index + 1, object?.name.length - 1); + const objectId = object?.id ?? ''; + nameTemplateOption.forEach((nameTemplateOption) => { + switch (nameTemplateOption) { + case NameTemplateOption.objectName: + rv += rv.length > nameStartingLength ? `_${name}` : name; + break; + case NameTemplateOption.objectId: + rv += rv.length > nameStartingLength ? `_${objectId}` : objectId; + break; + case NameTemplateOption.tabName: + rv += rv.length > nameStartingLength ? `_${tabName}` : tabName; + break; + case NameTemplateOption.runNumber: + rv += rv.length > nameStartingLength ? `_${runNumber}` : runNumber; + break; + } + }); + return rv; +} + +/** + * Get a ROOT file from a QCDB download request. + * @param {globalThis.Response} response - response from QCDB. + * @param {string} filename - filename for the resulting file. + * @returns {Promise} - ROOT file from response. + */ +async function _getFileFromResponse(response, filename) { + const contentType = response.headers.get(CONTENT_TYPE_HEADER); + const blob = await response.blob(); + const fullFilename = `${filename}.root`; + const file = new File([blob], fullFilename, { type: contentType ?? CONTENT_TYPE_DEFAULT }); + return file; +} diff --git a/QualityControl/lib/utils/download/enum/DownloadMode.js b/QualityControl/lib/utils/download/enum/DownloadMode.js new file mode 100644 index 000000000..8c4d87aa1 --- /dev/null +++ b/QualityControl/lib/utils/download/enum/DownloadMode.js @@ -0,0 +1,7 @@ +// eslint-disable-next-line no-var, init-declarations +export var DownloadMode; +(function (DownloadMode) { + DownloadMode[DownloadMode['object'] = 0] = 'object'; + DownloadMode[DownloadMode['tab'] = 1] = 'tab'; + DownloadMode[DownloadMode['layout'] = 2] = 'layout'; +})(DownloadMode || (DownloadMode = {})); diff --git a/QualityControl/lib/utils/download/enum/NameTemplateOption.js b/QualityControl/lib/utils/download/enum/NameTemplateOption.js new file mode 100644 index 000000000..ed62736d1 --- /dev/null +++ b/QualityControl/lib/utils/download/enum/NameTemplateOption.js @@ -0,0 +1,8 @@ +// eslint-disable-next-line no-var, init-declarations +export var NameTemplateOption; +(function (NameTemplateOption) { + NameTemplateOption[NameTemplateOption['objectName'] = 0] = 'objectName'; + NameTemplateOption[NameTemplateOption['objectId'] = 1] = 'objectId'; + NameTemplateOption[NameTemplateOption['tabName'] = 2] = 'tabName'; + NameTemplateOption[NameTemplateOption['runNumber'] = 3] = 'runNumber'; +})(NameTemplateOption || (NameTemplateOption = {})); diff --git a/QualityControl/lib/utils/download/tar/header.js b/QualityControl/lib/utils/download/tar/header.js new file mode 100644 index 000000000..d1814c89a --- /dev/null +++ b/QualityControl/lib/utils/download/tar/header.js @@ -0,0 +1,262 @@ +/* eslint-disable jsdoc/require-returns-description */ +/* eslint-disable jsdoc/require-param-description */ +/* eslint-disable jsdoc/require-description */ +// parse a 512-byte header block to a data object, or vice-versa +// encode returns `true` if a pax extended header is needed, because +// the data could not be faithfully encoded in a simple header. +// (Also, check header.needPax to see if it needs a pax header.) +// From https://github.com/isaacs/node-tar/blob/main/src/header.ts +import { posix as pathModule } from 'node:path'; +import * as large from './large-numbers.js'; + +export class Header { + cksumValid = false; + + needPax = false; + + nullBlock = false; + + block; + + path; + + mode; + + uid; + + gid; + + size; + + cksum; + + #type = 0; + + linkpath; + + uname; + + gname; + + devmaj = 0; + + devmin = 0; + + atime; + + ctime; + + mtime; + + charset; + + comment; + + /** + * @param {Buffer | HeaderData} [data] + */ + constructor(data) { + if (!Buffer.isBuffer(data) && data) { + this.#slurp(data); + } + } + + /** + * @param {HeaderData} ex + * @param {boolean} [gex=false] + * @returns {void} + */ + #slurp(ex, gex = false) { + Object.assign(this, Object.fromEntries(Object.entries(ex).filter(([k, v]) => + // we slurp in everything except for the path attribute in + // a global extended header, because that's weird. Also, any + // null/undefined values are ignored. + !(v === null || + v === undefined || + k === 'path' && gex || + k === 'linkpath' && gex || + k === 'global')))); + } + + /** + * @param {Buffer} [buf] + * @param {number} [off=0] + * @returns {boolean} + */ + encode(buf, off = 0) { + if (!buf) { + buf = this.block = Buffer.alloc(512); + } + if (!(buf.length >= off + 512)) { + throw new Error('need 512 bytes for header'); + } + const prefixSize = this.ctime || this.atime ? 130 : 155; + const split = splitPrefix(this.path || '', prefixSize); + const path = split[0]; + const prefix = split[1]; + this.needPax = Boolean(split[2]); + encString(buf, off, 100, path); + encNumber(buf, off + 100, 8, this.mode); + encNumber(buf, off + 108, 8, this.uid); + encNumber(buf, off + 116, 8, this.gid); + encNumber(buf, off + 124, 12, this.size); + encDate(buf, off + 136, 12, this.mtime); + // set type byte + buf[off + 156] = this.#type; + encString(buf, off + 157, 100, this.linkpath); + buf.write('ustar\u000000', off + 257, 8); + encString(buf, off + 265, 32, this.uname); + encString(buf, off + 297, 32, this.gname); + encNumber(buf, off + 329, 8, this.devmaj); + encNumber(buf, off + 337, 8, this.devmin); + encString(buf, off + 345, prefixSize, prefix); + if (buf[off + 475] !== 0) { + encString(buf, off + 345, 155, prefix); + } else { + encString(buf, off + 345, 130, prefix); + encDate(buf, off + 476, 12, this.atime); + encDate(buf, off + 488, 12, this.ctime); + } + let sum = 8 * 0x20; + for (let i = off; i < off + 148; i++) { + sum += buf[i]; + } + for (let i = off + 156; i < off + 512; i++) { + sum += buf[i]; + } + this.cksum = sum; + encNumber(buf, off + 148, 8, this.cksum); + this.cksumValid = true; + return this.needPax; + } +} + +// to handle prefix, whole path? only filename? depends on size. +/** + * @param {string} p + * @param {number} prefixSize + * @returns {[string, string, boolean]} + */ +const splitPrefix = (p, prefixSize) => { + const pathSize = 100; + let pp = p; + let prefix = ''; + let ret = undefined; + const root = pathModule.parse(p).root || '.'; + if (Buffer.byteLength(pp) < pathSize) { + ret = [pp, prefix, false]; + } else { + // first set prefix to the dir, and path to the base + prefix = pathModule.dirname(pp); + pp = pathModule.basename(pp); + do { + if (Buffer.byteLength(pp) <= pathSize && + Buffer.byteLength(prefix) <= prefixSize) { + // both fit! + ret = [pp, prefix, false]; + } else if (Buffer.byteLength(pp) > pathSize && + Buffer.byteLength(prefix) <= prefixSize) { + // prefix fits in prefix, but path doesn't fit in path + ret = [pp.slice(0, pathSize - 1), prefix, true]; + } else { + // make path take a bit from prefix + pp = pathModule.join(pathModule.basename(prefix), pp); + prefix = pathModule.dirname(prefix); + } + } while (prefix !== root && ret === undefined); + // at this point, found no resolution, just truncate + if (!ret) { + ret = [p.slice(0, pathSize - 1), '', true]; + } + } + return ret; +}; +// the maximum encodable as a null-terminated octal, by field size +const MAXNUM = { + 12: 0o77777777777, + 8: 0o7777777, +}; + +/** + * @param {Buffer} buf + * @param {number} off + * @param {12 | 8} size + * @param {number} [num] + * @returns {boolean} + */ +const encNumber = (buf, off, size, num) => num === undefined ? false + : num > MAXNUM[size] || num < 0 ? + (large.encode(num, buf.subarray(off, off + size)), true) + : (encSmallNumber(buf, off, size, num), false); + +/** + * @param {Buffer} buf + * @param {number} off + * @param {number} size + * @param {number} num + * @returns {number} + */ +const encSmallNumber = (buf, off, size, num) => buf.write(octalString(num, size), off, size, 'ascii'); + +/** + * @param {number} num + * @param {number} size + * @returns {string} + */ +const octalString = (num, size) => padOctal(Math.floor(num).toString(8), size); + +/** + * @param {string} str + * @param {number} size + * @returns {string} + */ +const padOctal = (str, size) => `${str.length === size - 1 ? + str + : `${new Array(size - str.length - 1).join('0') + str} `}\0`; + +/** + * @param {Buffer} buf + * @param {number} off + * @param {8 | 12} size + * @param {Date} [date] + * @returns {boolean} + */ +const encDate = (buf, off, size, date) => date === undefined ? false : encNumber(buf, off, size, date.getTime() / 1000); + +// enough to fill the longest string we've got +const NULLS = new Array(156).join('\0'); + +// pad with nulls, return true if it's longer or non-ascii +/** + * @param {Buffer} buf + * @param {number} off + * @param {number} size + * @param {string} [str] + * @returns {boolean} + */ +const encString = (buf, off, size, str) => str === undefined ? false : (buf.write(str + NULLS, off, size, 'utf8'), +str.length !== Buffer.byteLength(str) || str.length > size); + +/** + * @typedef {Object} HeaderData + * @property {string} [path] + * @property {number} [mode] + * @property {number} [uid] + * @property {number} [gid] + * @property {number} [size] + * @property {number} [cksum] + * @property {number | 0} [type] + * @property {string} [linkpath] + * @property {string} [uname] + * @property {string} [gname] + * @property {number} [devmaj] + * @property {number} [devmin] + * @property {Date} [atime] + * @property {Date} [ctime] + * @property {Date} [mtime] + * @property {string} [charset] + * @property {string} [comment] + * @property {number} [dev] + * @property {number} [ino] + * @property {number} [nlink] + */ diff --git a/QualityControl/lib/utils/download/tar/large-numbers.js b/QualityControl/lib/utils/download/tar/large-numbers.js new file mode 100644 index 000000000..2fe9a6e68 --- /dev/null +++ b/QualityControl/lib/utils/download/tar/large-numbers.js @@ -0,0 +1,133 @@ +/* eslint-disable jsdoc/require-param-description */ +/* eslint-disable jsdoc/require-description */ +// Tar can encode large and negative numbers using a leading byte of +// 0xff for negative, and 0x80 for positive. +// From https://github.com/isaacs/node-tar/blob/main/src/large-numbers.ts + +/** + * @param {number} num + * @param {Buffer} buf + * @returns {Buffer} number + */ +export const encode = (num, buf) => { + if (!Number.isSafeInteger(num)) { + // The number is so large that javascript cannot represent it with integer + // precision. + throw Error('cannot encode number outside of javascript safe integer range'); + } else if (num < 0) { + encodeNegative(num, buf); + } else { + encodePositive(num, buf); + } + return buf; +}; + +/** + * @param {number} num + * @param {Buffer} buf + * @returns {void} + */ +const encodePositive = (num, buf) => { + buf[0] = 0x80; + for (let i = buf.length; i > 1; i--) { + buf[i - 1] = num & 0xff; + num = Math.floor(num / 0x100); + } +}; + +/** + * @param {number} num + * @param {Buffer} buf + * @returns {void} + */ +const encodeNegative = (num, buf) => { + buf[0] = 0xff; + let flipped = false; + num = num * -1; + for (let i = buf.length; i > 1; i--) { + const byte = num & 0xff; + num = Math.floor(num / 0x100); + if (flipped) { + buf[i - 1] = onesComp(byte); + } else if (byte === 0) { + buf[i - 1] = 0; + } else { + flipped = true; + buf[i - 1] = twosComp(byte); + } + } +}; + +/** + * @param {Buffer} buf + * @returns {number} number + */ +export const parse = (buf) => { + const pre = buf[0]; + const value = pre === 0x80 ? pos(buf.subarray(1, buf.length)) + : pre === 0xff ? twos(buf) + : null; + if (value === null) { + throw Error('invalid base256 encoding'); + } + if (!Number.isSafeInteger(value)) { + // The number is so large that javascript cannot represent it with integer + // precision. + throw Error('parsed number outside of javascript safe integer range'); + } + return value; +}; + +/** + * @param {Buffer} buf + * @returns {number} number + */ +const twos = (buf) => { + let len = buf.length; + let sum = 0; + let flipped = false; + for (let i = len - 1; i > -1; i--) { + let byte = Number(buf[i]); + var f; + if (flipped) { + f = onesComp(byte); + } else if (byte === 0) { + f = byte; + } else { + flipped = true; + f = twosComp(byte); + } + if (f !== 0) { + sum -= f * Math.pow(256, len - i - 1); + } + } + return sum; +}; + +/** + * @param {Buffer} buf + * @returns {number} number + */ +const pos = (buf) => { + let len = buf.length; + let sum = 0; + for (let i = len - 1; i > -1; i--) { + let byte = Number(buf[i]); + if (byte !== 0) { + sum += byte * Math.pow(256, len - i - 1); + } + } + return sum; +}; + +/** + * @param {number} byte + * @returns {number} number + */ +const onesComp = (byte) => (0xff ^ byte) & 0xff; + +/** + * @param {number} byte + * @returns {number} number + */ +const twosComp = (byte) => (0xff ^ byte) + 1 & 0xff; diff --git a/QualityControl/lib/utils/download/tar/tar.js b/QualityControl/lib/utils/download/tar/tar.js new file mode 100644 index 000000000..15b72185a --- /dev/null +++ b/QualityControl/lib/utils/download/tar/tar.js @@ -0,0 +1,100 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +import { LogManager } from '@aliceo2/web-ui'; +import { Header } from './header.js'; +import { Buffer } from 'node:buffer'; + +const logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? 'qcg'}/download-tar`); + +/** + * Create a Tarball File, + * @param {Array} files - Files to add to Tarball. + * @param {string} tarName - Tarball file name. + * @returns {Promise} - Tarball file + */ +export async function createTar(files, tarName) { + // total size of our Tarball, always has two empty 512 blocks... + let totalSizeByte = 1024; + for (const file of files) { + const remainder = file.size % 512; + let bytesToPad = 0; + if (remainder !== 0) { + bytesToPad = 512 - remainder; + } + // header + file + padding + totalSizeByte += 512 + file.size + bytesToPad; + } + const buf = Buffer.alloc(totalSizeByte); + let offset = 0; + for (const file of files) { + const remainder = file.size % 512; + let bytesToPad = 0; + if (remainder !== 0) { + bytesToPad = 512 - remainder; + } + logger.debugMessage(`Remainder: ${remainder}`); + logger.debugMessage(`Bytes to padd: ${bytesToPad}`); + const header = new Header({ + path: file.name, + size: file.size, + mtime: new Date(file.lastModified), + mode: 0o00755, + }); + header.encode(buf, offset); + offset += 512; + // Write Root file to tar. + offset = await writeFile(file, buf, offset); + // Write padding + offset = padBlock(buf, bytesToPad, offset); + } + // fill in the last two empty blocks... + buf.fill('', offset); + // We assume it will be Gzipped later.... + // return new File([buf.buffer], `${tarName}.tar`) + return new File([buf.buffer], `${tarName}.tar.gz`); +} + +/** + * add file to tarball + * @param {File} file - file to add. + * @param {Buffer} buf - buffer to add to. + * @param {number} offset - current offset of buffer. + * @returns {Promise} offset. + */ +async function writeFile(file, buf, offset) { + // preserves the files structure. + const ab = await file.arrayBuffer(); + const fileBuf = Buffer.from(ab); + fileBuf.copy(buf, offset); + return offset + fileBuf.length; +} + +/** + * pad the block + * @param {Buffer} buf - buffer to pad. + * @param {number} bytesToPad - amount of bytes to pad buffer with. + * @param {number} offset - current offset. + * @returns {number} offset. + */ +function padBlock(buf, bytesToPad, offset) { + // No need to add padding + if (bytesToPad == 0) { + return offset; + } + logger.debugMessage(`bytes to pad = ${bytesToPad}`); + logger.debugMessage(`padding zeros from ${offset} to ${offset + bytesToPad}.`); + // fill buffer with zero's + buf.fill('', offset, offset + bytesToPad, 'utf-8'); + return offset + bytesToPad; +} diff --git a/QualityControl/test/api/download/api-post-download.test.js b/QualityControl/test/api/download/api-post-download.test.js new file mode 100644 index 000000000..f122b847e --- /dev/null +++ b/QualityControl/test/api/download/api-post-download.test.js @@ -0,0 +1,40 @@ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +import { suite, test } from 'node:test'; +import { OWNER_TEST_TOKEN, URL_ADDRESS } from '../config.js'; +import request from 'supertest'; +import { downloadMockLayout1 } from '../../demoData/layout/downloadLayout.mock.js'; +import { deepStrictEqual } from 'assert'; + +export const apiPostDownloadTests = () => { + suite('POST /download', () => { + test('should return a GUID key', async () => { + const layoutBody = downloadMockLayout1; + await request(`${URL_ADDRESS}/api/download`) + .post(`?token=${OWNER_TEST_TOKEN}&user_id=1`) + .send(layoutBody) + .expect(201) + .expect((res) => deepStrictEqual(res.text?.length, 36)); + }); + + test('should NOT return a GUID key', async () => { + await request(`${URL_ADDRESS}/api/download`) + .post(`?token=${OWNER_TEST_TOKEN}`) + .send({ hello: 'world' }) + .expect(400) + .expect((res) => deepStrictEqual(res.text, 'Could not save download data')); + }); + }); +}; diff --git a/QualityControl/test/common/library/download/downloadMappers.test.js b/QualityControl/test/common/library/download/downloadMappers.test.js new file mode 100644 index 000000000..176007696 --- /dev/null +++ b/QualityControl/test/common/library/download/downloadMappers.test.js @@ -0,0 +1,297 @@ +/* eslint-disable @stylistic/js/max-len */ +/** + * @license + * Copyright 2019-2020 CERN and copyright holders of ALICE O2. + * See http://alice-o2.web.cern.ch/copyright for details of the copyright holders. + * All rights not expressly granted are reserved. + * + * This software is distributed under the terms of the GNU General Public + * License v3 (GPL Version 3), copied verbatim in the file "COPYING". + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +import { suite, test } from 'node:test'; +import assert, { deepStrictEqual } from 'node:assert'; +import { ObjectData } from '../../../../lib/utils/download/classes/data/ObjectData.js'; +import { ObjectDomain } from '../../../../lib/utils/download/classes/domain/ObjectDomain.js'; +import { TabData } from '../../../../lib/utils/download/classes/data/TabData.js'; +import { TabDomain } from '../../../../lib/utils/download/classes/domain/TabDomain.js'; +import { LayoutData } from '../../../../lib/utils/download/classes/data/LayoutData.js'; +import { LayoutDomain } from '../../../../lib/utils/download/classes/domain/LayoutDomain.js'; +import { LayoutDomainStorage } from '../../../../lib/utils/download/classes/domain/LayoutDomainStorage.js'; + +// Typical example of an object, with all info included, should map just fine. +const plainFullObjectObject = { + id: 'a379543d-a93c-11ef-9af4-0aa14016a1a2', + x: 0, + y: 0, + h: 1, + w: 1, + name: 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2', + options: [], + autoSize: false, + ignoreDefaults: false, +}; +// data +const expectFullObject = new ObjectData('a379543d-a93c-11ef-9af4-0aa14016a1a2', 0, 0, 1, 1, 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2', [], false, false); +// domain +const expectDomainFullOject = new ObjectDomain('a379543d-a93c-11ef-9af4-0aa14016a1a2', 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2'); + +// Object that has just enough information to be mapped to domain. +const plainSlimObjectObject = { + id: 'a379543d-a93c-11ef-9af4-0aa14016a1a2', + name: 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2', +}; + +// data +const expectSlimObject = new ObjectData('a379543d-a93c-11ef-9af4-0aa14016a1a2', 0, 0, 0, 0, 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2', [], false, false); +// domain +const expectDomainSlimObject = new ObjectDomain('a379543d-a93c-11ef-9af4-0aa14016a1a2', 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2'); + +// Object that misses the required id and thus is invalid when mapped to domain. +const plainEmptybjectObject = { + // id: "a379543d-a93c-11ef-9af4-0aa14016a1a2", + // x: 0, + // y: 0, + // h: 1, + // w: 1, + name: 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2', + // options: [], + // autoSize: false, + // ignoreDefaults: false +}; + +const expectEmptyobject = new ObjectData(undefined, 0, 0, 0, 0, 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2', [], false, false); + +// ----------------------- +// Tab fixtures +// ----------------------- +const plainObjFull = { + id: 'obj-1', + x: 1, + y: 2, + h: 3, + w: 4, + name: 'some/name', + options: [], + autoSize: false, + ignoreDefaults: false, +}; + +const plainObjSlim = { + id: 'obj-2', + name: 'other/name', +}; + +const expectObjFull = new ObjectData('obj-1', 1, 2, 3, 4, 'some/name', [], false, false); +const expectObjSlim = new ObjectData('obj-2', 0, 0, 0, 0, 'other/name', [], false, false); + +const expectDomainObjFull = expectObjFull.mapToDomain(); +const expectDomainObjSlim = expectObjSlim.mapToDomain(); + +// Tab with full info +const plainFullTab = { + id: 'tab-1', + name: 'Tab One', + objects: [plainObjFull, plainObjSlim], + columns: 3, +}; + +const expectFullTab = new TabData('tab-1', 'Tab One', [expectObjFull, expectObjSlim], 3); +const expectDomainFullTab = new TabDomain('tab-1', 'Tab One', [expectDomainObjFull, expectDomainObjSlim]); + +// Tab with minimal info (objects empty) +const plainSlimTab = { + id: 'tab-2', + name: 'Tab Two', + columns: 0, +}; + +const expectSlimTab = new TabData('tab-2', 'Tab Two', [], 0); + +// Tab missing id (invalid for mapping to domain) +const plainInvalidTab = { + name: 'NoIdTab', + objects: [plainObjSlim], + columns: 1, +}; + +const expectInvalidTab = new TabData(undefined, 'NoIdTab', [expectObjSlim], 1); + +// ----------------------- +// Layout fixtures +// ----------------------- +const plainObj = { + id: 'obj-a', + name: 'name/a', +}; +const expectObj = new ObjectData('obj-a', 0, 0, 0, 0, 'name/a', [], false, false); + +const plainTab = { + id: 'tab-a', + name: 'Tab A', + objects: [plainObj], + columns: 2, +}; +const expectTab = new TabData('tab-a', 'Tab A', [expectObj], 2); +const expectDomainTab = expectTab.mapToDomain(); + +// layout with full info +const plainFullLayout = { + id: 'layout-1', + name: 'Layout One', + owner_id: 123, + owner_name: 'owner', + tabs: [plainTab], + collaborators: [{ id: 1 }], + displayTimestamp: true, + autoTabChange: 5, + isOfficial: false, +}; + +const expectFullLayout = new LayoutData('layout-1', 'Layout One', 123, 'owner', [expectTab], [{ id: 1 }], true, 5, false); +const expectDomainFullLayout = new LayoutDomain('layout-1', 'Layout One', [expectDomainTab]); + +// layout missing tabs (empty tabs array) - invalid for domain mapping +const plainLayoutEmptyTabs = { + id: 'layout-2', + name: 'Empty Tabs', + owner_id: 0, + owner_name: 'nobody', + tabs: [], + collaborators: [], + displayTimestamp: false, + autoTabChange: 0, + isOfficial: false, +}; +const expectEmptyTabsLayout = new LayoutData('layout-2', 'Empty Tabs', 0, 'nobody', [], [], false, 0, false); + +// layout missing id -> mapFromPlain should throw +const plainInvalidLayout = { + name: 'Bad Layout', + owner_id: 1, + owner_name: 'x', + tabs: [plainTab], +}; + +// ----------------------- +// LayoutDomainStorage fixtures +// ----------------------- +const objDomain = new ObjectDomain('od-1', 'obj/name'); +const tabDomain = new TabDomain('td-1', 'TabDomain', [objDomain]); + +const expectDomain = new LayoutDomain('lds-1', 'LD S', [tabDomain]); + +// Suites +export const downloadTestSuite = () => { + suite('downloadMappers - test suite', () => { + test('should successfully return Data object', () => { + const obj1 = ObjectData.mapFromPlain(plainFullObjectObject); + const obj2 = ObjectData.mapFromPlain(plainSlimObjectObject); + const obj3 = ObjectData.mapFromPlain(plainEmptybjectObject); + deepStrictEqual(obj1, expectFullObject); + deepStrictEqual(obj2, expectSlimObject); + deepStrictEqual(obj3, expectEmptyobject); + }); + + test('should handle domain mapping guards properly', () => { + assert.doesNotThrow(() => { + expectFullObject.mapToDomain(); + }); + assert.doesNotThrow(() => { + expectSlimObject.mapToDomain(); + }); + assert.throws(() => { + expectEmptyobject.mapToDomain(); + }); + }); + + test('should handle domain mapping properly', () => { + const domainFullObject = expectFullObject.mapToDomain(); + const domainSlimObject = expectSlimObject.mapToDomain(); + + deepStrictEqual(expectDomainFullOject, domainFullObject); + deepStrictEqual(expectDomainSlimObject, domainSlimObject); + }); + + // Tab tests + test('TabData: should successfully return TabData object from plain', () => { + const t1 = TabData.mapFromPlain(plainFullTab); + const t2 = TabData.mapFromPlain(plainSlimTab); + const t3 = TabData.mapFromPlain(plainInvalidTab); + + deepStrictEqual(t1, expectFullTab); + deepStrictEqual(t2, expectSlimTab); + deepStrictEqual(t3, expectInvalidTab); + }); + + test('TabData: should handle domain mapping guards properly', () => { + assert.doesNotThrow(() => { + expectFullTab.mapToDomain(); + }); + + // expectSlimTab has empty objects -> mapping to domain should throw + assert.throws(() => { + expectSlimTab.mapToDomain(); + }); + + // expectInvalidTab has undefined id -> mapping to domain should throw + assert.throws(() => { + expectInvalidTab.mapToDomain(); + }); + }); + + test('TabData: should handle domain mapping properly', () => { + const domain = expectFullTab.mapToDomain(); + deepStrictEqual(expectDomainFullTab, domain); + }); + + // Layout tests + test('LayoutData: should successfully return LayoutData object from plain', () => { + const l1 = LayoutData.mapFromPlain(plainFullLayout); + deepStrictEqual(l1, expectFullLayout); + + const l2 = LayoutData.mapFromPlain(plainLayoutEmptyTabs); + deepStrictEqual(l2, expectEmptyTabsLayout); + + assert.throws(() => { + LayoutData.mapFromPlain(plainInvalidLayout); + }); + }); + + test('LayoutData: should handle domain mapping guards properly', () => { + assert.doesNotThrow(() => { + expectFullLayout.mapToDomain(); + }); + + // layout with empty tabs should throw when mapping to domain + assert.throws(() => { + expectEmptyTabsLayout.mapToDomain(); + }); + }); + + test('LayoutData: should handle domain mapping properly', () => { + const domain = expectFullLayout.mapToDomain(); + deepStrictEqual(expectDomainFullLayout, domain); + }); + + // LayoutDomainStorage tests + test('LayoutDomainStorage: should construct when downloadUserId != 0', () => { + assert.doesNotThrow(() => { + new LayoutDomainStorage('lds-1', 'LD S', [tabDomain], 42); + }); + const ls = new LayoutDomainStorage('lds-1', 'LD S', [tabDomain], 42); + deepStrictEqual(ls.toSuper(), expectDomain); + deepStrictEqual(ls.downloadUserId, 42); + }); + + test('LayoutDomainStorage: should throw when downloadUserId == 0', () => { + assert.throws(() => { + new LayoutDomainStorage('lds-1', 'LD S', [tabDomain], 0); + }); + }); + }); +}; diff --git a/QualityControl/test/demoData/layout/downloadLayout.mock.js b/QualityControl/test/demoData/layout/downloadLayout.mock.js new file mode 100644 index 000000000..6b9d4c822 --- /dev/null +++ b/QualityControl/test/demoData/layout/downloadLayout.mock.js @@ -0,0 +1,104 @@ +export const downloadMockLayout1 = +{ + id: '68de787140445a815bed58ce', + name: 'jasper Layout', + owner_id: 883859, + owner_name: 'Jasper Houweling', + description: '', + displayTimestamp: false, + autoTabChange: 0, + tabs: [ + { + id: '68de7871b410c936e50d8a35', + name: 'main', + objects: [ + { + id: 'a379543d-a93c-11ef-9af4-0aa14016a1a2', + x: 0, + y: 0, + h: 1, + w: 1, + name: 'qc/CPV/MO/NoiseOnFLP/BadChannelMapM2', + options: [], + autoSize: false, + ignoreDefaults: false, + }, + { + id: '90c5335a-098d-11ee-ac92-0aa14016a1a2', + x: 1, + y: 0, + h: 1, + w: 1, + name: 'qc/CPV/MO/NoiseOnFLP/DigitFreqM3', + options: [], + autoSize: false, + ignoreDefaults: false, + }, + ], + columns: 2, + }, + { + id: '68f63c82360430da6e2796df', + name: 'mft', + objects: [ + { + id: 'f3accff9-ae7b-11f0-b850-0aa14520a1a2', + x: 0, + y: 0, + h: 1, + w: 1, + name: 'qc/CTP/MO/CcdbInspector/ObjectsStatus', + options: [], + autoSize: false, + ignoreDefaults: false, + }, + { + id: 'e16fe7d5-f618-11ed-a4d4-808d4d66a1a2', + x: 1, + y: 0, + h: 1, + w: 1, + name: 'qc/FOC/MO/RawTask/HitmapPadASIC_3', + options: [], + autoSize: false, + ignoreDefaults: false, + }, + { + id: 'e13de1c6-f618-11ed-a4d4-808d4d66a1a2', + x: 2, + y: 0, + h: 1, + w: 1, + name: 'qc/FOC/MO/RawTask/CRUcounter', + options: [], + autoSize: false, + ignoreDefaults: false, + }, + { + id: '208b98aa-ae7c-11f0-b866-0aa14414a1a2', + x: 0, + y: 1, + h: 1, + w: 1, + name: 'qc/ITS/MO/FHRTask/General/ErrorPlots', + options: [], + autoSize: false, + ignoreDefaults: false, + }, + { + id: 'be8c0940-2d1e-11ec-b08a-0aa14028a1a5', + x: 1, + y: 1, + h: 1, + w: 1, + name: 'qc/FV0/MO/CalibrationTask/Calibrated_time', + options: [], + autoSize: false, + ignoreDefaults: false, + }, + ], + columns: 3, + }, + ], + collaborators: [], +}; diff --git a/QualityControl/test/test-index.js b/QualityControl/test/test-index.js index 16405e1aa..1c1d2c6a1 100644 --- a/QualityControl/test/test-index.js +++ b/QualityControl/test/test-index.js @@ -98,6 +98,8 @@ import { runModeServiceTestSuite } from './lib/services/RunModeService.test.js'; import { apiGetRunStatusTests } from './api/filters/api-get-run-status.test.js'; import { runModeTests } from './public/features/runMode.test.js'; import { aliecsSynchronizerTestSuite } from './lib/services/external/AliEcsSynchronizer.test.js'; +import { apiPostDownloadTests } from './api/download/api-post-download.test.js'; +import { downloadTestSuite } from './common/library/download/downloadMappers.test.js'; const FRONT_END_PER_TEST_TIMEOUT = 5000; // each front-end test is allowed this timeout // remaining tests are based on the number of individual tests in each suite @@ -202,6 +204,7 @@ suite('All Tests - QCG', { timeout: FRONT_END_TIMEOUT + BACK_END_TIMEOUT }, asyn }); suite('Layout GET request test suite', async () => apiGetLayoutsTests()); + suite('Layout POST request test suite', async () => apiPostDownloadTests()); suite('Layout PUT request test suite', async () => apiPutLayoutTests()); suite('Layout PATCH request test suite', async () => apiPatchLayoutTests()); suite('Object GET request test suite', async () => apiGetObjectsTests()); @@ -214,6 +217,10 @@ suite('All Tests - QCG', { timeout: FRONT_END_TIMEOUT + BACK_END_TIMEOUT }, asyn suite('Utility "httpRequests" methods test suite', async () => await httpRequestsTestSuite()); }); + suite('Download - Test Suite', () => { + suite('Download mapper test suite', () => downloadTestSuite()); + }); + suite('Common Library - Test Suite', () => { suite('CL - Object Utility methods test suite', () => commonLibraryQcObjectUtilsTestSuite()); suite('CL - DateTime Utility methods test suite', () => commonLibraryUtilsDateTimeTestSuite());