diff --git a/client/modules/IDE/actions/project.js b/client/modules/IDE/actions/project.js index 7787f879bb..9577c8e427 100644 --- a/client/modules/IDE/actions/project.js +++ b/client/modules/IDE/actions/project.js @@ -306,6 +306,8 @@ export function cloneProject(project) { generateNewIdsForChildren(rootFile, newFiles); // duplicate all files hosted on S3 + const copiedS3Assets = []; + each( newFiles, (file, callback) => { @@ -316,24 +318,20 @@ export function cloneProject(project) { (file.url.includes(S3_BUCKET_URL_BASE) || file.url.includes(S3_BUCKET)) ) { - const formParams = { - url: file.url - }; - apiClient.post('/S3/copy', formParams).then((response) => { + apiClient.post('/S3/copy', { url: file.url }).then((response) => { file.url = response.data.url; + copiedS3Assets.push(response.data.url); callback(null); }); } else { callback(null); } }, - (err) => { - // if not errors in duplicating the files on S3, then duplicate it - const formParams = Object.assign( - {}, - { name: `${projectName} copy` }, - { files: newFiles } - ); + () => { + const formParams = { + name: `${projectName} copy`, + files: newFiles + }; apiClient .post('/projects', formParams) .then((response) => { @@ -343,6 +341,12 @@ export function cloneProject(project) { dispatch(setNewProject(response.data)); }) .catch((error) => { + copiedS3Assets.forEach((url) => { + const objectKey = url.split('/').pop(); + apiClient + .delete(`/S3/delete?objectKey=${objectKey}`) + .catch(() => {}); + }); dispatch({ type: ActionTypes.PROJECT_SAVE_FAIL, error: error?.response?.data diff --git a/server/controllers/aws.controller.js b/server/controllers/aws.controller.js index b6e03db13c..4effbef69d 100644 --- a/server/controllers/aws.controller.js +++ b/server/controllers/aws.controller.js @@ -32,7 +32,12 @@ function getExtension(filename) { } export function getObjectKey(url) { - const urlArray = url.split('/'); + const pendingMarker = '/pending/'; + const pendingIndex = url.indexOf(pendingMarker); + if (pendingIndex !== -1) { + return url.substring(pendingIndex + 1).split('?')[0]; + } + const urlArray = url.split('?')[0].split('/'); const objectKey = urlArray.pop(); const userId = urlArray.pop(); if (ObjectId.isValid(userId) && userId === new ObjectId(userId).toString()) { @@ -156,7 +161,7 @@ export async function signS3(req, res) { const acl = 'public-read'; const policy = S3Policy.generate({ acl, - key: `${req.body.userId}/${filename}`, + key: `pending/${req.user.id}/${filename}`, bucket: process.env.S3_BUCKET, contentType: req.body.type, region: process.env.AWS_REGION, diff --git a/server/controllers/project.controller.js b/server/controllers/project.controller.js index 57eda81381..91341a4a6f 100644 --- a/server/controllers/project.controller.js +++ b/server/controllers/project.controller.js @@ -11,6 +11,10 @@ import Project from '../models/project'; import { User } from '../models/user'; import { resolvePathToFile } from '../utils/filePath'; import { generateFileSystemSafeName } from '../utils/generateFileSystemSafeName'; +import { + commitPendingAssets, + rewritePendingFileUrls +} from '../utils/pendingAssets'; const s3Client = new S3Client({ credentials: { @@ -65,6 +69,12 @@ export async function updateProject(req, res) { updateData[field] = req.body[field]; } }); + + if (updateData.files) { + await commitPendingAssets(req.user.id, updateData.files); + updateData.files = rewritePendingFileUrls(updateData.files, req.user.id); + } + const updatedProject = await Project.findByIdAndUpdate( req.params.project_id, { @@ -77,6 +87,7 @@ export async function updateProject(req, res) { ) .populate('user', 'username') .exec(); + if ( req.body.files && updatedProject.files.length !== req.body.files.length diff --git a/server/controllers/project.controller/createProject.js b/server/controllers/project.controller/createProject.js index 001e007155..0315fec11d 100644 --- a/server/controllers/project.controller/createProject.js +++ b/server/controllers/project.controller/createProject.js @@ -4,26 +4,33 @@ import { FileValidationError, ProjectValidationError } from '../../domain-objects/Project'; +import { + commitPendingAssets, + rewritePendingFileUrls +} from '../../utils/pendingAssets'; -export default function createProject(req, res) { - const projectValues = Object.assign({}, req.body, { user: req.user._id }); +export default async function createProject(req, res) { + try { + const projectValues = Object.assign({}, req.body, { user: req.user._id }); - function sendFailure(err) { - res.status(400).json({ success: false }); - } + if (projectValues.files) { + await commitPendingAssets(req.user.id, projectValues.files); + projectValues.files = rewritePendingFileUrls( + projectValues.files, + req.user.id + ); + } - function populateUserData(newProject) { - return Project.populate(newProject, { + const newProject = await Project.create(projectValues); + const newProjectWithUser = await Project.populate(newProject, { path: 'user', select: 'username' - }).then((newProjectWithUser) => { - res.json(newProjectWithUser); }); - } - return Project.create(projectValues) - .then(populateUserData) - .catch(sendFailure); + res.json(newProjectWithUser); + } catch (err) { + res.status(400).json({ success: false }); + } } // TODO: What happens if you don't supply any files? @@ -86,7 +93,13 @@ export async function apiCreateProject(req, res) { throw error; } + if (model.files) { + await commitPendingAssets(req.user.id, model.files); + model.files = rewritePendingFileUrls(model.files, req.user.id); + } + const newProject = await model.save(); + res.status(201).json({ id: newProject.id }); } catch (err) { handleErrors(err); diff --git a/server/utils/pendingAssets.js b/server/utils/pendingAssets.js new file mode 100644 index 0000000000..a3cd0890d2 --- /dev/null +++ b/server/utils/pendingAssets.js @@ -0,0 +1,76 @@ +import { + S3Client, + CopyObjectCommand, + DeleteObjectsCommand +} from '@aws-sdk/client-s3'; + +const s3Client = new S3Client({ + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY, + secretAccessKey: process.env.AWS_SECRET_KEY + }, + region: process.env.AWS_REGION +}); + +function getPendingKeyFromUrl(url, userId) { + const marker = `pending/${userId}/`; + if (!url || !url.includes(marker)) { + return null; + } + const filename = url.split('?')[0].split('/').pop(); + return `pending/${userId}/${filename}`; +} + +export function rewritePendingFileUrls(files, userId) { + const marker = `pending/${userId}/`; + const replacement = `${userId}/`; + return files.map((file) => { + if (file.url && file.url.includes(marker)) { + return Object.assign({}, file, { + url: file.url.replace(marker, replacement) + }); + } + return file; + }); +} + +async function moveAssetFromPending(pendingKey, userId) { + const filename = pendingKey.split('/').pop(); + const destinationKey = `${userId}/${filename}`; + + await s3Client.send( + new CopyObjectCommand({ + Bucket: process.env.S3_BUCKET, + CopySource: `${process.env.S3_BUCKET}/${pendingKey}`, + Key: destinationKey, + ACL: 'public-read' + }) + ); + + await s3Client.send( + new DeleteObjectsCommand({ + Bucket: process.env.S3_BUCKET, + Delete: { Objects: [{ Key: pendingKey }] } + }) + ); + + return destinationKey; +} + +export async function commitPendingAssets(userId, files = []) { + const pendingKeys = [ + ...new Set( + files + .map((file) => getPendingKeyFromUrl(file.url, userId)) + .filter(Boolean) + ) + ]; + + if (pendingKeys.length === 0) { + return []; + } + + return Promise.all( + pendingKeys.map((key) => moveAssetFromPending(key, userId)) + ); +}