Orphaned Assets Fix

client/modules/IDE/actions/project.js

@@ -306,6 +306,8 @@ export function cloneProject(project) {
     generateNewIdsForChildren(rootFile, newFiles);
 
     // duplicate all files hosted on S3
+    const copiedS3Assets = [];
+
     each(
       newFiles,
       (file, callback) => {
@@ -316,24 +318,20 @@ export function cloneProject(project) {
           (file.url.includes(S3_BUCKET_URL_BASE) ||
             file.url.includes(S3_BUCKET))
         ) {
-          const formParams = {
-            url: file.url
-          };
-          apiClient.post('/S3/copy', formParams).then((response) => {
+          apiClient.post('/S3/copy', { url: file.url }).then((response) => {
             file.url = response.data.url;
+            copiedS3Assets.push(response.data.url);
             callback(null);
           });
         } else {
           callback(null);
         }
       },
-      (err) => {
-        // if not errors in duplicating the files on S3, then duplicate it
-        const formParams = Object.assign(
-          {},
-          { name: `${projectName} copy` },
-          { files: newFiles }
-        );
+      () => {
+        const formParams = {
+          name: `${projectName} copy`,
+          files: newFiles
+        };
         apiClient
           .post('/projects', formParams)
           .then((response) => {
@@ -343,6 +341,12 @@ export function cloneProject(project) {
             dispatch(setNewProject(response.data));
           })
           .catch((error) => {
+            copiedS3Assets.forEach((url) => {
+              const objectKey = url.split('/').pop();
+              apiClient
+                .delete(`/S3/delete?objectKey=${objectKey}`)
+                .catch(() => {});
+            });
             dispatch({
               type: ActionTypes.PROJECT_SAVE_FAIL,
               error: error?.response?.data

server/controllers/aws.controller.js

@@ -32,7 +32,12 @@ function getExtension(filename) {
 }
 
 export function getObjectKey(url) {
-  const urlArray = url.split('/');
+  const pendingMarker = '/pending/';
+  const pendingIndex = url.indexOf(pendingMarker);
+  if (pendingIndex !== -1) {
+    return url.substring(pendingIndex + 1).split('?')[0];
+  }
+  const urlArray = url.split('?')[0].split('/');
   const objectKey = urlArray.pop();
   const userId = urlArray.pop();
   if (ObjectId.isValid(userId) && userId === new ObjectId(userId).toString()) {
@@ -156,7 +161,7 @@ export async function signS3(req, res) {
     const acl = 'public-read';
     const policy = S3Policy.generate({
       acl,
-      key: `${req.body.userId}/${filename}`,
+      key: `pending/${req.user.id}/${filename}`,
       bucket: process.env.S3_BUCKET,
       contentType: req.body.type,
       region: process.env.AWS_REGION,

server/controllers/project.controller.js

@@ -11,6 +11,10 @@ import Project from '../models/project';
 import { User } from '../models/user';
 import { resolvePathToFile } from '../utils/filePath';
 import { generateFileSystemSafeName } from '../utils/generateFileSystemSafeName';
+import {
+  commitPendingAssets,
+  rewritePendingFileUrls
+} from '../utils/pendingAssets';
 
 const s3Client = new S3Client({
   credentials: {
@@ -65,6 +69,12 @@ export async function updateProject(req, res) {
         updateData[field] = req.body[field];
       }
     });
+
+    if (updateData.files) {
+      await commitPendingAssets(req.user.id, updateData.files);
+      updateData.files = rewritePendingFileUrls(updateData.files, req.user.id);
+    }
+
     const updatedProject = await Project.findByIdAndUpdate(
       req.params.project_id,
       {
@@ -77,6 +87,7 @@ export async function updateProject(req, res) {
     )
       .populate('user', 'username')
       .exec();
+
     if (
       req.body.files &&
       updatedProject.files.length !== req.body.files.length

server/controllers/project.controller/createProject.js

@@ -4,26 +4,33 @@ import {
   FileValidationError,
   ProjectValidationError
 } from '../../domain-objects/Project';
+import {
+  commitPendingAssets,
+  rewritePendingFileUrls
+} from '../../utils/pendingAssets';
 
-export default function createProject(req, res) {
-  const projectValues = Object.assign({}, req.body, { user: req.user._id });
+export default async function createProject(req, res) {
+  try {
+    const projectValues = Object.assign({}, req.body, { user: req.user._id });
 
-  function sendFailure(err) {
-    res.status(400).json({ success: false });
-  }
+    if (projectValues.files) {
+      await commitPendingAssets(req.user.id, projectValues.files);
+      projectValues.files = rewritePendingFileUrls(
+        projectValues.files,
+        req.user.id
+      );
+    }
 
-  function populateUserData(newProject) {
-    return Project.populate(newProject, {
+    const newProject = await Project.create(projectValues);
+    const newProjectWithUser = await Project.populate(newProject, {
       path: 'user',
       select: 'username'
-    }).then((newProjectWithUser) => {
-      res.json(newProjectWithUser);
     });
-  }
 
-  return Project.create(projectValues)
-    .then(populateUserData)
-    .catch(sendFailure);
+    res.json(newProjectWithUser);
+  } catch (err) {
+    res.status(400).json({ success: false });
+  }
 }
 
 // TODO: What happens if you don't supply any files?
@@ -86,7 +93,13 @@ export async function apiCreateProject(req, res) {
       throw error;
     }
 
+    if (model.files) {
+      await commitPendingAssets(req.user.id, model.files);
+      model.files = rewritePendingFileUrls(model.files, req.user.id);
+    }
+
     const newProject = await model.save();
+
     res.status(201).json({ id: newProject.id });
   } catch (err) {
     handleErrors(err);

server/utils/pendingAssets.js

@@ -0,0 +1,76 @@
+import {
+  S3Client,
+  CopyObjectCommand,
+  DeleteObjectsCommand
+} from '@aws-sdk/client-s3';
+
+const s3Client = new S3Client({
+  credentials: {
+    accessKeyId: process.env.AWS_ACCESS_KEY,
+    secretAccessKey: process.env.AWS_SECRET_KEY
+  },
+  region: process.env.AWS_REGION
+});
+
+function getPendingKeyFromUrl(url, userId) {
+  const marker = `pending/${userId}/`;
+  if (!url || !url.includes(marker)) {
+    return null;
+  }
+  const filename = url.split('?')[0].split('/').pop();
+  return `pending/${userId}/${filename}`;
+}
+
+export function rewritePendingFileUrls(files, userId) {
+  const marker = `pending/${userId}/`;
+  const replacement = `${userId}/`;
+  return files.map((file) => {
+    if (file.url && file.url.includes(marker)) {
+      return Object.assign({}, file, {
+        url: file.url.replace(marker, replacement)
+      });
+    }
+    return file;
+  });
+}
+
+async function moveAssetFromPending(pendingKey, userId) {
+  const filename = pendingKey.split('/').pop();
+  const destinationKey = `${userId}/${filename}`;
+
+  await s3Client.send(
+    new CopyObjectCommand({
+      Bucket: process.env.S3_BUCKET,
+      CopySource: `${process.env.S3_BUCKET}/${pendingKey}`,
+      Key: destinationKey,
+      ACL: 'public-read'
+    })
+  );
+
+  await s3Client.send(
+    new DeleteObjectsCommand({
+      Bucket: process.env.S3_BUCKET,
+      Delete: { Objects: [{ Key: pendingKey }] }
+    })
+  );
+
+  return destinationKey;
+}
+
+export async function commitPendingAssets(userId, files = []) {
+  const pendingKeys = [
+    ...new Set(
+      files
+        .map((file) => getPendingKeyFromUrl(file.url, userId))
+        .filter(Boolean)
+    )
+  ];
+
+  if (pendingKeys.length === 0) {
+    return [];
+  }
+
+  return Promise.all(
+    pendingKeys.map((key) => moveAssetFromPending(key, userId))
+  );
+}