feat: Make Python Virtual Environment Persistent: Add Environments to Left Panel

access-control-service/src/main/scala/org/apache/texera/service/resource/AccessControlResource.scala

@@ -23,7 +23,7 @@ import com.typesafe.scalalogging.LazyLogging
 import jakarta.annotation.security.{PermitAll, RolesAllowed}
 import jakarta.ws.rs.client.{Client, ClientBuilder, Entity}
 import jakarta.ws.rs.core._
-import jakarta.ws.rs.{Consumes, DELETE, GET, POST, Path, Produces}
+import jakarta.ws.rs.{Consumes, DELETE, GET, POST, PUT, Path, Produces}
 import org.apache.texera.auth.JwtParser.parseToken
 import org.apache.texera.auth.SessionUser
 import org.apache.texera.auth.util.{ComputingUnitAccess, HeaderField}
@@ -233,6 +233,16 @@ class AccessControlResource extends LazyLogging {
     AccessControlResource.authorize(uriInfo, headers, Option(body).map(_.trim).filter(_.nonEmpty))
   }
 
+  @PUT
+  @Path("/{path:.*}")
+  def authorizePut(
+      @Context uriInfo: UriInfo,
+      @Context headers: HttpHeaders,
+      body: String
+  ): Response = {
+    AccessControlResource.authorize(uriInfo, headers, Option(body).map(_.trim).filter(_.nonEmpty))
+  }
+
   @DELETE
   @Path("/{path:.*}")
   def authorizeDelete(

amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala

@@ -26,6 +26,10 @@ import scala.jdk.CollectionConverters._
 import scala.sys.process._
 import java.util.Comparator
 import org.apache.texera.amber.config.PythonUtils
+import org.apache.texera.dao.SqlServer
+import org.apache.texera.dao.jooq.generated.tables.daos.PythonVirtualEnvironmentsDao
+import org.apache.texera.dao.jooq.generated.tables.pojos.PythonVirtualEnvironments
+import org.jooq.JSONB
 
 /**
   * PveManager is responsible for managing Python Virtual Environments (PVEs)
@@ -47,6 +51,8 @@ object PveManager {
       userPackages: Seq[String]
   )
 
+  case class StoredPve(pveid: Int, name: String, packagesJson: String)
+
   private val VenvRoot: Path = Paths.get("/tmp/texera-pve/venvs")
 
   private val SafePveName = "^[A-Za-z0-9._-]+$".r
@@ -213,6 +219,72 @@ object PveManager {
     queue.put(s"[PVE] Created new environment for cuid = $cuid")
   }
 
+  // Returns every PVE row belonging to the given user.
+  def listPvesForUser(uid: Int): List[StoredPve] = {
+    import org.apache.texera.dao.jooq.generated.Tables.PYTHON_VIRTUAL_ENVIRONMENTS
+    SqlServer
+      .getInstance()
+      .createDSLContext()
+      .selectFrom(PYTHON_VIRTUAL_ENVIRONMENTS)
+      .where(PYTHON_VIRTUAL_ENVIRONMENTS.UID.eq(uid))
+      .fetchInto(classOf[PythonVirtualEnvironments])
+      .asScala
+      .map { row =>
+        val pkgsJson = Option(row.getPackages).map(_.data).getOrElse("{}")
+        StoredPve(row.getPveid, row.getName, pkgsJson)
+      }
+      .toList
+  }
+
+  // Deletes a PVE row owned by `uid`. Returns true if a row was deleted, false if no
+  // matching row was found (either the pveid doesn't exist or it belongs to another user).
+  def deletePveFromDb(pveid: Int, uid: Int): Boolean = {
+    import org.apache.texera.dao.jooq.generated.Tables.PYTHON_VIRTUAL_ENVIRONMENTS
+    val rows = SqlServer
+      .getInstance()
+      .createDSLContext()
+      .deleteFrom(PYTHON_VIRTUAL_ENVIRONMENTS)
+      .where(
+        PYTHON_VIRTUAL_ENVIRONMENTS.PVEID
+          .eq(pveid)
+          .and(PYTHON_VIRTUAL_ENVIRONMENTS.UID.eq(uid))
+      )
+      .execute()
+    rows > 0
+  }
+
+  // Updates an existing PVE row owned by `uid`. Returns true if a row was
+  // updated, false if no matching row was found.
+  def updatePve(pveid: Int, uid: Int, name: String, packagesJson: String): Boolean = {
+    import org.apache.texera.dao.jooq.generated.Tables.PYTHON_VIRTUAL_ENVIRONMENTS
+    val rows = SqlServer
+      .getInstance()
+      .createDSLContext()
+      .update(PYTHON_VIRTUAL_ENVIRONMENTS)
+      .set(PYTHON_VIRTUAL_ENVIRONMENTS.NAME, name)
+      .set(PYTHON_VIRTUAL_ENVIRONMENTS.PACKAGES, JSONB.valueOf(packagesJson))
+      .where(
+        PYTHON_VIRTUAL_ENVIRONMENTS.PVEID
+          .eq(pveid)
+          .and(PYTHON_VIRTUAL_ENVIRONMENTS.UID.eq(uid))
+      )
+      .execute()
+    rows > 0
+  }
+
+  // Persists a PVE spec (name + packages JSON) for the given user. Returns the new pveid.
+  def savePve(uid: Int, name: String, packagesJson: String): Int = {
+    val row = new PythonVirtualEnvironments()
+    row.setUid(uid)
+    row.setName(name)
+    row.setPackages(JSONB.valueOf(packagesJson))
+    val dao = new PythonVirtualEnvironmentsDao(
+      SqlServer.getInstance().createDSLContext().configuration
+    )
+    dao.insert(row)
+    row.getPveid
+  }
+
   // returns list of PVE names and corresponding user packages for a given CU
   def getEnvironments(cuid: Int): List[PvePackageResponse] = {
 

amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala

@@ -19,6 +19,11 @@
 
 package org.apache.texera.web.resource.pythonvirtualenvironment
 
+import com.fasterxml.jackson.core.`type`.TypeReference
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+import io.dropwizard.auth.Auth
+import org.apache.texera.auth.SessionUser
 import org.apache.texera.config.KubernetesConfig
 
 import javax.ws.rs._
@@ -29,9 +34,18 @@ import javax.ws.rs.DELETE
 import javax.ws.rs.PathParam
 import javax.ws.rs.core.Response
 
+object PveResource {
+  case class SavePvePayload(name: String, packages: Map[String, String])
+  case class PveListItem(pveid: Int, name: String, packages: Map[String, String])
+}
+
 @Path("/pve")
 @Consumes(Array(MediaType.APPLICATION_JSON))
 class PveResource {
+  import PveResource._
+
+  private val mapper: ObjectMapper = new ObjectMapper().registerModule(DefaultScalaModule)
+  private val packagesType = new TypeReference[java.util.Map[String, String]] {}
   // --------------------------------------------------
   // Get system packages
   // --------------------------------------------------
@@ -54,6 +68,90 @@ class PveResource {
     }
   }
 
+  // --------------------------------------------------
+  // List all PVEs for the current user from the database
+  // --------------------------------------------------
+  @GET
+  @Path("/db")
+  @Produces(Array(MediaType.APPLICATION_JSON))
+  def listPves(@Auth sessionUser: SessionUser): java.util.List[PveListItem] = {
+    PveManager
+      .listPvesForUser(sessionUser.getUid.intValue())
+      .map { stored =>
+        val packages: Map[String, String] =
+          try mapper.readValue(stored.packagesJson, packagesType).asScala.toMap
+          catch { case _: Throwable => Map.empty[String, String] }
+        PveListItem(stored.pveid, stored.name, packages)
+      }
+      .asJava
+  }
+
+  // --------------------------------------------------
+  // Update a PVE row owned by the current user
+  // --------------------------------------------------
+  @PUT
+  @Path("/db/{pveid}")
+  @Produces(Array(MediaType.APPLICATION_JSON))
+  def updatePve(
+      @PathParam("pveid") pveid: Int,
+      payload: SavePvePayload,
+      @Auth sessionUser: SessionUser
+  ): Response = {
+    val name = Option(payload.name).map(_.trim).getOrElse("")
+    if (name.isEmpty) {
+      return Response
+        .status(Response.Status.BAD_REQUEST)
+        .entity("name is required")
+        .build()
+    }
+    try {
+      val packagesJson = mapper.writeValueAsString(payload.packages)
+      val updated = PveManager.updatePve(pveid, sessionUser.getUid.intValue(), name, packagesJson)
+      if (updated) Response.ok(Map("pveid" -> pveid).asJava).build()
+      else Response.status(Response.Status.NOT_FOUND).build()
+    } catch {
+      case e: Exception =>
+        e.printStackTrace()
+        throw new InternalServerErrorException(s"Failed to update PVE: ${e.getMessage}")
+    }
+  }
+
+  // --------------------------------------------------
+  // Delete a PVE row owned by the current user
+  // --------------------------------------------------
+  @DELETE
+  @Path("/db/{pveid}")
+  def deletePveFromDb(@PathParam("pveid") pveid: Int, @Auth sessionUser: SessionUser): Response = {
+    val deleted = PveManager.deletePveFromDb(pveid, sessionUser.getUid.intValue())
+    if (deleted) Response.noContent().build()
+    else Response.status(Response.Status.NOT_FOUND).build()
+  }
+
+  // --------------------------------------------------
+  // Save a PVE (name + packages) to the database for the current user
+  // --------------------------------------------------
+  @POST
+  @Path("/db")
+  @Produces(Array(MediaType.APPLICATION_JSON))
+  def savePve(payload: SavePvePayload, @Auth sessionUser: SessionUser): Response = {
+    val name = Option(payload.name).map(_.trim).getOrElse("")
+    if (name.isEmpty) {
+      return Response
+        .status(Response.Status.BAD_REQUEST)
+        .entity("name is required")
+        .build()
+    }
+    try {
+      val packagesJson = mapper.writeValueAsString(payload.packages)
+      val pveid = PveManager.savePve(sessionUser.getUid.intValue(), name, packagesJson)
+      Response.ok(Map("pveid" -> pveid).asJava).build()
+    } catch {
+      case e: Exception =>
+        e.printStackTrace()
+        throw new InternalServerErrorException(s"Failed to save PVE: ${e.getMessage}")
+    }
+  }
+
   // --------------------------------------------------
   // Fetch PVEs and Installed User Packages
   // --------------------------------------------------

amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala

@@ -19,25 +19,58 @@
 
 package org.apache.texera.web.resource.pythonvirtualenvironment
 
-import org.scalatest.BeforeAndAfterEach
+import org.apache.texera.dao.MockTexeraDB
+import org.apache.texera.dao.jooq.generated.Tables.PYTHON_VIRTUAL_ENVIRONMENTS
+import org.apache.texera.dao.jooq.generated.tables.daos.UserDao
+import org.apache.texera.dao.jooq.generated.tables.pojos.User
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 
 import java.nio.file.{Files, Path, Paths}
+import java.util.UUID
 import java.util.concurrent.LinkedBlockingQueue
 import scala.jdk.CollectionConverters._
 
-class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach {
+class PveResourceSpec
+    extends AnyFlatSpec
+    with Matchers
+    with BeforeAndAfterAll
+    with BeforeAndAfterEach
+    with MockTexeraDB {
 
   private val testCuid = 256
+  // Randomised to avoid colliding with unrelated specs that may seed uids in the
+  // same embedded postgres if they run in parallel.
+  private val testUid = 8000 + scala.util.Random.nextInt(1000)
   private var testPveName: String = _
   private var testRoot: Path = _
   private var queue: LinkedBlockingQueue[String] = _
 
+  override protected def beforeAll(): Unit = {
+    initializeDBAndReplaceDSLContext()
+    // python_virtual_environments.uid has an FK to user(uid); seed one user
+    // for the DB-backed tests below to attach their rows to.
+    val userDao = new UserDao(getDSLContext.configuration())
+    val user = new User
+    user.setUid(testUid)
+    user.setName("pve_resource_spec_user")
+    user.setEmail(s"user_${UUID.randomUUID()}@example.com")
+    user.setPassword("password")
+    userDao.insert(user)
+  }
+
+  override protected def afterAll(): Unit = shutdownDB()
+
   override protected def beforeEach(): Unit = {
     testPveName = s"testenv${System.currentTimeMillis()}"
     testRoot = Paths.get("/tmp/texera-pve/venvs").resolve(testCuid.toString)
     queue = new LinkedBlockingQueue[String]()
+    // Clean any PVE rows left over from a prior test in this class.
+    getDSLContext
+      .deleteFrom(PYTHON_VIRTUAL_ENVIRONMENTS)
+      .where(PYTHON_VIRTUAL_ENVIRONMENTS.UID.eq(testUid))
+      .execute()
   }
 
   override protected def afterEach(): Unit = {
@@ -173,4 +206,44 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach
     PveManager.getPythonBin(testCuid, "name with spaces") shouldBe None
     PveManager.getPythonBin(testCuid, "name;rm") shouldBe None
   }
+
+  "PveManager.savePve + listPvesForUser" should "round-trip a row for the owning user" in {
+    val pveid = PveManager.savePve(testUid, "env-a", """{"numpy":"==1.26.0"}""")
+    pveid should be > 0
+
+    val rows = PveManager.listPvesForUser(testUid)
+    rows.map(_.name) should contain("env-a")
+    val row = rows.find(_.pveid == pveid).get
+    row.name shouldBe "env-a"
+    // Postgres JSONB normalises whitespace on read-back, so assert key/value separately
+    // rather than matching a literal JSON string.
+    row.packagesJson should include(""""numpy"""")
+    row.packagesJson should include(""""==1.26.0"""")
+  }
+
+  "PveManager.updatePve" should "mutate an owned row and refuse rows owned by someone else" in {
+    val pveid = PveManager.savePve(testUid, "env-b", "{}")
+
+    PveManager.updatePve(pveid, testUid, "env-b-renamed", """{"pandas":""}""") shouldBe true
+
+    val updated = PveManager.listPvesForUser(testUid).find(_.pveid == pveid).get
+    updated.name shouldBe "env-b-renamed"
+    updated.packagesJson should include(""""pandas"""")
+
+    // A different uid claiming the same pveid must not be able to update it.
+    val otherUid = testUid + 1
+    PveManager.updatePve(pveid, otherUid, "hijacked", "{}") shouldBe false
+    PveManager.listPvesForUser(testUid).find(_.pveid == pveid).get.name shouldBe "env-b-renamed"
+  }
+
+  "PveManager.deletePveFromDb" should "remove an owned row and return false for missing pveids" in {
+    val pveid = PveManager.savePve(testUid, "env-c", "{}")
+
+    PveManager.deletePveFromDb(pveid, testUid) shouldBe true
+    PveManager.listPvesForUser(testUid).map(_.pveid) should not contain pveid
+
+    // Already-deleted (or never-existed) pveid: deleter reports false, doesn't throw.
+    PveManager.deletePveFromDb(pveid, testUid) shouldBe false
+    PveManager.deletePveFromDb(-1, testUid) shouldBe false
+  }
 }

frontend/src/app/app-routing.constant.ts

@@ -35,6 +35,7 @@ export const USER_WORKFLOW = `${USER}/workflow`;
 export const USER_DATASET = `${USER}/dataset`;
 export const USER_DATASET_CREATE = `${USER_DATASET}/create`;
 export const USER_COMPUTING_UNIT = `${USER}/compute`;
+export const USER_PYTHON_VENV = `${USER}/python-venv`;
 export const USER_QUOTA = `${USER}/quota`;
 export const USER_DISCUSSION = `${USER}/discussion`;
 

frontend/src/app/app-routing.module.ts

@@ -25,6 +25,7 @@ import { UserQuotaComponent } from "./dashboard/component/user/user-quota/user-q
 import { UserProjectSectionComponent } from "./dashboard/component/user/user-project/user-project-section/user-project-section.component";
 import { UserProjectComponent } from "./dashboard/component/user/user-project/user-project.component";
 import { UserComputingUnitComponent } from "./dashboard/component/user/user-computing-unit/user-computing-unit.component";
+import { UserPythonVenvComponent } from "./dashboard/component/user/user-python-venv/user-python-venv.component";
 import { WorkspaceComponent } from "./workspace/component/workspace.component";
 import { AboutComponent } from "./hub/component/about/about.component";
 import { AuthGuardService } from "./common/service/user/auth-guard.service";
@@ -128,6 +129,10 @@ routes.push({
           path: "compute",
           component: UserComputingUnitComponent,
         },
+        {
+          path: "python-venv",
+          component: UserPythonVenvComponent,
+        },
         {
           path: "quota",
           component: UserQuotaComponent,

frontend/src/app/app.module.ts

@@ -192,6 +192,7 @@ import { NzCheckboxModule } from "ng-zorro-antd/checkbox";
 import { RegistrationRequestModalComponent } from "./common/service/user/registration-request-modal/registration-request-modal.component";
 import { UserComputingUnitComponent } from "./dashboard/component/user/user-computing-unit/user-computing-unit.component";
 import { UserComputingUnitListItemComponent } from "./dashboard/component/user/user-computing-unit/user-computing-unit-list-item/user-computing-unit-list-item.component";
+import { UserPythonVenvComponent } from "./dashboard/component/user/user-python-venv/user-python-venv.component";
 
 registerLocaleData(en);
 
@@ -361,6 +362,7 @@ registerLocaleData(en);
     MarkdownDescriptionComponent,
     UserComputingUnitComponent,
     UserComputingUnitListItemComponent,
+    UserPythonVenvComponent,
   ],
   providers: [
     provideNzI18n(en_US),