Memory use improvements

hypha/apply/dashboard/views.py

@@ -53,35 +53,31 @@ def paf_for_review(self):
         if not getattr(self.request.user, self.paf_reviewer_role, False):
             return {"count": None, "table": None}
         project_settings = ProjectSettings.for_request(self.request)
-        paf_approvals = list(
-            get_paf_for_review(
-                user=self.request.user,
-                is_paf_approval_sequential=project_settings.paf_approval_sequential,
-            ).select_related("project__submission__page", "paf_reviewer_role", "user")
-        )
+        paf_approvals = get_paf_for_review(
+            user=self.request.user,
+            is_paf_approval_sequential=project_settings.paf_approval_sequential,
+        ).select_related("project__submission__page", "paf_reviewer_role", "user")
         paf_table = PAFForReviewDashboardTable(
             paf_approvals, prefix="paf-review-", order_by="-date_requested"
         )
         RequestConfig(self.request, paginate=False).configure(paf_table)
         return {
-            "count": len(paf_approvals),
+            "count": paf_approvals.count(),
             "table": paf_table,
         }
 
 
 class HistoricalSubmissionMixin:
     def historical_submission_data(self):
-        historical_submissions = list(
-            ApplicationSubmission.objects.filter(
-                user=self.request.user,
-            )
+        qs = (
+            ApplicationSubmission.objects.filter(user=self.request.user)
             .inactive()
             .current()
             .for_table(self.request.user)
         )
         return {
-            "count": len(historical_submissions),
-            "table": SubmissionsTable(data=historical_submissions),
+            "count": qs.count(),
+            "table": SubmissionsTable(data=qs),
         }
 
 
@@ -120,47 +116,44 @@ def my_tasks(self):
         }
 
     def awaiting_reviews(self, submissions):
+        limit = 5
         submissions = submissions.in_review_for(self.request.user).order_by(
             "-submit_time"
         )
         count = submissions.count()
 
-        limit = 5
         return {
             "count": count,
             "display_more": count > limit,
             "submissions": submissions[:limit],
         }
 
     def active_invoices(self):
-        invoices = list(
+        qs = (
             Invoice.objects.filter(
                 project__lead=self.request.user,
             )
             .in_progress()
             .select_related("project")
         )
-
         return {
-            "count": len(invoices),
-            "table": InvoiceDashboardTable(invoices),
+            "count": qs.count(),
+            "table": InvoiceDashboardTable(qs),
         }
 
     def projects(self):
+        limit = 10
         projects = Project.objects.filter(lead=self.request.user).for_table()
-
         filterset = ProjectListFilter(
             data=self.request.GET or None, request=self.request, queryset=projects
         )
-
-        limit = 10
-        projects_count = projects.count()
+        count = projects.count()
 
         return {
-            "count": projects_count,
+            "count": count,
             "filterset": filterset,
             "table": ProjectsDashboardTable(data=projects[:limit], prefix="project-"),
-            "display_more": projects_count > limit,
+            "display_more": count > limit,
             "url": reverse("apply:projects:all"),
         }
 
@@ -206,21 +199,21 @@ def my_tasks(self):
         }
 
     def active_invoices(self):
-        invoices = list(Invoice.objects.for_finance_1().select_related("project"))
+        qs = Invoice.objects.for_finance_1().select_related("project")
         return {
-            "count": len(invoices),
-            "table": InvoiceDashboardTable(invoices),
+            "count": qs.count(),
+            "table": InvoiceDashboardTable(qs),
         }
 
     def invoices_for_approval(self):
-        invoices = list(Invoice.objects.approved_by_staff().select_related("project"))
-        return {"count": len(invoices), "table": InvoiceDashboardTable(invoices)}
+        qs = Invoice.objects.approved_by_staff().select_related("project")
+        return {"count": qs.count(), "table": InvoiceDashboardTable(qs)}
 
     def invoices_to_convert(self):
-        invoices = list(Invoice.objects.waiting_to_convert().select_related("project"))
+        qs = Invoice.objects.waiting_to_convert().select_related("project")
         return {
-            "count": len(invoices),
-            "table": InvoiceDashboardTable(invoices),
+            "count": qs.count(),
+            "table": InvoiceDashboardTable(qs),
         }
 
 
@@ -274,12 +267,12 @@ def get_context_data(self, **kwargs):
         return context
 
     def awaiting_reviews(self, submissions):
+        limit = 5
         submissions = submissions.in_review_for(self.request.user).order_by(
             "-submit_time"
         )
         count = submissions.count()
 
-        limit = 5
         return {
             "count": count,
             "display_more": count > limit,
@@ -324,22 +317,24 @@ def projects_in_contracting(self):
                 },
             }
         projects_in_contracting = Project.objects.in_contracting()
-        waiting_for_contract = list(
-            projects_in_contracting.filter(contracts__isnull=True).for_table()
-        )
-        waiting_for_contract_approval = list(
-            projects_in_contracting.filter(contracts__isnull=False).for_table()
-        )
+        waiting_for_contract = projects_in_contracting.filter(
+            contracts__isnull=True
+        ).for_table()
+        waiting_for_contract_approval = projects_in_contracting.filter(
+            contracts__isnull=False
+        ).for_table()
+        wfc_count = waiting_for_contract.count()
+        wfca_count = waiting_for_contract_approval.count()
         return {
-            "count": len(waiting_for_contract) + len(waiting_for_contract_approval),
+            "count": wfc_count + wfca_count,
             "waiting_for_contract": {
-                "count": len(waiting_for_contract),
+                "count": wfc_count,
                 "table": ProjectsDashboardTable(
                     data=waiting_for_contract, prefix="project-waiting-contract-"
                 ),
             },
             "waiting_for_contract_approval": {
-                "count": len(waiting_for_contract_approval),
+                "count": wfca_count,
                 "table": ProjectsDashboardTable(
                     data=waiting_for_contract_approval,
                     prefix="project-waiting-approval-",
@@ -415,22 +410,18 @@ def my_tasks(self):
         }
 
     def active_invoices(self):
-        active_invoices = list(
+        qs = (
             Invoice.objects.filter(project__user=self.request.user)
             .exclude(status__in=[PAID, DECLINED])
             .order_by("-requested_at")
         )
-        return {"count": len(active_invoices), "data": active_invoices}
+        return {"count": qs.count(), "data": qs}
 
     def historical_project_data(self):
-        historical_projects = list(
-            Project.objects.filter(user=self.request.user).complete().for_table()
-        )
+        qs = Project.objects.filter(user=self.request.user).complete().for_table()
         return {
-            "count": len(historical_projects),
-            "table": ProjectsDashboardTable(
-                data=historical_projects, prefix="past-project-"
-            ),
+            "count": qs.count(),
+            "table": ProjectsDashboardTable(data=qs, prefix="past-project-"),
         }
 
 

hypha/apply/funds/models/mixins.py

@@ -226,11 +226,9 @@ def file_field_ids(self):
 
     @property
     def question_text_field_ids(self):
-        file_fields = list(self.file_field_ids)
+        file_fields = set(self.file_field_ids)
         for field_id, field in self.fields.items():
-            if field_id in file_fields:
-                pass
-            elif isinstance(field.block, FormFieldBlock):
+            if field_id not in file_fields and isinstance(field.block, FormFieldBlock):
                 yield field_id
 
     @property

hypha/apply/funds/tables.py

@@ -166,13 +166,19 @@ def get_round_leads(request):
 
 
 def get_screening_statuses(request):
+    cache_attr = "_cache_screening_statuses"
+    if request is not None and hasattr(request, cache_attr):
+        return getattr(request, cache_attr)
     sub_filter = Q(
         id__in=ApplicationSubmission.objects.all()
         .values("screening_statuses__id")
         .distinct("screening_statuses__id")
     )
     anonymized_filter = Q(anonymized_submissions__isnull=False)
-    return ScreeningStatus.objects.filter(sub_filter | anonymized_filter)
+    qs = ScreeningStatus.objects.filter(sub_filter | anonymized_filter)
+    if request is not None:
+        setattr(request, cache_attr, qs)
+    return qs
 
 
 def get_meta_terms(request):

hypha/apply/funds/tasks.py

@@ -31,7 +31,9 @@ def generate_submission_csv(
         request_user_id: The ID of the user issuing the export request
     """
     try:
-        qs = ApplicationSubmission.objects.filter(id__in=qs_ids)
+        qs = ApplicationSubmission.objects.filter(id__in=qs_ids).only(
+            "id", "form_data", "form_fields"
+        )
         request_user = User.objects.get(pk=request_user_id)
 
         # If the user already has an existing export, delete it to begin the new one
@@ -41,8 +43,7 @@ def generate_submission_csv(
         export_manager = SubmissionExportManager.objects.create(
             user=request_user, total_export=len(qs_ids)
         )
-        csv_string = export_submissions_to_csv(qs, base_uri)
-        export_manager.export_data = "".join(csv_string.readlines())
+        export_manager.export_data = export_submissions_to_csv(qs.iterator(), base_uri)
         export_manager.set_completed_and_save()
 
         user_task = DOWNLOAD_SUBMISSIONS_EXPORT

hypha/apply/funds/utils.py

@@ -115,36 +115,36 @@ def export_submissions_to_csv(
 ):
     csv_stream = StringIO()
     header_row = [gettext_lazy("Application #"), gettext_lazy("URL")]
+    header_set = set(header_row)
     index = 2
     data_list = []
 
     for submission in submissions_list:
-        values = {}
-        values[_("Application #")] = submission.id
-        values[_("URL")] = f"{base_uri}{submission.get_absolute_url().lstrip('/')}"
+        values = {
+            _("Application #"): submission.id,
+            _("URL"): f"{base_uri}{submission.get_absolute_url().lstrip('/')}",
+        }
+        named = submission.named_blocks
         for field_id in submission.question_text_field_ids:
             question_field = submission.serialize(field_id)
             field_name = question_field["question"]
             field_value = question_field["answer"]
             if field_id == "address" and isinstance(field_value, dict):
-                address = []
-                for key, value in field_value.items():
-                    address.append(f"{key}: {value}")
-                field_value = "\n".join(address)
-            if field_name not in header_row:
-                if field_id not in submission.named_blocks:
+                field_value = "\n".join(f"{k}: {v}" for k, v in field_value.items())
+            if field_name not in header_set:
+                header_set.add(field_name)
+                if field_id not in named:
                     header_row.append(field_name)
                 else:
                     header_row.insert(index, field_name)
-                    index = index + 1
+                    index += 1
             values[field_name] = strip_tags(field_value)
         data_list.append(values)
     writer = csv.DictWriter(csv_stream, fieldnames=header_row, restval="")
     writer.writeheader()
     for data in data_list:
         writer.writerow(data)
-    csv_stream.seek(0)
-    return csv_stream
+    return csv_stream.getvalue()
 
 
 def get_copied_form_name(original_form_name: str) -> str:

hypha/apply/funds/views/all.py

@@ -27,7 +27,6 @@
     has_final_determination,
     outcome_from_actions,
 )
-from hypha.apply.funds.models.screening import ScreeningStatus
 from hypha.apply.funds.tasks import generate_submission_csv
 from hypha.apply.funds.views.partials import submission_export_download
 from hypha.apply.funds.workflows import (
@@ -51,13 +50,14 @@
 )
 from ..tables import (
     SubmissionFilter,
+    get_screening_statuses,
 )
 from ..utils import check_submissions_same_determination_form, get_export_polling_time
 
 User = get_user_model()
 
 
-def screening_decision_context(selected_screening_statuses: list) -> dict:
+def screening_decision_context(request, selected_screening_statuses: list) -> dict:
     screening_options = [
         {
             "slug": "null",
@@ -70,11 +70,7 @@ def screening_decision_context(selected_screening_statuses: list) -> dict:
             "title": item.title,
             "selected": str(item.id) in selected_screening_statuses,
         }
-        for item in ScreeningStatus.objects.filter(
-            id__in=ApplicationSubmission.objects.all()
-            .values("screening_statuses__id")
-            .distinct("screening_statuses__id")
-        )
+        for item in get_screening_statuses(request)
     ]
 
     selected_screening_statuses_objects = filter(
@@ -133,9 +129,9 @@ def submissions_all(
     start = time.time()
 
     if can_view_archives and show_archived:
-        qs = ApplicationSubmission.objects.include_archive().for_table(request.user)
+        qs = ApplicationSubmission.objects.include_archive()
     else:
-        qs = ApplicationSubmission.objects.current().for_table(request.user)
+        qs = ApplicationSubmission.objects.current()
 
     # Reviewers also have access to this view but should only see a subset of submissions.
     if request.user.is_reviewer:
@@ -248,8 +244,7 @@ def submissions_all(
         ]
     )
 
-    qs = filters.qs
-    qs = qs.prefetch_related("meta_terms")
+    qs = filters.qs.for_table(request.user).prefetch_related("meta_terms")
 
     sort_options_raw = {
         "submitted-desc": ("-submit_time", _("Newest")),
@@ -344,7 +339,7 @@ def submissions_all(
         "can_bulk_anonymize": permissions.can_bulk_delete_submissions(request.user),
         "can_export_submissions": permissions.can_export_submissions(request.user),
         "enable_selection": permissions.can_bulk_update_submissions(request.user),
-    } | screening_decision_context(selected_screening_statuses)
+    } | screening_decision_context(request, selected_screening_statuses)
     return render(request, template_name, ctx)