From 1c149fbdb4fdc2910e579c58975bdaca39bd9a3d Mon Sep 17 00:00:00 2001 From: Fredrik Jonsson Date: Wed, 6 May 2026 09:55:34 +0200 Subject: [PATCH 1/4] Revert change made for fewer db queries. Uses more memory and db cheap python expensive is a good rule. --- hypha/apply/dashboard/views.py | 93 +++++++++++++++------------------- 1 file changed, 42 insertions(+), 51 deletions(-) diff --git a/hypha/apply/dashboard/views.py b/hypha/apply/dashboard/views.py index 8d4e7e9bf1..e8505088f4 100644 --- a/hypha/apply/dashboard/views.py +++ b/hypha/apply/dashboard/views.py @@ -53,35 +53,31 @@ def paf_for_review(self): if not getattr(self.request.user, self.paf_reviewer_role, False): return {"count": None, "table": None} project_settings = ProjectSettings.for_request(self.request) - paf_approvals = list( - get_paf_for_review( - user=self.request.user, - is_paf_approval_sequential=project_settings.paf_approval_sequential, - ).select_related("project__submission__page", "paf_reviewer_role", "user") - ) + paf_approvals = get_paf_for_review( + user=self.request.user, + is_paf_approval_sequential=project_settings.paf_approval_sequential, + ).select_related("project__submission__page", "paf_reviewer_role", "user") paf_table = PAFForReviewDashboardTable( paf_approvals, prefix="paf-review-", order_by="-date_requested" ) RequestConfig(self.request, paginate=False).configure(paf_table) return { - "count": len(paf_approvals), + "count": paf_approvals.count(), "table": paf_table, } class HistoricalSubmissionMixin: def historical_submission_data(self): - historical_submissions = list( - ApplicationSubmission.objects.filter( - user=self.request.user, - ) + qs = ( + ApplicationSubmission.objects.filter(user=self.request.user) .inactive() .current() .for_table(self.request.user) ) return { - "count": len(historical_submissions), - "table": SubmissionsTable(data=historical_submissions), + "count": qs.count(), + "table": SubmissionsTable(data=qs), } @@ -120,12 +116,12 @@ def my_tasks(self): } def awaiting_reviews(self, submissions): + limit = 5 submissions = submissions.in_review_for(self.request.user).order_by( "-submit_time" ) count = submissions.count() - limit = 5 return { "count": count, "display_more": count > limit, @@ -133,34 +129,31 @@ def awaiting_reviews(self, submissions): } def active_invoices(self): - invoices = list( + qs = ( Invoice.objects.filter( project__lead=self.request.user, ) .in_progress() .select_related("project") ) - return { - "count": len(invoices), - "table": InvoiceDashboardTable(invoices), + "count": qs.count(), + "table": InvoiceDashboardTable(qs), } def projects(self): + limit = 10 projects = Project.objects.filter(lead=self.request.user).for_table() - filterset = ProjectListFilter( data=self.request.GET or None, request=self.request, queryset=projects ) - - limit = 10 - projects_count = projects.count() + count = projects.count() return { - "count": projects_count, + "count": count, "filterset": filterset, "table": ProjectsDashboardTable(data=projects[:limit], prefix="project-"), - "display_more": projects_count > limit, + "display_more": count > limit, "url": reverse("apply:projects:all"), } @@ -206,21 +199,21 @@ def my_tasks(self): } def active_invoices(self): - invoices = list(Invoice.objects.for_finance_1().select_related("project")) + qs = Invoice.objects.for_finance_1().select_related("project") return { - "count": len(invoices), - "table": InvoiceDashboardTable(invoices), + "count": qs.count(), + "table": InvoiceDashboardTable(qs), } def invoices_for_approval(self): - invoices = list(Invoice.objects.approved_by_staff().select_related("project")) - return {"count": len(invoices), "table": InvoiceDashboardTable(invoices)} + qs = Invoice.objects.approved_by_staff().select_related("project") + return {"count": qs.count(), "table": InvoiceDashboardTable(qs)} def invoices_to_convert(self): - invoices = list(Invoice.objects.waiting_to_convert().select_related("project")) + qs = Invoice.objects.waiting_to_convert().select_related("project") return { - "count": len(invoices), - "table": InvoiceDashboardTable(invoices), + "count": qs.count(), + "table": InvoiceDashboardTable(qs), } @@ -274,12 +267,12 @@ def get_context_data(self, **kwargs): return context def awaiting_reviews(self, submissions): + limit = 5 submissions = submissions.in_review_for(self.request.user).order_by( "-submit_time" ) count = submissions.count() - limit = 5 return { "count": count, "display_more": count > limit, @@ -324,22 +317,24 @@ def projects_in_contracting(self): }, } projects_in_contracting = Project.objects.in_contracting() - waiting_for_contract = list( - projects_in_contracting.filter(contracts__isnull=True).for_table() - ) - waiting_for_contract_approval = list( - projects_in_contracting.filter(contracts__isnull=False).for_table() - ) + waiting_for_contract = projects_in_contracting.filter( + contracts__isnull=True + ).for_table() + waiting_for_contract_approval = projects_in_contracting.filter( + contracts__isnull=False + ).for_table() + wfc_count = waiting_for_contract.count() + wfca_count = waiting_for_contract_approval.count() return { - "count": len(waiting_for_contract) + len(waiting_for_contract_approval), + "count": wfc_count + wfca_count, "waiting_for_contract": { - "count": len(waiting_for_contract), + "count": wfc_count, "table": ProjectsDashboardTable( data=waiting_for_contract, prefix="project-waiting-contract-" ), }, "waiting_for_contract_approval": { - "count": len(waiting_for_contract_approval), + "count": wfca_count, "table": ProjectsDashboardTable( data=waiting_for_contract_approval, prefix="project-waiting-approval-", @@ -415,22 +410,18 @@ def my_tasks(self): } def active_invoices(self): - active_invoices = list( + qs = ( Invoice.objects.filter(project__user=self.request.user) .exclude(status__in=[PAID, DECLINED]) .order_by("-requested_at") ) - return {"count": len(active_invoices), "data": active_invoices} + return {"count": qs.count(), "data": qs} def historical_project_data(self): - historical_projects = list( - Project.objects.filter(user=self.request.user).complete().for_table() - ) + qs = Project.objects.filter(user=self.request.user).complete().for_table() return { - "count": len(historical_projects), - "table": ProjectsDashboardTable( - data=historical_projects, prefix="past-project-" - ), + "count": qs.count(), + "table": ProjectsDashboardTable(data=qs, prefix="past-project-"), } From c023c8fd6d2938250b9655d1474bdebcf45d2ea4 Mon Sep 17 00:00:00 2001 From: Fredrik Jonsson Date: Wed, 6 May 2026 14:38:23 +0200 Subject: [PATCH 2/4] Improve cvs export,part 1. --- hypha/apply/funds/tasks.py | 3 +-- hypha/apply/funds/utils.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hypha/apply/funds/tasks.py b/hypha/apply/funds/tasks.py index de7c717df3..fb8b57b622 100644 --- a/hypha/apply/funds/tasks.py +++ b/hypha/apply/funds/tasks.py @@ -41,8 +41,7 @@ def generate_submission_csv( export_manager = SubmissionExportManager.objects.create( user=request_user, total_export=len(qs_ids) ) - csv_string = export_submissions_to_csv(qs, base_uri) - export_manager.export_data = "".join(csv_string.readlines()) + export_manager.export_data = export_submissions_to_csv(qs, base_uri) export_manager.set_completed_and_save() user_task = DOWNLOAD_SUBMISSIONS_EXPORT diff --git a/hypha/apply/funds/utils.py b/hypha/apply/funds/utils.py index 22a73bade7..a89aa203a6 100644 --- a/hypha/apply/funds/utils.py +++ b/hypha/apply/funds/utils.py @@ -143,8 +143,7 @@ def export_submissions_to_csv( writer.writeheader() for data in data_list: writer.writerow(data) - csv_stream.seek(0) - return csv_stream + return csv_stream.getvalue() def get_copied_form_name(original_form_name: str) -> str: From b6104b0d629c12271370114523eefa1861ea2a6f Mon Sep 17 00:00:00 2001 From: Fredrik Jonsson Date: Wed, 6 May 2026 14:54:50 +0200 Subject: [PATCH 3/4] Improve cvs export,part 2. --- hypha/apply/funds/models/mixins.py | 6 ++---- hypha/apply/funds/tasks.py | 6 ++++-- hypha/apply/funds/utils.py | 21 +++++++++++---------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/hypha/apply/funds/models/mixins.py b/hypha/apply/funds/models/mixins.py index dc9d1f6d69..f5e9e676e6 100644 --- a/hypha/apply/funds/models/mixins.py +++ b/hypha/apply/funds/models/mixins.py @@ -226,11 +226,9 @@ def file_field_ids(self): @property def question_text_field_ids(self): - file_fields = list(self.file_field_ids) + file_fields = set(self.file_field_ids) for field_id, field in self.fields.items(): - if field_id in file_fields: - pass - elif isinstance(field.block, FormFieldBlock): + if field_id not in file_fields and isinstance(field.block, FormFieldBlock): yield field_id @property diff --git a/hypha/apply/funds/tasks.py b/hypha/apply/funds/tasks.py index fb8b57b622..761614e6ab 100644 --- a/hypha/apply/funds/tasks.py +++ b/hypha/apply/funds/tasks.py @@ -31,7 +31,9 @@ def generate_submission_csv( request_user_id: The ID of the user issuing the export request """ try: - qs = ApplicationSubmission.objects.filter(id__in=qs_ids) + qs = ApplicationSubmission.objects.filter(id__in=qs_ids).only( + "id", "form_data", "form_fields" + ) request_user = User.objects.get(pk=request_user_id) # If the user already has an existing export, delete it to begin the new one @@ -41,7 +43,7 @@ def generate_submission_csv( export_manager = SubmissionExportManager.objects.create( user=request_user, total_export=len(qs_ids) ) - export_manager.export_data = export_submissions_to_csv(qs, base_uri) + export_manager.export_data = export_submissions_to_csv(qs.iterator(), base_uri) export_manager.set_completed_and_save() user_task = DOWNLOAD_SUBMISSIONS_EXPORT diff --git a/hypha/apply/funds/utils.py b/hypha/apply/funds/utils.py index a89aa203a6..60cc4f0a78 100644 --- a/hypha/apply/funds/utils.py +++ b/hypha/apply/funds/utils.py @@ -115,28 +115,29 @@ def export_submissions_to_csv( ): csv_stream = StringIO() header_row = [gettext_lazy("Application #"), gettext_lazy("URL")] + header_set = set(header_row) index = 2 data_list = [] for submission in submissions_list: - values = {} - values[_("Application #")] = submission.id - values[_("URL")] = f"{base_uri}{submission.get_absolute_url().lstrip('/')}" + values = { + _("Application #"): submission.id, + _("URL"): f"{base_uri}{submission.get_absolute_url().lstrip('/')}", + } + named = submission.named_blocks for field_id in submission.question_text_field_ids: question_field = submission.serialize(field_id) field_name = question_field["question"] field_value = question_field["answer"] if field_id == "address" and isinstance(field_value, dict): - address = [] - for key, value in field_value.items(): - address.append(f"{key}: {value}") - field_value = "\n".join(address) - if field_name not in header_row: - if field_id not in submission.named_blocks: + field_value = "\n".join(f"{k}: {v}" for k, v in field_value.items()) + if field_name not in header_set: + header_set.add(field_name) + if field_id not in named: header_row.append(field_name) else: header_row.insert(index, field_name) - index = index + 1 + index += 1 values[field_name] = strip_tags(field_value) data_list.append(values) writer = csv.DictWriter(csv_stream, fieldnames=header_row, restval="") From 2742e3c143720e3d92495b8289b9033e08068571 Mon Sep 17 00:00:00 2001 From: Fredrik Jonsson Date: Wed, 6 May 2026 16:19:18 +0200 Subject: [PATCH 4/4] Improve submission all view,part 1. --- hypha/apply/funds/tables.py | 8 +++++++- hypha/apply/funds/views/all.py | 19 +++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/hypha/apply/funds/tables.py b/hypha/apply/funds/tables.py index 9dce4ff221..077f2fb47a 100644 --- a/hypha/apply/funds/tables.py +++ b/hypha/apply/funds/tables.py @@ -166,13 +166,19 @@ def get_round_leads(request): def get_screening_statuses(request): + cache_attr = "_cache_screening_statuses" + if request is not None and hasattr(request, cache_attr): + return getattr(request, cache_attr) sub_filter = Q( id__in=ApplicationSubmission.objects.all() .values("screening_statuses__id") .distinct("screening_statuses__id") ) anonymized_filter = Q(anonymized_submissions__isnull=False) - return ScreeningStatus.objects.filter(sub_filter | anonymized_filter) + qs = ScreeningStatus.objects.filter(sub_filter | anonymized_filter) + if request is not None: + setattr(request, cache_attr, qs) + return qs def get_meta_terms(request): diff --git a/hypha/apply/funds/views/all.py b/hypha/apply/funds/views/all.py index 90e4c37037..3288f4aeef 100644 --- a/hypha/apply/funds/views/all.py +++ b/hypha/apply/funds/views/all.py @@ -27,7 +27,6 @@ has_final_determination, outcome_from_actions, ) -from hypha.apply.funds.models.screening import ScreeningStatus from hypha.apply.funds.tasks import generate_submission_csv from hypha.apply.funds.views.partials import submission_export_download from hypha.apply.funds.workflows import ( @@ -51,13 +50,14 @@ ) from ..tables import ( SubmissionFilter, + get_screening_statuses, ) from ..utils import check_submissions_same_determination_form, get_export_polling_time User = get_user_model() -def screening_decision_context(selected_screening_statuses: list) -> dict: +def screening_decision_context(request, selected_screening_statuses: list) -> dict: screening_options = [ { "slug": "null", @@ -70,11 +70,7 @@ def screening_decision_context(selected_screening_statuses: list) -> dict: "title": item.title, "selected": str(item.id) in selected_screening_statuses, } - for item in ScreeningStatus.objects.filter( - id__in=ApplicationSubmission.objects.all() - .values("screening_statuses__id") - .distinct("screening_statuses__id") - ) + for item in get_screening_statuses(request) ] selected_screening_statuses_objects = filter( @@ -133,9 +129,9 @@ def submissions_all( start = time.time() if can_view_archives and show_archived: - qs = ApplicationSubmission.objects.include_archive().for_table(request.user) + qs = ApplicationSubmission.objects.include_archive() else: - qs = ApplicationSubmission.objects.current().for_table(request.user) + qs = ApplicationSubmission.objects.current() # Reviewers also have access to this view but should only see a subset of submissions. if request.user.is_reviewer: @@ -248,8 +244,7 @@ def submissions_all( ] ) - qs = filters.qs - qs = qs.prefetch_related("meta_terms") + qs = filters.qs.for_table(request.user).prefetch_related("meta_terms") sort_options_raw = { "submitted-desc": ("-submit_time", _("Newest")), @@ -344,7 +339,7 @@ def submissions_all( "can_bulk_anonymize": permissions.can_bulk_delete_submissions(request.user), "can_export_submissions": permissions.can_export_submissions(request.user), "enable_selection": permissions.can_bulk_update_submissions(request.user), - } | screening_decision_context(selected_screening_statuses) + } | screening_decision_context(request, selected_screening_statuses) return render(request, template_name, ctx)