From 935dfe5fa8967030ebcea4d24d443ab1ac4103ec Mon Sep 17 00:00:00 2001 From: DarKylX Date: Wed, 21 Oct 2026 01:54:49 +0300 Subject: [PATCH] feat(analytics): export a consolidated program workbook --- partner_programs/report_views.py | 26 ++ partner_programs/services/program_report.py | 390 ++++++++++++++++++ partner_programs/tests/test_program_report.py | 242 +++++++++++ partner_programs/urls.py | 2 + 4 files changed, 660 insertions(+) create mode 100644 partner_programs/report_views.py create mode 100644 partner_programs/services/program_report.py create mode 100644 partner_programs/tests/test_program_report.py diff --git a/partner_programs/report_views.py b/partner_programs/report_views.py new file mode 100644 index 00000000..94ab4e31 --- /dev/null +++ b/partner_programs/report_views.py @@ -0,0 +1,26 @@ +from django.db import connection, transaction +from rest_framework.permissions import IsAuthenticated +from rest_framework.views import APIView + +from core.utils import build_xlsx_download_response +from partner_programs.services.management import managed_program +from partner_programs.services.program_report import build_program_report + + +class ProgramReportView(APIView): + permission_classes = [IsAuthenticated] + + def get(self, request, pk): + with transaction.atomic(): + if connection.vendor == "postgresql": + with connection.cursor() as cursor: + cursor.execute( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY" + ) + program = managed_program(pk, request.user) + data = build_program_report(program) + response = build_xlsx_download_response( + data, base_name=f"Программа - {program.name}" + ) + response["Cache-Control"] = "private, no-store" + return response diff --git a/partner_programs/services/program_report.py b/partner_programs/services/program_report.py new file mode 100644 index 00000000..df58956f --- /dev/null +++ b/partner_programs/services/program_report.py @@ -0,0 +1,390 @@ +"""A single, program-scoped workbook with stable IDs between its sheets.""" + +from collections import Counter +from datetime import datetime +from decimal import Decimal, InvalidOperation +from io import BytesIO +import json +from zoneinfo import ZoneInfo + +from django.db.models import Prefetch +from django.utils import timezone +from openpyxl import Workbook +from openpyxl.cell import WriteOnlyCell +from openpyxl.styles import Font, PatternFill +from openpyxl.utils import get_column_letter + +from partner_programs.models import PartnerProgramFieldValue, PartnerProgramProject +from partner_programs.services.analytics import build_program_manager_analytics +from partner_programs.services.assignment_analytics import build_assignments +from partner_programs.services.case_analytics import project_case_links +from partner_programs.services.case_fields import get_program_case_field +from partner_programs.services.exports import _literal_excel_cell +from partner_programs.services.management import participant_rows, person_name +from project_rates.models import Criteria, ProjectScore +from projects.models import Collaborator + +ASSIGNMENT_LABELS = { + "completed": "Выполнено", + "pending": "Ожидает оценки", + "in_progress": "В работе", + "not_ready": "Решение еще не сдано", +} + + +def text_value(value): + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False) + return value + + +def score_value(value, kind): + if value is not None and value != "" and kind in ("int", "float"): + try: + number = Decimal(str(value)) + if number.is_finite(): + return number + except InvalidOperation: + pass + return value + + +class Report: + def __init__(self): + self.book = Workbook(write_only=True) + + def sheet(self, name, headers): + sheet = self.book.create_sheet(name) + sheet.freeze_panes = "A2" + for index, title in enumerate(headers, 1): + sheet.column_dimensions[get_column_letter(index)].width = ( + 30 if "ID" not in title else 16 + ) + cells = [] + for title in headers: + cell = _literal_excel_cell(sheet, title) + cell.font = Font(bold=True) + cell.fill = PatternFill("solid", fgColor="E8E0FA") + cells.append(cell) + sheet.append(cells) + return sheet + + def append(self, sheet, values): + cells = [] + for value in values: + if isinstance(value, datetime): + value = value.astimezone(ZoneInfo("Europe/Moscow")).replace(tzinfo=None) + cell = WriteOnlyCell(sheet, value) + cell.number_format = "dd.mm.yyyy hh:mm" + elif isinstance(value, Decimal): + cell = WriteOnlyCell(sheet, value) + else: + cell = _literal_excel_cell(sheet, text_value(value)) + cells.append(cell) + sheet.append(cells) + + def finish(self): + output = BytesIO() + self.book.save(output) + return output.getvalue() + + +def build_program_report(program): + report = Report() + overview = build_program_manager_analytics(program) + summary = report.sheet("Сводка", ["Показатель", "Значение"]) + for row in [ + ["ID программы", program.pk], + ["Программа", program.name], + ["Сформировано (МСК)", timezone.now()], + ["Описание", program.description], + ["Начало программы (МСК)", program.datetime_started], + ["Окончание программы (МСК)", program.datetime_finished], + ["Окончание регистрации (МСК)", program.datetime_registration_ends], + ["Окончание приема решений (МСК)", program.get_project_submission_deadline()], + ["Окончание оценивания (МСК)", program.datetime_evaluation_ends], + ["Участники", overview["summary"]["participants"]["total"]], + ["Проекты", overview["summary"]["projects"]["total"]], + ["Сдано", overview["solution_funnel"]["submitted"]], + ["Оценено проектов", overview["solution_funnel"]["evaluated"]], + ["Заходили на платформу", overview["participant_funnel"]["visited_participants"]], + ["Вступили в команду", overview["participant_funnel"]["with_team"]], + [ + "Формат оценивания", + "Распределенное" if program.is_distributed_evaluation else "Открытое", + ], + [ + "Связи между листами", + "ID участника = ID пользователя. Проекты, команды, назначения и оценки связаны по ID проекта.", + ], + ]: + report.append(summary, row) + + people = report.sheet( + "Участники", + [ + "ID участника", + "Участник", + "Почта", + "Регион", + "Дата регистрации (МСК)", + "Последняя активность (МСК)", + "Последний вход (МСК)", + ], + ) + answers = report.sheet( + "Анкеты", + ["ID регистрации", "ID участника", "Участник", "ID поля", "Поле анкеты", "Ответ"], + ) + seen = set() + for profile in participant_rows(program).iterator(chunk_size=500): + user = profile.user + if user.pk not in seen: + report.append( + people, + [ + user.pk, + person_name(user), + user.email, + profile.participant_region, + profile.datetime_created, + user.last_activity, + user.last_login, + ], + ) + seen.add(user.pk) + schema = ( + profile.registration_schema + if isinstance(profile.registration_schema, dict) + else {} + ) + fields = { + field["id"]: field.get("label", field["id"]) + for field in schema.get("fields", []) + if isinstance(field, dict) and "id" in field + } + data = ( + profile.partner_program_data + if isinstance(profile.partner_program_data, dict) + else {} + ) + for key, value in data.items(): + report.append( + answers, + [ + profile.pk, + user.pk, + person_name(user), + key, + fields.get(key, key), + value, + ], + ) + + fields = list(program.fields.order_by("pk")) + projects = report.sheet( + "Проекты", + [ + "ID проекта", + "ID связи с программой", + "Проект", + "Описание", + "Кейс", + "Регион", + "ID лидера", + "Лидер", + "Размер команды", + "Презентация", + "Сдача решения", + "Добавлен (МСК)", + "Сдан (МСК)", + ] + + [f"{field.label} [ID {field.pk}]" for field in fields], + ) + teams = report.sheet( + "Команды", ["ID проекта", "Проект", "ID участника", "Участник", "Роль"] + ) + case_field = get_program_case_field(program) + links = ( + project_case_links(program, field=case_field) + .select_related("project", "project__leader") + .prefetch_related( + Prefetch( + "project__collaborator_set", + queryset=Collaborator.objects.select_related("user"), + to_attr="report_collaborators", + ), + Prefetch( + "field_values", + queryset=PartnerProgramFieldValue.objects.filter( + field__partner_program=program + ), + to_attr="report_values", + ), + ) + .order_by("pk") + ) + for link in links.iterator(chunk_size=500): + project = link.project + team = {} + if project.leader_id: + team[project.leader_id] = (project.leader, "Лидер") + for member in project.report_collaborators: + team.setdefault(member.user_id, (member.user, "Участник")) + values = {value.field_id: value.value_text for value in link.report_values} + report.append( + projects, + [ + project.pk, + link.pk, + project.name, + project.description, + link.case_name or "Без выбранного кейса", + project.region, + project.leader_id, + person_name(project.leader) if project.leader_id else "", + len(team), + project.presentation_address, + ("Сдано" if link.submitted else "Не сдано") + if program.is_competitive + else "Не требуется", + link.datetime_created, + link.datetime_submitted if link.submitted else None, + ] + + [values.get(field.pk, "") for field in fields], + ) + for user, role in team.values(): + report.append( + teams, [project.pk, project.name, user.pk, person_name(user), role] + ) + + cases = report.sheet("Кейсы", ["Кейс", "Проектов", "Сдано", "Участников"]) + for case in overview["cases"]["items"]: + report.append( + cases, + [ + case["name"], + case["projects_total"], + case["submitted"], + case["participants_total"], + ], + ) + case = overview["cases"]["without_case"] + report.append( + cases, + [ + "Без выбранного кейса", + case["projects_total"], + case["submitted"], + case["participants_total"], + ], + ) + + assignment_data = build_assignments(program.pk) + total = Counter(row["expert"]["expert_id"] for row in assignment_data) + completed = Counter( + row["expert"]["expert_id"] + for row in assignment_data + if row["status"] == "completed" + ) + experts = report.sheet( + "Эксперты", + ["ID эксперта", "ID пользователя", "Эксперт", "Почта", "Назначений", "Выполнено"], + ) + for expert in program.experts.select_related("user").order_by("pk"): + report.append( + experts, + [ + expert.pk, + expert.user_id, + person_name(expert.user), + expert.user.email, + total[expert.pk], + completed[expert.pk], + ], + ) + assignments = report.sheet( + "Назначения", + [ + "ID назначения", + "ID проекта", + "Проект", + "ID эксперта", + "ID пользователя", + "Эксперт", + "Статус", + "Назначено (МСК)", + "Сдано (МСК)", + ], + ) + for row in assignment_data: + report.append( + assignments, + [ + row["assignment_id"], + row["project"]["id"], + row["project"]["name"], + row["expert"]["expert_id"], + row["expert"]["user_id"], + row["expert"]["full_name"], + ASSIGNMENT_LABELS[row["status"]], + row["assigned_at"], + row["project_submitted_at"], + ], + ) + criteria = report.sheet( + "Критерии", ["ID критерия", "Критерий", "Описание", "Тип", "Минимум", "Максимум"] + ) + for criterion in Criteria.objects.filter(partner_program=program).order_by("pk"): + report.append( + criteria, + [ + criterion.pk, + criterion.name, + criterion.description, + criterion.type, + criterion.min_value, + criterion.max_value, + ], + ) + scores = report.sheet( + "Оценки", + [ + "ID оценки", + "ID проекта", + "Проект", + "ID пользователя эксперта", + "Эксперт", + "ID критерия", + "Критерий", + "Тип", + "Значение", + "Исходное значение", + ], + ) + for score in ( + ProjectScore.objects.filter( + criteria__partner_program=program, + project_id__in=PartnerProgramProject.objects.filter( + partner_program=program + ).values("project_id"), + ) + .select_related("criteria", "project", "user") + .order_by("project_id", "user_id", "criteria_id") + ): + report.append( + scores, + [ + score.pk, + score.project_id, + score.project.name, + score.user_id, + person_name(score.user), + score.criteria_id, + score.criteria.name, + score.criteria.type, + score_value(score.value, score.criteria.type), + score.value, + ], + ) + return report.finish() diff --git a/partner_programs/tests/test_program_report.py b/partner_programs/tests/test_program_report.py new file mode 100644 index 00000000..8b505201 --- /dev/null +++ b/partner_programs/tests/test_program_report.py @@ -0,0 +1,242 @@ +"""The consolidated report preserves actual data and program boundaries.""" + +from datetime import datetime, timezone +from io import BytesIO + +from django.db import connection +from django.test import TestCase, TransactionTestCase +from django.test.utils import CaptureQueriesContext +from openpyxl import load_workbook +from rest_framework.test import APIClient + +from partner_programs.models import PartnerProgramFieldValue, PartnerProgramUserProfile +from partner_programs.services.program_report import build_program_report +from partner_programs.tests.helpers import ( + create_partner_program, + create_program_field, + create_program_member, + create_program_project, + create_project, + create_user, +) +from partner_programs.tests.test_case_fields import create_case_field +from project_rates.models import Criteria, ProjectExpertAssignment, ProjectScore +from project_rates.tests.helpers import create_rate_expert +from projects.models import Collaborator + + +def rows(book, sheet): + return list(book[sheet].values)[1:] + + +class ProgramReportTests(TestCase): + def setUp(self): + self.program = create_partner_program( + is_competitive=True, is_distributed_evaluation=True, max_project_rates=1 + ) + + def report(self): + return load_workbook(BytesIO(build_program_report(self.program))) + + def test_empty_report_includes_all_sheets_and_unscored_criteria(self): + book = self.report() + self.assertEqual( + book.sheetnames, + [ + "Сводка", + "Участники", + "Анкеты", + "Проекты", + "Команды", + "Кейсы", + "Эксперты", + "Назначения", + "Критерии", + "Оценки", + ], + ) + self.assertEqual(rows(book, "Проекты"), []) + self.assertEqual(rows(book, "Оценки"), []) + self.assertEqual(len(rows(book, "Критерии")), self.program.criterias.count()) + self.assertEqual(book["Проекты"].freeze_panes, "A2") + + def test_linked_projects_cases_teams_and_program_specific_fields(self): + case = create_case_field(self.program, options=["Кейс А", "Пустой кейс"]) + extra = create_program_field(self.program, label="Сайт проекта") + leader = create_user(first_name="Алёна") + member = create_user() + link = create_program_project( + self.program, + project=create_project( + leader=leader, + name="Ёлка", + presentation_address="https://example.com/slides", + ), + submitted=True, + ) + # Historical duplicates are valid input to the read-only report. + Collaborator.objects.bulk_create( + [Collaborator(project=link.project, user=member)] + ) + PartnerProgramFieldValue.objects.bulk_create( + [ + PartnerProgramFieldValue( + program_project=link, field=case, value_text="Кейс А" + ), + PartnerProgramFieldValue( + program_project=link, field=extra, value_text="=HYPERLINK(1)" + ), + ] + ) + other = create_partner_program() + create_program_project(other) + book = self.report() + project = rows(book, "Проекты")[0] + self.assertEqual(len(rows(book, "Проекты")), 1) + self.assertEqual(project[:3], (link.project_id, link.pk, "Ёлка")) + self.assertEqual(project[4], "Кейс А") + self.assertEqual(project[8:11], (2, "https://example.com/slides", "Сдано")) + self.assertEqual(project[-1], "=HYPERLINK(1)") + self.assertEqual(book["Проекты"].cell(2, len(project)).data_type, "s") + self.assertEqual( + {row[2] for row in rows(book, "Команды")}, {leader.pk, member.pk} + ) + self.assertEqual([r[4] for r in rows(book, "Команды")].count("Лидер"), 1) + self.assertIn(("Пустой кейс", 0, 0, 0), rows(book, "Кейсы")) + + def test_participants_deduplicated_but_all_registration_answers_preserved(self): + user = create_user(first_name="Алёна", last_name="Ёлкина") + first = create_program_member( + self.program, user=user, data={"why": "=SUM(1)", "legacy": ["А", "Б"]} + ) + PartnerProgramUserProfile.objects.filter(pk=first.pk).update( + registration_schema={"fields": [{"id": "why", "label": "Мотивация"}]}, + datetime_created=datetime(2026, 10, 1, 12, tzinfo=timezone.utc), + ) + second = create_program_member(self.program, data={"extra": "Другой ответ"}) + create_program_member(create_partner_program(), data={"secret": "Чужая анкета"}) + book = self.report() + people = rows(book, "Участники") + self.assertEqual(len(people), 2) + self.assertIn("Алёна", people[0][1]) + self.assertEqual(people[0][4], datetime(2026, 10, 1, 15)) + answers = rows(book, "Анкеты") + self.assertEqual({r[0] for r in answers}, {first.pk, second.pk}) + self.assertIn("Мотивация", [r[4] for r in answers]) + self.assertIn('["А", "Б"]', [r[5] for r in answers]) + self.assertNotIn("Чужая анкета", str(answers)) + self.assertEqual(book["Анкеты"]["F2"].data_type, "s") + + def test_scores_zero_comments_assignments_and_cross_program_isolation(self): + Criteria.objects.filter(partner_program=self.program).delete() + numeric = Criteria.objects.create( + partner_program=self.program, + name="Баллы", + type="int", + min_value=0, + max_value=10, + ) + comment = Criteria.objects.create( + partner_program=self.program, name="Комментарий", type="str" + ) + expert = create_rate_expert(program=self.program) + link = create_program_project(self.program, submitted=True) + assignment = ProjectExpertAssignment.objects.create( + partner_program=self.program, project=link.project, expert=expert.expert + ) + zero = ProjectScore.objects.create( + project=link.project, user=expert, criteria=numeric, value="0" + ) + ProjectScore.objects.create( + project=link.project, user=expert, criteria=comment, value="=не формула" + ) + other = create_partner_program() + foreign_criterion = Criteria.objects.filter(partner_program=other).first() + foreign_link = create_program_project(other) + ProjectScore.objects.create( + project=link.project, + user=expert, + criteria=foreign_criterion, + value="Чужой критерий", + ) + ProjectScore.objects.create( + project=foreign_link.project, user=expert, criteria=numeric, value="9" + ) + book = self.report() + scores = rows(book, "Оценки") + self.assertEqual(len(scores), 2) + self.assertEqual(scores[0][0], zero.pk) + self.assertEqual(scores[0][8:10], (0, "0")) + self.assertEqual(book["Оценки"]["I2"].data_type, "n") + self.assertEqual(book["Оценки"]["I3"].data_type, "s") + assigned = rows(book, "Назначения")[0] + self.assertEqual(assigned[:2], (assignment.pk, link.project_id)) + self.assertEqual(assigned[6], "Выполнено") + self.assertEqual(rows(book, "Эксперты")[0][4:], (1, 1)) + + def test_open_evaluation_preserves_real_scores_without_assignments(self): + self.program.is_distributed_evaluation = False + self.program.save(update_fields=["is_distributed_evaluation"]) + link = create_program_project(self.program, submitted=True) + criterion = Criteria.objects.filter(partner_program=self.program).first() + ProjectScore.objects.create( + project=link.project, + user=create_rate_expert(program=self.program), + criteria=criterion, + value="Открытая оценка", + ) + book = self.report() + self.assertEqual(len(rows(book, "Оценки")), 1) + self.assertEqual(rows(book, "Назначения"), []) + + def test_exports_beyond_page_size_without_per_row_queries(self): + link = create_program_project(self.program) + with CaptureQueriesContext(connection) as small: + self.report() + for _ in range(27): + create_program_member(self.program) + create_program_project( + self.program, project=create_project(leader=link.project.leader) + ) + with CaptureQueriesContext(connection) as large: + book = self.report() + self.assertEqual(len(rows(book, "Проекты")), 28) + self.assertEqual(len(rows(book, "Участники")), 27) + self.assertLessEqual(len(large), len(small) + 5) + + +class ProgramReportAccessTests(TransactionTestCase): + def setUp(self): + self.client = APIClient() + self.manager = create_user() + self.program = create_partner_program(name="=Сохраняем название") + self.program.managers.add(self.manager) + self.url = f"/programs/{self.program.pk}/manager-overview/export/" + + def test_auth_and_manager_permission_checked_before_report(self): + self.assertEqual(self.client.get(self.url).status_code, 401) + self.client.force_authenticate(create_user()) + self.assertEqual(self.client.get(self.url).status_code, 403) + self.client.force_authenticate(self.manager) + self.assertEqual( + self.client.get("/programs/99999999/manager-overview/export/").status_code, + 404, + ) + + def test_read_only_download_has_private_headers_and_literal_program_name(self): + self.client.force_authenticate(self.manager) + with CaptureQueriesContext(connection) as captured: + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + self.assertEqual(response["Cache-Control"], "private, no-store") + self.assertIn("attachment", response["Content-Disposition"]) + self.assertIn("spreadsheetml", response["Content-Type"]) + book = load_workbook(BytesIO(response.content)) + self.assertEqual(book["Сводка"]["B3"].value, "=Сохраняем название") + self.assertEqual(book["Сводка"]["B3"].data_type, "s") + writes = [ + q["sql"] + for q in captured + if q["sql"].lstrip().upper().startswith(("INSERT", "UPDATE", "DELETE")) + ] + self.assertEqual(writes, []) diff --git a/partner_programs/urls.py b/partner_programs/urls.py index 02744147..a2a724e3 100644 --- a/partner_programs/urls.py +++ b/partner_programs/urls.py @@ -1,5 +1,6 @@ from django.urls import path from partner_programs.attention_views import AttentionActionView +from partner_programs.report_views import ProgramReportView from partner_programs.registration_views import ( RegistrationSettingsView, RegistrationPublishView, @@ -83,6 +84,7 @@ app_name = "partner_programs" urlpatterns = [ + path("/manager-overview/export/", ProgramReportView.as_view()), path( "/manager-overview/attention///", AttentionActionView.as_view(),