diff --git a/docs/analytics-case-drilldown-api.md b/docs/analytics-case-drilldown-api.md new file mode 100644 index 00000000..a14a749e --- /dev/null +++ b/docs/analytics-case-drilldown-api.md @@ -0,0 +1,122 @@ +# Analytics → Case drilldown: production API + +## Основание и целевая ветка + +Выборочный перенос принятого на DEV PR #760 в `master`. +Production base после fetch: `7b2918ed3d4914e50a5a5fc684706044ba7f5c06`. + +Реализация сопоставлена с сохранённым вариантом для master `608bf09` и финальным DEV `2603312`. +Сохранены production-модули `project_case_analytics.py`, `serializers/project_analytics_attention.py` +и `ProjectAnalyticsAttentionPagination`. Сводка и attention остаются на `/project-analytics/`. +Из DEV перенесено усиленное независимое assertion заголовков XLSX. + +Модели, endpoints и миграции не добавлялись. Прежние контракты и permissions сохранены; +прочие DEV-функции не переносились. +Кнопка общей выгрузки в Angular Cases удалена по PR #390; API `case_scope=all` сохранён. + +## Запросы + +```text +GET /programs/{id}/projects/?view=case_analytics&case_scope=selected&case_name={encodedExactOption}&limit=25&offset=0&search={query} +GET /programs/{id}/projects/?view=case_analytics&case_scope=without_case +GET /programs/{id}/projects/?view=case_analytics&case_scope=all + +GET /programs/{id}/export-projects/?view=case_analytics&case_scope=selected&case_name={encodedExactOption} +GET /programs/{id}/export-projects/?view=case_analytics&case_scope=without_case +GET /programs/{id}/export-projects/?view=case_analytics&case_scope=all +``` + +- Case_scope обязателен; case_name обязателен только для selected, в других scopes запрещён. + Имя — точный текущий option, без trim и изменения регистра. +- Неправильные/повторные/неизвестные параметры, selector без правильного view, неизвестный option, + пустые/невалидные числа дают 400. Fallback на all отсутствует. +- List: limit по умолчанию 25, диапазон 1–100; offset по умолчанию 0, минимум 0. + Search ищет подстроку в имени проекта без учёта регистра; внешние пробелы удаляются. + Порядок: `datetime_created ASC, pk ASC` связи. +- Export не принимает search, limit, offset, only_submitted, включая пустые значения. + Выгружается полный bucket независимо от поиска и страницы. +- Без параметров нового режима прежние serializer, pagination и XLSX, включая only_submitted, + сохраняются. Generic filter и rates export не затронуты. + +## Ответ списка + +Поля страницы: count, next, previous, results, selection, cases_configured, +submission_applicable, case_metrics. Count учитывает search; case_metrics описывают весь bucket: +projects_total, participants_total, submitted, not_submitted. Для all case_metrics = null. +Selection = `{scope, case_name}`; имя вне selected — null. + +Пример строки (данные условные): + +```json +{ + "program_project_id": 421, + "project": { + "id": 81, + "name": "Название проекта", + "presentation_address": "https://example.org/presentation.pdf", + "region": "Москва" + }, + "case": {"kind": "selected", "name": "Точный option"}, + "leader": {"user_id": 15, "full_name": "Имя Фамилия"}, + "team_size": 3, + "linked_at": "2026-09-01T10:00:00Z", + "submitted": true, + "submitted_at": "2026-09-20T10:00:00Z" +} +``` + +Without_case: `{kind: "without_case", name: null}`. Презентация/регион nullable и могут быть пустыми. +Отсутствующий leader — null. Для несданной связи/отсутствующей исторической даты submitted_at — null. +Презентация берётся исключительно из `Project.presentation_address`. + +## Семантика, права и запросы + +Общий `project_case_links()` в `services/project_case_analytics.py` обслуживает overview/list/export. +Он использует field и links выбранной программы. Missing, blank, whitespace, obsolete и nonexact +попадают в without_case. Настоящий option «Без выбранного кейса» остаётся selected. +Без definition/options все существующие связи попадают в without_case; synthetic rows не создаются. +Cases_configured означает наличие field, как в прежнем overview. + +Project/leader загружаются через select_related; team_size — коррелированный aggregate. +Сохраняется legacy формула `1 + число строк Collaborator`, включая сохранённую строку лидера; +она не равна числу зарегистрированных уникальных участников bucket. ProjectDetail fetch и N+1 отсутствуют. + +Server-side `IsAdminOrManagerOfProgram` и прежняя дополнительная проверка export сохранены. +Anonymous — 401; participant, leader/team member, expert-only, outsider, foreign manager — 403. +Manager текущей программы и staff/superuser сохраняют API-доступ. +Несуществующая программа: 403 обычному пользователю, 404 staff/superuser, по прежнему порядку +permission/lookup этих endpoints. UI gate staff/superuser не расширяется; контакты/auth-поля не выдаются. + +## XLSX + +Лист «Проекты». Порядок колонок: №; Название проекта; Кейс; Ссылка на презентацию; Лидер; +Регион; Размер команды; Сдача решения; Дата привязки к программе; Дата сдачи решения. + +Without_case отображается как «Без выбранного кейса». Полный URL — literal text, без HYPERLINK. +Пользовательские строки имеют явный string cell type: =, +, -, @ и #N/A остаются текстом. +Прежняя очистка управляющих символов/длины Excel сохранена. Даты — ISO 8601 с timezone, null — пустая ячейка. +Noncompetitive — «Не требуется», raw submitted не меняется. +Если проекты исчезли к экспорту, результат — header-only workbook. + +Base name: `projects_{all_cases | without_case | case - NAME} - PROGRAM - DD.MM.YY`. +Content-Disposition использует прежний NFKD sanitizer и UTF-8 filename*. +Angular повторяет этот формат, используя название программы и UTC-дату. +Legacy workbook builders не изменены. + +## Production rollout и smoke-check + +1. Пользователь мержит backend PR в `master`. +2. Пользователь публикует production release или запускает существующий **Release Prod** + с соответствующими `image_tag` / `deploy_ref`. +3. Проверяет доступность API/admin и здоровье Celery, затем manager selected/without_case list, + отсутствие доступа у участника/менеджера другой программы, presentation URL/null и scoped XLSX. +4. После успешного API smoke-check пользователь мержит отдельный Angular production PR; + push в Angular `master` запускает существующий frontend deploy workflow. +5. Проверяет Team, drilldown, отсутствие общей кнопки в Cases, поиск/страницы/презентацию/XLSX, + mobile и lifecycle карточек. + +List и export выпускаются одной backend-версией. Новая версия совместима со старым Angular. +Angular проверяет DTO/scope и не подменяет case export старой общей выгрузкой. +Feature не записывает данные; rollback схемы не требуется. При откате backend после Angular +сначала откатить Angular либо принять временную недоступность case drilldown. +Release/deploy и проверку живой production-среды выполняет пользователь. diff --git a/partner_programs/serializers/project_case_drilldown.py b/partner_programs/serializers/project_case_drilldown.py new file mode 100644 index 00000000..84c2e84f --- /dev/null +++ b/partner_programs/serializers/project_case_drilldown.py @@ -0,0 +1,89 @@ +"""Opt-in контракт менеджера, отдельный от прежнего сериализатора проектов.""" + +from rest_framework import serializers + +from .project_analytics_attention import participant_name + + +def case_analytics_requested(params): + if not {"view", "case_scope", "case_name"}.intersection(params): + return False + if params.get("view") != "case_analytics": + raise serializers.ValidationError({"view": "Укажите view=case_analytics."}) + return True + + +class ProjectCaseQuerySerializer(serializers.Serializer): + view = serializers.ChoiceField(choices=("case_analytics",)) + case_scope = serializers.ChoiceField(choices=("all", "selected", "without_case")) + case_name = serializers.CharField(required=False, trim_whitespace=False) + + def to_internal_value(self, data): + errors = { + key: "Параметр не поддерживается в этом режиме." + for key in set(data) - set(self.fields) + } + if hasattr(data, "getlist"): + for key in data: + if len(data.getlist(key)) != 1: + errors[key] = "Укажите параметр только один раз." + if errors: + raise serializers.ValidationError(errors) + # DRF считает QueryDict HTML-формой: пустой integer иначе становится + # отсутствующим полем и незаметно получает default. + return super().to_internal_value({key: data[key] for key in data}) + + def validate(self, attrs): + if attrs["case_scope"] == "selected": + if attrs.get("case_name") not in self.context["case_options"]: + raise serializers.ValidationError( + {"case_name": "Выберите точный текущий вариант кейса."} + ) + elif "case_name" in attrs: + raise serializers.ValidationError( + {"case_name": "Название допустимо только для case_scope=selected."} + ) + return attrs + + +class ProjectCaseListQuerySerializer(ProjectCaseQuerySerializer): + limit = serializers.IntegerField(min_value=1, max_value=100, default=25) + offset = serializers.IntegerField(min_value=0, default=0) + search = serializers.CharField(allow_blank=True, default="", trim_whitespace=True) + + +class CaseProjectSerializer(serializers.Serializer): + id = serializers.IntegerField() + name = serializers.CharField() + presentation_address = serializers.CharField(allow_null=True, allow_blank=True) + region = serializers.CharField(allow_null=True, allow_blank=True) + + +class CaseLeaderSerializer(serializers.Serializer): + user_id = serializers.IntegerField(source="pk") + full_name = serializers.SerializerMethodField() + + def get_full_name(self, user): + return participant_name(user.pk, user.first_name, user.last_name) + + +class ProjectCaseRowSerializer(serializers.Serializer): + program_project_id = serializers.IntegerField(source="pk") + project = CaseProjectSerializer() + case = serializers.SerializerMethodField() + leader = CaseLeaderSerializer(source="project.leader", allow_null=True) + team_size = serializers.IntegerField() + linked_at = serializers.DateTimeField(source="datetime_created") + submitted = serializers.BooleanField() + submitted_at = serializers.SerializerMethodField() + + def get_case(self, link): + return { + "kind": "selected" if link.case_name is not None else "without_case", + "name": link.case_name, + } + + def get_submitted_at(self, link): + if not link.submitted or link.datetime_submitted is None: + return None + return serializers.DateTimeField().to_representation(link.datetime_submitted) diff --git a/partner_programs/services/exports.py b/partner_programs/services/exports.py index e2c153ef..ef9143cd 100644 --- a/partner_programs/services/exports.py +++ b/partner_programs/services/exports.py @@ -6,6 +6,7 @@ from django.db.models import Prefetch from django.utils import timezone from openpyxl import Workbook +from openpyxl.cell import WriteOnlyCell from core.utils import XlsxFileToExport, sanitize_excel_value from partner_programs.models import ( @@ -15,6 +16,8 @@ PartnerProgramProject, PartnerProgramUserProfile, ) +from partner_programs.serializers.project_analytics_attention import participant_name +from partner_programs.services.project_case_drilldown import case_project_rows from project_rates.models import Criteria, ProjectScore from projects.models import Collaborator @@ -340,6 +343,73 @@ def build_program_project_scores_export_file( ) +CASE_PROJECT_COLUMNS = [ + "№", + "Название проекта", + "Кейс", + "Ссылка на презентацию", + "Лидер", + "Регион", + "Размер команды", + "Сдача решения", + "Дата привязки к программе", + "Дата сдачи решения", +] + + +def _literal_excel_cell(worksheet, value): + value = sanitize_excel_value(value) + cell = WriteOnlyCell(worksheet, value=value) + if isinstance(value, str): + # openpyxl распознаёт '=' / '#N/A' как формулу/ошибку. + # Сохраняем очищенный текст буквально, без апострофа и вычислений. + cell.data_type = "s" + return cell + + +def build_case_projects_export_file(*, program, field, selection) -> ProgramExportFile: + """Выгружаем весь bucket; поиск и пагинация не входят в этот контракт.""" + workbook = Workbook(write_only=True) + worksheet = workbook.create_sheet(title="Проекты") + worksheet.append(CASE_PROJECT_COLUMNS) + links = case_project_rows(program, field=field, selection=selection) + for number, link in enumerate(links.iterator(chunk_size=1000), start=1): + project = link.project + leader = project.leader + submitted_at = link.datetime_submitted if link.submitted else None + values = [ + number, + project.name, + link.case_name if link.case_name is not None else "Без выбранного кейса", + project.presentation_address, + participant_name(leader.pk, leader.first_name, leader.last_name) + if leader + else "", + project.region, + link.team_size, + ("Сдано" if link.submitted else "Не сдано") + if program.is_competitive + else "Не требуется", + link.datetime_created.isoformat(), + submitted_at.isoformat() if submitted_at else "", + ] + worksheet.append([_literal_excel_cell(worksheet, value) for value in values]) + + buffer = io.BytesIO() + workbook.save(buffer) + scope = selection["case_scope"] + scope_label = { + "all": "all_cases", + "without_case": "without_case", + "selected": f"case - {selection.get('case_name', '')}", + }[scope] + date_suffix = timezone.now().strftime("%d.%m.%y") + return ProgramExportFile( + binary_data=buffer.getvalue(), + base_name=f"projects_{scope_label} - {program.name or 'program'} - {date_suffix}", + ) + + def prepare_project_scores_export_data(program_id: int) -> list[dict]: """ Готовит данные для выгрузки оценок проектов. diff --git a/partner_programs/services/project_case_analytics.py b/partner_programs/services/project_case_analytics.py index 979f5ebc..be05b210 100644 --- a/partner_programs/services/project_case_analytics.py +++ b/partner_programs/services/project_case_analytics.py @@ -1,6 +1,6 @@ """Read-only case analytics for legacy Project links in one program.""" -from django.db.models import Count, OuterRef, Subquery, TextField, Value +from django.db.models import Case, Count, F, OuterRef, Subquery, TextField, Value, When from partner_programs.models import ( PartnerProgramFieldValue, @@ -19,20 +19,16 @@ def _empty_metrics(): } -def build_project_case_analytics(program) -> dict: - """Group current-program links by exact current system case options. +_UNSET = object() - Four bounded SELECTs load the case definition, grouped link counts, - registered leaders and registered collaborators. Every link belongs to one - project bucket; participants are unique inside each bucket. + +def project_case_links(program, *, field): + """Классифицируем связь Project × Program по точным текущим options. + + NULL обозначает without_case, включая отсутствующие, пустые и устаревшие + значения. Сводка, список и экспорт используют одно выражение без записи в БД. """ - field = get_program_case_field(program) options = field.get_options_list() if field else [] - buckets = {name: _empty_metrics() for name in options} - without_case = _empty_metrics() - participant_ids = {name: set() for name in buckets} - participant_ids[None] = set() - case_value = ( Subquery( PartnerProgramFieldValue.objects.filter( @@ -42,13 +38,38 @@ def build_project_case_analytics(program) -> dict: if field else Value(None, output_field=TextField()) ) - links = ( + return ( PartnerProgramProject.objects.filter(partner_program_id=program.pk) .order_by() .annotate(case_value=case_value) + .annotate( + case_name=Case( + When(case_value__in=options, then=F("case_value")), + default=Value(None), + output_field=TextField(), + ) + ) ) - for row in links.values("case_value", "submitted").annotate(total=Count("pk")): - metrics = buckets.get(row["case_value"], without_case) + + +def build_project_case_analytics(program, *, field=_UNSET) -> dict: + """Group current-program links by exact current system case options. + + Four bounded SELECTs load the case definition, grouped link counts, + registered leaders and registered collaborators. Every link belongs to one + project bucket; participants are unique inside each bucket. + """ + if field is _UNSET: + field = get_program_case_field(program) + options = field.get_options_list() if field else [] + buckets = {name: _empty_metrics() for name in options} + without_case = _empty_metrics() + participant_ids = {name: set() for name in buckets} + participant_ids[None] = set() + + links = project_case_links(program, field=field) + for row in links.values("case_name", "submitted").annotate(total=Count("pk")): + metrics = buckets.get(row["case_name"], without_case) metrics["projects_total"] += row["total"] metrics["submitted" if row["submitted"] else "not_submitted"] += row["total"] @@ -58,7 +79,7 @@ def build_project_case_analytics(program) -> dict: for user_path in ("project__leader_id", "project__collaborator__user_id"): pairs = ( links.filter(**{f"{user_path}__in": registered_users}) - .values_list("case_value", user_path) + .values_list("case_name", user_path) .distinct() ) for value, user_id in pairs: diff --git a/partner_programs/services/project_case_drilldown.py b/partner_programs/services/project_case_drilldown.py new file mode 100644 index 00000000..7dd818c3 --- /dev/null +++ b/partner_programs/services/project_case_drilldown.py @@ -0,0 +1,68 @@ +"""Read-only проекция связей выбранной программы для списка кейса и XLSX.""" + +from django.db.models import Count, IntegerField, OuterRef, Subquery, Value +from django.db.models.functions import Coalesce + +from projects.models import Collaborator + +from .project_case_analytics import project_case_links + + +def case_project_rows(program, *, field, selection): + links = project_case_links(program, field=field) + if selection["case_scope"] == "selected": + links = links.filter(case_name=selection["case_name"]) + elif selection["case_scope"] == "without_case": + links = links.filter(case_name__isnull=True) + + collaborators = ( + Collaborator.objects.filter(project_id=OuterRef("project_id")) + .order_by() + .values("project_id") + .annotate(total=Count("pk")) + .values("total") + ) + return ( + links.select_related("project", "project__leader") + .only( + "id", + "project_id", + "datetime_created", + "submitted", + "datetime_submitted", + "project__id", + "project__name", + "project__presentation_address", + "project__region", + "project__leader_id", + "project__leader__id", + "project__leader__first_name", + "project__leader__last_name", + ) + .annotate( + # Сохраняем legacy export: лидер + строки Collaborator. + # Это не число уникальных зарегистрированных участников bucket. + team_size=Coalesce( + Subquery(collaborators, output_field=IntegerField()), Value(0) + ) + + Value(1) + ) + .order_by("datetime_created", "pk") + ) + + +def case_list_metadata(cases, selection): + scope = selection["case_scope"] + name = selection.get("case_name") + metrics = None + if scope == "without_case": + metrics = cases["without_case"] + elif scope == "selected": + bucket = next(item for item in cases["items"] if item["name"] == name) + metrics = {key: value for key, value in bucket.items() if key != "name"} + return { + "selection": {"scope": scope, "case_name": name}, + "cases_configured": cases["configured"], + "submission_applicable": cases["submission_applicable"], + "case_metrics": metrics, + } diff --git a/partner_programs/tests/test_project_case_drilldown.py b/partner_programs/tests/test_project_case_drilldown.py new file mode 100644 index 00000000..499f2648 --- /dev/null +++ b/partner_programs/tests/test_project_case_drilldown.py @@ -0,0 +1,590 @@ +"""Контракт, права, XLSX и число запросов нового режима аналитики кейсов.""" + +import io +from urllib.parse import parse_qs, unquote, urlsplit + +from django.db import connection +from django.test import TestCase +from django.test.utils import CaptureQueriesContext +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from openpyxl import load_workbook +from rest_framework.test import APIClient + +from partner_programs.models import ( + PartnerProgramField, + PartnerProgramFieldValue, + PartnerProgramProject, +) +from partner_programs.serializers.project_case_drilldown import ProjectCaseRowSerializer +from partner_programs.services.exports import ( + BASE_COLUMNS, + build_program_projects_export_file, +) +from partner_programs.services.project_case_analytics import build_project_case_analytics +from partner_programs.services.project_case_drilldown import case_project_rows +from partner_programs.tests.helpers import ( + create_partner_program, + create_program_member, + create_program_project, + create_project, + create_user, +) +from partner_programs.tests.test_case_fields import create_case_field +from project_rates.tests.helpers import create_rate_expert +from projects.models import Collaborator +from projects.serializers import ProjectListSerializer + + +class ProjectCaseDrilldownTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.program = create_partner_program(is_competitive=True) + cls.manager = create_user(password=None) + cls.program.managers.add(cls.manager) + cls.field = create_case_field( + cls.program, + options=["A", "B", "Empty", "Без выбранного кейса", "Кейс + &=100%"], + ) + + def setUp(self): + self.client = APIClient() + self.client.force_authenticate(self.manager) + + def url(self, export=False, program=None): + program = program or self.program + suffix = "export-projects" if export else "projects" + return f"/programs/{program.pk}/{suffix}/" + + def get(self, scope="selected", *, name="A", export=False, program=None, **params): + query = {"view": "case_analytics", "case_scope": scope, **params} + if scope == "selected": + query["case_name"] = name + return self.client.get(self.url(export, program), query) + + def link( + self, value=None, *, program=None, field=None, project=None, submitted=False + ): + link = create_program_project( + program or self.program, + project=project or create_project(leader=self.manager), + submitted=submitted, + ) + if value is not None: + # Исторические невалидные значения читаются без ослабления правил записи. + PartnerProgramFieldValue.objects.bulk_create( + [ + PartnerProgramFieldValue( + program_project=link, field=field or self.field, value_text=value + ) + ] + ) + return link + + def page(self, *args, **kwargs): + response = self.get(*args, **kwargs) + self.assertEqual(response.status_code, 200, response.data) + return response.json() + + def workbook(self, *args, **kwargs): + response = self.get(*args, export=True, **kwargs) + self.assertEqual(response.status_code, 200, response.content[:300]) + self.assertEqual( + response["Content-Type"], + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + book = load_workbook(io.BytesIO(response.content), data_only=False) + self.assertEqual(book.sheetnames, ["Проекты"]) + self.addCleanup(book.close) + rows = list(book.active.iter_rows(values_only=True)) + self.assertEqual( + list(rows[0]), + [ + "№", + "Название проекта", + "Кейс", + "Ссылка на презентацию", + "Лидер", + "Регион", + "Размер команды", + "Сдача решения", + "Дата привязки к программе", + "Дата сдачи решения", + ], + ) + return book, rows[1:], response + + def test_selected_projection_and_full_bucket_metrics(self): + create_program_member(self.program, user=self.manager) + member = create_user(password=None) + create_program_member(self.program, user=member) + unregistered = create_user(password=None) + project = create_project( + leader=self.manager, + name="Selected", + region="Москва", + presentation_address="https://example.org/project.pdf", + draft=True, + ) + Collaborator.objects.create(project=project, user=member) + Collaborator.objects.create(project=project, user=unregistered) + link = self.link("A", project=project, submitted=True) + self.link("A") + self.link("B") + page = self.page(search=" Selected ") + self.assertEqual(page["count"], 1) + self.assertEqual(page["selection"], {"scope": "selected", "case_name": "A"}) + self.assertTrue(page["cases_configured"]) + self.assertTrue(page["submission_applicable"]) + self.assertEqual( + page["case_metrics"], + { + "projects_total": 2, + "participants_total": 2, + "submitted": 1, + "not_submitted": 1, + }, + ) + row = page["results"][0] + self.assertEqual( + set(row), + { + "program_project_id", + "project", + "case", + "leader", + "team_size", + "linked_at", + "submitted", + "submitted_at", + }, + ) + self.assertEqual(row["program_project_id"], link.pk) + self.assertEqual( + row["project"], + { + "id": project.pk, + "name": "Selected", + "region": "Москва", + "presentation_address": "https://example.org/project.pdf", + }, + ) + self.assertEqual(row["case"], {"kind": "selected", "name": "A"}) + self.assertEqual( + row["leader"], + { + "user_id": self.manager.pk, + "full_name": "Program User", + }, + ) + # Создание Project также добавляет лидера в Collaborator. Сохраняем + # формулу существующего экспорта (1 + строки), включая эту запись. + self.assertEqual(row["team_size"], 4) + self.assertTrue(row["submitted"]) + self.assertEqual(parse_datetime(row["submitted_at"]), link.datetime_submitted) + self.assertEqual(parse_datetime(row["linked_at"]), link.datetime_created) + + def test_classifier_parity_for_missing_blank_obsolete_and_nonexact_values(self): + values = [ + "A", + "B", + "Без выбранного кейса", + None, + "", + " \t\n", + "old", + "a", + " A", + "A ", + ] + links = [self.link(value) for value in values] + expected_without = {link.pk for link in links[3:]} + overview = build_project_case_analytics(self.program) + without = self.page("without_case") + self.assertEqual(without["case_metrics"], overview["without_case"]) + self.assertEqual( + {r["program_project_id"] for r in without["results"]}, expected_without + ) + for row in without["results"]: + self.assertEqual(row["case"], {"kind": "without_case", "name": None}) + for bucket in overview["items"]: + page = self.page(name=bucket["name"]) + self.assertEqual(page["count"], bucket["projects_total"]) + self.assertEqual( + page["case_metrics"], {k: v for k, v in bucket.items() if k != "name"} + ) + all_page = self.page("all") + self.assertEqual(all_page["count"], len(values)) + self.assertIsNone(all_page["case_metrics"]) + self.assertEqual(all_page["selection"], {"scope": "all", "case_name": None}) + _, rows, _ = self.workbook("without_case") + self.assertEqual({r[1] for r in rows}, {link.project.name for link in links[3:]}) + self.assertTrue(all(r[2] == "Без выбранного кейса" for r in rows)) + _, literal_rows, _ = self.workbook(name="Без выбранного кейса") + self.assertEqual([r[1] for r in literal_rows], [links[2].project.name]) + self.assertEqual( + PartnerProgramFieldValue.objects.get(program_project=links[-1]).value_text, + "A ", + ) + + def test_without_case_without_definition_or_options_and_empty_program(self): + program = create_partner_program() + program.managers.add(self.manager) + page = self.page("without_case", program=program) + self.assertEqual(page["count"], 0) + self.assertFalse(page["cases_configured"]) + self.link(program=program) + page = self.page("without_case", program=program) + self.assertEqual(page["count"], 1) + self.assertFalse(page["cases_configured"]) + _, rows, _ = self.workbook("without_case", program=program) + self.assertEqual(len(rows), 1) + field = create_case_field(program) + PartnerProgramField.objects.filter(pk=field.pk).update(options="") + page = self.page("without_case", program=program) + self.assertTrue(page["cases_configured"]) + self.assertEqual(page["case_metrics"]["projects_total"], 1) + self.assertEqual(self.get(program=program).status_code, 400) + _, rows, _ = self.workbook("without_case", program=program) + self.assertEqual(len(rows), 1) + + def test_removed_option_is_rejected_and_its_links_move_to_without_case(self): + link = self.link("A") + self.assertEqual(self.page()["count"], 1) + PartnerProgramField.objects.filter(pk=self.field.pk).update(options="B") + for export in (False, True): + self.assertEqual(self.get(export=export).status_code, 400) + page = self.page("without_case") + self.assertEqual( + [row["program_project_id"] for row in page["results"]], [link.pk] + ) + self.assertEqual(page["case_metrics"]["projects_total"], 1) + _, rows, _ = self.workbook("without_case") + self.assertEqual(rows[0][2], "Без выбранного кейса") + self.assertEqual( + PartnerProgramFieldValue.objects.get(program_project=link).value_text, "A" + ) + + def test_same_project_in_two_programs_has_independent_case_submission_and_dates(self): + other = create_partner_program(is_competitive=True) + other.managers.add(self.manager) + other_field = create_case_field(other, options=["A", "B"]) + shared = create_project( + leader=self.manager, presentation_address="https://example.org/shared" + ) + link_a = self.link("A", project=shared, submitted=True) + link_b = self.link("B", project=shared, program=other, field=other_field) + created = timezone.now() - timezone.timedelta(days=10) + PartnerProgramProject.objects.filter(pk=link_b.pk).update( + datetime_created=created + ) + row_a = self.page()["results"][0] + row_b = self.page(name="B", program=other)["results"][0] + self.assertNotEqual(row_a["program_project_id"], row_b["program_project_id"]) + self.assertEqual(row_a["program_project_id"], link_a.pk) + self.assertEqual(row_b["program_project_id"], link_b.pk) + self.assertEqual(row_a["project"], row_b["project"]) + self.assertTrue(row_a["submitted"]) + self.assertFalse(row_b["submitted"]) + self.assertIsNone(row_b["submitted_at"]) + self.assertEqual(parse_datetime(row_b["linked_at"]), created) + self.assertEqual(self.page(program=other)["count"], 0) + _, rows_a, _ = self.workbook() + _, rows_b, _ = self.workbook(name="B", program=other) + self.assertEqual(rows_a[0][2], "A") + self.assertEqual(rows_b[0][2], "B") + self.assertEqual(rows_a[0][7], "Сдано") + self.assertEqual(rows_b[0][7], "Не сдано") + self.assertIsNone(rows_b[0][9]) + # Даже исторически некорректные значения другой программы игнорируются. + self.link("A", field=other_field) + self.assertEqual(self.page("without_case")["count"], 1) + + def test_pagination_search_exact_encoded_option_and_full_exports(self): + name = "Кейс + &=100%" + links = [ + self.link( + name, + project=create_project( + leader=self.manager, name=f"{'Find' if i < 3 else 'Project'} {i:02d}" + ), + ) + for i in range(30) + ] + self.link("B") + self.link() + PartnerProgramProject.objects.filter(pk__in=[x.pk for x in links]).update( + datetime_created=timezone.now() + ) + first = self.page(name=name) + self.assertEqual(len(first["results"]), 25) + self.assertEqual(first["count"], 30) + self.assertIsNone(first["previous"]) + self.assertEqual(parse_qs(urlsplit(first["next"]).query)["case_name"], [name]) + second = self.client.get(first["next"]).json() + ids = [r["program_project_id"] for r in first["results"] + second["results"]] + self.assertEqual(ids, [link.pk for link in links]) + self.assertIsNone(second["next"]) + self.assertIsNotNone(second["previous"]) + search = self.page(name=name, search=" fInD ", limit=2) + self.assertEqual(search["count"], 3) + self.assertEqual(search["case_metrics"]["projects_total"], 30) + self.assertEqual(self.page(name=name, search="missing")["count"], 0) + self.assertEqual(self.page(name=name, offset=100)["results"], []) + _, selected, _ = self.workbook(name=name) + self.assertEqual([r[1] for r in selected], [link.project.name for link in links]) + _, all_rows, _ = self.workbook("all") + self.assertEqual(len(all_rows), 32) + self.assertEqual(len({r[1] for r in all_rows}), 32) + self.assertEqual([r[0] for r in all_rows], list(range(1, 33))) + + def test_noncompetitive_null_presentation_and_historical_submission_dates(self): + self.program.is_competitive = False + self.program.save(update_fields=["is_competitive"]) + historical = self.link("A", submitted=True) + unsent = self.link("A") + PartnerProgramProject.objects.filter(pk=historical.pk).update( + datetime_submitted=None + ) + PartnerProgramProject.objects.filter(pk=unsent.pk).update( + datetime_submitted=timezone.now() + ) + page = self.page() + self.assertFalse(page["submission_applicable"]) + self.assertEqual([r["submitted"] for r in page["results"]], [True, False]) + for row in page["results"]: + self.assertIsNone(row["submitted_at"]) + self.assertIsNone(row["project"]["presentation_address"]) + _, rows, _ = self.workbook() + for row in rows: + self.assertEqual(row[7], "Не требуется") + self.assertIsNone(row[3]) + self.assertIsNone(row[9]) + self.assertIsNotNone(parse_datetime(row[8]).tzinfo) + + def test_empty_case_returns_zero_metadata_and_header_only_workbook(self): + self.assertEqual(self.page(name="Empty")["count"], 0) + self.assertEqual(self.page(name="Empty")["case_metrics"]["projects_total"], 0) + _, rows, _ = self.workbook(name="Empty") + self.assertEqual(rows, []) + + def test_query_validation_rejects_invalid_and_ambiguous_scope_without_fallback(self): + self.link("A") + invalid = [ + {}, + {"case_scope": "unknown"}, + {"case_scope": "selected"}, + {"case_scope": "selected", "case_name": "a"}, + {"case_scope": "selected", "case_name": " A "}, + {"case_scope": "selected", "case_name": "old"}, + {"case_scope": "selected", "case_name": ""}, + {"case_scope": "selected", "case_name": " \t "}, + {"case_scope": "all", "case_name": "A"}, + {"case_scope": "without_case", "case_name": "A"}, + {"case_scope": "all", "unknown": "1"}, + {"case_scope": "all", "only_submitted": "1"}, + ] + for export in (False, True): + for query in invalid: + with self.subTest(export=export, query=query): + response = self.client.get( + self.url(export), {"view": "case_analytics", **query} + ) + self.assertEqual(response.status_code, 400) + self.assertNotIn("Content-Disposition", response) + for query in ( + "view=wrong&case_scope=all", + "case_scope=selected&case_name=A", + "view=case_analytics&view=wrong&case_scope=all", + "view=case_analytics&case_scope=all&case_scope=selected", + "view=case_analytics&case_scope=selected&case_name=A&case_name=B", + ): + self.assertEqual( + self.client.get(self.url(export) + "?" + query).status_code, 400 + ) + for params in ( + {"limit": 0}, + {"limit": 101}, + {"limit": "x"}, + {"offset": -1}, + {"offset": "x"}, + {"limit": ""}, + {"limit": "1.5"}, + {"offset": ""}, + ): + self.assertEqual(self.get(**params).status_code, 400) + for key in ("search", "limit", "offset", "only_submitted"): + for value in ("", "1"): + self.assertEqual(self.get(export=True, **{key: value}).status_code, 400) + self.assertEqual(self.page(limit=100)["count"], 1) + + def test_permission_matrix_missing_program_and_read_only_methods(self): + self.link( + "A", project=create_project(leader=self.manager, name="Private case project") + ) + foreign = create_partner_program() + foreign_manager = create_user(password=None) + foreign.managers.add(foreign_manager) + participant = create_program_member(self.program).user + expert = create_rate_expert(program=self.program) + outsider = create_user(password=None) + leader = create_program_member(self.program).user + teammate = create_program_member(self.program).user + owned = self.link("A", project=create_project(leader=leader)) + Collaborator.objects.create(project=owned.project, user=teammate) + staff = create_user(password=None, is_staff=True) + superuser = create_user(password=None, is_superuser=True) + for user, expected in ( + (None, 401), + (participant, 403), + (expert, 403), + (outsider, 403), + (leader, 403), + (teammate, 403), + (foreign_manager, 403), + (self.manager, 200), + (staff, 200), + (superuser, 200), + ): + self.client.force_authenticate(user) + for export in (False, True): + with self.subTest(user=user, export=export): + response = self.get(export=export) + self.assertEqual(response.status_code, expected) + if expected != 200: + self.assertNotIn(b"Private case project", response.content) + self.assertNotIn("Content-Disposition", response) + missing_url = f"/programs/{foreign.pk + 100000}/{'export-projects' if export else 'projects'}/" + missing = self.client.get( + missing_url, {"view": "case_analytics", "case_scope": "all"} + ) + self.assertEqual( + missing.status_code, + 401 + if user is None + else 404 + if user in (staff, superuser) + else 403, + ) + self.client.force_authenticate(self.manager) + for export in (False, True): + for method in ("post", "put", "patch", "delete"): + self.assertEqual( + getattr(self.client, method)(self.url(export)).status_code, 405 + ) + + def test_legacy_list_and_all_submitted_xlsx_contracts_stay_unchanged(self): + links = [self.link("A", submitted=i == 0) for i in range(12)] + legacy = self.client.get(self.url()) + self.assertEqual(legacy.status_code, 200) + self.assertEqual(set(legacy.data), {"count", "next", "previous", "results"}) + self.assertEqual(len(legacy.data["results"]), 10) + expected = ProjectListSerializer( + [x.project for x in reversed(links[-10:])], + many=True, + context={"request": legacy.wsgi_request}, + ).data + self.assertEqual(legacy.data["results"], expected) + for submitted in (False, True): + response = self.client.get( + self.url(True), {"only_submitted": "1"} if submitted else {} + ) + self.assertEqual(response.status_code, 200) + actual = load_workbook(io.BytesIO(response.content)) + reference = build_program_projects_export_file( + program=self.program, only_submitted=submitted + ) + baseline = load_workbook(io.BytesIO(reference.binary_data)) + self.addCleanup(actual.close) + self.addCleanup(baseline.close) + actual_rows = list(actual.active.values) + self.assertEqual(actual_rows, list(baseline.active.values)) + self.assertEqual( + list(actual_rows[0]), + [title for _, title in BASE_COLUMNS] + [self.field.label], + ) + self.assertEqual(len(actual_rows) - 1, 1 if submitted else 12) + + def test_xlsx_user_values_are_literal_text_and_filename_is_safe(self): + option = '=HYPERLINK("https://example.org", "case")' + PartnerProgramField.objects.filter(pk=self.field.pk).update(options=option) + self.program.name = 'Program / unsafe\r\n"name' + self.program.save(update_fields=["name"]) + self.manager.first_name = "=1+1" + self.manager.last_name = "" + self.manager.save(update_fields=["first_name", "last_name"]) + names = [ + "=1+1", + "+SUM(1,2)", + "-1+2", + "@SUM(1,2)", + "#N/A", + "Текст & + %", + "\t=1+1", + ] + url = "https://example.org/presentation?x=1&token=a+b%20c#slide-15" + for name in names: + self.link( + option, + project=create_project( + leader=self.manager, + name=name, + region="=1+1", + presentation_address=url, + ), + ) + book, rows, response = self.workbook(name=option) + self.assertEqual([r[1] for r in rows], names) + for cells in list(book.active.iter_rows())[1:]: + for index in (1, 2, 3, 4, 5): + self.assertEqual(cells[index].data_type, "s") + self.assertIsNone(cells[index].hyperlink) + self.assertEqual(cells[2].value, option) + self.assertEqual(cells[3].value, url) + self.assertEqual(cells[4].value, "=1+1") + disposition = unquote(response["Content-Disposition"]) + for unsafe in ("\r", "\n", "/", "\\"): + self.assertNotIn(unsafe, disposition) + self.assertIn("projects_case", disposition) + self.assertIn(".xlsx", disposition) + for scope, label in (("all", "all_cases"), ("without_case", "without_case")): + _, _, response = self.workbook(scope) + self.assertIn(label, response["Content-Disposition"]) + + def test_constant_queries_for_one_and_one_hundred_rows_and_no_writes(self): + counts = {"list": [], "export": [], "overview": []} + self.link("A") + collaborator = create_user(password=None) + for size in (1, 100): + if size == 100: + for index in range(99): + project = create_project(leader=self.manager, name=f"Load {index}") + Collaborator.objects.create(project=project, user=collaborator) + self.link("A", project=project) + for operation in counts: + with CaptureQueriesContext(connection) as queries: + if operation == "overview": + payload = build_project_case_analytics(self.program) + self.assertEqual(payload["items"][0]["projects_total"], size) + elif operation == "list": + self.assertEqual(len(self.page(limit=100)["results"]), size) + else: + _, rows, _ = self.workbook() + self.assertEqual(len(rows), size) + counts[operation].append(len(queries)) + for query in queries: + self.assertNotRegex( + query["sql"], r"(?i)\b(INSERT INTO|UPDATE|DELETE FROM)\b" + ) + self.assertEqual(counts["overview"], [4, 4]) + self.assertEqual(counts["list"][0], counts["list"][1], counts) + self.assertEqual(counts["export"][0], counts["export"][1], counts) + self.assertLessEqual(counts["list"][1], 10, counts) + self.assertLessEqual(counts["export"][1], 7, counts) + rows = list( + case_project_rows( + self.program, field=self.field, selection={"case_scope": "all"} + ) + ) + with self.assertNumQueries(0): + self.assertEqual(len(ProjectCaseRowSerializer(rows, many=True).data), 100) diff --git a/partner_programs/views.py b/partner_programs/views.py index 7cb99e96..9b47b22d 100644 --- a/partner_programs/views.py +++ b/partner_programs/views.py @@ -18,7 +18,10 @@ PartnerProgramProject, PartnerProgramUserProfile, ) -from partner_programs.pagination import PartnerProgramPagination +from partner_programs.pagination import ( + PartnerProgramPagination, + ProjectAnalyticsAttentionPagination, +) from partner_programs.permissions import ( IsAdminOrManagerOfProgram, IsAdminOrManagerOrExpertOfProgram, @@ -56,6 +59,19 @@ submit_program_project, update_program_link_fields, ) +from partner_programs.serializers.project_case_drilldown import ( + ProjectCaseListQuerySerializer, + ProjectCaseQuerySerializer, + ProjectCaseRowSerializer, + case_analytics_requested, +) +from partner_programs.services.case_fields import get_program_case_field +from partner_programs.services.exports import build_case_projects_export_file +from partner_programs.services.project_case_analytics import build_project_case_analytics +from partner_programs.services.project_case_drilldown import ( + case_list_metadata, + case_project_rows, +) from projects.models import Project from projects.serializers import ProjectListSerializer @@ -407,6 +423,30 @@ class PartnerProgramProjectsAPIView(generics.ListAPIView): permission_classes = [IsAuthenticated, IsAdminOrManagerOfProgram] pagination_class = PartnerProgramPagination + def list(self, request, *args, **kwargs): + if not case_analytics_requested(request.query_params): + return super().list(request, *args, **kwargs) + + program = get_object_or_404(PartnerProgram, pk=self.kwargs["pk"]) + field = get_program_case_field(program) + query = ProjectCaseListQuerySerializer( + data=request.query_params, + context={"case_options": field.get_options_list() if field else []}, + ) + query.is_valid(raise_exception=True) + selection = query.validated_data + rows = case_project_rows(program, field=field, selection=selection) + if selection["search"]: + rows = rows.filter(project__name__icontains=selection["search"]) + paginator = ProjectAnalyticsAttentionPagination(selection) + page = paginator.paginate_queryset(rows, request, view=self) + response = paginator.get_paginated_response( + ProjectCaseRowSerializer(page, many=True).data + ) + cases = build_project_case_analytics(program, field=field) + response.data.update(case_list_metadata(cases, selection)) + return response + def get_queryset(self): if "pk" not in self.kwargs: return Project.objects.none() @@ -475,6 +515,20 @@ def get(self, request, pk: int): {"detail": "Недостаточно прав."}, status=status.HTTP_403_FORBIDDEN ) + if case_analytics_requested(request.query_params): + field = get_program_case_field(program) + query = ProjectCaseQuerySerializer( + data=request.query_params, + context={"case_options": field.get_options_list() if field else []}, + ) + query.is_valid(raise_exception=True) + export_file = build_case_projects_export_file( + program=program, field=field, selection=query.validated_data + ) + return build_xlsx_download_response( + export_file.binary_data, base_name=export_file.base_name + ) + only_submitted = request.query_params.get("only_submitted") in ( "1", "true",