diff --git a/partner_programs/results_views.py b/partner_programs/results_views.py new file mode 100644 index 00000000..4d9aef18 --- /dev/null +++ b/partner_programs/results_views.py @@ -0,0 +1,36 @@ +from django.http import HttpResponse +from rest_framework.response import Response + +from partner_programs.management_views import ManagementBase +from partner_programs.services.management import managed_program +from partner_programs.services.results import ResultsQuery, results_data +from partner_programs.services.results_excel import export_results + + +class ManagementResultsView(ManagementBase): + def get(self, request, pk): + program = managed_program(pk, request.user) + query = ResultsQuery(data=request.query_params) + query.is_valid(raise_exception=True) + values = dict(query.validated_data) + values["scope"] = "current" + response = Response(results_data(program, values)) + response["Cache-Control"] = "private, no-store" + return response + + +class ManagementResultsExportView(ManagementBase): + def get(self, request, pk): + program = managed_program(pk, request.user) + query = ResultsQuery(data=request.query_params) + query.is_valid(raise_exception=True) + data = results_data(program, query.validated_data) + response = HttpResponse( + export_results(data), + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + response[ + "Content-Disposition" + ] = f'attachment; filename="program-{pk}-results.xlsx"' + response["Cache-Control"] = "private, no-store" + return response diff --git a/partner_programs/services/results.py b/partner_programs/services/results.py new file mode 100644 index 00000000..45916943 --- /dev/null +++ b/partner_programs/services/results.py @@ -0,0 +1,192 @@ +"""Organizer results: one calculation for the ranking and its Excel export.""" + +from collections import defaultdict +from decimal import Decimal, ROUND_HALF_UP + +from django.utils import timezone +from rest_framework import serializers + +from partner_programs.services.case_analytics import project_case_links +from partner_programs.services.case_fields import get_program_case_field +from partner_programs.services.management import person_name +from project_rates.models import Criteria, ProjectExpertAssignment, ProjectScore +from project_rates.validators import ProjectScoreValidator + + +class ResultsQuery(serializers.Serializer): + search = serializers.CharField( + required=False, default="", allow_blank=True, max_length=200 + ) + case_scope = serializers.ChoiceField( + choices=["all", "selected", "without_case"], default="all" + ) + case_name = serializers.CharField(required=False, allow_blank=True, max_length=500) + order = serializers.ChoiceField(choices=["asc", "desc"], default="desc") + scope = serializers.ChoiceField(choices=["current", "all"], default="current") + + +def rounded(value): + return value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) + + +def numeric(value): + return float(value) if value is not None else None + + +def valid_score(criterion, value): + try: + ProjectScoreValidator.validate( + criteria_type=criterion.type, + value=value, + criteria_min_value=criterion.min_value, + criteria_max_value=criterion.max_value, + ) + return True + except (TypeError, ValueError): + return False + + +def results_data(program, query): + criteria = list(Criteria.objects.filter(partner_program=program).order_by("pk")) + numeric_criteria = [c for c in criteria if c.type in ("int", "float")] + field = get_program_case_field(program) + options = field.get_options_list() if field else [] + if query["case_scope"] == "selected" and query.get("case_name") not in options: + raise serializers.ValidationError({"case_name": "Выберите существующий кейс."}) + links = list( + project_case_links(program, field=field) + .filter(submitted=True) + .select_related("project") + .order_by("pk") + ) + project_ids = [link.project_id for link in links] + values, people = defaultdict(dict), defaultdict(dict) + for score in ProjectScore.objects.filter( + criteria__partner_program=program, project_id__in=project_ids + ).select_related("user"): + values[(score.project_id, score.user_id)][score.criteria_id] = score.value + people[score.project_id][score.user_id] = score.user + for assignment in ProjectExpertAssignment.objects.filter( + partner_program=program, project_id__in=project_ids + ).select_related("expert__user"): + people[assignment.project_id][assignment.expert.user_id] = assignment.expert.user + required = program.required_evaluators or program.max_project_rates or 1 + maximum = ( + sum(Decimal(str(c.max_value)) for c in numeric_criteria) + if numeric_criteria and all(c.max_value is not None for c in numeric_criteria) + else None + ) + buckets = {name: [] for name in [*options, None]} + for link in links: + experts = [] + for user_id, user in sorted(people[link.project_id].items()): + scores = values[(link.project_id, user_id)] + complete = bool(criteria) and all( + c.pk in scores and valid_score(c, scores[c.pk]) for c in criteria + ) + total = ( + sum(Decimal(scores[c.pk]) for c in numeric_criteria) + if complete and numeric_criteria + else None + ) + experts.append( + { + "user_id": user_id, + "name": person_name(user), + "email": user.email, + "avatar": user.avatar or None, + "complete": complete, + "total": numeric(total), + "values": [ + {"criterion_id": c.pk, "value": scores.get(c.pk)} + for c in criteria + ], + } + ) + completed = [e for e in experts if e["complete"]] + score = ( + rounded(sum(Decimal(str(e["total"])) for e in completed) / len(completed)) + if completed and numeric_criteria + else None + ) + averages = { + str(c.pk): numeric( + rounded( + sum( + Decimal(values[(link.project_id, e["user_id"])][c.pk]) + for e in completed + ) + / len(completed) + ) + ) + for c in numeric_criteria + if completed + } + buckets.setdefault(link.case_name, []).append( + { + "id": link.pk, + "project_id": link.project_id, + "name": link.project.name, + "case_name": link.case_name, + "completed": len(completed), + "required": required, + "ready": len(completed) >= required and score is not None, + "score": numeric(score), + "place": None, + "averages": averages, + "experts": experts, + } + ) + groups = [] + for case_name, projects in buckets.items(): + ready = sorted( + [p for p in projects if p["ready"]], + key=lambda p: (-p["score"], p["name"].lower(), p["id"]), + ) + last_score, place = None, 0 + for index, project in enumerate(ready, 1): + if project["score"] != last_score: + place = index + project["place"] = place + last_score = project["score"] + if query["scope"] != "all": + if query["case_scope"] == "selected" and case_name != query.get("case_name"): + continue + if query["case_scope"] == "without_case" and case_name is not None: + continue + projects = [ + p for p in projects if query["search"].lower() in p["name"].lower() + ] + sign = -1 if query["order"] == "desc" else 1 + projects.sort( + key=lambda p: ( + not p["ready"], + p["score"] is None, + sign * (p["score"] or 0), + p["name"].lower(), + p["id"], + ) + ) + if projects: + groups.append({"case_name": case_name, "projects": projects}) + return { + "program_id": program.pk, + "program_name": program.name, + "generated_at": timezone.now(), + "required_evaluators": required, + "maximum": numeric(maximum), + "has_numeric_criteria": bool(numeric_criteria), + "case_options": options, + "groups": groups, + "criteria": [ + { + "id": c.pk, + "name": c.name, + "description": c.description, + "type": c.type, + "min_value": c.min_value, + "max_value": c.max_value, + } + for c in criteria + ], + } diff --git a/partner_programs/services/results_excel.py b/partner_programs/services/results_excel.py new file mode 100644 index 00000000..467cb2de --- /dev/null +++ b/partner_programs/services/results_excel.py @@ -0,0 +1,143 @@ +"""Typed cells and real formulas, from the same results snapshot as the UI.""" + +from io import BytesIO +from math import isfinite + +from openpyxl import Workbook +from openpyxl.styles import Font, PatternFill +from openpyxl.utils import get_column_letter +from openpyxl.workbook.properties import CalcProperties +from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE + + +def text_cell(sheet, row, column, value): + cell = sheet.cell( + row, + column, + ILLEGAL_CHARACTERS_RE.sub("", str(value)) if value is not None else "", + ) + cell.data_type = "s" # User strings must never become executable Excel formulas. + + +def export_results(data): + workbook = Workbook() + workbook.calculation = CalcProperties(fullCalcOnLoad=True) + rating = workbook.active + rating.title = "Рейтинг" + experts = workbook.create_sheet("Оценки экспертов") + criteria = data["criteria"] + numeric = [c for c in criteria if c["type"] in ("int", "float")] + other = [c for c in criteria if c not in numeric] + rating.append( + [ + "Место", + "Проект", + "Кейс", + "Получено оценок", + "Требуется оценок", + *[c["name"] + " (среднее)" for c in numeric], + "Итог", + "Статус", + "ID проекта", + ] + ) + experts.append( + [ + "Проект", + "Кейс", + "Эксперт", + "Email", + "Статус", + *[c["name"] for c in numeric], + "Сумма", + *[c["name"] for c in other], + "ID проекта", + "ID эксперта", + ] + ) + total_col = 6 + len(numeric) + for group in data["groups"]: + for project in group["projects"]: + expert_rows = [] + start_row = experts.max_row + 1 + for expert in project["experts"]: + row = experts.max_row + 1 + for col, value in enumerate( + [ + project["name"], + project["case_name"], + expert["name"], + expert["email"], + "Полная" if expert["complete"] else "Неполная", + ], + 1, + ): + text_cell(experts, row, col, value) + values = {v["criterion_id"]: v["value"] for v in expert["values"]} + for col, criterion in enumerate(numeric, 6): + value = values.get(criterion["id"]) + if value is not None and value != "": + try: + number = float(value) + if not isfinite(number): + raise ValueError() + experts.cell(row, col, number) + except (TypeError, ValueError): + text_cell(experts, row, col, value) + if expert["complete"]: + expert_rows.append(row) + if numeric: + experts.cell( + row, + total_col, + f"=SUM(F{row}:{get_column_letter(total_col - 1)}{row})", + ) + for col, criterion in enumerate(other, total_col + 1): + value = values.get(criterion["id"]) + if criterion["type"] == "bool" and value in ("True", "False"): + value = "Да" if value == "True" else "Нет" + text_cell(experts, row, col, value) + experts.cell(row, total_col + len(other) + 1, project["project_id"]) + experts.cell(row, total_col + len(other) + 2, expert["user_id"]) + row = rating.max_row + 1 + if project["place"] is not None: + rating.cell(row, 1, project["place"]) + text_cell(rating, row, 2, project["name"]) + text_cell(rating, row, 3, project["case_name"] or "Без кейса") + rating.cell(row, 4, project["completed"]) + rating.cell(row, 5, project["required"]) + for index in range(len(numeric) + 1): + if not expert_rows or not numeric: + continue + col = 6 + index + last_row = experts.max_row + statuses = f"'Оценки экспертов'!$E${start_row}:$E${last_row}" + column = get_column_letter(col) + refs = f"'Оценки экспертов'!{column}{start_row}:{column}{last_row}" + formula = f'AVERAGEIF({statuses},"Полная",{refs})' + if index == len(numeric): + formula = f"ROUND({formula},2)" + rating.cell(row, col, "=" + formula).number_format = "0.00" + status = ( + "Нет балльных критериев" + if not numeric + else "В рейтинге" + if project["ready"] + else "Предварительно" + if project["completed"] + else "Нет оценок" + ) + text_cell(rating, row, total_col + 1, status) + rating.cell(row, total_col + 2, project["project_id"]) + for sheet in (rating, experts): + sheet.freeze_panes = "D2" + sheet.auto_filter.ref = sheet.dimensions + for cell in sheet[1]: + cell.data_type = "s" + cell.font = Font(bold=True, color="FFFFFF") + cell.fill = PatternFill("solid", fgColor="8A63E6") + for col in range(1, sheet.max_column + 1): + sheet.column_dimensions[get_column_letter(col)].width = 24 + output = BytesIO() + workbook.save(output) + return output.getvalue() diff --git a/partner_programs/tests/test_management_results_api.py b/partner_programs/tests/test_management_results_api.py new file mode 100644 index 00000000..73a1e06c --- /dev/null +++ b/partner_programs/tests/test_management_results_api.py @@ -0,0 +1,207 @@ +from io import BytesIO + +from django.test import TestCase +from django.urls import reverse +from openpyxl import load_workbook +from rest_framework.test import APIClient + +from partner_programs.models import PartnerProgramField, PartnerProgramFieldValue +from project_rates.models import Criteria, ProjectScore +from project_rates.tests.helpers import ( + create_rate_expert, + create_rate_program, + create_rate_project, + create_rate_user, + link_project_to_program, +) + + +class ManagementResultsTests(TestCase): + @classmethod + def setUpTestData(cls): + cls.manager = create_rate_user() + cls.program = create_rate_program(is_competitive=True, required_evaluators=3) + cls.program.managers.add(cls.manager) + cls.program.criterias.all().delete() + cls.criteria = [ + Criteria.objects.create( + partner_program=cls.program, + name=name, + type=kind, + min_value=minimum, + max_value=maximum, + ) + for name, kind, minimum, maximum in [ + ("Аналитика", "int", 0, 10), + ("Проработка", "int", 0, 20), + ("Условия", "bool", None, None), + ("Комментарий", "str", None, None), + ] + ] + cls.experts = [create_rate_expert(program=cls.program) for _ in range(4)] + cls.field = PartnerProgramField.objects.create( + partner_program=cls.program, + name="case", + label="Кейс", + field_type="select", + is_required=True, + show_filter=True, + options='Сбер|Т-Банк|Кейс "А"', + ) + cls.projects = [] + for name, case in [ + ("Город", "Сбер"), + ("Маршрут", "Сбер"), + ("Кампус", "Сбер"), + ("Река", "Сбер"), + ("Район", "Сбер"), + ("Без оценок", "Сбер"), + ("Ноль", "Т-Банк"), + ("Черновик", "Сбер"), + ]: + project = create_rate_project() + project.name = name + project.save() + cls.program.users.add( + project.leader, through_defaults={"partner_program_data": {}} + ) + link = link_project_to_program(cls.program, project) + PartnerProgramFieldValue.objects.create( + program_project=link, field=cls.field, value_text=case + ) + type(link).objects.filter(pk=link.pk).update(submitted=name != "Черновик") + cls.projects.append(project) + for index, pairs in enumerate( + [ + [(9, 18), (7, 14), (8, 16)], + [(8, 14), (7, 15), (8, 15)], + [(8, 14), (7, 15), (8, 15)], + [(7, 13), (7, 14), (7, 14)], + [(9, 18), (8, 18)], + [], + [(0, 0), (0, 0), (0, 0)], + [], + ] + ): + for expert, pair in zip(cls.experts, pairs): + cls.score(cls.projects[index], expert, pair) + + @classmethod + def score(cls, project, expert, pair): + for criterion, value in zip(cls.criteria, [*pair, "False", ""]): + ProjectScore.objects.create( + criteria=criterion, user=expert, project=project, value=str(value) + ) + + def setUp(self): + self.client = APIClient() + self.client.force_authenticate(self.manager) + + def url(self, export=False, pk=None): + return reverse( + "partner_programs:management-results" + ("-export" if export else ""), + kwargs={"pk": pk or self.program.pk}, + ) + + def rows(self, **query): + response = self.client.get(self.url(), query) + self.assertEqual(response.status_code, 200, response.data) + return [p for g in response.data["groups"] for p in g["projects"]] + + def test_rounding_ties_case_ranks_and_pending(self): + rows = {p["name"]: p for p in self.rows()} + self.assertEqual((rows["Город"]["score"], rows["Город"]["place"]), (24, 1)) + self.assertEqual((rows["Кампус"]["score"], rows["Кампус"]["place"]), (22.33, 2)) + self.assertEqual(rows["Маршрут"]["place"], 2) + self.assertEqual(rows["Река"]["place"], 4) + self.assertEqual((rows["Район"]["score"], rows["Район"]["place"]), (26.5, None)) + self.assertEqual((rows["Ноль"]["score"], rows["Ноль"]["place"]), (0, 1)) + self.assertIsNone(rows["Без оценок"]["score"]) + self.assertNotIn("Черновик", rows) + + def test_sort_and_search_preserve_rank(self): + rows = self.rows(case_scope="selected", case_name="Сбер", order="asc") + self.assertEqual([p["place"] for p in rows if p["ready"]], [4, 2, 2, 1]) + self.assertEqual(self.rows(search="РЕКА")[0]["place"], 4) + self.assertEqual(len(self.rows(case_scope="selected", case_name="Т-Банк")), 1) + self.assertEqual(self.client.get(self.url(), {"order": "bad"}).status_code, 400) + + def test_partial_evaluation_excluded_and_surplus_retained(self): + ProjectScore.objects.create( + criteria=self.criteria[0], + project=self.projects[0], + user=self.experts[3], + value="10", + ) + row = self.rows(search="Город")[0] + self.assertEqual((row["score"], row["completed"]), (24, 3)) + ProjectScore.objects.filter(user=self.experts[3]).delete() + self.score(self.projects[0], self.experts[3], (0, 0)) + row = self.rows(search="Город")[0] + self.assertEqual((row["score"], row["completed"]), (18, 4)) + type(self.program).objects.filter(pk=self.program.pk).update( + required_evaluators=5 + ) + row = self.rows(search="Город")[0] + self.assertEqual((row["score"], row["place"]), (18, None)) + + def test_access_and_foreign_program(self): + for user, expected in [ + (None, 401), + (self.experts[0], 403), + (create_rate_user(), 403), + ]: + self.client.force_authenticate(user) + for export in (False, True): + self.assertEqual(self.client.get(self.url(export)).status_code, expected) + self.client.force_authenticate(self.manager) + foreign = create_rate_program() + self.assertEqual(self.client.get(self.url(pk=foreign.pk)).status_code, 403) + + def test_xlsx_types_formulas_scope_and_text_injection(self): + self.projects[0].name = "=1+1" + self.projects[0].save() + response = self.client.get(self.url(True), {"search": "=1+1"}) + self.assertEqual(response.status_code, 200) + book = load_workbook(BytesIO(response.content)) + self.assertEqual(book.sheetnames, ["Рейтинг", "Оценки экспертов"]) + rating, experts = book.worksheets + self.assertEqual(rating.max_row, 2) + self.assertEqual(rating["B2"].data_type, "s") + self.assertEqual(rating["B2"].value, "=1+1") + self.assertEqual(experts["F2"].data_type, "n") + self.assertEqual(experts["H2"].value, "=SUM(F2:G2)") + self.assertIn("ROUND(AVERAGEIF(", rating["H2"].value) + self.assertIn('"Полная"', rating["H2"].value) + self.assertEqual(experts["I2"].value, "Нет") + response = self.client.get(self.url(True), {"search": "nothing", "scope": "all"}) + self.assertEqual( + load_workbook(BytesIO(response.content)).worksheets[0].max_row, 8 + ) + + def test_no_numeric_criteria_produces_no_score(self): + program = create_rate_program(required_evaluators=1) + program.managers.add(self.manager) + program.criterias.all().delete() + criterion = Criteria.objects.create( + partner_program=program, name="Условия", type="bool" + ) + project = create_rate_project() + program.users.add(project.leader, through_defaults={"partner_program_data": {}}) + link = link_project_to_program(program, project) + type(link).objects.filter(pk=link.pk).update(submitted=True) + ProjectScore.objects.create( + criteria=criterion, project=project, user=self.experts[0], value="True" + ) + data = self.client.get(self.url(pk=program.pk)).data + self.assertFalse(data["has_numeric_criteria"]) + row = data["groups"][0]["projects"][0] + self.assertIsNone(row["score"]) + self.assertIsNone(row["place"]) + + def test_invalid_historical_value_does_not_break_export(self): + ProjectScore.objects.filter( + criteria=self.criteria[0], project=self.projects[0], user=self.experts[0] + ).update(value="bad") + self.assertEqual(self.rows(search="Город")[0]["completed"], 2) + self.assertEqual(self.client.get(self.url(True)).status_code, 200) diff --git a/partner_programs/urls.py b/partner_programs/urls.py index c429f052..3ea5239c 100644 --- a/partner_programs/urls.py +++ b/partner_programs/urls.py @@ -54,9 +54,24 @@ EvaluatorsView, ) +from partner_programs.results_views import ( + ManagementResultsView, + ManagementResultsExportView, +) + app_name = "partner_programs" urlpatterns = [ + path( + "/management/results/", + ManagementResultsView.as_view(), + name="management-results", + ), + path( + "/management/results/export/", + ManagementResultsExportView.as_view(), + name="management-results-export", + ), path("/management/experts/", ExpertsView.as_view()), path( "/management/experts//cases/",