From 1f6b326af1e51d6ac13f0dd33c92c1d4d78fa12f Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 24 Mar 2026 10:21:55 +1100 Subject: [PATCH 001/199] chore: reset submission date when deleting portfolio --- app/api/submission/portfolio_api.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/api/submission/portfolio_api.rb b/app/api/submission/portfolio_api.rb index 611616813e..aec64c65e1 100644 --- a/app/api/submission/portfolio_api.rb +++ b/app/api/submission/portfolio_api.rb @@ -55,6 +55,10 @@ class PortfolioApi < Grape::API # Remove file or portfolio? if params[:idx].nil? && params[:name].nil? && params[:kind].nil? + project.update!({ + portfolio_submission_date: nil, + portfolio_production_date: nil + }) project.remove_portfolio # returns details of file elsif !(params[:idx].nil? || params[:name].nil? || params[:kind].nil?) idx = params[:idx] From c997ab04afa53e33572c0b8a478fedc494175996 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 24 Mar 2026 10:22:10 +1100 Subject: [PATCH 002/199] chore: show portfolio submission date in grades csv --- app/models/unit.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index d31d84fb78..1a65420aa9 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -2474,9 +2474,9 @@ def student_task_completion_stats def student_grades_csv CSV.generate do |row| - row << %w(unit_code username student_id target_grade submitted_grade portfolio_production_date has_portfolio spec_con_days grade rationale assessor assessor_id) + row << %w(unit_code username student_id target_grade submitted_grade portfolio_submission_date portfolio_production_date has_portfolio spec_con_days grade rationale assessor assessor_id) active_projects.each do |project| - row << [project.unit.code, project.student.username, project.student.student_id, project.target_grade, project.submitted_grade, project.portfolio_production_date, project.portfolio_exists?, project.spec_con_days, project.grade, project.grade_rationale, project.assessor&.name, project.assessor&.id] + row << [project.unit.code, project.student.username, project.student.student_id, project.target_grade, project.submitted_grade, project.portfolio_submission_date, project.portfolio_production_date, project.portfolio_exists?, project.spec_con_days, project.grade, project.grade_rationale, project.assessor&.name, project.assessor&.id] end end end From 5f49f462cefa8ee386a2f767b05d098106376450 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:16:46 +1100 Subject: [PATCH 003/199] test: attempt fix for flaky test --- test/models/task_test.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/models/task_test.rb b/test/models/task_test.rb index cab193b681..e3662b6ab5 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -822,7 +822,9 @@ def test_accept_files_checks_they_all_exist task_definition = unit.task_definitions.first task_definition.start_date = Time.zone.now - 1.week + task_definition.target_date = Time.zone.now + 1.day task_definition.due_date = Time.zone.now + 1.week + task_definition.target_grade = 0 task_definition.upload_requirements = [ { "key" => 'file0', From a607486e9953e28b2bd6b04e9a135a4190b87c6e Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:29:47 +1100 Subject: [PATCH 004/199] fix: normalise test to ensure its consistent (#594) * fix: normalise test to ensure its consistent * refactor: ensure test is consistent --- test/api/tasks_api_test.rb | 87 +++++++++++++++----------------------- 1 file changed, 34 insertions(+), 53 deletions(-) diff --git a/test/api/tasks_api_test.rb b/test/api/tasks_api_test.rb index 4d2f996016..75e6b578dc 100644 --- a/test/api/tasks_api_test.rb +++ b/test/api/tasks_api_test.rb @@ -930,17 +930,30 @@ def test_require_comment_for_feedback_submission_assess_in_portfolio def test_resubmission_doesnt_change_submission_date Sidekiq::Testing.inline! do - unit = FactoryBot.create(:unit, task_count: 2, student_count: 0) + unit = FactoryBot.create( + :unit, + with_students: false, + student_count: 0, + task_count: 0, + tutorials: 0, + stream_count: 0, + staff_count: 0, + campus_count: 0, + outcome_count: 0 + ) tutor = FactoryBot.create(:user, :tutor) unit_role = unit.employ_staff(tutor, Role.tutor) tutorial_stream = FactoryBot.create(:tutorial_stream, unit: unit) tutorial = FactoryBot.create(:tutorial, unit: unit, tutorial_stream: tutorial_stream, campus: nil, unit_role: unit_role) - - td = unit.task_definitions.first - - td.update!(due_date: Time.zone.today + 1.day, tutorial_stream: tutorial_stream) - assert_not td.nil? + td = FactoryBot.create( + :task_definition, + unit: unit, + tutorial_stream: tutorial_stream, + target_grade: 0, + outcome_count: 0 + ) + td.update!(due_date: Time.zone.today + 1.day) student1 = FactoryBot.create(:user, :student) student2 = FactoryBot.create(:user, :student) @@ -951,29 +964,20 @@ def test_resubmission_doesnt_change_submission_date project1.enrol_in(tutorial) project2.enrol_in(tutorial) + task1 = project1.task_for_task_definition(td) + task2 = project2.task_for_task_definition(td) + tasks = unit.tasks_for_task_inbox(tutor, false) assert tasks.to_a.empty? # Submit a task before the due date (student 1) - add_auth_header_for(user: student1) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project1.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task1.submit(student1) travel 10.minutes # Submit a task before the due date (student 2) - add_auth_header_for(user: student2) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project2.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task2.submit(student2) tasks = unit.tasks_for_task_inbox(tutor, false) @@ -982,56 +986,40 @@ def test_resubmission_doesnt_change_submission_date assert_equal project1.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project2.id, tasks.second.project.id, "Second task in inbox should be project2's task" - task1 = project1.task_for_task_definition(td) - task2 = project2.task_for_task_definition(td) - assert_equal TaskStatus.ready_for_feedback, task1.task_status assert_equal TaskStatus.ready_for_feedback, task2.task_status assert task2.submission_date > task1.submission_date + original_submission_date = task1.submission_date # Submit the task again, ensure the submission_date hasn't changed (student1) travel 10.minutes - # Submit a task before the due date (student 1) - add_auth_header_for(user: student1) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project1.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task1.submit(student1) tasks = unit.tasks_for_task_inbox(tutor, false) assert_equal project1.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project2.id, tasks.second.project.id, "Second task in inbox should be project2's task" - task1 = project1.task_for_task_definition(td) - task2 = project2.task_for_task_definition(td) + task1.reload assert task2.submission_date > task1.submission_date + assert_equal original_submission_date, task1.submission_date assert TaskStatus.ready_for_feedback, task1.task_status # Submit the task again after the duedate, ensure the submission_date hasn't changed (student1) travel 2.days - # Submit a task before the due date (student 1) - add_auth_header_for(user: student1) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project1.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task1.submit(student1) tasks = unit.tasks_for_task_inbox(tutor, false) assert_equal project1.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project2.id, tasks.second.project.id, "Second task in inbox should be project2's task" - task1 = project1.task_for_task_definition(td) - task2 = project2.task_for_task_definition(td) + task1.reload assert task2.submission_date > task1.submission_date + assert_equal original_submission_date, task1.submission_date assert TaskStatus.ready_for_feedback, task1.task_status task1.update(task_status_id: TaskStatus.fix_and_resubmit.id) @@ -1039,23 +1027,16 @@ def test_resubmission_doesnt_change_submission_date # Submit the task again, now expecting submission date to update travel 10.minutes - # Submit a task before the due date (student 1) - add_auth_header_for(user: student1) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project1.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task1.submit(student1) tasks = unit.tasks_for_task_inbox(tutor, false) assert_equal project2.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project1.id, tasks.second.project.id, "Second task in inbox should be project2's task" - task1 = project1.task_for_task_definition(td) - task2 = project2.task_for_task_definition(td) + task1.reload assert task1.submission_date > task2.submission_date + assert task1.submission_date > original_submission_date end end From e5f128d12c908954121fd638952a30c7ff724183 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:39:13 +1100 Subject: [PATCH 005/199] Merge pull request #592 from b0ink/refactor/confirm-recursive-fix refactor: require confirmation for recursive fix and resubmit --- app/api/tasks_api.rb | 3 ++- app/models/task.rb | 10 +++++----- test/models/task_test.rb | 10 +++++----- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/app/api/tasks_api.rb b/app/api/tasks_api.rb index ac3a636fdd..86790affa2 100644 --- a/app/api/tasks_api.rb +++ b/app/api/tasks_api.rb @@ -158,6 +158,7 @@ class TasksApi < Grape::API optional :grade, type: Integer, desc: 'Grade value if task is a graded task (required if task definition is a graded task)' optional :quality_pts, type: Integer, desc: 'Quality points value if task has quality assessment' optional :discussed, type: Boolean, desc: 'Mark task as discussed' + optional :trigger_recursive_fix, desc: 'If marking fix and resubmit, recursively update preqreuisite submissions to fix' end put '/projects/:id/task_def_id/:task_definition_id' do project = Project.find(params[:id]) @@ -203,7 +204,7 @@ class TasksApi < Grape::API end logger.info "#{current_user.username} assessing task #{task.id} to #{params[:trigger]}" - result = task.trigger_transition(trigger: params[:trigger], by_user: current_user, quality: params[:quality_pts]) + result = task.trigger_transition(trigger: params[:trigger], by_user: current_user, quality: params[:quality_pts], recursive_fix: params[:trigger_recursive_fix]) if result.nil? && task.task_definition.restrict_status_updates error!({ error: 'This task can only be updated by your tutor.' }, 403) end diff --git a/app/models/task.rb b/app/models/task.rb index ee3e76564c..619c06e5db 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -522,7 +522,7 @@ def ensured_group_submission group.create_submission self, '', group.projects.map { |proj| { project: proj, pct: 100 / group.projects.count } } end - def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: false, quality: 1) + def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: false, quality: 1, recursive_fix: false) # # Ensure that assessor is allowed to update the task in the indicated way # @@ -594,7 +594,7 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: lc = comments.last # Prevent duplicate status comments during feedback unless lc && lc.user == by_user && lc.comment == status.name && (lc.content_type != 'status' || lc.task_status == status) - assess status, by_user + assess status, by_user, Time.zone.now, recursive_fix # Add a status comment for new assessments - only recorded on submitter's task in groups add_status_comment(by_user, status) @@ -674,7 +674,7 @@ def grade_task(new_grade, ui = nil, grading_group = false) end end - def assess(task_status, assessor, assess_date = Time.zone.now) + def assess(task_status, assessor, assess_date = Time.zone.now, recursive_fix = false) # Set the task's status to the assessment outcome status # and flag it as no longer awaiting signoff self.task_status = task_status @@ -725,7 +725,7 @@ def assess(task_status, assessor, assess_date = Time.zone.now) end end - if task_status == TaskStatus.fix_and_resubmit + if task_status == TaskStatus.fix_and_resubmit && recursive_fix # Look for other submitted tasks from this student that has this task as a prerequisite # If they are ready for feedback, automatically assess them to fix and resubmit dependents = TaskPrerequisite.where(prerequisite_id: task_definition.id) @@ -738,7 +738,7 @@ def assess(task_status, assessor, assess_date = Time.zone.now) next unless task.task_status == TaskStatus.ready_for_feedback # Since we are calling this assess method again, we recursively check for more dependent tasks that need to be updated - task.assess(TaskStatus.fix_and_resubmit, assessor, assess_date) + task.assess(TaskStatus.fix_and_resubmit, assessor, assess_date, recursive_fix) task.add_status_comment(assessor, TaskStatus.fix_and_resubmit) task.add_text_comment(assessor, "**Automated comment**: A prerequisite task was updated to Fix and Resubmit, so this task was updated as well. You may need to review and update the prerequisite before resubmitting.") end diff --git a/test/models/task_test.rb b/test/models/task_test.rb index e3662b6ab5..27ee3670f2 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -1473,7 +1473,7 @@ def test_prerequisite_tasks_change_to_fix_and_resubmit assert_equal TaskStatus.ready_for_feedback, task3.task_status assert_equal TaskStatus.ready_for_feedback, task4.task_status - # Test case 1: Ensure parent prerequisite is not affected + # Test case 1: Without recursive_fix, dependent tasks should not be affected task2.assess(TaskStatus.fix_and_resubmit, tutor) task1.reload @@ -1481,10 +1481,10 @@ def test_prerequisite_tasks_change_to_fix_and_resubmit task3.reload task4.reload - # Task 1 should not be affected if the dependent task is assessed to fix and resubmit + # Task 1 should not be affected, and recursive fixes should not run by default assert_equal TaskStatus.ready_for_feedback, task1.task_status, "Parent prerequisite should not be affected" assert_equal TaskStatus.fix_and_resubmit, task2.task_status, "Task should have updated to Fix and Resubmit" - assert_equal TaskStatus.fix_and_resubmit, task3.task_status, "Dependent task should have automatically moved to Fix and Resubmit" + assert_equal TaskStatus.ready_for_feedback, task3.task_status, "Dependent task should not change without recursive_fix" assert_equal TaskStatus.ready_for_feedback, task4.task_status # Task 4 has no prerequsite links # Reset status @@ -1501,7 +1501,7 @@ def test_prerequisite_tasks_change_to_fix_and_resubmit task4.reload # Test case 2: Ensure dependent tasks are recursively moved to fix and resubmit - task1.assess(TaskStatus.fix_and_resubmit, tutor) + task1.assess(TaskStatus.fix_and_resubmit, tutor, Time.zone.now, true) task1.reload task2.reload @@ -1529,7 +1529,7 @@ def test_prerequisite_tasks_change_to_fix_and_resubmit task4.reload # Test case 3: Ensure tasks that are not Ready for Feedback are not moved to Fix and resubmit - task1.assess(TaskStatus.fix_and_resubmit, tutor) + task1.assess(TaskStatus.fix_and_resubmit, tutor, Time.zone.now, true) task1.reload task2.reload From 18b6ea7d04d50d024d47c8b814636470e34b477d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:40:03 +1100 Subject: [PATCH 006/199] chore(release): 10.0.0-95 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34e30380ce..63c2d35d0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-95](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-94...v10.0.0-95) (2026-03-24) + + +### Bug Fixes + +* normalise test to ensure its consistent ([#594](https://github.com/b0ink/doubtfire-deploy/issues/594)) ([a607486](https://github.com/b0ink/doubtfire-deploy/commit/a607486e9953e28b2bd6b04e9a135a4190b87c6e)) + ## [10.0.0-94](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-93...v10.0.0-94) (2026-03-23) From 7d36efc5ab42e0380ad9bd4a6fe77860e028834c Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Tue, 24 Mar 2026 15:39:55 +1100 Subject: [PATCH 007/199] test: skip pdf gen to improve test speeds (#596) --- test/api/tasks_api_test.rb | 317 ++++++++++++++------------------ test/models/task_status_test.rb | 32 +--- 2 files changed, 146 insertions(+), 203 deletions(-) diff --git a/test/api/tasks_api_test.rb b/test/api/tasks_api_test.rb index 75e6b578dc..ccdca9140f 100644 --- a/test/api/tasks_api_test.rb +++ b/test/api/tasks_api_test.rb @@ -564,96 +564,93 @@ def test_download_task_pdf end def test_cant_submit_until_prerequisites_submitted - Sidekiq::Testing.inline! do - # Create a unit and two task definitions - unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) - td1 = unit.task_definitions.first - td2 = unit.task_definitions.second - project = unit.active_projects.first - - task = project.task_for_task_definition(td2) - - td1.update( - upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], - target_grade: 0, # Pass - start_date: Time.zone.now - 2.weeks, - target_date: Time.zone.now + 1.week - ) + # Create a unit and two task definitions + unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) + td1 = unit.task_definitions.first + td2 = unit.task_definitions.second + project = unit.active_projects.first - td2.update( - upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], - target_grade: 3, # HD - start_date: Time.zone.now - 2.weeks, - target_date: Time.zone.now + 1.week - ) + task = project.task_for_task_definition(td2) - # Create a prerequisite on the second taskDef that adds the first taskDef as a prereq - prereq = TaskPrerequisite.create!( - task_definition: td2, # Before you can submit td2... - prerequisite: td1, # You need to submit td1 - task_status_id: TaskStatus.ready_for_feedback.id - ) + td1.update( + upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], + target_grade: 0, # Pass + start_date: Time.zone.now - 2.weeks, + target_date: Time.zone.now + 1.week + ) - assert prereq.valid? + td2.update( + upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], + target_grade: 3, # HD + start_date: Time.zone.now - 2.weeks, + target_date: Time.zone.now + 1.week + ) - # Add username and auth_token to Header - add_auth_header_for(user: project.user) + # Create a prerequisite on the second taskDef that adds the first taskDef as a prereq + prereq = TaskPrerequisite.create!( + task_definition: td2, # Before you can submit td2... + prerequisite: td1, # You need to submit td1 + task_status_id: TaskStatus.ready_for_feedback.id + ) - data_to_post = { - trigger: 'ready_for_feedback' - } + assert prereq.valid? - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) + # Add username and auth_token to Header + add_auth_header_for(user: project.user) - # Attempt to make a submission that has an unsubmitted prerequisite - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 409, last_response.status, last_response_body - task = project.task_for_task_definition(td2) - # Ensure the submission was denied - assert_equal TaskStatus.not_started, task.task_status - assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been submitted" + data_to_post = { + trigger: 'ready_for_feedback' + } - prereq.update(task_status_id: TaskStatus.discuss.id) - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 409, last_response.status, last_response_body - task.reload - # Ensure the submission was denied - assert_equal TaskStatus.not_started, task.task_status - assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been discussed" + data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - prereq.update(task_status_id: TaskStatus.demonstrate.id) - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 409, last_response.status, last_response_body - task.reload - # Ensure the submission was denied - assert_equal TaskStatus.not_started, task.task_status - assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been demonstrated" + # Attempt to make a submission that has an unsubmitted prerequisite + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 409, last_response.status, last_response_body + task = project.task_for_task_definition(td2) + # Ensure the submission was denied + assert_equal TaskStatus.not_started, task.task_status + assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been submitted" - prereq.update(task_status_id: TaskStatus.complete.id) - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 409, last_response.status, last_response_body - task.reload - # Ensure the submission was denied - assert_equal TaskStatus.not_started, task.task_status - assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been completed" + prereq.update(task_status_id: TaskStatus.discuss.id) + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 409, last_response.status, last_response_body + task.reload + # Ensure the submission was denied + assert_equal TaskStatus.not_started, task.task_status + assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been discussed" - prereq.update(task_status_id: TaskStatus.ready_for_feedback.id) + prereq.update(task_status_id: TaskStatus.demonstrate.id) + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 409, last_response.status, last_response_body + task.reload + # Ensure the submission was denied + assert_equal TaskStatus.not_started, task.task_status + assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been demonstrated" - # Make a submission to the prerequsite task - post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body - task1 = project.task_for_task_definition(td1) - assert_equal TaskStatus.ready_for_feedback, task1.task_status + prereq.update(task_status_id: TaskStatus.complete.id) + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 409, last_response.status, last_response_body + task.reload + # Ensure the submission was denied + assert_equal TaskStatus.not_started, task.task_status + assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been completed" - # Re-attempt to make a submission (Prerequisite status is ready for feedback, expecting complete) - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body - task.reload - assert_equal TaskStatus.ready_for_feedback, task.task_status + prereq.update(task_status_id: TaskStatus.ready_for_feedback.id) - prereq.destroy - unit.destroy - end + # Use a direct status change here to avoid waiting for submission processing. + task1 = project.task_for_task_definition(td1) + task1.submit(project.user) + assert_equal TaskStatus.ready_for_feedback, task1.task_status + + # Re-attempt to make a submission (Prerequisite status is ready for feedback, expecting complete) + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 201, last_response.status, last_response_body + task.reload + assert_equal TaskStatus.ready_for_feedback, task.task_status + + prereq.destroy + unit.destroy end def test_prerequisites_task_status @@ -678,13 +675,6 @@ def test_prerequisites_task_status project = unit.active_projects.first - # Add username and auth_token to Header - add_auth_header_for(user: project.user) - - data_to_post = { - trigger: 'ready_for_feedback' - } - # Create a prerequisite on the second taskDef that adds the first taskDef as a prereq prereq = TaskPrerequisite.create!( task_definition: td2, # Before you can submit td2... @@ -697,78 +687,53 @@ def test_prerequisites_task_status tests = [ { prerequisite_status: TaskStatus.ready_for_feedback, - required_status: TaskStatus.ready_for_feedback, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.ready_for_feedback }, { prerequisite_status: TaskStatus.discuss, - required_status: TaskStatus.discuss, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.discuss }, { prerequisite_status: TaskStatus.demonstrate, - required_status: TaskStatus.demonstrate, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.demonstrate }, { prerequisite_status: TaskStatus.discuss, - required_status: TaskStatus.demonstrate, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.demonstrate }, { prerequisite_status: TaskStatus.demonstrate, - required_status: TaskStatus.discuss, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.discuss }, { prerequisite_status: TaskStatus.complete, - required_status: TaskStatus.complete, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.complete }, { prerequisite_status: TaskStatus.complete, - required_status: TaskStatus.ready_for_feedback, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.ready_for_feedback }, { prerequisite_status: TaskStatus.discuss, - required_status: TaskStatus.ready_for_feedback, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.ready_for_feedback } ] - Sidekiq::Testing.inline! do - prereq_task = project.task_for_task_definition(td1) - task = project.task_for_task_definition(td2) - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - - tests.each do |test| - prereq_task.update(task_status_id: test[:prerequisite_status].id) - task.update(task_status_id: TaskStatus.not_started.id) - - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal test[:expected_status], last_response.status, last_response_body - task.reload - if test[:expected_status] == 201 - # Ensure submission was accepted - assert_equal TaskStatus.ready_for_feedback, task.task_status - else - # Ensure submission was denied - assert_equal TaskStatus.not_started, task.task_status - end - end + prereq_task = project.task_for_task_definition(td1) + task = project.task_for_task_definition(td2) + + tests.each do |test| + prereq.update!(task_status_id: test[:required_status].id) + prereq_task.update!(task_status_id: test[:prerequisite_status].id) + task.update!(task_status_id: TaskStatus.not_started.id, submission_date: nil) - prereq.destroy - unit.destroy + task.submit(project.user) + task.reload + assert_equal TaskStatus.ready_for_feedback, task.task_status end + + prereq.destroy + unit.destroy end def test_check_in_comment @@ -866,66 +831,60 @@ def test_requires_discussion_blocks_complete_until_discussed_comment_added end def test_require_comment_for_feedback_submission_assess_in_portfolio - Sidekiq::Testing.inline! do - unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) - td1 = unit.task_definitions.first - project = unit.active_projects.first - - task = project.task_for_task_definition(td1) - - td1.update( - upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], - target_grade: 0, # Pass - start_date: Time.zone.now - 2.weeks, - target_date: Time.zone.now + 1.week, - assess_in_portfolio_only: false - ) + unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) + td1 = unit.task_definitions.first + project = unit.active_projects.first - add_auth_header_for(user: project.user) + task = project.task_for_task_definition(td1) - # Make a submission where a comment isn't required - post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", - with_file('test_files/submissions/program.cs', 'application/json', { - trigger: 'ready_for_feedback' - }) - assert_equal 201, last_response.status, last_response_body - task.reload - assert_equal TaskStatus.ready_for_feedback, task.task_status + td1.update( + upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], + target_grade: 0, # Pass + start_date: Time.zone.now - 2.weeks, + target_date: Time.zone.now + 1.week, + assess_in_portfolio_only: false + ) + + add_auth_header_for(user: project.user) - task.update(task_status: TaskStatus.not_started) + # Use a direct submit here so the test can focus on the comment requirement. + task.submit(project.user) + task.reload + assert_equal TaskStatus.ready_for_feedback, task.task_status - td1.update(assess_in_portfolio_only: true) + task.update!(task_status: TaskStatus.not_started, submission_date: nil) - # Make a submission where a comment is required - post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", - with_file('test_files/submissions/program.cs', 'application/json', { - trigger: 'ready_for_feedback' - }) - assert_equal 422, last_response.status, last_response_body - task.reload - assert_equal TaskStatus.not_started, task.task_status + td1.update(assess_in_portfolio_only: true) - comment = 'I would like feedback with my code..' + # Make a submission where a comment is required + post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", + with_file('test_files/submissions/program.cs', 'application/json', { + trigger: 'ready_for_feedback' + }) + assert_equal 422, last_response.status, last_response_body + task.reload + assert_equal TaskStatus.not_started, task.task_status - # Make a submission with comment - post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", - with_file('test_files/submissions/program.cs', 'application/json', { - trigger: 'ready_for_feedback', - comment: comment - }) - assert_equal 201, last_response.status, last_response_body - task.reload - assert_equal TaskStatus.ready_for_feedback, task.task_status + comment = 'I would like feedback with my code..' - status_comment = task.comments.last - text_comment = task.comments.second_to_last + # Make a submission with comment + post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", + with_file('test_files/submissions/program.cs', 'application/json', { + trigger: 'ready_for_feedback', + comment: comment + }) + assert_equal 201, last_response.status, last_response_body + task.reload + assert_equal TaskStatus.ready_for_feedback, task.task_status - assert_not status_comment.nil? - assert_not text_comment.nil? + status_comment = task.comments.last + text_comment = task.comments.second_to_last - assert_equal TaskStatus.ready_for_feedback.name, status_comment.comment - assert_equal comment, text_comment.comment - end + assert_not status_comment.nil? + assert_not text_comment.nil? + + assert_equal TaskStatus.ready_for_feedback.name, status_comment.comment + assert_equal comment, text_comment.comment end def test_resubmission_doesnt_change_submission_date diff --git a/test/models/task_status_test.rb b/test/models/task_status_test.rb index e5081c59d3..0c61e627ba 100644 --- a/test/models/task_status_test.rb +++ b/test/models/task_status_test.rb @@ -95,15 +95,10 @@ def test_status_changed_task_definition_assess_in_portfolio_only task_status: TaskStatus.not_started ) - data_to_post = { - trigger: 'ready_for_feedback', - comment: "I would like feedback for my task" - } + late_submission_time = td.due_date + 1.minute - add_auth_header_for(user: project.student) - - # Make a submission for this student - post "/api/projects/#{project.id}/task_def_id/#{td.id}/submission", data_to_post + # Make a late submission for this student. + tc.submit(project.student, late_submission_time) # Get the exceeded exceeded task and check it is now assess_in_portfolio tc.reload @@ -144,14 +139,10 @@ def test_status_changed_unit_has_assess_in_portfolio_tasks task_status: TaskStatus.not_started ) - data_to_post = { - trigger: 'ready_for_feedback' - } - - add_auth_header_for(user: project.student) + late_submission_time = td.due_date + 1.minute - # Make a submission for this student - post "/api/projects/#{project.id}/task_def_id/#{td.id}/submission", data_to_post + # Make a late submission for this student. + tc.submit(project.student, late_submission_time) tc.reload assert_equal TaskStatus.assess_in_portfolio, tc.task_status end @@ -190,15 +181,8 @@ def test_tutor_cant_signoff_tasks_complete_assess_in_portfolio_only task_status: TaskStatus.not_started ) - data_to_post = { - trigger: 'ready_for_feedback', - comment: 'I would like feedback for my task' - } - - add_auth_header_for(user: project.student) - - # Make a submission for this student - post "/api/projects/#{project.id}/task_def_id/#{td.id}/submission", data_to_post + # Move the task to ready_for_feedback without waiting for submission processing. + tc.submit(project.student) tc.reload From eb2195a18d148264eb0570cdfc3ecfa6b577e1f9 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Wed, 25 Mar 2026 11:18:18 +1100 Subject: [PATCH 008/199] feat: allow admin users to view any unit analytics (#597) --- app/api/marking_sessions_api.rb | 4 ++-- app/models/unit.rb | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/api/marking_sessions_api.rb b/app/api/marking_sessions_api.rb index 54edc83e7f..36d16af2f5 100644 --- a/app/api/marking_sessions_api.rb +++ b/app/api/marking_sessions_api.rb @@ -27,7 +27,7 @@ class MarkingSessionsApi < Grape::API end unit_role = unit.unit_role_for(current_user) - unless unit_role + unless unit_role || current_user.role == Role.admin error!({ error: "You are not authorised to view marking sessions for this unit" }, 403) end @@ -51,7 +51,7 @@ class MarkingSessionsApi < Grape::API .where(unit: unit) .where(start_time: start_date..end_date) - sessions = sessions.where(user_id: current_user.id) if unit_role.role != Role.convenor + sessions = sessions.where(user_id: current_user.id) if unit_role && unit_role&.role != Role.convenor present sessions, with: Entities::MarkingSessionEntity end diff --git a/app/models/unit.rb b/app/models/unit.rb index 1a65420aa9..64e753b67a 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -91,7 +91,8 @@ def self.permissions :grant_spec_con, :download_jplag_report, :get_marking_sessions, - :get_staff_notes + :get_staff_notes, + :get_tutor_times ] # What can auditors do with units? From 0aa220863329083b1fe7d53fc2a4f9ee9a3f9386 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:05:57 +1100 Subject: [PATCH 009/199] feat: pause days awaiting feedback during breaks (#599) --- app/api/entities/teaching_period_entity.rb | 2 +- app/api/teaching_periods_public_api.rb | 4 +- app/models/task.rb | 28 ++++++++ app/models/unit.rb | 78 ++++++++++++---------- app/models/unit_role.rb | 18 +++-- test/models/task_test.rb | 58 ++++++++++++++++ 6 files changed, 143 insertions(+), 45 deletions(-) diff --git a/app/api/entities/teaching_period_entity.rb b/app/api/entities/teaching_period_entity.rb index 67a9b570e6..772e2b3e89 100644 --- a/app/api/entities/teaching_period_entity.rb +++ b/app/api/entities/teaching_period_entity.rb @@ -9,7 +9,7 @@ class TeachingPeriodEntity < Grape::Entity expose :active do |teaching_period, options| object.active_until > DateTime.now end - expose :breaks, if: :full_details, using: Entities::BreakEntity + expose :breaks, if: :include_breaks, using: Entities::BreakEntity expose :units, if: :full_details do |teaching_period, options| Entities::UnitEntity.represent teaching_period.units, summary_only: true, user: options[:user] end diff --git a/app/api/teaching_periods_public_api.rb b/app/api/teaching_periods_public_api.rb index 9f17860541..704527fd96 100644 --- a/app/api/teaching_periods_public_api.rb +++ b/app/api/teaching_periods_public_api.rb @@ -4,12 +4,12 @@ class TeachingPeriodsPublicApi < Grape::API desc "Get a teaching period's details" get '/teaching_periods/:id' do teaching_period = TeachingPeriod.find(params[:id]) - present teaching_period, with: Entities::TeachingPeriodEntity, full_details: true, user: current_user + present teaching_period, with: Entities::TeachingPeriodEntity, full_details: true, include_breaks: true, user: current_user end desc 'Get all the Teaching Periods' get '/teaching_periods' do teaching_periods = TeachingPeriod.all - present teaching_periods, with: Entities::TeachingPeriodEntity + present teaching_periods, with: Entities::TeachingPeriodEntity, include_breaks: true end end diff --git a/app/models/task.rb b/app/models/task.rb index 619c06e5db..d0b397f38e 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -408,6 +408,19 @@ def due_date return extension_date end + def days_awaiting_feedback(now_time = Time.zone.now) + return 0 if submission_date.blank? + + submission_time = submission_date.to_f + current_time = now_time.to_f + return 0 if current_time <= submission_time + + teaching_breaks = unit&.teaching_period&.breaks || [] + paused_seconds = break_overlap_seconds(submission_time, current_time, teaching_breaks) + + ([0, current_time - submission_time - paused_seconds].max / 1.day).floor + end + def complete? status == :complete end @@ -1695,6 +1708,21 @@ def mark_as_moderated(moderation_type: :random_sample) private + def break_overlap_seconds(start_time, end_time, teaching_breaks) + teaching_breaks.sum do |teaching_break| + break_start = teaching_break.start_date.to_f + break_duration = teaching_break.number_of_weeks.to_i.weeks + break_end = break_start + break_duration + + next 0 unless break_start.finite? && break_duration.positive? + + overlap_start = [start_time, break_start].max + overlap_end = [end_time, break_end].min + + [0, overlap_end - overlap_start].max + end + end + def delete_associated_files if group_submission && group_submission.tasks.count <= 1 group_submission.destroy diff --git a/app/models/unit.rb b/app/models/unit.rb index 64e753b67a..6da145b9fb 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -1646,7 +1646,7 @@ def tutorial_stream_abbr def times_tasks_have_been_assessed CSV.generate() do |csv| # Add headers - csv << ([ + csv << [ 'Tutorial', 'Tutor', 'Student Username', @@ -1662,7 +1662,7 @@ def times_tasks_have_been_assessed 'demonstrate', 'ready_for_feedback', 'discussed_in_class' - ]) + ] tasks .joins("LEFT JOIN task_engagements ON task_engagements.task_id = tasks.id") @@ -1716,7 +1716,7 @@ def times_tasks_have_been_assessed def days_awaiting_feedback_by_tutorial_csv CSV.generate() do |csv| # Add headers - csv << ([ + csv << [ 'Tutorial', 'Tutor', 'Username', @@ -1725,41 +1725,47 @@ def days_awaiting_feedback_by_tutorial_csv 'Task Definition', 'Task ID', 'Days Awaiting Feedback' - ]) + ] # Add data - tasks - .joins(:task_definition) - .joins('INNER JOIN users ON users.id = projects.user_id') - .joins("LEFT OUTER JOIN (#{tutorial_enrolment_subquery}) as sq ON sq.project_id = projects.id AND (sq.tutorial_stream_id = task_definitions.tutorial_stream_id OR sq.tutorial_stream_id IS NULL)") - .joins('LEFT JOIN tutorials ON tutorials.id = sq.tutorial_id') - .select( - 'users.username AS username', - 'users.first_name AS first_name', - 'users.last_name AS last_name', - 'tasks.id as task_id', - 'task_definitions.abbreviation as task_abbr', - 'tasks.project_id as project_id', - 'DATEDIFF(CURDATE(),submission_date) AS days_since_submission', - 'tutorial_id', - 'tutorials.unit_role_id as unit_role_id', - 'tutorials.abbreviation AS tutorial_abbreviation' - ) - .group('tasks.id', 'task_definitions.abbreviation', 'tasks.project_id', 'tutorial_id', 'unit_role_id', 'submission_date') - .order('unit_role_id', 'days_since_submission DESC') - .where(projects: { enrolled: true }) - .where(task_status: TaskStatus.ready_for_feedback) - .each do |row| - csv << ([ + rows = tasks + .includes(project: { unit: { teaching_period: :breaks } }) + .joins(:task_definition) + .joins('INNER JOIN users ON users.id = projects.user_id') + .joins("LEFT OUTER JOIN (#{tutorial_enrolment_subquery}) as sq ON sq.project_id = projects.id AND (sq.tutorial_stream_id = task_definitions.tutorial_stream_id OR sq.tutorial_stream_id IS NULL)") + .joins('LEFT JOIN tutorials ON tutorials.id = sq.tutorial_id') + .select( + 'users.username AS username', + 'users.first_name AS first_name', + 'users.last_name AS last_name', + 'tasks.id as task_id', + 'task_definitions.abbreviation as task_abbr', + 'tasks.project_id as project_id', + 'tasks.submission_date', + 'tutorial_id', + 'tutorials.unit_role_id as unit_role_id', + 'tutorials.abbreviation AS tutorial_abbreviation' + ) + .group('tasks.id', 'task_definitions.abbreviation', 'tasks.project_id', 'tutorial_id', 'unit_role_id', 'tasks.submission_date') + .where(projects: { enrolled: true }) + .where(task_status: TaskStatus.ready_for_feedback) + .to_a + + unit_roles_by_id = UnitRole.includes(:user).where(id: rows.map(&:unit_role_id).compact.uniq).index_by(&:id) + + rows + .sort_by { |row| [row.unit_role_id || Float::INFINITY, -row.days_awaiting_feedback] } + .each do |row| + csv << [ row['tutorial_abbreviation'], - row['unit_role_id'].present? ? UnitRole.find(row['unit_role_id']).user.name : '', + row.unit_role_id.present? ? unit_roles_by_id[row.unit_role_id]&.user&.name.to_s : '', row['username'], "#{row['first_name']} #{row['last_name']}", row['project_id'], row['task_abbr'], row['task_id'], - row['days_since_submission'] - ]) + row.days_awaiting_feedback + ] end end end @@ -1860,7 +1866,7 @@ def task_completion_csv def staff_notes_csv CSV.generate() do |csv| # Add headers - csv << ([ + csv << [ 'Student Username', 'Project ID', 'Student Name', @@ -1868,7 +1874,7 @@ def staff_notes_csv 'Created', 'Author Name', 'Author Username', - ]) + ] StaffNote.joins(project: :unit) .where(units: { id: id }) @@ -3110,23 +3116,23 @@ def get_tutor_times_csv(start_date: nil, end_date: nil, timezone: nil, ignore_se CSV.generate() do |csv| # Add headers - csv << ([ + csv << [ 'User ID', 'Tutor', 'Total Minutes', 'Assessments', 'Comments', - ]) + ] summary.each do |row| - csv << ([ + csv << [ row[:user_id].to_s, row[:tutor_name], row[:total_minutes].to_s, row[:assessments_made].to_s, row[:comments_made].to_s, - ]) + ] end end end diff --git a/app/models/unit_role.rb b/app/models/unit_role.rb index 765bf117f8..dd0e0c7b48 100644 --- a/app/models/unit_role.rb +++ b/app/models/unit_role.rb @@ -38,7 +38,9 @@ def tasks_awaiting_feedback end def oldest_task_awaiting_feedback - tasks_awaiting_feedback.order("submission_date ASC").first + tasks_awaiting_feedback + .includes(project: { unit: { teaching_period: :breaks } }) + .max_by(&:days_awaiting_feedback) end # @@ -147,7 +149,11 @@ def populate_summary_stats(summary_stats, tutorial_stream, tutorial, row) .distinct if tutorial_tasks.count > 0 - data[:oldest_task_days] = (Time.zone.now - tutorial_tasks.order("submission_date ASC").first.submission_date.to_time).to_i / 1.day + oldest_task = tutorial_tasks + .includes(project: { unit: { teaching_period: :breaks } }) + .max_by(&:days_awaiting_feedback) + + data[:oldest_task_days] = oldest_task&.days_awaiting_feedback || 0 data[:tasks_awaiting_feedback_count] = tutorial_tasks.count else data[:oldest_task_days] = 0 @@ -258,7 +264,7 @@ def get_marking_sessions_csv(start_date: nil, end_date: nil, timezone: nil) CSV.generate do |csv| # Add headers - csv << ([ + csv << [ 'Start Date', 'Start Time', 'End Date', @@ -270,13 +276,13 @@ def get_marking_sessions_csv(start_date: nil, end_date: nil, timezone: nil) 'Comments Added', 'Assessments Made', 'During Tutorial' - ]) + ] result.each do |row| start_time = row[:start_time].in_time_zone(tz) end_time = row[:end_time].in_time_zone(tz) - csv << ([ + csv << [ start_time.strftime('%Y-%m-%d %A'), start_time.strftime('%H:%M'), end_time.strftime('%Y-%m-%d %A'), @@ -288,7 +294,7 @@ def get_marking_sessions_csv(start_date: nil, end_date: nil, timezone: nil) row[:comments_added], row[:assessments], row[:during_tutorial] ? 'TRUE' : 'FALSE' - ]) + ] end end diff --git a/test/models/task_test.rb b/test/models/task_test.rb index 27ee3670f2..7321e2e4c0 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -9,6 +9,7 @@ class TaskTest < ActiveSupport::TestCase include TestHelpers::TestFileHelper include TestHelpers::AuthHelper include TestHelpers::JsonHelper + include ActiveSupport::Testing::TimeHelpers def error!(msg, _code) raise StandardError, msg @@ -48,6 +49,63 @@ def test_comments_for_user end end + def test_days_awaiting_feedback_pauses_during_break + travel_to Time.zone.parse('2026-04-10 00:00:00 UTC') do + teaching_period = FactoryBot.create( + :teaching_period, + start_date: Time.zone.parse('2026-03-01 00:00:00 UTC'), + end_date: Time.zone.parse('2026-04-30 00:00:00 UTC'), + active_until: Time.zone.parse('2026-05-31 00:00:00 UTC') + ) + teaching_period.add_break(Time.zone.parse('2026-04-01 00:00:00 UTC'), 2) + + unit = FactoryBot.create(:unit, teaching_period: teaching_period, with_students: false) + task = FactoryBot.create(:task, project: FactoryBot.create(:project, unit: unit)) + task.update!(submission_date: Time.zone.parse('2026-03-29 00:00:00 UTC')) + + assert_equal 3.0, task.days_awaiting_feedback + end + travel_back + end + + def test_days_awaiting_feedback_resumes_after_break + travel_to Time.zone.parse('2026-04-18 00:00:00 UTC') do + teaching_period = FactoryBot.create( + :teaching_period, + start_date: Time.zone.parse('2026-03-01 00:00:00 UTC'), + end_date: Time.zone.parse('2026-04-30 00:00:00 UTC'), + active_until: Time.zone.parse('2026-05-31 00:00:00 UTC') + ) + teaching_period.add_break(Time.zone.parse('2026-04-01 00:00:00 UTC'), 2) + + unit = FactoryBot.create(:unit, teaching_period: teaching_period, with_students: false) + task = FactoryBot.create(:task, project: FactoryBot.create(:project, unit: unit)) + task.update!(submission_date: Time.zone.parse('2026-03-29 00:00:00 UTC')) + + assert_equal 6.0, task.days_awaiting_feedback + end + travel_back + end + + def test_days_awaiting_feedback_stays_at_zero_for_submissions_made_during_break + travel_to Time.zone.parse('2026-04-10 00:00:00 UTC') do + teaching_period = FactoryBot.create( + :teaching_period, + start_date: Time.zone.parse('2026-03-01 00:00:00 UTC'), + end_date: Time.zone.parse('2026-04-30 00:00:00 UTC'), + active_until: Time.zone.parse('2026-05-31 00:00:00 UTC') + ) + teaching_period.add_break(Time.zone.parse('2026-04-01 00:00:00 UTC'), 2) + + unit = FactoryBot.create(:unit, teaching_period: teaching_period, with_students: false) + task = FactoryBot.create(:task, project: FactoryBot.create(:project, unit: unit)) + task.update!(submission_date: Time.zone.parse('2026-04-05 00:00:00 UTC')) + + assert_equal 0.0, task.days_awaiting_feedback + end + travel_back + end + def test_pdf_creation_with_gif unit = Unit.first td = TaskDefinition.new({ From 73203a25cf9c73b6b97565df7a6c9af6f71895dd Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:43:21 +1100 Subject: [PATCH 010/199] chore(release): 10.0.0-96 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63c2d35d0b..9f18ecdbc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-96](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-95...v10.0.0-96) (2026-03-26) + + +### Features + +* allow admin users to view any unit analytics ([#597](https://github.com/b0ink/doubtfire-api/issues/597)) ([eb2195a](https://github.com/b0ink/doubtfire-api/commit/eb2195a18d148264eb0570cdfc3ecfa6b577e1f9)) +* pause days awaiting feedback during breaks ([#599](https://github.com/b0ink/doubtfire-api/issues/599)) ([0aa2208](https://github.com/b0ink/doubtfire-api/commit/0aa220863329083b1fe7d53fc2a4f9ee9a3f9386)) + ## [10.0.0-95](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-94...v10.0.0-95) (2026-03-24) From b1946bd8105e70e4baff1e0547b1b728d8a731c8 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Fri, 27 Mar 2026 11:10:04 +1100 Subject: [PATCH 011/199] feat: prefer encrypted rails credentials over env vars (#600) --- .gitignore | 3 +++ config/application.rb | 27 ++++++++++++++++++++------- config/database.yml | 40 ++++++++++++++++++++-------------------- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 977065f4e4..35a46c3ae7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ student-work/ .DS_Store .env .env* +/config/credentials/*.yml.enc +/config/credentials/*.key +/config/master.key # IDE configs .idea/ diff --git a/config/application.rb b/config/application.rb index fe280a2899..761103f765 100644 --- a/config/application.rb +++ b/config/application.rb @@ -17,6 +17,8 @@ module Doubtfire # class Application < Rails::Application config.load_defaults 7.0 + config.credentials.content_path = Rails.root.join('config/credentials/credentials.yml.enc') + config.credentials.key_path = Rails.root.join('config/credentials/master.key') # Remove Action Mailbox and Active Storage routes - not used initializer(:remove_action_mailbox_and_activestorage_routes, after: :add_routing_paths) do |app| @@ -64,6 +66,17 @@ def self.fetch_boolean_env(name) %w'true 1'.include?(ENV.fetch(name, 'false').downcase) end + def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) + credential_value = + if credential_path.length == 1 && credentials.respond_to?(credential_path.first) + credentials.public_send(credential_path.first) + else + credentials.dig(*credential_path) + end + + credential_value.nil? ? ENV.fetch(env_key, default) : credential_value + end + # ==> Log to stdout config.log_to_stdout = Application.fetch_boolean_env('DF_LOG_TO_STDOUT') @@ -83,19 +96,19 @@ def self.fetch_boolean_env(name) # Defaults to 10MB (10,000,000 bytes) config.max_file_size = ENV.fetch('DF_MAX_FILE_SIZE', 10_000_000) - # ==> Load credentials from env - credentials.secret_key_base = ENV.fetch('DF_SECRET_KEY_BASE', Rails.env.production? ? nil : '9e010ee2f52af762916406fd2ac488c5694a6cc784777136e657511f8bbc7a73f96d59c0a9a778a0d7cf6406f8ecbf77efe4701dfbd63d8248fc7cc7f32dea97') - credentials.secret_key_attr = ENV.fetch('DF_SECRET_KEY_ATTR', Rails.env.production? ? nil : 'e69fc5960ca0e8700844a3a25fe80373b41c0a265d342eba06950113f3766fd983bad9ec51bf36eb615d9711bfe1dd90b8e35f01841b323f604ffee857e32055') - credentials.secret_key_devise = ENV.fetch('DF_SECRET_KEY_DEVISE', Rails.env.production? ? nil : 'f4e23c4388dc600e503a09ad057b8271d8fcf4c2cd6723b44f33db638e49075fe96bc545eed9110ded0c5df505625d4e1c838b718349eecf1d39270d0829d5b9') - credentials.secret_key_aaf = ENV.fetch('DF_SECRET_KEY_AAF', Rails.env.production? ? nil : 'secretsecret12345') - credentials.secret_key_moss = ENV.fetch('DF_SECRET_KEY_MOSS', nil) + # Prefer encrypted Rails credentials, while keeping env vars as a safe fallback. + credentials.secret_key_base = Application.fetch_credential_or_env(:secret_key_base, env_key: 'DF_SECRET_KEY_BASE', default: Rails.env.production? ? nil : '9e010ee2f52af762916406fd2ac488c5694a6cc784777136e657511f8bbc7a73f96d59c0a9a778a0d7cf6406f8ecbf77efe4701dfbd63d8248fc7cc7f32dea97') + credentials.secret_key_attr = Application.fetch_credential_or_env(:secret_key_attr, env_key: 'DF_SECRET_KEY_ATTR', default: Rails.env.production? ? nil : 'e69fc5960ca0e8700844a3a25fe80373b41c0a265d342eba06950113f3766fd983bad9ec51bf36eb615d9711bfe1dd90b8e35f01841b323f604ffee857e32055') + credentials.secret_key_devise = Application.fetch_credential_or_env(:secret_key_devise, env_key: 'DF_SECRET_KEY_DEVISE', default: Rails.env.production? ? nil : 'f4e23c4388dc600e503a09ad057b8271d8fcf4c2cd6723b44f33db638e49075fe96bc545eed9110ded0c5df505625d4e1c838b718349eecf1d39270d0829d5b9') + credentials.secret_key_aaf = Application.fetch_credential_or_env(:aaf, :secret_key, env_key: 'DF_SECRET_KEY_AAF', default: Rails.env.production? ? nil : 'secretsecret12345') + credentials.secret_key_moss = Application.fetch_credential_or_env(:moss, :secret_key, env_key: 'DF_SECRET_KEY_MOSS') # ==> LTI settings # If enabled, mounts the LTI routes and enables LTI authentication. config.lti_enabled = ENV.fetch('LTI_ENABLED', false).to_s.downcase == "true" # Shared secret between Ruby on Rails API and the LTI.js API # LTI.js will send signed JWT tokens using this secret - config.lti_api_secret = ENV.fetch('LTI_SHARED_API_SECRET', nil) + config.lti_api_secret = Application.fetch_credential_or_env(:lti, :shared_api_secret, env_key: 'LTI_SHARED_API_SECRET') # ==> Moderation settings config.moderation_score_factor = Float(ENV.fetch('MODERATION_SCORE_FACTOR', 1.0)) diff --git a/config/database.yml b/config/database.yml index 06faaed683..eaaf83f653 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,29 +1,29 @@ development: - adapter: <%= ENV['DF_DEV_DB_ADAPTER'] %> - database: <%= ENV['DF_DEV_DB_DATABASE'] %> - username: <%= ENV['DF_DEV_DB_USERNAME'] %> - password: <%= ENV['DF_DEV_DB_PASSWORD'] %> - host: <%= ENV['DF_DEV_DB_HOST'] %> + adapter: <%= Rails.application.credentials.dig(:database, :development, :adapter) || ENV['DF_DEV_DB_ADAPTER'] %> + database: <%= Rails.application.credentials.dig(:database, :development, :database) || ENV['DF_DEV_DB_DATABASE'] %> + username: <%= Rails.application.credentials.dig(:database, :development, :username) || ENV['DF_DEV_DB_USERNAME'] %> + password: <%= Rails.application.credentials.dig(:database, :development, :password) || ENV['DF_DEV_DB_PASSWORD'] %> + host: <%= Rails.application.credentials.dig(:database, :development, :host) || ENV['DF_DEV_DB_HOST'] %> min_messages: warning test: - adapter: <%= ENV['DF_TEST_DB_ADAPTER'] %> - database: <%= ENV['DF_TEST_DB_DATABASE'] %> - username: <%= ENV['DF_TEST_DB_USERNAME'] %> - password: <%= ENV['DF_TEST_DB_PASSWORD'] %> - host: <%= ENV['DF_TEST_DB_HOST'] %> + adapter: <%= Rails.application.credentials.dig(:database, :test, :adapter) || ENV['DF_TEST_DB_ADAPTER'] %> + database: <%= Rails.application.credentials.dig(:database, :test, :database) || ENV['DF_TEST_DB_DATABASE'] %> + username: <%= Rails.application.credentials.dig(:database, :test, :username) || ENV['DF_TEST_DB_USERNAME'] %> + password: <%= Rails.application.credentials.dig(:database, :test, :password) || ENV['DF_TEST_DB_PASSWORD'] %> + host: <%= Rails.application.credentials.dig(:database, :test, :host) || ENV['DF_TEST_DB_HOST'] %> min_messages: warning staging: - adapter: <%= ENV['DF_STAGING_DB_ADAPTER'] %> - host: <%= ENV['DF_STAGING_DB_HOST'] %> - database: <%= ENV['DF_STAGING_DB_DATABASE'] %> - username: <%= ENV['DF_STAGING_DB_USERNAME'] %> - password: <%= ENV['DF_STAGING_DB_PASSWORD'] %> + adapter: <%= Rails.application.credentials.dig(:database, :staging, :adapter) || ENV['DF_STAGING_DB_ADAPTER'] %> + host: <%= Rails.application.credentials.dig(:database, :staging, :host) || ENV['DF_STAGING_DB_HOST'] %> + database: <%= Rails.application.credentials.dig(:database, :staging, :database) || ENV['DF_STAGING_DB_DATABASE'] %> + username: <%= Rails.application.credentials.dig(:database, :staging, :username) || ENV['DF_STAGING_DB_USERNAME'] %> + password: <%= Rails.application.credentials.dig(:database, :staging, :password) || ENV['DF_STAGING_DB_PASSWORD'] %> production: - adapter: <%= ENV['DF_PRODUCTION_DB_ADAPTER'] %> - host: <%= ENV['DF_PRODUCTION_DB_HOST'] %> - database: <%= ENV['DF_PRODUCTION_DB_DATABASE'] %> - username: <%= ENV['DF_PRODUCTION_DB_USERNAME'] %> - password: <%= ENV['DF_PRODUCTION_DB_PASSWORD'] %> + adapter: <%= Rails.application.credentials.dig(:database, :production, :adapter) || ENV['DF_PRODUCTION_DB_ADAPTER'] %> + host: <%= Rails.application.credentials.dig(:database, :production, :host) || ENV['DF_PRODUCTION_DB_HOST'] %> + database: <%= Rails.application.credentials.dig(:database, :production, :database) || ENV['DF_PRODUCTION_DB_DATABASE'] %> + username: <%= Rails.application.credentials.dig(:database, :production, :username) || ENV['DF_PRODUCTION_DB_USERNAME'] %> + password: <%= Rails.application.credentials.dig(:database, :production, :password) || ENV['DF_PRODUCTION_DB_PASSWORD'] %> From 2104123214aaa5ee0d16c7c984fa32f482ef5497 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 27 Mar 2026 11:11:13 +1100 Subject: [PATCH 012/199] chore(release): 10.0.0-97 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f18ecdbc9..84b9d2a3ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-97](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-96...v10.0.0-97) (2026-03-27) + + +### Features + +* prefer encrypted rails credentials over env vars ([#600](https://github.com/b0ink/doubtfire-api/issues/600)) ([b1946bd](https://github.com/b0ink/doubtfire-api/commit/b1946bd8105e70e4baff1e0547b1b728d8a731c8)) + ## [10.0.0-96](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-95...v10.0.0-96) (2026-03-26) From 4357f5c17db2d937aa0ab573567d45f41cadcf5b Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Fri, 27 Mar 2026 11:57:04 +1100 Subject: [PATCH 013/199] feat: move tii and active record secrets to rails credentials (#601) --- app/helpers/turn_it_in.rb | 2 +- config/environments/development.rb | 6 +++--- config/environments/production.rb | 6 +++--- config/environments/test.rb | 6 +++--- test/helpers/tii_test_helper.rb | 3 ++- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/app/helpers/turn_it_in.rb b/app/helpers/turn_it_in.rb index 7bf73e4f7a..fdcafd788a 100644 --- a/app/helpers/turn_it_in.rb +++ b/app/helpers/turn_it_in.rb @@ -31,7 +31,7 @@ def self.load_config(config) # Setup authorization TCAClient.configure do |tii_config| # Configure API key authorization: api_key - tii_config.api_key['api_key'] = ENV.fetch('TCA_API_KEY', nil) + tii_config.api_key['api_key'] = Doubtfire::Application.fetch_credential_or_env(:tii, :api_key, env_key: 'TCA_API_KEY') # Uncomment the following line to set a prefix for the API key, e.g. 'Bearer' (defaults to nil) tii_config.api_key_prefix['api_key'] = 'Bearer' tii_config.host = ENV.fetch('TCA_HOST', nil) diff --git a/config/environments/development.rb b/config/environments/development.rb index 7dd6f8112f..05d01df74c 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -106,7 +106,7 @@ # pdfgen log verbosity config.pdfgen_quiet = false - config.active_record.encryption.key_derivation_salt = ENV['DF_ENCRYPTION_KEY_DERIVATION_SALT'] || 'U9jurHMfZbMpzlbDTMe5OSAhUJYHla9Z' - config.active_record.encryption.deterministic_key = ENV['DF_ENCRYPTION_DETERMINISTIC_KEY'] || 'zYtzYUlLFaWdvdUO5eIINRT6ZKDddcgx' - config.active_record.encryption.primary_key = ENV['DF_ENCRYPTION_PRIMARY_KEY'] || '92zoF7RJaQ01JEExOgHbP9bRWldNQUz5' + config.active_record.encryption.key_derivation_salt = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :key_derivation_salt, env_key: 'DF_ENCRYPTION_KEY_DERIVATION_SALT', default: 'U9jurHMfZbMpzlbDTMe5OSAhUJYHla9Z') + config.active_record.encryption.deterministic_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :deterministic_key, env_key: 'DF_ENCRYPTION_DETERMINISTIC_KEY', default: 'zYtzYUlLFaWdvdUO5eIINRT6ZKDddcgx') + config.active_record.encryption.primary_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :primary_key, env_key: 'DF_ENCRYPTION_PRIMARY_KEY', default: '92zoF7RJaQ01JEExOgHbP9bRWldNQUz5') end diff --git a/config/environments/production.rb b/config/environments/production.rb index 37b429654a..b673b52f68 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -50,7 +50,7 @@ config.action_mailer.smtp_settings[:authentication] = nil if %w[no_auth none].include?(config.action_mailer.smtp_settings[:authentication]) end - config.active_record.encryption.key_derivation_salt = ENV.fetch('DF_ENCRYPTION_KEY_DERIVATION_SALT', nil) - config.active_record.encryption.deterministic_key = ENV.fetch('DF_ENCRYPTION_DETERMINISTIC_KEY', nil) - config.active_record.encryption.primary_key = ENV.fetch('DF_ENCRYPTION_PRIMARY_KEY', nil) + config.active_record.encryption.key_derivation_salt = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :key_derivation_salt, env_key: 'DF_ENCRYPTION_KEY_DERIVATION_SALT') + config.active_record.encryption.deterministic_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :deterministic_key, env_key: 'DF_ENCRYPTION_DETERMINISTIC_KEY') + config.active_record.encryption.primary_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :primary_key, env_key: 'DF_ENCRYPTION_PRIMARY_KEY') end diff --git a/config/environments/test.rb b/config/environments/test.rb index 24fcb3d4cc..fd2b7ce8f9 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -37,9 +37,9 @@ # Logging level (:debug, :info, :warn, :error, :fatal) config.log_level = :warn - config.active_record.encryption.key_derivation_salt = ENV['DF_ENCRYPTION_KEY_DERIVATION_SALT'] || 'U9jurHMfZbMpzlbDTMe5OSAhUJYHla9Z' - config.active_record.encryption.deterministic_key = ENV['DF_ENCRYPTION_KEY_DERIVATION_SALT'] || 'zYtzYUlLFaWdvdUO5eIINRT6ZKDddcgx' - config.active_record.encryption.primary_key = ENV['DF_ENCRYPTION_KEY_DERIVATION_SALT'] || '92zoF7RJaQ01JEExOgHbP9bRWldNQUz5' + config.active_record.encryption.key_derivation_salt = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :key_derivation_salt, env_key: 'DF_ENCRYPTION_KEY_DERIVATION_SALT', default: 'U9jurHMfZbMpzlbDTMe5OSAhUJYHla9Z') + config.active_record.encryption.deterministic_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :deterministic_key, env_key: 'DF_ENCRYPTION_DETERMINISTIC_KEY', default: 'zYtzYUlLFaWdvdUO5eIINRT6ZKDddcgx') + config.active_record.encryption.primary_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :primary_key, env_key: 'DF_ENCRYPTION_PRIMARY_KEY', default: '92zoF7RJaQ01JEExOgHbP9bRWldNQUz5') # Set turn it in environment ENV.store('TCA_SIGNING_KEY', 'test') diff --git a/test/helpers/tii_test_helper.rb b/test/helpers/tii_test_helper.rb index 0a76ab35c5..92a5f234c1 100644 --- a/test/helpers/tii_test_helper.rb +++ b/test/helpers/tii_test_helper.rb @@ -8,9 +8,10 @@ module TiiTestHelper module_function def tii_headers(base = {}) + api_key = Doubtfire::Application.fetch_credential_or_env(:tii, :api_key, env_key: 'TCA_API_KEY') base["headers"] = { 'Accept'=>'application/json', - 'Authorization'=>"Bearer #{ENV['TCA_API_KEY']}", + 'Authorization'=>"Bearer #{api_key}", 'Content-Type'=>'application/json', 'X-Turnitin-Integration-Name'=>'formatif-tii', 'X-Turnitin-Integration-Version'=>'1.0' From 89cad868c7d6eb3b8f9ed971b2724d10d3e0609e Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 27 Mar 2026 11:57:17 +1100 Subject: [PATCH 014/199] chore(release): 10.0.0-98 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b9d2a3ce..110b058595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-98](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-97...v10.0.0-98) (2026-03-27) + + +### Features + +* move tii and active record secrets to rails credentials ([#601](https://github.com/b0ink/doubtfire-api/issues/601)) ([4357f5c](https://github.com/b0ink/doubtfire-api/commit/4357f5c17db2d937aa0ab573567d45f41cadcf5b)) + ## [10.0.0-97](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-96...v10.0.0-97) (2026-03-27) From 4d1c6ee8d227b1d4b0ac57bbeea30f20f47e8b71 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sun, 29 Mar 2026 16:55:04 +1100 Subject: [PATCH 015/199] feat: add enforce feedback unit setting --- app/api/entities/unit_entity.rb | 2 ++ app/api/units_api.rb | 8 ++++++-- ...1457_add_enforce_feedback_before_discussed_in_class.rb | 5 +++++ db/schema.rb | 3 ++- 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20260327041457_add_enforce_feedback_before_discussed_in_class.rb diff --git a/app/api/entities/unit_entity.rb b/app/api/entities/unit_entity.rb index 4743f67511..6a9e0aa8b9 100644 --- a/app/api/entities/unit_entity.rb +++ b/app/api/entities/unit_entity.rb @@ -67,5 +67,7 @@ def can_read_unit_config?(my_role) expose :feedback_warning_threshold_days, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) } expose :feedback_overflow_threshold_days, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) } + + expose :enforce_feedback_before_discussed_in_class, if: lambda { |unit, options| is_staff?(options[:my_role]) } end end diff --git a/app/api/units_api.rb b/app/api/units_api.rb index a20c8a3d7a..8741ffb32e 100644 --- a/app/api/units_api.rb +++ b/app/api/units_api.rb @@ -92,6 +92,7 @@ class UnitsApi < Grape::API optional :assessment_enabled, type: Boolean optional :feedback_warning_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its highlighted in the tutors inbox' optional :feedback_overflow_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its added to overflow marking' + optional :enforce_feedback_before_discussed_in_class, type: Boolean, desc: 'Require feedback to be completed before tasks can be marked discussed in class' mutually_exclusive :teaching_period_id, :start_date mutually_exclusive :teaching_period_id, :end_date @@ -126,7 +127,8 @@ class UnitsApi < Grape::API :overseer_image_id, :assessment_enabled, :feedback_warning_threshold_days, - :feedback_overflow_threshold_days + :feedback_overflow_threshold_days, + :enforce_feedback_before_discussed_in_class ) if unit.teaching_period_id.present? && (unit_parameters.key?(:start_date) || unit_parameters['teaching_period_id'] == -1) @@ -174,6 +176,7 @@ class UnitsApi < Grape::API optional :allow_student_change_tutorial, type: Boolean, desc: 'Can turn on/off student ability to change tutorials', default: true optional :feedback_warning_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its highlighted in the tutors inbox' optional :feedback_overflow_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its added to overflow marking' + optional :enforce_feedback_before_discussed_in_class, type: Boolean, desc: 'Require feedback to be completed before tasks can be marked discussed in class', default: false mutually_exclusive :teaching_period_id, :start_date mutually_exclusive :teaching_period_id, :end_date @@ -205,7 +208,8 @@ class UnitsApi < Grape::API :portfolio_auto_generation_date, :allow_student_change_tutorial, :feedback_warning_threshold_days, - :feedback_overflow_threshold_days + :feedback_overflow_threshold_days, + :enforce_feedback_before_discussed_in_class ) # Ensure the user is authorised to convene units diff --git a/db/migrate/20260327041457_add_enforce_feedback_before_discussed_in_class.rb b/db/migrate/20260327041457_add_enforce_feedback_before_discussed_in_class.rb new file mode 100644 index 0000000000..ba60168e01 --- /dev/null +++ b/db/migrate/20260327041457_add_enforce_feedback_before_discussed_in_class.rb @@ -0,0 +1,5 @@ +class AddEnforceFeedbackBeforeDiscussedInClass < ActiveRecord::Migration[8.0] + def change + add_column :units, :enforce_feedback_before_discussed_in_class, :boolean, null: false, default: false + end +end diff --git a/db/schema.rb b/db/schema.rb index 73fc5ebd58..1e49bb2022 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_03_22_230239) do +ActiveRecord::Schema[8.0].define(version: 2026_03_27_041457) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -743,6 +743,7 @@ t.boolean "mark_late_submissions_as_assess_in_portfolio", default: false, null: false t.integer "feedback_warning_threshold_days", default: 5 t.integer "feedback_overflow_threshold_days", default: 7 + t.boolean "enforce_feedback_before_discussed_in_class", default: false, null: false t.index ["draft_task_definition_id"], name: "index_units_on_draft_task_definition_id" t.index ["main_convenor_id"], name: "index_units_on_main_convenor_id" t.index ["overseer_image_id"], name: "index_units_on_overseer_image_id" From b494ed8a35d6ad7f5b5d98f71b96e5df9ce02ed1 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 31 Mar 2026 13:30:01 +1100 Subject: [PATCH 016/199] refactor: use portfolio submission date and time instead of pdf gen date - the timezone of the project's campus will also be used to format the time --- .../project_compile_portfolio_module.rb | 26 +++++++++++++++++++ app/views/portfolio/portfolio_pdf.pdf.erb | 7 ++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/app/models/pdf_generation/project_compile_portfolio_module.rb b/app/models/pdf_generation/project_compile_portfolio_module.rb index f5e2c8bba5..311bcaff57 100644 --- a/app/models/pdf_generation/project_compile_portfolio_module.rb +++ b/app/models/pdf_generation/project_compile_portfolio_module.rb @@ -42,6 +42,9 @@ class ProjectAppController < ApplicationController :base_path, :image_path, :learning_summary_report, + :submission_date, + :formatted_submission_date, + :formatted_submission_time, :ordered_tasks, :portfolio_tasks, :task_defs, @@ -55,6 +58,8 @@ def init(project, is_retry) @student = project.student @project = project @learning_summary_report = project.learning_summary_report_path + @submission_date = project.portfolio_submission_date + @formatted_submission_date, @formatted_submission_time = format_submission_date(project) @files = project.portfolio_files(ensure_valid: true, force_ascii: is_retry) @base_path = project.portfolio_temp_path @image_path = Rails.root.join('public/assets/images') @@ -73,6 +78,27 @@ def make_pdf logger.debug 'Running make_pdf: (portfolio)' generate_pdf(template: '/portfolio/portfolio_pdf') end + + private + + def format_submission_date(project) + return [nil, nil] if @submission_date.blank? + + campus_timezone = project.campus&.timezone.presence + + submission_time = + if campus_timezone.present? + @submission_date.in_time_zone(campus_timezone) + else + @submission_date.to_time.getlocal + end + + timezone_label = campus_timezone || ENV['TZ'].presence || submission_time.zone || Time.zone.name + [ + submission_time.strftime('%d %b %Y'), + "#{submission_time.strftime('%I:%M %p')} #{timezone_label}" + ] + end end # A custom error to capture the log message from the latex error diff --git a/app/views/portfolio/portfolio_pdf.pdf.erb b/app/views/portfolio/portfolio_pdf.pdf.erb index 64704249c1..edf3f4fda7 100644 --- a/app/views/portfolio/portfolio_pdf.pdf.erb +++ b/app/views/portfolio/portfolio_pdf.pdf.erb @@ -96,7 +96,12 @@ No Tutor % DATE SECTION %---------------------------------------------------------------------------------------- -{\large \today}\\[3cm] % Date, change the \today to a set date if you want to be precise +<% if @formatted_submission_date.present? %> +{\large <%= lesc @formatted_submission_date %>}\\[0.2cm] +{\large \textcolor{gray}{<%= lesc @formatted_submission_time %>}}\\[3cm] +<% else %> +\vspace{3cm} +<% end %> %---------------------------------------------------------------------------------------- % LOGO SECTION From 43fcae7aec1ac9925bf8214e5257102ad844c0ec Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 31 Mar 2026 15:20:27 +1100 Subject: [PATCH 017/199] chore: ensure overseer doesnt change task status if already aip --- app/sidekiq/accept_overseer_job.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/sidekiq/accept_overseer_job.rb b/app/sidekiq/accept_overseer_job.rb index 5f176e2dfd..1a9eb782ba 100644 --- a/app/sidekiq/accept_overseer_job.rb +++ b/app/sidekiq/accept_overseer_job.rb @@ -96,7 +96,9 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment end else oa.update!(status: :failed) - unless failure_status.nil? + preserve_status_on_failure = [TaskStatus.time_exceeded.id, TaskStatus.assess_in_portfolio.id].include?(task.task_status_id) + + unless failure_status.nil? || preserve_status_on_failure # TODO: have an override status setting for the step? eg. if the task is overdue, let it remain overdue, otherwise use this task status task.update!(task_status: failure_status) task.add_status_comment(task.project.tutor_for(task.task_definition), failure_status) From c1420a8980f0953207f66d845f9196797ef6a61d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 13 Apr 2026 14:56:22 +1000 Subject: [PATCH 018/199] test: improve reliability of test --- test/api/tasks_api_test.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/api/tasks_api_test.rb b/test/api/tasks_api_test.rb index ccdca9140f..475db1e3d0 100644 --- a/test/api/tasks_api_test.rb +++ b/test/api/tasks_api_test.rb @@ -912,7 +912,7 @@ def test_resubmission_doesnt_change_submission_date target_grade: 0, outcome_count: 0 ) - td.update!(due_date: Time.zone.today + 1.day) + td.update!(due_date: Time.zone.today + 1.week) student1 = FactoryBot.create(:user, :student) student2 = FactoryBot.create(:user, :student) @@ -940,14 +940,14 @@ def test_resubmission_doesnt_change_submission_date tasks = unit.tasks_for_task_inbox(tutor, false) + assert_equal TaskStatus.ready_for_feedback, task1.task_status + assert_equal TaskStatus.ready_for_feedback, task2.task_status + assert_equal 2, tasks.to_a.count assert_equal project1.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project2.id, tasks.second.project.id, "Second task in inbox should be project2's task" - assert_equal TaskStatus.ready_for_feedback, task1.task_status - assert_equal TaskStatus.ready_for_feedback, task2.task_status - assert task2.submission_date > task1.submission_date original_submission_date = task1.submission_date @@ -987,9 +987,12 @@ def test_resubmission_doesnt_change_submission_date travel 10.minutes task1.submit(student1) + task1.reload + assert TaskStatus.ready_for_feedback, task1.task_status tasks = unit.tasks_for_task_inbox(tutor, false) + assert_operator tasks.to_a.count, :>=, 2, "Expected 2 or more tasks in tutors inbox" assert_equal project2.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project1.id, tasks.second.project.id, "Second task in inbox should be project2's task" From 0bba8279da40a8674f802b961302b84560855e76 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 13 Apr 2026 15:53:48 +1000 Subject: [PATCH 019/199] feat: ensure tutorials are reassigned before deleting a unit role --- app/api/unit_roles_api.rb | 25 +++++++++++++- test/api/unit_roles_test.rb | 66 +++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/app/api/unit_roles_api.rb b/app/api/unit_roles_api.rb index ff0d9f9004..8a726e428a 100644 --- a/app/api/unit_roles_api.rb +++ b/app/api/unit_roles_api.rb @@ -25,6 +25,10 @@ class UnitRolesApi < Grape::API end desc 'Delete a unit role' + params do + requires :id, type: Integer, desc: 'The id of the unit role to delete' + optional :reassign_to_unit_role_id, type: Integer, desc: 'The unit role to reassign tutorials to before deletion' + end delete '/unit_roles/:id' do unit_role = UnitRole.find(params[:id]) @@ -32,7 +36,26 @@ class UnitRolesApi < Grape::API error!({ error: "You do not have permission to perform this action" }, 403) end - unit_role.destroy! + tutorials = unit_role.unit.tutorials.where(unit_role_id: unit_role.id) + + if tutorials.exists? + if params[:reassign_to_unit_role_id].blank? + error!({ error: 'Unable to delete this unit role while tutorials are assigned without providing a reassignment target' }, 400) + end + + reassignment_role = unit_role.unit.staff.find_by(id: params[:reassign_to_unit_role_id]) + + if reassignment_role.nil? || reassignment_role.id == unit_role.id + error!({ error: 'Unable to delete this unit role with the provided reassignment target' }, 400) + end + + ActiveRecord::Base.transaction do + tutorials.update_all(unit_role_id: reassignment_role.id) + unit_role.destroy! + end + else + unit_role.destroy! + end end desc 'Employ a user as a teaching role in a unit' diff --git a/test/api/unit_roles_test.rb b/test/api/unit_roles_test.rb index 5e24941fd0..db0736418d 100644 --- a/test/api/unit_roles_test.rb +++ b/test/api/unit_roles_test.rb @@ -145,6 +145,72 @@ def test_delete_main_convenor refute UnitRole.where(id: initial_id).present? end + def test_delete_unit_role_with_assigned_tutorials_requires_reassignment + unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + tutor_user = FactoryBot.create :user, :tutor + tutor_role = unit.employ_staff tutor_user, Role.tutor + tutorial = FactoryBot.create :tutorial, unit: unit, unit_role: tutor_role + + add_auth_header_for(user: unit.main_convenor_user) + + delete "/api/unit_roles/#{tutor_role.id}" + + assert_equal 400, last_response.status + assert_equal tutor_role.id, tutorial.reload.unit_role_id + assert UnitRole.exists?(tutor_role.id) + end + + def test_delete_unit_role_with_assigned_tutorials_reassigns_and_deletes + unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + tutor_user = FactoryBot.create :user, :tutor + replacement_user = FactoryBot.create :user, :tutor + tutor_role = unit.employ_staff tutor_user, Role.tutor + replacement_role = unit.employ_staff replacement_user, Role.tutor + tutorial = FactoryBot.create :tutorial, unit: unit, unit_role: tutor_role + + add_auth_header_for(user: unit.main_convenor_user) + + delete "/api/unit_roles/#{tutor_role.id}", { reassign_to_unit_role_id: replacement_role.id } + + assert_equal 200, last_response.status + assert_equal replacement_role.id, tutorial.reload.unit_role_id + refute UnitRole.exists?(tutor_role.id) + end + + def test_delete_unit_role_with_assigned_tutorials_rejects_invalid_reassignment_target + unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + other_unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + tutor_user = FactoryBot.create :user, :tutor + tutor_role = unit.employ_staff tutor_user, Role.tutor + invalid_role = other_unit.staff.first + tutorial = FactoryBot.create :tutorial, unit: unit, unit_role: tutor_role + + add_auth_header_for(user: unit.main_convenor_user) + + delete "/api/unit_roles/#{tutor_role.id}", { reassign_to_unit_role_id: invalid_role.id } + + assert_equal 400, last_response.status + assert_equal tutor_role.id, tutorial.reload.unit_role_id + assert UnitRole.exists?(tutor_role.id) + end + + def test_delete_main_convenor_with_reassignment_rolls_back_tutorial_updates + unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + replacement_user = FactoryBot.create :user, :convenor + replacement_role = unit.employ_staff replacement_user, Role.convenor + tutorial = FactoryBot.create :tutorial, unit: unit, unit_role: unit.main_convenor + initial_main_convenor_id = unit.main_convenor_id + + add_auth_header_for(user: unit.main_convenor_user) + + delete "/api/unit_roles/#{initial_main_convenor_id}", { reassign_to_unit_role_id: replacement_role.id } + + assert_equal 400, last_response.status, last_response.inspect + assert_equal initial_main_convenor_id, tutorial.reload.unit_role_id + assert UnitRole.exists?(initial_main_convenor_id) + assert UnitRole.exists?(replacement_role.id) + end + def test_observer_unit_role unit = FactoryBot.create(:unit, with_students: true, task_count: 2, tutorials: 1, outcome_count: 0, staff_count: 0, campus_count: 0) From 654144d670035d21c244b5fbfa98c9ecda80344c Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 13 Apr 2026 16:46:48 +1000 Subject: [PATCH 020/199] chore: manually update each tutorial --- app/api/unit_roles_api.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api/unit_roles_api.rb b/app/api/unit_roles_api.rb index 8a726e428a..9b03699c05 100644 --- a/app/api/unit_roles_api.rb +++ b/app/api/unit_roles_api.rb @@ -50,7 +50,9 @@ class UnitRolesApi < Grape::API end ActiveRecord::Base.transaction do - tutorials.update_all(unit_role_id: reassignment_role.id) + tutorials.find_each do |tutorial| + tutorial.update!(unit_role: reassignment_role) + end unit_role.destroy! end else From f41d2ae6741a02552fee17d801ddefbfe99e7c75 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 13 Apr 2026 18:45:44 +1000 Subject: [PATCH 021/199] chore(release): 10.0.0-99 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 110b058595..3e8bea5f24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-99](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-98...v10.0.0-99) (2026-04-13) + + +### Features + +* ensure tutorials are reassigned before deleting a unit role ([0bba827](https://github.com/b0ink/doubtfire-api/commit/0bba8279da40a8674f802b961302b84560855e76)) + ## [10.0.0-98](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-97...v10.0.0-98) (2026-03-27) From c0d66151c22ad0ff48ec15be391d49771b9c9f8e Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 15 Apr 2026 11:40:15 +1000 Subject: [PATCH 022/199] chore(release): 10.0.0-100 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e8bea5f24..144c41c457 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-100](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-99...v10.0.0-100) (2026-04-15) + + +### Features + +* add enforce feedback unit setting ([4d1c6ee](https://github.com/b0ink/doubtfire-api/commit/4d1c6ee8d227b1d4b0ac57bbeea30f20f47e8b71)) + ## [10.0.0-99](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-98...v10.0.0-99) (2026-04-13) From 7a10685138a99eac0719aa2a37f8f883167edc45 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:15:35 +1000 Subject: [PATCH 023/199] feat: query pinned tasks in task explorer --- app/api/task_definitions_api.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/api/task_definitions_api.rb b/app/api/task_definitions_api.rb index 8da40c69ad..814561628b 100644 --- a/app/api/task_definitions_api.rb +++ b/app/api/task_definitions_api.rb @@ -559,6 +559,7 @@ class TaskDefinitionsApi < Grape::API .joins("LEFT JOIN task_comments ON task_comments.task_id = tasks.id AND (task_comments.type IS NULL OR task_comments.type <> 'TaskStatusComment')") .joins("LEFT OUTER JOIN (#{subquery}) as sq ON sq.project_id = projects.id") .joins('LEFT OUTER JOIN task_similarities ON tasks.id = task_similarities.task_id') + .joins("LEFT JOIN task_pins ON task_pins.task_id = tasks.id AND task_pins.user_id = #{current_user.id}") .select( 'sq.tutorial_stream_id as tutorial_stream_id', 'sq.tutorial_id as tutorial_id', @@ -572,7 +573,8 @@ class TaskDefinitionsApi < Grape::API 'grade', 'quality_pts', "SUM(case when task_comments.date_extension_assessed IS NULL AND task_comments.type = 'ExtensionComment' AND NOT task_comments.id IS NULL THEN 1 ELSE 0 END) > 0 as has_extensions", - 'SUM(case when task_similarities.flagged then 1 else 0 end) as similar_to_count' + 'SUM(case when task_similarities.flagged then 1 else 0 end) as similar_to_count', + 'COUNT(distinct task_pins.task_id) != 0 as pinned' ) .where('task_definition_id = :id', id: params[:task_def_id]) .group( @@ -603,7 +605,8 @@ class TaskDefinitionsApi < Grape::API similarity_flag: t.similar_to_count > 0, grade: t.grade, quality_pts: t.quality_pts, - has_extensions: t.has_extensions + has_extensions: t.has_extensions, + pinned: t.pinned } end From 5baa171baff4343161e13e79613b81eb985acc29 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Sat, 18 Apr 2026 16:10:11 +1000 Subject: [PATCH 024/199] feat: batch upload feedback csv (#609) * feat: batch upload feedback csv * refactor: support zip upload of submission files * refactor: allow feedback without file submission --- app/api/submission/batch_task_api.rb | 17 ++ app/models/unit.rb | 433 +++++++++++++++++++++++++-- 2 files changed, 426 insertions(+), 24 deletions(-) diff --git a/app/api/submission/batch_task_api.rb b/app/api/submission/batch_task_api.rb index c53d721cdd..d01c3a7f80 100644 --- a/app/api/submission/batch_task_api.rb +++ b/app/api/submission/batch_task_api.rb @@ -61,5 +61,22 @@ class BatchTaskApi < Grape::API # present unit.upload_batch_task_zip_or_csv(current_user, params[:file]), with: Grape::Presenters::Presenter # end # post + + desc 'Upload a batch feedback CSV or zip package for a selected task definition.' + params do + requires :file, type: File, desc: 'Batch feedback csv or zip upload' + requires :unit_id, type: Integer, desc: 'Unit ID to upload marked submissions to.' + requires :task_definition_id, type: Integer, desc: 'Task definition ID the uploaded CSV relates to.' + end + post '/submission/batch_feedback_csv/' do + unit = Unit.find(params[:unit_id]) + task_definition = unit.task_definitions.find(params[:task_definition_id]) + + unless authorise? current_user, unit, :provide_bulk_feedback + error!({ error: 'Not authorised to batch upload feedback csv' }, 401) + end + + present unit.upload_batch_feedback_csv(current_user, task_definition, params[:file]), with: Grape::Presenters::Presenter + end end end diff --git a/app/models/unit.rb b/app/models/unit.rb index 6da145b9fb..3cbaa09530 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -52,6 +52,7 @@ def self.permissions :add_tutorial, :add_task_def, :provide_feedback, + :provide_bulk_feedback, :change_project_enrolment, :download_stats, :download_grades, @@ -2695,15 +2696,28 @@ def update_task_status_from_csv(user, csv_str, success, _ignored, errors) end begin - task.trigger_transition(trigger: task_entry['status'], by_user: user, quality: task_entry['new quality'].to_i) # saves task - task.grade_task(task_entry['new grade']) # try to grade task if need be + requested_status = TaskStatus.status_for_name(task_entry['status'].to_s) + raise "Unable to update task status to '#{task_entry['status']}'." if requested_status.nil? - if task_entry['new comment'].blank? - success << { row: task_entry, message: "Updated task #{task.task_definition.abbreviation} for #{owner_text}" } - else - task.add_text_comment user, task_entry['new comment'] - success << { row: task_entry, message: "Updated task #{task.task_definition.abbreviation} for #{owner_text}" } - success << { row: {}, message: "Added comment to #{task.task_definition.abbreviation} for #{owner_text}" } + status_changed = false + comment_added = false + + unless task.task_status == requested_status + task.trigger_transition(trigger: task_entry['status'], by_user: user, quality: task_entry['new quality'].to_i) # saves task + task.grade_task(task_entry['new grade']) # try to grade task if need be + status_changed = true + end + + new_comment = task_entry['new comment'].to_s.strip + last_comment = task.comments.last&.comment.to_s.strip + if new_comment.present? && last_comment != new_comment + task.add_text_comment user, new_comment + comment_added = true + end + + if status_changed || comment_added + success << { row: task_entry, message: "Updated task #{task.task_definition.abbreviation} for #{owner_text}" } if status_changed + success << { row: {}, message: "Added comment to #{task.task_definition.abbreviation} for #{owner_text}" } if comment_added end rescue Exception => e errors << { row: task_entry, message: e.message } @@ -2806,49 +2820,49 @@ def upload_batch_task_zip_or_csv(user, file) # Copy over the updated/marked files to the file system zip.each do |file| # Skip processing marking file - next if ['marks.csv', 'readme.txt'].include?(File.basename(file[:name])) + next if ['marks.csv', 'readme.txt'].include?(File.basename(file.name)) # Test filename pattern - if (/.*-\d+.pdf/i =~ File.basename(file[:name])) != 0 - if file[:name][-1] != '/' - ignored << { row: "File #{file[:name]}", message: 'Does not appear to be a task PDF.' } + if (/.*-\d+.pdf/i =~ File.basename(file.name)) != 0 + if file.name[-1] != '/' + ignored << { row: "File #{file.name}", message: 'Does not appear to be a task PDF.' } end next end - if (/\._.*/ =~ File.basename(file[:name])) == 0 - ignored << { row: "File #{file[:name]}", message: 'Does not appear to be a task PDF.' } + if (/\._.*/ =~ File.basename(file.name)) == 0 + ignored << { row: "File #{file.name}", message: 'Does not appear to be a task PDF.' } next end # Extract the id from the filename - task_id_from_filename = File.basename(file[:name], '.pdf').split('-').last + task_id_from_filename = File.basename(file.name, '.pdf').split('-').last task = Task.find_by(id: task_id_from_filename) if task.nil? - ignored << { row: "File #{file[:name]}", message: 'Unable to find associated task.' } + ignored << { row: "File #{file.name}", message: 'Unable to find associated task.' } next end # Ensure that this task's id is inside entry_data task_entry = entry_data.select { |t| t['task'] == task.task_definition.abbreviation.tr(',', '_') && t['username'] == task.project.user.username }.first if task_entry.nil? - # error!({"error" => "File #{file[:name]} has a mismatch of task id ##{task.id} (this task id does not exist in marks.csv)"}, 403) - errors << { row: "File #{file[:name]}", message: "Task id #{task.id} not in marks.csv" } + # error!({"error" => "File #{file.name} has a mismatch of task id ##{task.id} (this task id does not exist in marks.csv)"}, 403) + errors << { row: "File #{file.name}", message: "Task id #{task.id} not in marks.csv" } next end if task.unit != self - errors << { row: "File #{file[:name]}", message: 'This task does not relate to this unit.' } + errors << { row: "File #{file.name}", message: 'This task does not relate to this unit.' } next end # Can the user assess this task? unless AuthorisationHelpers.authorise? user, task, :put - errors << { row: "File #{file[:name]}", error: "You do not have permission to assess task with id #{task.id}" } + errors << { row: "File #{file.name}", error: "You do not have permission to assess task with id #{task.id}" } next end # Read into the task's final pdf path the new file - tmp_file = File.join(tmp_dir, File.basename(file[:name])) + tmp_file = File.join(tmp_dir, File.basename(file.name)) # get file out of zip... to tmp_file file.extract(tmp_file) { true } @@ -2856,13 +2870,13 @@ def upload_batch_task_zip_or_csv(user, file) # copy tmp_file to dest if FileHelper.copy_pdf(tmp_file, task.final_pdf_path) if task.group.nil? - success << { row: "File #{file[:name]}", message: "Replace PDF of task #{task.task_definition.abbreviation} for #{task.student.name}" } + success << { row: "File #{file.name}", message: "Replace PDF of task #{task.task_definition.abbreviation} for #{task.student.name}" } else - success << { row: "File #{file[:name]}", message: "Replace PDF of group task #{task.task_definition.abbreviation} for #{task.group.name}" } + success << { row: "File #{file.name}", message: "Replace PDF of group task #{task.task_definition.abbreviation} for #{task.group.name}" } end FileUtils.rm tmp_file else - errors << { row: "File #{file[:name]}", message: 'The file does not appear to be a valid PDF.' } + errors << { row: "File #{file.name}", message: 'The file does not appear to be a valid PDF.' } next end end @@ -2879,6 +2893,377 @@ def upload_batch_task_zip_or_csv(user, file) } end + def batch_feedback_csv_required_headers + ['username', 'student id', 'status', 'comment'] + end + + def parse_batch_feedback_csv(csv_str, return_headers: false) + CSV.parse( + csv_str, + headers: true, + return_headers: return_headers, + header_converters: [->(body) { body&.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')&.downcase }], + converters: [->(body) { body&.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') }] + ) + end + + def find_project_for_batch_feedback_csv_entry(task_entry) + username = task_entry['username'].to_s.strip + student_id = task_entry['student id'].to_s.strip + + return [nil, 'Provide either Username or Student ID.'] if username.blank? && student_id.blank? + + username_project = if username.present? + projects.joins(:user).where('LOWER(users.username) = ?', username.downcase).first + end + student_id_project = if student_id.present? + projects.joins(:user).where(users: { student_id: student_id }).first + end + + if username.present? && username_project.nil? && student_id.blank? + return [nil, "Unable to find student with username '#{username}'."] + end + + if student_id.present? && student_id_project.nil? && username.blank? + return [nil, "Unable to find student with student ID '#{student_id}'."] + end + + if username_project.present? && student_id_project.present? && username_project != student_id_project + return [nil, "Username '#{username}' and student ID '#{student_id}' refer to different students."] + end + + project = username_project || student_id_project + return [nil, 'Unable to find student project for this row.'] if project.nil? + + [project, nil] + end + + def update_task_status_from_batch_feedback_csv(user, task_definition, csv_str, success, ignored, errors) + done = {} + + csv_str.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') + csv_str.tr!("\r", "\n") + csv_str.gsub!("\n\n", "\n") + + parse_batch_feedback_csv(csv_str, return_headers: true).each do |task_entry| + if task_entry.header_row? + batch_feedback_csv_required_headers.each do |expect_header| + unless task_entry.to_hash.keys.include?(expect_header) + errors << { row: task_entry, message: "Missing header '#{expect_header}', ensure first row has header information." } + return false + end + end + next + end + + next if task_entry.to_hash.values.all? { |value| value.to_s.strip.blank? } + + project, project_error = find_project_for_batch_feedback_csv_entry(task_entry) + if project_error.present? + errors << { row: task_entry, message: project_error } + next + end + + task = project.task_for_task_definition(task_definition) + if task.nil? + errors << { row: task_entry, message: "Unable to find task for #{task_definition.abbreviation}" } + next + end + + unless AuthorisationHelpers.authorise? user, task, :put + errors << { row: task_entry, error: 'You do not have permission to assess this task.' } + next + end + + status = task_entry['status'].to_s.strip + comment = task_entry['comment'].to_s.strip + + if status.blank? + errors << { row: task_entry, message: 'Status cannot be blank.' } + next + end + + begin + requested_status = TaskStatus.status_for_name(status) + raise "Unable to update task status to '#{status}'." if requested_status.nil? + + status_changed = false + comment_added = false + + unless task.task_status == requested_status + updated = task.trigger_transition( + trigger: status, + by_user: user, + quality: task.quality_pts || task.task_definition.max_quality_pts + ) + raise "Unable to update task status to '#{status}'." unless updated + + status_changed = true + end + + last_comment = task.comments.last&.comment.to_s.strip + if comment.present? && last_comment != comment + task.add_text_comment(user, comment) + comment_added = true + end + + unless status_changed || comment_added + ignored << { + row: task_entry, + message: "No changes required for #{task.task_definition.abbreviation} and #{project.user.name}" + } + next + end + + message = [] + message << "Updated task #{task.task_definition.abbreviation} for #{project.user.name}" if status_changed + message << "Added comment to #{task.task_definition.abbreviation} for #{project.user.name}" if comment_added + success << { + row: task_entry, + message: message.join(' and ') + } + rescue StandardError => e + errors << { row: task_entry, message: e.message } + next + end + + done[project] = [] if done[project].nil? + done[project] << task unless done[project].include?(task) + end + + begin + done.each do |project, tasks| + logger.info "Checking feedback email for project #{project.id}" + if project.student.receive_feedback_notifications + logger.info "Emailing feedback notification to #{project.student.name}" + PortfolioEvidenceMailer.task_feedback_ready(project, tasks).deliver + end + end + rescue => e + logger.error "Failed to send emails from feedback submission. Rescued with error: #{e.message}" + end + + true + end + + def upload_batch_feedback_csv(user, task_definition, file) + success = [] + errors = [] + ignored = [] + + type = mime_type(file["tempfile"].path) + + unless mime_in_list?(file["tempfile"].path, ['text/', 'text/plain', 'text/csv', 'application/zip', 'multipart/x-gzip', 'multipart/x-zip', 'application/x-gzip', 'application/octet-stream']) + errors << { row: {}, message: "File given is not a csv or zip file - detected #{type}" } + return { + success: success, + ignored: ignored, + errors: errors + } + end + + if type.start_with?('text/', 'text/plain', 'text/csv') + update_task_status_from_batch_feedback_csv( + user, + task_definition, + File.read(file["tempfile"].path), + success, + ignored, + errors + ) + else + return upload_batch_feedback_zip(user, task_definition, file) + end + + { + success: success, + ignored: ignored, + errors: errors + } + end + + def batch_feedback_zip_requirement_candidates(requirement) + name = requirement['name'].to_s.strip + return [] if name.blank? + + candidates = [name] + if File.extname(name).blank? && requirement['type'] == 'document' + candidates << "#{name}.pdf" + end + candidates.uniq + end + + def find_batch_feedback_requirement_entry(zip, username, requirement) + candidates = batch_feedback_zip_requirement_candidates(requirement) + + zip.find do |entry| + next false if entry.name_is_directory? + + path_parts = entry.name.split('/').reject(&:blank?) + next false unless path_parts[0...-1].any? { |part| part.casecmp(username).zero? } + + candidates.any? { |candidate| File.basename(entry.name).casecmp(candidate).zero? } + end + end + + def batch_feedback_entries_for_username(zip, username) + zip.select do |entry| + path_parts = entry.name.split('/').reject(&:blank?) + next false if path_parts.empty? + + if entry.name_is_directory? + path_parts.any? { |part| part.casecmp(username).zero? } + else + path_parts[0...-1].any? { |part| part.casecmp(username).zero? } + end + end + end + + def build_batch_feedback_legacy_marks_csv(task_rows) + CSV.generate do |csv| + csv << check_mark_csv_headers.split(',') + + task_rows.each do |task_row| + task = task_row[:task] + project = task_row[:project] + task_entry = task_row[:task_entry] + tutorial = project.tutorial_for(task.task_definition)&.abbreviation.to_s + + csv << [ + project.user.username, + project.user.name, + tutorial, + task.task_definition.abbreviation, + task_entry['status'].to_s.strip, + '', + '', + task_entry['comment'].to_s.strip + ] + end + end + end + + def upload_batch_feedback_zip(user, task_definition, file) + success = [] + errors = [] + ignored = [] + task_rows = [] + repacked_zip = Tempfile.new(["batch_feedback_#{id}_", '.zip']) + + Zip::File.open(file["tempfile"].path) do |zip| + marking_file = zip.glob('**/marks.csv').first + if marking_file.nil? + errors << { row: {}, message: 'No marks.csv contained in zip.' } + return { + success: success, + ignored: ignored, + errors: errors + } + end + + csv_str = marking_file.get_input_stream.read + csv_str.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') unless csv_str.nil? + + parse_batch_feedback_csv(csv_str, return_headers: true).each do |task_entry| + if task_entry.header_row? + batch_feedback_csv_required_headers.each do |expect_header| + unless task_entry.to_hash.keys.include?(expect_header) + errors << { row: task_entry, message: "Missing header '#{expect_header}', ensure first row has header information." } + next + end + end + next + end + + next if task_entry.to_hash.values.all? { |value| value.to_s.strip.blank? } + + project, project_error = find_project_for_batch_feedback_csv_entry(task_entry) + if project_error.present? + errors << { row: task_entry, message: project_error } + next + end + + task = project.task_for_task_definition(task_definition) + if task.nil? + errors << { row: task_entry, message: "Unable to find task for #{task_definition.abbreviation}" } + next + end + + unless AuthorisationHelpers.authorise? user, task, :put + errors << { row: task_entry, error: 'You do not have permission to assess this task.' } + next + end + + if task.group_task? + errors << { row: task_entry, message: 'Batch feedback zip upload does not support group tasks.' } + next + end + + if task.upload_requirements.length != 1 || task.upload_requirements.first['type'] != 'document' + errors << { row: task_entry, message: 'Batch feedback zip upload currently requires exactly one document upload requirement.' } + next + end + + student_entries = batch_feedback_entries_for_username(zip, project.user.username) + requirement_entry = nil + + if student_entries.any? + requirement = task.upload_requirements.first + expected_candidates = batch_feedback_zip_requirement_candidates(requirement) + + requirement_entry = find_batch_feedback_requirement_entry( + zip, + project.user.username, + requirement + ) + + if requirement_entry.nil? + expected = expected_candidates.join(' or ') + errors << { + row: task_entry, + message: "Missing required file for #{project.user.username}. Expected #{expected} inside that student's folder." + } + next + end + end + + task_rows << { + task: task, + project: project, + task_entry: task_entry, + requirement_entry: requirement_entry + } + end + + if errors.any? + return { + success: success, + ignored: ignored, + errors: errors + } + end + + Zip::File.open(repacked_zip.path, Zip::File::CREATE) do |output_zip| + output_zip.get_output_stream('marks.csv') do |f| + f.write(build_batch_feedback_legacy_marks_csv(task_rows)) + end + + task_rows.each do |task_row| + next if task_row[:requirement_entry].nil? + + output_name = "#{task_row[:task].task_definition.abbreviation}-#{task_row[:task].id}.pdf" + output_zip.get_output_stream(output_name) do |f| + f.write(task_row[:requirement_entry].get_input_stream.read) + end + end + end + end + + upload_batch_task_zip_or_csv(user, { 'tempfile' => repacked_zip }) + ensure + repacked_zip.close! if defined?(repacked_zip) && repacked_zip.present? + end + def send_weekly_status_emails(summary_stats) return unless send_notifications From d5127ea02828b62051244ffa4d7b42d9e49d3f2d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 18 Apr 2026 16:15:09 +1000 Subject: [PATCH 025/199] chore(release): 10.0.0-101 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 144c41c457..24eedbd63d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-101](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-100...v10.0.0-101) (2026-04-18) + + +### Features + +* batch upload feedback csv ([#609](https://github.com/b0ink/doubtfire-api/issues/609)) ([5baa171](https://github.com/b0ink/doubtfire-api/commit/5baa171baff4343161e13e79613b81eb985acc29)) +* query pinned tasks in task explorer ([7a10685](https://github.com/b0ink/doubtfire-api/commit/7a10685138a99eac0719aa2a37f8f883167edc45)) + ## [10.0.0-100](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-99...v10.0.0-100) (2026-04-15) From 479bb6600b3f84f251cea150b1cbae34d3856c03 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 18 Apr 2026 17:17:05 +1000 Subject: [PATCH 026/199] refactor: ensure single pdf submission to use as final pdf --- app/models/unit.rb | 233 +++++++++++++-------------------------------- 1 file changed, 68 insertions(+), 165 deletions(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index 3cbaa09530..7a37415bdd 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -2868,7 +2868,10 @@ def upload_batch_task_zip_or_csv(user, file) file.extract(tmp_file) { true } # copy tmp_file to dest - if FileHelper.copy_pdf(tmp_file, task.final_pdf_path) + destination_path = task.final_pdf_path(ignore_portfolio_evidence: true) + + if FileHelper.copy_pdf(tmp_file, destination_path) + task.update(portfolio_evidence: nil) if task.portfolio_evidence.present? if task.group.nil? success << { row: "File #{file.name}", message: "Replace PDF of task #{task.task_definition.abbreviation} for #{task.student.name}" } else @@ -2938,8 +2941,8 @@ def find_project_for_batch_feedback_csv_entry(task_entry) [project, nil] end - def update_task_status_from_batch_feedback_csv(user, task_definition, csv_str, success, ignored, errors) - done = {} + def build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: nil) + task_rows = [] csv_str.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') csv_str.tr!("\r", "\n") @@ -2950,7 +2953,7 @@ def update_task_status_from_batch_feedback_csv(user, task_definition, csv_str, s batch_feedback_csv_required_headers.each do |expect_header| unless task_entry.to_hash.keys.include?(expect_header) errors << { row: task_entry, message: "Missing header '#{expect_header}', ensure first row has header information." } - return false + return nil end end next @@ -2970,80 +2973,59 @@ def update_task_status_from_batch_feedback_csv(user, task_definition, csv_str, s next end - unless AuthorisationHelpers.authorise? user, task, :put - errors << { row: task_entry, error: 'You do not have permission to assess this task.' } + if task.group_task? + errors << { row: task_entry, message: 'Batch feedback upload does not support group tasks.' } next end status = task_entry['status'].to_s.strip - comment = task_entry['comment'].to_s.strip - if status.blank? errors << { row: task_entry, message: 'Status cannot be blank.' } next end - begin - requested_status = TaskStatus.status_for_name(status) - raise "Unable to update task status to '#{status}'." if requested_status.nil? - - status_changed = false - comment_added = false - - unless task.task_status == requested_status - updated = task.trigger_transition( - trigger: status, - by_user: user, - quality: task.quality_pts || task.task_definition.max_quality_pts - ) - raise "Unable to update task status to '#{status}'." unless updated + if TaskStatus.status_for_name(status).nil? + errors << { row: task_entry, message: "Unable to update task status to '#{status}'." } + next + end - status_changed = true - end + student_entries = zip.nil? ? [] : batch_feedback_entries_for_username(zip, project.user.username) + pdf_entries = zip.nil? ? [] : batch_feedback_named_pdf_entries_for_username(zip, project.user.username) - last_comment = task.comments.last&.comment.to_s.strip - if comment.present? && last_comment != comment - task.add_text_comment(user, comment) - comment_added = true + if zip.present? + if student_entries.any? && pdf_entries.empty? + errors << { + row: task_entry, + message: "Expected a PDF named #{project.user.username}.pdf inside #{project.user.username}'s folder." + } + next end - unless status_changed || comment_added - ignored << { + if pdf_entries.length > 1 + errors << { row: task_entry, - message: "No changes required for #{task.task_definition.abbreviation} and #{project.user.name}" + message: "Found multiple PDFs named #{project.user.username}.pdf inside #{project.user.username}'s folder." } next end - - message = [] - message << "Updated task #{task.task_definition.abbreviation} for #{project.user.name}" if status_changed - message << "Added comment to #{task.task_definition.abbreviation} for #{project.user.name}" if comment_added - success << { - row: task_entry, - message: message.join(' and ') - } - rescue StandardError => e - errors << { row: task_entry, message: e.message } - next end - done[project] = [] if done[project].nil? - done[project] << task unless done[project].include?(task) + task_rows << { + task: task, + project: project, + task_entry: task_entry, + pdf_entry: pdf_entries.first + } end - begin - done.each do |project, tasks| - logger.info "Checking feedback email for project #{project.id}" - if project.student.receive_feedback_notifications - logger.info "Emailing feedback notification to #{project.student.name}" - PortfolioEvidenceMailer.task_feedback_ready(project, tasks).deliver - end - end - rescue => e - logger.error "Failed to send emails from feedback submission. Rescued with error: #{e.message}" - end + task_rows + end - true + def write_batch_feedback_csv_file(task_rows) + file = Tempfile.new(["batch_feedback_#{id}_", '.csv']) + file.write(build_batch_feedback_legacy_marks_csv(task_rows)) + file.rewind + file end def upload_batch_feedback_csv(user, task_definition, file) @@ -3063,47 +3045,27 @@ def upload_batch_feedback_csv(user, task_definition, file) end if type.start_with?('text/', 'text/plain', 'text/csv') - update_task_status_from_batch_feedback_csv( - user, + task_rows = build_batch_feedback_task_rows( task_definition, File.read(file["tempfile"].path), - success, - ignored, errors ) - else - return upload_batch_feedback_zip(user, task_definition, file) - end - { - success: success, - ignored: ignored, - errors: errors - } - end - - def batch_feedback_zip_requirement_candidates(requirement) - name = requirement['name'].to_s.strip - return [] if name.blank? - - candidates = [name] - if File.extname(name).blank? && requirement['type'] == 'document' - candidates << "#{name}.pdf" - end - candidates.uniq - end - - def find_batch_feedback_requirement_entry(zip, username, requirement) - candidates = batch_feedback_zip_requirement_candidates(requirement) - - zip.find do |entry| - next false if entry.name_is_directory? - - path_parts = entry.name.split('/').reject(&:blank?) - next false unless path_parts[0...-1].any? { |part| part.casecmp(username).zero? } + return { + success: success, + ignored: ignored, + errors: errors + } if task_rows.nil? || task_rows.empty? - candidates.any? { |candidate| File.basename(entry.name).casecmp(candidate).zero? } + converted_csv = write_batch_feedback_csv_file(task_rows) + result = upload_batch_task_zip_or_csv(user, { 'tempfile' => converted_csv }) + result[:errors] = errors + result[:errors] + return result + else + return upload_batch_feedback_zip(user, task_definition, file) end + ensure + converted_csv.close! if defined?(converted_csv) && converted_csv.present? end def batch_feedback_entries_for_username(zip, username) @@ -3119,6 +3081,15 @@ def batch_feedback_entries_for_username(zip, username) end end + def batch_feedback_named_pdf_entries_for_username(zip, username) + batch_feedback_entries_for_username(zip, username).select do |entry| + next false if entry.name_is_directory? + next false unless File.extname(entry.name).casecmp('.pdf').zero? + + File.basename(entry.name, '.pdf').casecmp(username).zero? + end + end + def build_batch_feedback_legacy_marks_csv(task_rows) CSV.generate do |csv| csv << check_mark_csv_headers.split(',') @@ -3147,7 +3118,6 @@ def upload_batch_feedback_zip(user, task_definition, file) success = [] errors = [] ignored = [] - task_rows = [] repacked_zip = Tempfile.new(["batch_feedback_#{id}_", '.zip']) Zip::File.open(file["tempfile"].path) do |zip| @@ -3164,78 +3134,9 @@ def upload_batch_feedback_zip(user, task_definition, file) csv_str = marking_file.get_input_stream.read csv_str.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') unless csv_str.nil? - parse_batch_feedback_csv(csv_str, return_headers: true).each do |task_entry| - if task_entry.header_row? - batch_feedback_csv_required_headers.each do |expect_header| - unless task_entry.to_hash.keys.include?(expect_header) - errors << { row: task_entry, message: "Missing header '#{expect_header}', ensure first row has header information." } - next - end - end - next - end - - next if task_entry.to_hash.values.all? { |value| value.to_s.strip.blank? } - - project, project_error = find_project_for_batch_feedback_csv_entry(task_entry) - if project_error.present? - errors << { row: task_entry, message: project_error } - next - end - - task = project.task_for_task_definition(task_definition) - if task.nil? - errors << { row: task_entry, message: "Unable to find task for #{task_definition.abbreviation}" } - next - end - - unless AuthorisationHelpers.authorise? user, task, :put - errors << { row: task_entry, error: 'You do not have permission to assess this task.' } - next - end - - if task.group_task? - errors << { row: task_entry, message: 'Batch feedback zip upload does not support group tasks.' } - next - end - - if task.upload_requirements.length != 1 || task.upload_requirements.first['type'] != 'document' - errors << { row: task_entry, message: 'Batch feedback zip upload currently requires exactly one document upload requirement.' } - next - end - - student_entries = batch_feedback_entries_for_username(zip, project.user.username) - requirement_entry = nil + task_rows = build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: zip) - if student_entries.any? - requirement = task.upload_requirements.first - expected_candidates = batch_feedback_zip_requirement_candidates(requirement) - - requirement_entry = find_batch_feedback_requirement_entry( - zip, - project.user.username, - requirement - ) - - if requirement_entry.nil? - expected = expected_candidates.join(' or ') - errors << { - row: task_entry, - message: "Missing required file for #{project.user.username}. Expected #{expected} inside that student's folder." - } - next - end - end - - task_rows << { - task: task, - project: project, - task_entry: task_entry, - requirement_entry: requirement_entry - } - end - - if errors.any? + if task_rows.nil? || task_rows.empty? return { success: success, ignored: ignored, @@ -3249,17 +3150,19 @@ def upload_batch_feedback_zip(user, task_definition, file) end task_rows.each do |task_row| - next if task_row[:requirement_entry].nil? + next if task_row[:pdf_entry].nil? output_name = "#{task_row[:task].task_definition.abbreviation}-#{task_row[:task].id}.pdf" output_zip.get_output_stream(output_name) do |f| - f.write(task_row[:requirement_entry].get_input_stream.read) + f.write(task_row[:pdf_entry].get_input_stream.read) end end end end - upload_batch_task_zip_or_csv(user, { 'tempfile' => repacked_zip }) + result = upload_batch_task_zip_or_csv(user, { 'tempfile' => repacked_zip }) + result[:errors] = errors + result[:errors] + result ensure repacked_zip.close! if defined?(repacked_zip) && repacked_zip.present? end From b266498e38739c50ebe77aea13f3ad4932f26d62 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 18 Apr 2026 17:19:27 +1000 Subject: [PATCH 027/199] chore(release): 10.0.0-102 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24eedbd63d..403cf471f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-102](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-101...v10.0.0-102) (2026-04-18) + ## [10.0.0-101](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-100...v10.0.0-101) (2026-04-18) From 90945ca36d2187e4ee8d1f178f046fadc046b8f5 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 18 Apr 2026 17:29:57 +1000 Subject: [PATCH 028/199] chore: fix rubocop --- app/models/unit.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index 7a37415bdd..a77fca14c4 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -3051,18 +3051,20 @@ def upload_batch_feedback_csv(user, task_definition, file) errors ) - return { - success: success, - ignored: ignored, - errors: errors - } if task_rows.nil? || task_rows.empty? + if task_rows.blank? + return { + success: success, + ignored: ignored, + errors: errors + } + end converted_csv = write_batch_feedback_csv_file(task_rows) result = upload_batch_task_zip_or_csv(user, { 'tempfile' => converted_csv }) result[:errors] = errors + result[:errors] - return result + result else - return upload_batch_feedback_zip(user, task_definition, file) + upload_batch_feedback_zip(user, task_definition, file) end ensure converted_csv.close! if defined?(converted_csv) && converted_csv.present? From 76afc3c1eb06e15b3a14c7dbf694869d36713671 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 18 Apr 2026 17:33:41 +1000 Subject: [PATCH 029/199] chore: fix rubocop --- app/models/unit.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index a77fca14c4..607a552a6c 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -3138,7 +3138,7 @@ def upload_batch_feedback_zip(user, task_definition, file) task_rows = build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: zip) - if task_rows.nil? || task_rows.empty? + if task_rows.blank? return { success: success, ignored: ignored, From 9960e313676a3053227f4dbfa75e2b000dad5cdc Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 18 Apr 2026 18:22:34 +1000 Subject: [PATCH 030/199] refactor: move batch tasks upload to sidekiq job --- app/api/submission/batch_task_api.rb | 21 +++++++++++-- app/models/unit.rb | 39 +++++++++++++++++++----- app/sidekiq/import_batch_feedback_job.rb | 39 ++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 app/sidekiq/import_batch_feedback_job.rb diff --git a/app/api/submission/batch_task_api.rb b/app/api/submission/batch_task_api.rb index d01c3a7f80..1ab664dabd 100644 --- a/app/api/submission/batch_task_api.rb +++ b/app/api/submission/batch_task_api.rb @@ -5,6 +5,7 @@ class BatchTaskApi < Grape::API helpers GenerateHelpers helpers AuthenticationHelpers helpers AuthorisationHelpers + helpers SidekiqHelper before do authenticated? @@ -70,13 +71,29 @@ class BatchTaskApi < Grape::API end post '/submission/batch_feedback_csv/' do unit = Unit.find(params[:unit_id]) - task_definition = unit.task_definitions.find(params[:task_definition_id]) + unit.task_definitions.find(params[:task_definition_id]) unless authorise? current_user, unit, :provide_bulk_feedback error!({ error: 'Not authorised to batch upload feedback csv' }, 401) end - present unit.upload_batch_feedback_csv(current_user, task_definition, params[:file]), with: Grape::Presenters::Presenter + error!({ error: "No file uploaded" }, 403) if params[:file].blank? + + import_dir = Rails.root.join(FileHelper.tmp_file_dir, 'batch-feedback') + FileUtils.mkdir_p(import_dir) + + extension = File.extname(params[:file][:filename].to_s) + extension = '.upload' if extension.blank? + file_name = File.join( + import_dir, + "batch-feedback-#{unit.id}-#{params[:task_definition_id]}-#{Process.pid}-#{Thread.current.object_id}-#{current_user.id}#{extension}" + ) + + FileUtils.cp(params[:file][:tempfile].path, file_name) + + job_id = ImportBatchFeedbackJob.perform_async(unit.id, current_user.id, params[:task_definition_id], file_name) + job = setup_job(job_id) + present job, with: Entities::SidekiqJobEntity end end end diff --git a/app/models/unit.rb b/app/models/unit.rb index 607a552a6c..c40d412e13 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -2941,14 +2941,28 @@ def find_project_for_batch_feedback_csv_entry(task_entry) [project, nil] end - def build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: nil) + def build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: nil, progress_callback: nil) task_rows = [] csv_str.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') csv_str.tr!("\r", "\n") csv_str.gsub!("\n\n", "\n") - parse_batch_feedback_csv(csv_str, return_headers: true).each do |task_entry| + entries = parse_batch_feedback_csv(csv_str, return_headers: true) + total_rows = 0 + + entries.each do |task_entry| + next if task_entry.header_row? + next if task_entry.to_hash.values.all? { |value| value.to_s.strip.blank? } + + total_rows += 1 + end + + rows_processed = 0 + + progress_callback&.call(message: 'Validating batch feedback rows', total_rows: total_rows, rows_processed: 0) + + entries.each do |task_entry| if task_entry.header_row? batch_feedback_csv_required_headers.each do |expect_header| unless task_entry.to_hash.keys.include?(expect_header) @@ -2961,6 +2975,9 @@ def build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: nil) next if task_entry.to_hash.values.all? { |value| value.to_s.strip.blank? } + rows_processed += 1 + progress_callback&.call(message: 'Validating batch feedback rows', rows_processed: rows_processed) + project, project_error = find_project_for_batch_feedback_csv_entry(task_entry) if project_error.present? errors << { row: task_entry, message: project_error } @@ -3028,7 +3045,7 @@ def write_batch_feedback_csv_file(task_rows) file end - def upload_batch_feedback_csv(user, task_definition, file) + def upload_batch_feedback_csv(user, task_definition, file, progress_callback: nil) success = [] errors = [] ignored = [] @@ -3048,7 +3065,8 @@ def upload_batch_feedback_csv(user, task_definition, file) task_rows = build_batch_feedback_task_rows( task_definition, File.read(file["tempfile"].path), - errors + errors, + progress_callback: progress_callback ) if task_rows.blank? @@ -3060,11 +3078,12 @@ def upload_batch_feedback_csv(user, task_definition, file) end converted_csv = write_batch_feedback_csv_file(task_rows) + progress_callback&.call(message: 'Applying batch feedback updates', total_rows: task_rows.count, rows_processed: 0) result = upload_batch_task_zip_or_csv(user, { 'tempfile' => converted_csv }) result[:errors] = errors + result[:errors] result else - upload_batch_feedback_zip(user, task_definition, file) + upload_batch_feedback_zip(user, task_definition, file, progress_callback: progress_callback) end ensure converted_csv.close! if defined?(converted_csv) && converted_csv.present? @@ -3116,7 +3135,7 @@ def build_batch_feedback_legacy_marks_csv(task_rows) end end - def upload_batch_feedback_zip(user, task_definition, file) + def upload_batch_feedback_zip(user, task_definition, file, progress_callback: nil) success = [] errors = [] ignored = [] @@ -3136,7 +3155,7 @@ def upload_batch_feedback_zip(user, task_definition, file) csv_str = marking_file.get_input_stream.read csv_str.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') unless csv_str.nil? - task_rows = build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: zip) + task_rows = build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: zip, progress_callback: progress_callback) if task_rows.blank? return { @@ -3151,13 +3170,17 @@ def upload_batch_feedback_zip(user, task_definition, file) f.write(build_batch_feedback_legacy_marks_csv(task_rows)) end - task_rows.each do |task_row| + progress_callback&.call(message: 'Preparing PDF replacements', total_rows: task_rows.count, rows_processed: 0) + + task_rows.each_with_index do |task_row, index| next if task_row[:pdf_entry].nil? output_name = "#{task_row[:task].task_definition.abbreviation}-#{task_row[:task].id}.pdf" output_zip.get_output_stream(output_name) do |f| f.write(task_row[:pdf_entry].get_input_stream.read) end + + progress_callback&.call(message: 'Preparing PDF replacements', rows_processed: index + 1) end end end diff --git a/app/sidekiq/import_batch_feedback_job.rb b/app/sidekiq/import_batch_feedback_job.rb new file mode 100644 index 0000000000..e8504d6eb2 --- /dev/null +++ b/app/sidekiq/import_batch_feedback_job.rb @@ -0,0 +1,39 @@ +class ImportBatchFeedbackJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { args.first(3) }, + on_conflict: :reject, + retry: false + + def perform(unit_id, assessor_id, task_definition_id, path_to_upload) + logger.info "Starting batch feedback import..." + + at(0, 'Preparing batch feedback import') + total(1) + + unit = Unit.find(unit_id) + task_definition = unit.task_definitions.find(task_definition_id) + assessor = User.find(assessor_id) + + file = File.open(path_to_upload, 'rb') + result = unit.upload_batch_feedback_csv( + assessor, + task_definition, + { 'tempfile' => file }, + progress_callback: lambda { |message: nil, total_rows: nil, rows_processed: nil| + total(total_rows) if total_rows + at(rows_processed, message) unless rows_processed.nil? + } + ) + + store(result: result.to_json) + + logger.info "Completed batch feedback import!" + ensure + file&.close if defined?(file) && file.present? + FileUtils.rm_f(path_to_upload) if path_to_upload.present? + end +end From e3af0005dd9b432fb904ae405595da017e9f5044 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 18 Apr 2026 18:25:45 +1000 Subject: [PATCH 031/199] chore(release): 10.0.0-103 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 403cf471f4..e681b78c5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-103](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-102...v10.0.0-103) (2026-04-18) + ## [10.0.0-102](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-101...v10.0.0-102) (2026-04-18) ## [10.0.0-101](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-100...v10.0.0-101) (2026-04-18) From 51f20a75012c72be41940cf5a028c8918b8e6012 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:39:58 +1000 Subject: [PATCH 032/199] chore: rename outdated package --- texlive.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/texlive.Dockerfile b/texlive.Dockerfile index c68816ce24..79d502a0c5 100644 --- a/texlive.Dockerfile +++ b/texlive.Dockerfile @@ -53,7 +53,7 @@ RUN tlmgr install \ paralist \ pdfcol \ pdflscape \ - pdfmanagement-testphase \ + pdfmanagement \ pdfpages \ tagpdf \ tcolorbox \ From 1d056ca36aca7d9de8977b7a96783ed7a8d05367 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 23 Apr 2026 12:28:17 +1000 Subject: [PATCH 033/199] fix: prevent duplicate submission requests (#611) --- app/models/task.rb | 11 +++++++++++ app/sidekiq/accept_overseer_job.rb | 3 +-- app/sidekiq/accept_submission_job.rb | 5 +++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/models/task.rb b/app/models/task.rb index d0b397f38e..86439b3d08 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -1519,11 +1519,17 @@ def create_submission_and_trigger_state_change(user, propagate = true, contribut # Checks to make sure that the files match what we expect # def accept_submission(current_user, files, ui, contributions, trigger, alignments, accepted_tii_eula: false, test_submission: false) + submission_lock_target.with_lock do # Ensure there is not a submission already in process if processing_pdf? ui.error!({ 'error' => 'A submission is already being processed. Please wait for the current submission process to complete.' }, 403) end + if !test_submission && (overseer_enabled? || task_definition.assessment_enabled) && + overseer_assessments.where(status: OverseerAssessment.statuses[:pre_queued]).exists? + ui.error!({ 'error' => 'A submission is already waiting for automated feedback. Please wait for the current Overseer job to complete before submitting again.' }, 403) + end + # Ensure all of the files are present if files.nil? || files.length != task_definition.number_of_uploaded_files ui.error!({ 'error' => 'Some files are missing from the submission upload' }, 403) @@ -1618,6 +1624,11 @@ def accept_submission(current_user, files, ui, contributions, trigger, alignment # Trigger processing of new submission - async AcceptSubmissionJob.perform_async(id, current_user.id, accepted_tii_eula, test_submission) + end + end + + def submission_lock_target + group_task? ? group : self end # The name that should be used for the uploaded file (based on index of upload requirements) diff --git a/app/sidekiq/accept_overseer_job.rb b/app/sidekiq/accept_overseer_job.rb index 1a9eb782ba..7d00f9017c 100644 --- a/app/sidekiq/accept_overseer_job.rb +++ b/app/sidekiq/accept_overseer_job.rb @@ -9,8 +9,7 @@ class AcceptOverseerJob include FileHelper sidekiq_options lock: :until_executed, - # TODO: should students be allowed to submit a new task submission when the previous overseer job has not started/completed? - lock_args_method: ->(args) { [args.first, 'overseer-assessment'] }, + lock_args_method: ->(args) { [args.first, args.last, 'overseer-assessment'] }, on_conflict: :reject, retry: 1 diff --git a/app/sidekiq/accept_submission_job.rb b/app/sidekiq/accept_submission_job.rb index 15a92d084e..99dfdc04d9 100644 --- a/app/sidekiq/accept_submission_job.rb +++ b/app/sidekiq/accept_submission_job.rb @@ -2,6 +2,11 @@ class AcceptSubmissionJob include Sidekiq::Job include LogHelper + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first] }, + on_conflict: :reject, + retry: false + def perform(task_id, user_id, accepted_tii_eula, test_submission) begin # Ensure cwd is valid... From cdb7eb6ec0c8ab73c5e9b79b850382e2327d0726 Mon Sep 17 00:00:00 2001 From: Prishnee Nuckcheddy Date: Thu, 23 Apr 2026 13:33:03 +1000 Subject: [PATCH 034/199] fix: tighten API CORS policy to trusted origins Replace wildcard CORS behavior with whitelist validation and restrict allowed headers/methods so only trusted frontend origins can make cross-origin API requests. Made-with: Cursor --- app/api/api_root.rb | 3 --- config/application.rb | 21 +++++++++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/app/api/api_root.rb b/app/api/api_root.rb index e36e21226e..6f2e2fe704 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -10,9 +10,6 @@ class ApiRoot < Grape::API format :json before do - header['Access-Control-Allow-Origin'] = '*' - header['Access-Control-Request-Method'] = '*' - Thread.current.thread_variable_set(:ip, request.ip) end diff --git a/config/application.rb b/config/application.rb index fe280a2899..36e403e43c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -244,10 +244,27 @@ def self.fetch_boolean_env(name) Rails.root.join('app/models/d2l') # CORS config + # Configure a strict allowlist. Override per environment via: + # CORS_ALLOWED_ORIGINS="http://localhost:4200,https://frontend.example.edu" + default_cors_origins = [ + 'http://localhost:4200', + "https://#{config.institution[:host]}" + ].uniq + allowed_cors_origins = ENV.fetch('CORS_ALLOWED_ORIGINS', default_cors_origins.join(',')) + .split(',') + .map(&:strip) + .reject(&:empty?) + .uniq + config.middleware.insert_before Warden::Manager, Rack::Cors do allow do - origins '*' - resource '*', headers: :any, methods: %i(get post put delete options) + origins do |source, _env| + allowed_cors_origins.include?(source) + end + + resource '*', + headers: %w[Content-Type Authorization Accept], + methods: %i[get post put delete options] end end From 5dbd57588c06cb69eae35c5eab12d757bbe0bec5 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:47:56 +1000 Subject: [PATCH 035/199] feat: import/export overseer steps via task definition csv (#604) * feat: import/export overseer steps via task definition csv * chore: add overseer steps task csv test file * chore: add bash overseer image * refactor: use .cpp extension and add overseer steps * feat: rollover overseer steps * chore: add prerequisites to tasks csv * ci: enable overseer to init docker config * chore: always init docker config * test: expect .cpp extension * chore: update run command * fix: fix csv task import when replacing task prerequisites * fix: ignore last pulled date --- .github/workflows/push.yml | 2 +- app/models/task_definition.rb | 75 +++++++++++++++- app/models/unit.rb | 9 +- config/application.rb | 14 +-- lib/helpers/database_populator.rb | 13 +++ test/api/csv_test.rb | 2 +- test/api/overseer_image_api_test.rb | 2 +- test/models/task_definition_test.rb | 90 ++++++++++++++++++- test/models/unit_model_test.rb | 56 ++++++++++++ .../COS10001-ImportTasksWithOverseerSteps.csv | 2 + ...COS10001-ImportTasksWithTutorialStream.csv | 2 +- ...10001-ImportTasksWithoutTutorialStream.csv | 2 +- test_files/COS10001-Tasks.csv | 76 ++++++++-------- .../COS10001-TasksUnorderedUploadReqs.csv | 2 +- 14 files changed, 291 insertions(+), 56 deletions(-) create mode 100644 test_files/COS10001-ImportTasksWithOverseerSteps.csv diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2640e2f00c..1ec93a0761 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -22,7 +22,7 @@ env: DF_TEST_DB_DATABASE: "doubtfire-test" DF_TEST_DB_USERNAME: "dfire" DF_TEST_DB_PASSWORD: "pwd" - OVERSEER_ENABLED: "0" + OVERSEER_ENABLED: "true" DF_ENCRYPTION_PRIMARY_KEY: "AMLOMYA5GV8B4fTK3VKMhVGn8WdvUW8g" DF_ENCRYPTION_DETERMINISTIC_KEY: "anlmuJ6cB3bN3biXRbYvmPsC5ALPFqGG" DF_ENCRYPTION_KEY_DERIVATION_SALT: "hzPR8D4qpOnAg7VeAhkhWw6JmmzKJB10" diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index 8d2fa7c5cd..4e4b222c64 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -226,6 +226,11 @@ def copy_to(other_unit) end new_td.save! + overseer_steps.find_each do |step| + new_td.overseer_steps.create!( + step.attributes.except('id', 'task_definition_id', 'created_at', 'updated_at') + ) + end new_td end @@ -510,7 +515,7 @@ def propogate_date_changes date_diff def to_csv_row TaskDefinition.csv_columns - .reject { |col| [:start_week, :start_day, :target_week, :target_day, :due_week, :due_day, :upload_requirements, :group_set, :tutorial_stream, :assess_in_portfolio_only, :task_prerequisites, :discussion_prompts].include? col} + .reject { |col| [:start_week, :start_day, :target_week, :target_day, :due_week, :due_day, :upload_requirements, :group_set, :tutorial_stream, :assess_in_portfolio_only, :task_prerequisites, :discussion_prompts, :overseer_steps].include? col} .map { |column| attributes[column.to_s] } + [ group_set.nil? ? "" : group_set.name, @@ -535,6 +540,30 @@ def to_csv_row content: prompt.content, priority: prompt.priority } + end.to_json, + overseer_steps.map do |step| + { + name: step.name, + description: step.description, + display_name: step.display_name, + display_description: step.display_description, + run_command: step.run_command, + timeout: step.timeout, + sort_order: step.sort_order, + step_type: step.step_type, + partial_output_diff: step.partial_output_diff, + stdin_input_file: step.stdin_input_file, + expected_output_file: step.expected_output_file, + feedback_message: step.feedback_message, + status_on_success: TaskStatus.find_by(id: step.status_on_success_id)&.status_key, + status_on_failure: TaskStatus.find_by(id: step.status_on_failure_id)&.status_key, + halt_on_success: step.halt_on_success, + halt_on_failure: step.halt_on_failure, + show_expected_output: step.show_expected_output, + show_stdin: step.show_stdin, + show_stdout: step.show_stdout, + enabled: step.enabled + } end.to_json ] # [target_date.strftime('%d-%m-%Y')] + @@ -545,7 +574,11 @@ def self.csv_columns [:name, :abbreviation, :description, :weighting, :target_grade, :restrict_status_updates, :max_quality_pts, :is_graded, :plagiarism_warn_pct, :scorm_enabled, :scorm_allow_review, :scorm_bypass_test, :scorm_time_delay_enabled, :scorm_attempt_limit, :group_set, :upload_requirements, :start_week, :start_day, :target_week, :target_day, - :due_week, :due_day, :tutorial_stream, :assess_in_portfolio_only, :task_prerequisites, :discussion_prompts] + :due_week, :due_day, :tutorial_stream, :assess_in_portfolio_only, :task_prerequisites, :discussion_prompts, :overseer_steps] + end + + def self.required_csv_columns + csv_columns - [:overseer_steps] end def self.task_def_for_csv_row(unit, row) @@ -611,6 +644,7 @@ def self.task_def_for_csv_row(unit, row) end import_discussion_prompts_from_csv_row(result, row) + import_overseer_steps_from_csv_row(result, row) result.assess_in_portfolio_only = %w(Yes y Y yes true TRUE 1).include? "#{row[:assess_in_portfolio_only]}".strip @@ -658,6 +692,43 @@ def self.import_discussion_prompts_from_csv_row(task_definition, row) end end + def self.import_overseer_steps_from_csv_row(task_definition, row) + task_definition.overseer_steps.destroy_all + return if row[:overseer_steps].blank? + + JSON.parse(row[:overseer_steps]).each do |step| + OverseerStep.create!( + task_definition: task_definition, + name: step['name'], + description: step['description'], + display_name: step['display_name'], + display_description: step['display_description'], + run_command: step['run_command'], + timeout: step['timeout'], + sort_order: step['sort_order'], + step_type: step['step_type'], + partial_output_diff: step['partial_output_diff'], + stdin_input_file: step['stdin_input_file'], + expected_output_file: step['expected_output_file'], + feedback_message: step['feedback_message'], + status_on_success_id: status_id_from_csv(step['status_on_success']), + status_on_failure_id: status_id_from_csv(step['status_on_failure']), + halt_on_success: step['halt_on_success'], + halt_on_failure: step['halt_on_failure'], + show_expected_output: step['show_expected_output'], + show_stdin: step['show_stdin'], + show_stdout: step['show_stdout'], + enabled: step.key?('enabled') ? step['enabled'] : true + ) + end + end + + def self.status_id_from_csv(value) + return nil if value.blank? + + TaskStatus.status_for_name(value)&.id || TaskStatus.find_by(id: value.to_i)&.id + end + def is_group_task? !group_set.nil? end diff --git a/app/models/unit.rb b/app/models/unit.rb index c40d412e13..f877e4b4a8 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -1578,20 +1578,25 @@ def import_tasks_from_csv(file) next if row[0] =~ /^(Task Name)|(name)/ # Skip header begin - missing = missing_headers(row, TaskDefinition.csv_columns) + missing = missing_headers(row, TaskDefinition.required_csv_columns) if missing.count > 0 errors << { row: row, message: "Missing headers: #{missing.join(', ')}" } next end + existing_task_definition = task_definitions.find_by(abbreviation: row[:abbreviation]&.strip) + existing_task_definition ||= task_definitions.find_by(name: row[:name]&.strip) + existing_task_definition&.task_prerequisites&.destroy_all + task_definition, new_task, message = TaskDefinition.task_def_for_csv_row(self, row) - prerequisites_by_task[task_definition.abbreviation] = JSON.parse(row[:task_prerequisites]) unless row[:task_prerequisites].nil? if task_definition.nil? errors << { row: row, message: message } next end + prerequisites_by_task[task_definition.abbreviation] = JSON.parse(row[:task_prerequisites]) unless row[:task_prerequisites].nil? + success << { row: row, message: message } rescue Exception => e errors << { row: row, message: e.message } diff --git a/config/application.rb b/config/application.rb index 761103f765..23567a6463 100644 --- a/config/application.rb +++ b/config/application.rb @@ -282,14 +282,14 @@ def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) config.sm_instance = nil config.overseer_enabled = ENV['OVERSEER_ENABLED'].present? && ENV['OVERSEER_ENABLED'].to_s.downcase != "false" && ENV['OVERSEER_ENABLED'].to_i != 0 - if (config.overseer_enabled) - config.docker_config = { - DOCKER_REGISTRY_URL: ENV.fetch('DOCKER_REGISTRY_URL', nil), - DOCKER_PROXY_URL: ENV.fetch('DOCKER_PROXY_URL', nil), - DOCKER_TOKEN: ENV.fetch('DOCKER_TOKEN', nil), - DOCKER_USER: ENV.fetch('DOCKER_USER', nil) - } + config.docker_config = { + DOCKER_REGISTRY_URL: ENV.fetch('DOCKER_REGISTRY_URL', nil), + DOCKER_PROXY_URL: ENV.fetch('DOCKER_PROXY_URL', nil), + DOCKER_TOKEN: ENV.fetch('DOCKER_TOKEN', nil), + DOCKER_USER: ENV.fetch('DOCKER_USER', nil) + } + if (config.overseer_enabled) # Path to a physical directory on the host used for mounting overseer task work directories. # # Example (macOS development): diff --git a/lib/helpers/database_populator.rb b/lib/helpers/database_populator.rb index 57911a98bc..b415a30ec2 100644 --- a/lib/helpers/database_populator.rb +++ b/lib/helpers/database_populator.rb @@ -64,6 +64,7 @@ def initialize(scale = :small) generate_global_learning_outcomes() generate_campuses generate_activity_types + generate_overseer_images end def generate_teaching_periods @@ -166,6 +167,18 @@ def generate_activity_types ActivityType.create! data end + def generate_overseer_images + echo_line "--> Generating overseer images" + + overseer_image = OverseerImage.create!( + name: 'Bash', + tag: 'bash:latest' + ) + + echo_line "---> Pulling overseer image #{overseer_image.tag}" + overseer_image.pull_from_docker + end + # # Generate some users. Pass in an optional filter(s) for: # Role.admin, Role.convenor, Role.tutor, Role.student diff --git a/test/api/csv_test.rb b/test/api/csv_test.rb index 39845dcd38..0b45e7d761 100644 --- a/test/api/csv_test.rb +++ b/test/api/csv_test.rb @@ -181,7 +181,7 @@ def test_csv_upload_normalises_bad_upload_requirement_file_keys # Ensure our test csv has unordered or duplicate file keys # This scenario can happen when moving upload requirements around from different tasks expected_upload_requirements_by_task = { - '1.1P' => '[{"key":"file1","name":"HelloWorld.pas","type":"code"},{"key":"file1","name":"Screenshot","type":"image"}]', + '1.1P' => '[{"key":"file1","name":"HelloWorld.cpp","type":"code"},{"key":"file1","name":"Screenshot","type":"image"}]', '1.2P' => '[{"key":"file0","name":"PictureDrawing.pas","type":"code"},{"key":"file5","name":"Screenshot","type":"image"}]', '1.3P' => '[{"key":"file0","name":"PictureDrawing.pas","type":"code"},{"key":"file3","name":"Screenshot","type":"image"},{"key":"file2","name":"Screenshot","type":"image"}]' } diff --git a/test/api/overseer_image_api_test.rb b/test/api/overseer_image_api_test.rb index 65025a8803..99e45cd4d8 100644 --- a/test/api/overseer_image_api_test.rb +++ b/test/api/overseer_image_api_test.rb @@ -26,7 +26,7 @@ def test_get_all_overseer_images assert_equal 200, last_response.status assert_equal expected_data.count, last_response_body.count, last_response_body - response_keys = %w[name tag pulled_image_text pulled_image_status last_pulled_date] + response_keys = %w[name tag pulled_image_text pulled_image_status] last_response_body.each do |data| expected_data = OverseerImage.find(data['id']) diff --git a/test/models/task_definition_test.rb b/test/models/task_definition_test.rb index 77f9d9ed5f..ba62646455 100644 --- a/test/models/task_definition_test.rb +++ b/test/models/task_definition_test.rb @@ -141,11 +141,34 @@ def test_group_tasks def test_export_task_definitions_csv unit = FactoryBot.create(:unit, with_students: false) stream_1 = FactoryBot.create(:tutorial_stream, unit: unit) + task_def_with_steps = unit.task_definitions.first + task_def_with_steps.overseer_steps.create!( + name: 'compile', + description: 'Compile the submission', + display_name: 'Compile', + display_description: 'Compile step', + run_command: 'make test', + timeout: 45, + sort_order: 0, + step_type: 'run', + partial_output_diff: true, + stdin_input_file: 'stdin.txt', + expected_output_file: 'expected.txt', + feedback_message: 'Compilation failed', + status_on_success_id: TaskStatus.complete.id, + status_on_failure_id: TaskStatus.fix_and_resubmit.id, + halt_on_success: false, + halt_on_failure: true, + show_expected_output: true, + show_stdin: false, + show_stdout: true, + enabled: true + ) task_defs_csv = CSV.parse unit.task_definitions_csv, headers: true task_defs_csv.each do |task_def_csv| task_def = unit.task_definitions.find_by(abbreviation: task_def_csv['abbreviation']) - keys_to_ignore = %w[tutorial_stream start_week start_day target_week target_day due_week due_day upload_requirements task_prerequisites discussion_prompts] + keys_to_ignore = %w[tutorial_stream start_week start_day target_week target_day due_week due_day upload_requirements task_prerequisites discussion_prompts overseer_steps] task_def_csv.each do |key, value| unless keys_to_ignore.include?(key) assert_equal(task_def[key].to_s, value) @@ -170,9 +193,74 @@ def test_export_task_definitions_csv end.to_json assert_equal prerequisites, task_def_csv['task_prerequisites'] + + overseer_steps = task_def.overseer_steps.map do |step| + { + 'name' => step.name, + 'description' => step.description, + 'display_name' => step.display_name, + 'display_description' => step.display_description, + 'run_command' => step.run_command, + 'timeout' => step.timeout, + 'sort_order' => step.sort_order, + 'step_type' => step.step_type, + 'partial_output_diff' => step.partial_output_diff, + 'stdin_input_file' => step.stdin_input_file, + 'expected_output_file' => step.expected_output_file, + 'feedback_message' => step.feedback_message, + 'status_on_success' => TaskStatus.find_by(id: step.status_on_success_id)&.status_key&.to_s, + 'status_on_failure' => TaskStatus.find_by(id: step.status_on_failure_id)&.status_key&.to_s, + 'halt_on_success' => step.halt_on_success, + 'halt_on_failure' => step.halt_on_failure, + 'show_expected_output' => step.show_expected_output, + 'show_stdin' => step.show_stdin, + 'show_stdout' => step.show_stdout, + 'enabled' => step.enabled + } + end + + assert_equal overseer_steps, JSON.parse(task_def_csv['overseer_steps']) end end + def test_import_overseer_steps_from_csv_fixture + target_unit = Unit.create!( + code: 'CSVSTEP1', + name: 'CSV Import With Overseer Steps', + description: 'Import target', + teaching_period: TeachingPeriod.find(3) + ) + + result = target_unit.import_tasks_from_csv( + File.open(Rails.root.join("test_files/COS10001-ImportTasksWithOverseerSteps.csv")) + ) + + assert_empty result[:errors], result + + imported_task_def = target_unit.task_definitions.find_by(abbreviation: '1.1P') + assert_not_nil imported_task_def + assert_equal 1, imported_task_def.overseer_steps.count + + imported_step = imported_task_def.overseer_steps.first + assert_equal 'Step 1', imported_step.name + assert_equal 'Step 1 student', imported_step.display_name + assert_equal 'b64:IyEvYmluL2Jhc2gKCmVjaG8gIkhlbGxvIHdvcmxkISI', imported_step.run_command + assert_equal 30, imported_step.timeout + assert_equal 'status_check', imported_step.step_type + assert_nil imported_step.status_on_success_id + assert_nil imported_step.status_on_failure_id + assert_nil imported_step.partial_output_diff + assert_nil imported_step.stdin_input_file + assert_nil imported_step.expected_output_file + assert_nil imported_step.feedback_message + assert_nil imported_step.halt_on_success + assert_nil imported_step.halt_on_failure + assert imported_step.show_expected_output + assert_nil imported_step.show_stdin + assert imported_step.show_stdout + assert imported_step.enabled + end + def test_export_without_tutorial_stream data = { code: 'COS10001', diff --git a/test/models/unit_model_test.rb b/test/models/unit_model_test.rb index 6a54c9b284..39144c51d7 100644 --- a/test/models/unit_model_test.rb +++ b/test/models/unit_model_test.rb @@ -353,6 +353,62 @@ def test_rollover_of_discussion_prompts assert_equal new_prompt3.priority, 3 end + def test_rollover_of_overseer_steps + unit = FactoryBot.create(:unit, with_students: false, task_count: 2) + td = unit.task_definitions.first + + td.overseer_steps.create!( + name: 'compile', + description: 'Compile the submission', + display_name: 'Compile', + display_description: 'Compile step', + run_command: 'make test', + timeout: 45, + sort_order: 0, + step_type: 'run', + partial_output_diff: true, + stdin_input_file: 'stdin.txt', + expected_output_file: 'expected.txt', + feedback_message: 'Compilation failed', + status_on_success_id: TaskStatus.complete.id, + status_on_failure_id: TaskStatus.fix_and_resubmit.id, + halt_on_success: false, + halt_on_failure: true, + show_expected_output: true, + show_stdin: false, + show_stdout: true, + enabled: true + ) + + unit2 = unit.rollover(TeachingPeriod.find(2), nil, nil, nil) + new_td = unit2.task_definitions.find_by!(abbreviation: td.abbreviation) + new_step = new_td.overseer_steps.first + + assert_equal 1, new_td.overseer_steps.count + assert_not_nil new_step, 'Overseer step should be duplicated in rollover' + assert_equal new_td.id, new_step.task_definition_id + assert_equal 'compile', new_step.name + assert_equal 'Compile the submission', new_step.description + assert_equal 'Compile', new_step.display_name + assert_equal 'Compile step', new_step.display_description + assert_equal 'make test', new_step.run_command + assert_equal 45, new_step.timeout + assert_equal 0, new_step.sort_order + assert_equal 'run', new_step.step_type + assert_equal true, new_step.partial_output_diff + assert_equal 'stdin.txt', new_step.stdin_input_file + assert_equal 'expected.txt', new_step.expected_output_file + assert_equal 'Compilation failed', new_step.feedback_message + assert_equal TaskStatus.complete.id, new_step.status_on_success_id + assert_equal TaskStatus.fix_and_resubmit.id, new_step.status_on_failure_id + assert_equal false, new_step.halt_on_success + assert_equal true, new_step.halt_on_failure + assert_equal true, new_step.show_expected_output + assert_equal false, new_step.show_stdin + assert_equal true, new_step.show_stdout + assert_equal true, new_step.enabled + end + def test_rollover_of_task_prerequisites unit = FactoryBot.create(:unit, with_students: false, task_count: 4) td1 = unit.task_definitions.first diff --git a/test_files/COS10001-ImportTasksWithOverseerSteps.csv b/test_files/COS10001-ImportTasksWithOverseerSteps.csv new file mode 100644 index 0000000000..d990f899e9 --- /dev/null +++ b/test_files/COS10001-ImportTasksWithOverseerSteps.csv @@ -0,0 +1,2 @@ +name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts,overseer_steps +Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,FALSE,FALSE,FALSE,FALSE,0,FALSE,[],[],"[{""name"":""Step 1"",""description"":""Step 1"",""display_name"":""Step 1 student"",""display_description"":""Step 1 student"",""run_command"":""b64:IyEvYmluL2Jhc2gKCmVjaG8gIkhlbGxvIHdvcmxkISI"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":null,""halt_on_success"":null,""halt_on_failure"":null,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" diff --git a/test_files/COS10001-ImportTasksWithTutorialStream.csv b/test_files/COS10001-ImportTasksWithTutorialStream.csv index bc92146d61..3e6d413711 100644 --- a/test_files/COS10001-ImportTasksWithTutorialStream.csv +++ b/test_files/COS10001-ImportTasksWithTutorialStream.csv @@ -1,5 +1,5 @@ name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] Credit Task 1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] diff --git a/test_files/COS10001-ImportTasksWithoutTutorialStream.csv b/test_files/COS10001-ImportTasksWithoutTutorialStream.csv index a5e4a89700..e23a686ac5 100644 --- a/test_files/COS10001-ImportTasksWithoutTutorialStream.csv +++ b/test_files/COS10001-ImportTasksWithoutTutorialStream.csv @@ -1,5 +1,5 @@ name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] Credit Task 1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] diff --git a/test_files/COS10001-Tasks.csv b/test_files/COS10001-Tasks.csv index 54036fe3c2..f1a1b36659 100644 --- a/test_files/COS10001-Tasks.csv +++ b/test_files/COS10001-Tasks.csv @@ -1,38 +1,38 @@ -name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 2.1 - Hand Execute Assignment,2.1P,"Using the assignment statement, you can assign a value to a variable. In this task you will demonstrate how this action works within the computer.",2,0,FALSE,"[{""key"":""file0"",""name"":""Program Execution 1"",""type"":""image""},{""key"":""file1"",""name"":""Program Execution 2"",""type"":""image""},{""key"":""file2"",""name"":""Program Execution 3"",""type"":""image""},{""key"":""file3"",""name"":""Program Execution 4"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 2.2 - Hello User,2.2P,Now that we have variables we can create a program that reads in the users name from the Terminal and echoes back a welcome message.,4,0,FALSE,"[{""key"":""file0"",""name"":""HelloUser.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 2.3 - My Drawing Procedure,2.3P,Procedures are a great way of encapsulating the instructions needed to perform a task. In most cases the task will need some input data for it to work with. Use parameters to provide data to your procedures.,2,0,FALSE,"[{""key"":""file0"",""name"":""Shape Drawing Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 2.4 - My Functions,2.4P,Using functions you can now create artefacts to encapsulate the steps needed to calculate a value.,4,0,FALSE,"[{""key"":""file0"",""name"":""My Function Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 2.5 - Concept Maps,2.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",2,Tue,3,Tue,,,5,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 3.1 - Hand Execution of Control Flow,3.1P,In this task you will use the hand execution process to demonstrate how the control flow constructs operate within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Button Code"",""type"":""code""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 3.2 - Name Tester,3.2P,Control flow enables you to easily add conditions and loops to your programs. In this task you will create a small program that uses conditions and loops to output custom messages to users.,4,0,FALSE,"[{""key"":""file0"",""name"":""Name Tester code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 3.3 - Circle Moving,3.3P,In this task you will create a small program that allows the user to move a circle around on the screen.,4,0,FALSE,"[{""key"":""file0"",""name"":""Circle Mover code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 3.4 - User Input Functions,3.4C,So far we have provided you with a unit to read and check values entered by the user: the Terminal User Input unit. In this task you will extend this library so that it has a number of additional functions.,4,1,FALSE,"[{""key"":""file0"",""name"":""User Input unit code"",""type"":""code""},{""key"":""file1"",""name"":""Program code"",""type"":""code""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 3.5 - Concept Map,3.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Distinction Task 3.6 - Mandelbrot,3.6D,The Mandelbrot provides an interesting challenge in order to determine how to zoom in to and out of the section of the Mandelbrot being shown to the user.,4,2,FALSE,"[{""key"":""file0"",""name"":""Mandelbrot code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 4.1 - Using Records and Enumerations,4.1P,Effectively organising your data makes programs much easier to develop. By using records and enumerations you can start to model the entities associated with your programs.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 4.2 - Fruit Punch,4.2C,Create a program using the concepts covered so far.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 4.3 - Concept Map,4.3C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Test 1,T1,Test 1 covers weeks 1 to 3,1,0,TRUE,[],5,Fri,5,Fri,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 5.1 - Hand Execution of Arrays,5.1P,Demonstrate how arrays work within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 5.2 - Arrays of Records,5.2P,Add an array of records to your program that uses records.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 5.3 - Food Hunter,5.3C,Extend a small game to make use of arrays.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 5.4 - Concept Map,5.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Distinction Task 5.5 - Sort Visualiser,5.5D,Create a program to demonstrate sorting working within the computer.,4,2,FALSE,"[{""key"":""file0"",""name"":""Sort Visualiser"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,TRUE,"[{""abbreviation"":""1.4C"",""task_status_id"":2},{""abbreviation"":""3.3P"",""task_status_id"":9}]",[] -Pass Task 6.1 - Structure Charts,6.1P,Illustrate the structure of your program using a structure chart.,2,0,FALSE,"[{""key"":""file0"",""name"":""Program structrue chart"",""type"":""image""}]",6,Tue,7,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 7.1 - Programming Principles,7.1P,"Describe the principles of structured, procedural, programming.",4,0,FALSE,"[{""key"":""file0"",""name"":""Program Principles Description"",""type"":""document""}]",7,Tue,8,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Distinction Task 7.2 - Game of Life,7.2D,Create the Game of Life,4,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",7,Tue,8,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 8.1 - Language Reference Sheet,8.1P,Create a reference sheet for C or C#,4,0,FALSE,"[{""key"":""file0"",""name"":""Reference Sheet"",""type"":""document""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 8.2 - Circle Moving 2,8.2P,Recreate your circle moving program using C,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Test 2,T2,Covers all core concepts.,1,0,TRUE,[],9,Fri,9,Fri,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 9.1 - Reading Another Language,9.1P,Demonstrate how programs written in C work within the computer,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Credit Task 9.2 - Another Language,9.2C,Create a program with C using the concepts covered.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -High Distinction Task 10.1 - Custom Program,10.1H,Extend your custom program to meet the High Distinction criteria.,4,3,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -High Distinction Task 10.2 - Research Report,10.2H,Start working on a research project,8,3,FALSE,"[{""key"":""file0"",""name"":""Research Report Document"",""type"":""document""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 11.1 - Learning Summary Report,11.1P,Summarise your learning from the unit.,4,0,FALSE,"[{""key"":""file0"",""name"":""Learning Summary Report"",""type"":""document""}]",11,Tue,12,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[] -Distinction Task 6.2 - Custom Program,6.2D,Start working on your custom program!,16,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",6,Tue,13,Tue,,,5,TRUE,90,,,import-tasks,FALSE,[],[] -Test 10,T10,Test 10 tests the import task bug,1,0,TRUE,[],10,Fri,10,Fri,,,0,FALSE,90,,,import-tasks,FALSE,[],[] +name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts,overseer_steps +Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],,"[{""name"":""Hello World"",""description"":""Hello World"",""display_name"":""Hello World"",""display_description"":""Hello World"",""run_command"":""b64:IyEvYmluL3NoCgplY2hvICJIZWxsbyB3b3JsZCEi"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":null,""halt_on_success"":null,""halt_on_failure"":null,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" +Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.1P"",""task_status_id"":9}]",,"[{""name"":""Picture Drawing"",""description"":""Picture Drawing"",""display_name"":""Picture Drawing"",""display_description"":""Picture Drawing"",""run_command"":""b64:IyEvYmluL3NoCgplY2hvICJIZWxsbyB3b3JsZCEi"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":""fix_and_resubmit"",""halt_on_success"":null,""halt_on_failure"":true,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" +Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.2P"",""task_status_id"":9}]",, +Credit Task 1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.3P"",""task_status_id"":9}]",, +Pass Task 2.1 - Hand Execute Assignment,2.1P,"Using the assignment statement, you can assign a value to a variable. In this task you will demonstrate how this action works within the computer.",2,0,FALSE,"[{""key"":""file0"",""name"":""Program Execution 1"",""type"":""image""},{""key"":""file1"",""name"":""Program Execution 2"",""type"":""image""},{""key"":""file2"",""name"":""Program Execution 3"",""type"":""image""},{""key"":""file3"",""name"":""Program Execution 4"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.3P"",""task_status_id"":9}]",, +Pass Task 2.2 - Hello User,2.2P,Now that we have variables we can create a program that reads in the users name from the Terminal and echoes back a welcome message.,4,0,FALSE,"[{""key"":""file0"",""name"":""HelloUser.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.1P"",""task_status_id"":9}]",, +Pass Task 2.3 - My Drawing Procedure,2.3P,Procedures are a great way of encapsulating the instructions needed to perform a task. In most cases the task will need some input data for it to work with. Use parameters to provide data to your procedures.,2,0,FALSE,"[{""key"":""file0"",""name"":""Shape Drawing Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.2P"",""task_status_id"":9}]",, +Pass Task 2.4 - My Functions,2.4P,Using functions you can now create artefacts to encapsulate the steps needed to calculate a value.,4,0,FALSE,"[{""key"":""file0"",""name"":""My Function Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.3P"",""task_status_id"":9}]",, +Credit Task 2.5 - Concept Maps,2.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",2,Tue,3,Tue,,,5,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.4C"",""task_status_id"":2}]",, +Pass Task 3.1 - Hand Execution of Control Flow,3.1P,In this task you will use the hand execution process to demonstrate how the control flow constructs operate within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Button Code"",""type"":""code""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.4P"",""task_status_id"":9}]",, +Pass Task 3.2 - Name Tester,3.2P,Control flow enables you to easily add conditions and loops to your programs. In this task you will create a small program that uses conditions and loops to output custom messages to users.,4,0,FALSE,"[{""key"":""file0"",""name"":""Name Tester code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.1P"",""task_status_id"":9}]",, +Pass Task 3.3 - Circle Moving,3.3P,In this task you will create a small program that allows the user to move a circle around on the screen.,4,0,FALSE,"[{""key"":""file0"",""name"":""Circle Mover code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.2P"",""task_status_id"":9}]",, +Credit Task 3.4 - User Input Functions,3.4C,So far we have provided you with a unit to read and check values entered by the user: the Terminal User Input unit. In this task you will extend this library so that it has a number of additional functions.,4,1,FALSE,"[{""key"":""file0"",""name"":""User Input unit code"",""type"":""code""},{""key"":""file1"",""name"":""Program code"",""type"":""code""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.5C"",""task_status_id"":9}]",, +Credit Task 3.5 - Concept Map,3.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.4C"",""task_status_id"":8}]",, +Distinction Task 3.6 - Mandelbrot,3.6D,The Mandelbrot provides an interesting challenge in order to determine how to zoom in to and out of the section of the Mandelbrot being shown to the user.,4,2,FALSE,"[{""key"":""file0"",""name"":""Mandelbrot code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.5C"",""task_status_id"":2}]",, +Pass Task 4.1 - Using Records and Enumerations,4.1P,Effectively organising your data makes programs much easier to develop. By using records and enumerations you can start to model the entities associated with your programs.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.3P"",""task_status_id"":9}]",, +Credit Task 4.2 - Fruit Punch,4.2C,Create a program using the concepts covered so far.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Credit Task 4.3 - Concept Map,4.3C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Test 1,T1,Test 1 covers weeks 1 to 3,1,0,TRUE,[],5,Fri,5,Fri,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Pass Task 5.1 - Hand Execution of Arrays,5.1P,Demonstrate how arrays work within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Pass Task 5.2 - Arrays of Records,5.2P,Add an array of records to your program that uses records.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Credit Task 5.3 - Food Hunter,5.3C,Extend a small game to make use of arrays.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Credit Task 5.4 - Concept Map,5.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Distinction Task 5.5 - Sort Visualiser,5.5D,Create a program to demonstrate sorting working within the computer.,4,2,FALSE,"[{""key"":""file0"",""name"":""Sort Visualiser"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,TRUE,"[]",[],,,,"[{""abbreviation"":""1.4C"",""task_status_id"":2},{""abbreviation"":""3.3P"",""task_status_id"":9}]",, +Pass Task 6.1 - Structure Charts,6.1P,Illustrate the structure of your program using a structure chart.,2,0,FALSE,"[{""key"":""file0"",""name"":""Program structrue chart"",""type"":""image""}]",6,Tue,7,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Pass Task 7.1 - Programming Principles,7.1P,"Describe the principles of structured, procedural, programming.",4,0,FALSE,"[{""key"":""file0"",""name"":""Program Principles Description"",""type"":""document""}]",7,Tue,8,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Distinction Task 7.2 - Game of Life,7.2D,Create the Game of Life,4,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",7,Tue,8,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Pass Task 8.1 - Language Reference Sheet,8.1P,Create a reference sheet for C or C#,4,0,FALSE,"[{""key"":""file0"",""name"":""Reference Sheet"",""type"":""document""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Pass Task 8.2 - Circle Moving 2,8.2P,Recreate your circle moving program using C,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Test 2,T2,Covers all core concepts.,1,0,TRUE,[],9,Fri,9,Fri,,,0,FALSE,90,,,import-tasks,FALSE,"[]",[],,,,"[{""abbreviation"":""T1"",""task_status_id"":2},{""abbreviation"":""8.1P"",""task_status_id"":9},{""abbreviation"":""8.2P"",""task_status_id"":9}]",, +Pass Task 9.1 - Reading Another Language,9.1P,Demonstrate how programs written in C work within the computer,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Credit Task 9.2 - Another Language,9.2C,Create a program with C using the concepts covered.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +High Distinction Task 10.1 - Custom Program,10.1H,Extend your custom program to meet the High Distinction criteria.,4,3,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +High Distinction Task 10.2 - Research Report,10.2H,Start working on a research project,8,3,FALSE,"[{""key"":""file0"",""name"":""Research Report Document"",""type"":""document""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Pass Task 11.1 - Learning Summary Report,11.1P,Summarise your learning from the unit.,4,0,FALSE,"[{""key"":""file0"",""name"":""Learning Summary Report"",""type"":""document""}]",11,Tue,12,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +Distinction Task 6.2 - Custom Program,6.2D,Start working on your custom program!,16,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",6,Tue,13,Tue,,,5,TRUE,90,,,import-tasks,FALSE,[],[],,,,[],, +Test 10,T10,Test 10 tests the import task bug,1,0,TRUE,[],10,Fri,10,Fri,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, diff --git a/test_files/COS10001-TasksUnorderedUploadReqs.csv b/test_files/COS10001-TasksUnorderedUploadReqs.csv index 7a4d27adc4..e41817add2 100644 --- a/test_files/COS10001-TasksUnorderedUploadReqs.csv +++ b/test_files/COS10001-TasksUnorderedUploadReqs.csv @@ -1,4 +1,4 @@ name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file1"",""name"":""HelloWorld.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] +Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file1"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file5"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file3"",""name"":""Screenshot"",""type"":""image""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] From 3feb02c6137ff260d3de98f0cab33f4ee11df82f Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 23 Apr 2026 17:37:47 +1000 Subject: [PATCH 036/199] chore(release): 10.0.0-104 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e681b78c5b..1d1d509292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-104](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-103...v10.0.0-104) (2026-04-23) + + +### Features + +* import/export overseer steps via task definition csv ([#604](https://github.com/b0ink/doubtfire-api/issues/604)) ([5dbd575](https://github.com/b0ink/doubtfire-api/commit/5dbd57588c06cb69eae35c5eab12d757bbe0bec5)) + + +### Bug Fixes + +* prevent duplicate submission requests ([#611](https://github.com/b0ink/doubtfire-api/issues/611)) ([1d056ca](https://github.com/b0ink/doubtfire-api/commit/1d056ca36aca7d9de8977b7a96783ed7a8d05367)) + ## [10.0.0-103](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-102...v10.0.0-103) (2026-04-18) ## [10.0.0-102](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-101...v10.0.0-102) (2026-04-18) From 78d4f11d43acc29d542077a64ce3b1c66390cf12 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:06:21 +1000 Subject: [PATCH 037/199] refactor: allow either student id or username for batch pdf upload --- app/models/unit.rb | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index f877e4b4a8..7647c50915 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -3011,22 +3011,24 @@ def build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: nil, p next end - student_entries = zip.nil? ? [] : batch_feedback_entries_for_username(zip, project.user.username) - pdf_entries = zip.nil? ? [] : batch_feedback_named_pdf_entries_for_username(zip, project.user.username) + student_entries = zip.nil? ? [] : batch_feedback_entries_for_project(zip, project) + pdf_entries = zip.nil? ? [] : batch_feedback_named_pdf_entries_for_project(zip, project) if zip.present? if student_entries.any? && pdf_entries.empty? + expected_identifier = batch_feedback_primary_identifier_for_project(project) errors << { row: task_entry, - message: "Expected a PDF named #{project.user.username}.pdf inside #{project.user.username}'s folder." + message: "Expected a PDF named #{expected_identifier}.pdf inside #{expected_identifier}'s folder." } next end if pdf_entries.length > 1 + expected_identifier = batch_feedback_primary_identifier_for_project(project) errors << { row: task_entry, - message: "Found multiple PDFs named #{project.user.username}.pdf inside #{project.user.username}'s folder." + message: "Found multiple PDFs named #{expected_identifier}.pdf inside #{expected_identifier}'s folder." } next end @@ -3094,28 +3096,51 @@ def upload_batch_feedback_csv(user, task_definition, file, progress_callback: ni converted_csv.close! if defined?(converted_csv) && converted_csv.present? end - def batch_feedback_entries_for_username(zip, username) + def batch_feedback_primary_identifier_for_project(project) + project.user.student_id.to_s.strip.presence || project.user.username.to_s.strip + end + + def batch_feedback_identifiers_for_project(project) + [ + project.user.student_id.to_s.strip.presence, + project.user.username.to_s.strip.presence + ].compact.uniq + end + + def batch_feedback_entries_for_identifier(zip, identifier) zip.select do |entry| path_parts = entry.name.split('/').reject(&:blank?) next false if path_parts.empty? if entry.name_is_directory? - path_parts.any? { |part| part.casecmp(username).zero? } + path_parts.any? { |part| part.casecmp(identifier).zero? } else - path_parts[0...-1].any? { |part| part.casecmp(username).zero? } + path_parts[0...-1].any? { |part| part.casecmp(identifier).zero? } end end end - def batch_feedback_named_pdf_entries_for_username(zip, username) - batch_feedback_entries_for_username(zip, username).select do |entry| + def batch_feedback_entries_for_project(zip, project) + batch_feedback_identifiers_for_project(project).flat_map do |identifier| + batch_feedback_entries_for_identifier(zip, identifier) + end.uniq + end + + def batch_feedback_named_pdf_entries_for_identifier(zip, identifier) + batch_feedback_entries_for_identifier(zip, identifier).select do |entry| next false if entry.name_is_directory? next false unless File.extname(entry.name).casecmp('.pdf').zero? - File.basename(entry.name, '.pdf').casecmp(username).zero? + File.basename(entry.name, '.pdf').casecmp(identifier).zero? end end + def batch_feedback_named_pdf_entries_for_project(zip, project) + batch_feedback_identifiers_for_project(project).flat_map do |identifier| + batch_feedback_named_pdf_entries_for_identifier(zip, identifier) + end.uniq + end + def build_batch_feedback_legacy_marks_csv(task_rows) CSV.generate do |csv| csv << check_mark_csv_headers.split(',') From 34d258e965dae13e6a251dec07ed61122de752a2 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:06:39 +1000 Subject: [PATCH 038/199] chore(release): 10.0.0-105 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d1d509292..6ccb51acce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-105](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-104...v10.0.0-105) (2026-04-27) + ## [10.0.0-104](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-103...v10.0.0-104) (2026-04-23) From 4096f8716b63f4ca5239f1f0ff7a4591096ee10d Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Tue, 28 Apr 2026 11:42:50 +1000 Subject: [PATCH 039/199] feat: enable comment editing (#613) --- app/api/task_comments_api.rb | 48 +++++++++++++++++++++++++++ app/models/session_activity.rb | 2 +- test/api/comments/comment_test.rb | 55 +++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) diff --git a/app/api/task_comments_api.rb b/app/api/task_comments_api.rb index 51eaf2f5da..1307051e96 100644 --- a/app/api/task_comments_api.rb +++ b/app/api/task_comments_api.rb @@ -191,6 +191,54 @@ class TaskCommentsApi < Grape::API present false end + desc 'Edit a comment' + params do + requires :comment, type: String, desc: 'The updated comment text' + end + put '/projects/:project_id/task_def_id/:task_definition_id/comments/:id' do + project = Project.find(params[:project_id]) + task_definition = project.unit.task_definitions.find(params[:task_definition_id]) + + unless authorise? current_user, project, :make_submission + error!({ error: 'Not authorised to edit this comment' }, 403) + end + + task = project.task_for_task_definition(task_definition) + task_comment = task.all_comments.find(params[:id]) + + unless task_comment.user == current_user + error!({ error: 'You can only edit your own comments' }, 403) + end + + unless task_comment.content_type.blank? || task_comment.content_type == 'text' + error!({ error: 'Only text comments can be edited' }, 403) + end + + if task_comment.created_at < 10.minutes.ago + error!({ error: 'Comments can only be edited within 10 minutes of being created' }, 403) + end + + if params[:comment].blank? + error!({ error: 'Comment text is empty, unable to update comment' }, 403) + end + + task_comment.comment = params[:comment] + + unless task_comment.save + error!({ error: task_comment.errors.full_messages.to_sentence.presence || 'Unable to update comment' }, 403) + end + + SessionTracker.record_assessment_activity( + action: 'edit-comment', + user: current_user, + project: project, + ip_address: request.ip, + task: task + ) + + present task_comment.serialize(current_user), with: Grape::Presenters::Presenter + end + desc 'Mark a comment as unread' post '/projects/:project_id/task_def_id/:task_definition_id/comments/:id' do project = Project.find(params[:project_id]) diff --git a/app/models/session_activity.rb b/app/models/session_activity.rb index 22c135923d..18e6a55a65 100644 --- a/app/models/session_activity.rb +++ b/app/models/session_activity.rb @@ -4,6 +4,6 @@ class SessionActivity < ApplicationRecord belongs_to :task, optional: true belongs_to :task_definition, optional: true - VALID_ACTIONS = %w[inbox GET PUT assessing add-comment get-comments delete-comment get-comment-attachment mark-comment-unread get-submission-details get-submission-files].freeze + VALID_ACTIONS = %w[inbox GET PUT assessing add-comment edit-comment get-comments delete-comment get-comment-attachment mark-comment-unread get-submission-details get-submission-files].freeze validates :action, presence: true, inclusion: { in: VALID_ACTIONS } end diff --git a/test/api/comments/comment_test.rb b/test/api/comments/comment_test.rb index 1e1fc558a1..8b2ce97204 100644 --- a/test/api/comments/comment_test.rb +++ b/test/api/comments/comment_test.rb @@ -2,6 +2,7 @@ class CommentTest < ActiveSupport::TestCase include Rack::Test::Methods + include ActiveSupport::Testing::TimeHelpers include TestHelpers::AuthHelper include TestHelpers::JsonHelper include TestHelpers::TestFileHelper @@ -221,6 +222,60 @@ def test_student_reply_to_themselve_in_different_task_same_project assert_equal 404, last_response.status end + def test_student_can_edit_own_comment_within_10_minutes + project = Project.first + user = project.student + task_definition = project.unit.task_definitions.first + + add_auth_header_for(user: user) + post_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments", comment: 'Original comment' + assert_equal 201, last_response.status + + comment_id = last_response_body['id'] + + travel_to 9.minutes.from_now do + put_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments/#{comment_id}", comment: 'Edited comment' + assert_equal 200, last_response.status, last_response.body + end + + assert_equal 'Edited comment', TaskComment.find(comment_id).read_attribute(:comment) + assert_equal 'Edited comment', last_response_body['comment'] + end + + def test_student_cannot_edit_own_comment_after_10_minutes + project = Project.first + user = project.student + task_definition = project.unit.task_definitions.first + + add_auth_header_for(user: user) + post_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments", comment: 'Original comment' + assert_equal 201, last_response.status + + comment_id = last_response_body['id'] + + travel_to 11.minutes.from_now do + put_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments/#{comment_id}", comment: 'Too late' + assert_equal 403, last_response.status, last_response.body + end + + assert_equal 'Original comment', TaskComment.find(comment_id).read_attribute(:comment) + end + + def test_student_cannot_edit_other_users_comment + project = Project.first + task_definition = project.unit.task_definitions.first + tutor = project.tutor_for(task_definition) + user = project.student + task = project.task_for_task_definition(task_definition) + comment = task.add_text_comment(tutor, 'Tutor comment') + + add_auth_header_for(user: user) + put_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments/#{comment.id}", comment: 'Edited by student' + + assert_equal 403, last_response.status, last_response.body + assert_equal 'Tutor comment', comment.reload.read_attribute(:comment) + end + def test_student_reply_to_other_student_in_same_group unit = FactoryBot.create :unit From 9dc7436a2bdb8042a1afb408b4e356069e2f16be Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 28 Apr 2026 12:15:07 +1000 Subject: [PATCH 040/199] chore(release): 10.0.0-106 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ccb51acce..3cd48b901a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-106](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-105...v10.0.0-106) (2026-04-28) + + +### Features + +* enable comment editing ([#613](https://github.com/b0ink/doubtfire-api/issues/613)) ([4096f87](https://github.com/b0ink/doubtfire-api/commit/4096f8716b63f4ca5239f1f0ff7a4591096ee10d)) + ## [10.0.0-105](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-104...v10.0.0-105) (2026-04-27) ## [10.0.0-104](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-103...v10.0.0-104) (2026-04-23) From f883c37a4102d41b01b684350298fe073c21fcce Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Tue, 5 May 2026 11:16:27 +1000 Subject: [PATCH 041/199] refactor: check for feedback on task status change server side (#616) * refactor: check for feedback on task status change server side * chore: only check for feedback comment if updating from api * test: add tutor feedback --- app/api/tasks_api.rb | 11 ++++++- app/models/task.rb | 24 +++++++++++++- test/api/marking_sessions_api_test.rb | 2 ++ test/api/tasks_api_test.rb | 2 ++ test/models/task_test.rb | 47 +++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 2 deletions(-) diff --git a/app/api/tasks_api.rb b/app/api/tasks_api.rb index 86790affa2..d08c43ca8b 100644 --- a/app/api/tasks_api.rb +++ b/app/api/tasks_api.rb @@ -204,7 +204,16 @@ class TasksApi < Grape::API end logger.info "#{current_user.username} assessing task #{task.id} to #{params[:trigger]}" - result = task.trigger_transition(trigger: params[:trigger], by_user: current_user, quality: params[:quality_pts], recursive_fix: params[:trigger_recursive_fix]) + result = task.trigger_transition( + trigger: params[:trigger], + by_user: current_user, + quality: params[:quality_pts], + recursive_fix: params[:trigger_recursive_fix], + check_feedback: true + ) + if result.nil? && task.errors.any? + error!({ error: task.errors.full_messages.to_sentence }, 403) + end if result.nil? && task.task_definition.restrict_status_updates error!({ error: 'This task can only be updated by your tutor.' }, 403) end diff --git a/app/models/task.rb b/app/models/task.rb index 86439b3d08..51af0a327c 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -535,7 +535,8 @@ def ensured_group_submission group.create_submission self, '', group.projects.map { |proj| { project: proj, pct: 100 / group.projects.count } } end - def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: false, quality: 1, recursive_fix: false) + def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: false, quality: 1, recursive_fix: false, + check_feedback: false) # # Ensure that assessor is allowed to update the task in the indicated way # @@ -589,6 +590,12 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: return nil end + if check_feedback && [TaskStatus.complete, TaskStatus.fix_and_resubmit, TaskStatus.redo].include?(status) && + !has_manual_feedback_since_first_ready_for_feedback? + errors.add(:task_status, "cannot be moved to '#{status.name}' until feedback has been given") + return nil + end + if task_definition.assess_in_portfolio_only # Block assess_in_portfolio_only tasks from being signed off as complete if status == TaskStatus.complete @@ -633,6 +640,21 @@ def has_discussed_in_class_comment? comments.where(content_type: 'discussed_in_class').exists? end + def has_manual_feedback_since_first_ready_for_feedback? + first_ready_for_feedback_at = comments + .where(content_type: 'status', task_status_id: TaskStatus.ready_for_feedback.id) + .order(:created_at) + .pick(:created_at) + + feedback_comments = comments + .where(content_type: %w[text audio image pdf discussion]) + .where(user_id: unit.staff.select(:user_id)) + + feedback_comments = feedback_comments.where('created_at >= ?', first_ready_for_feedback_at) if first_ready_for_feedback_at + + feedback_comments.where.not("COALESCE(comment, '') LIKE ?", '**Automated Message:%').exists? + end + def grade_desc grade_for(grade) end diff --git a/test/api/marking_sessions_api_test.rb b/test/api/marking_sessions_api_test.rb index 7a0615f7db..a680de77cb 100644 --- a/test/api/marking_sessions_api_test.rb +++ b/test/api/marking_sessions_api_test.rb @@ -223,6 +223,8 @@ def test_assesment_activities assert_equal project.id, last_activity.project.id assert_equal "delete-comment", last_activity.action + project.task_for_task_definition(td).add_text_comment(tutor, 'Manual tutor feedback') + # Test asessment put "/api/projects/#{project.id}/task_def_id/#{td.id}", { trigger: 'complete' } assert_equal 200, last_response.status diff --git a/test/api/tasks_api_test.rb b/test/api/tasks_api_test.rb index 475db1e3d0..75a6fb005f 100644 --- a/test/api/tasks_api_test.rb +++ b/test/api/tasks_api_test.rb @@ -824,6 +824,8 @@ def test_requires_discussion_blocks_complete_until_discussed_comment_added assert_instance_of TaskDiscussedComment, discussed_comment assert_equal 'discussed_in_class', discussed_comment.content_type + task.add_text_comment(tutor, 'Manual tutor feedback') + put "/api/projects/#{project.id}/task_def_id/#{td.id}", { trigger: 'complete' } assert_equal 200, last_response.status task.reload diff --git a/test/models/task_test.rb b/test/models/task_test.rb index 7321e2e4c0..dd8e72b3be 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -49,6 +49,53 @@ def test_comments_for_user end end + def test_trigger_transition_allows_assessment_outcomes_without_feedback_check_by_default + project = FactoryBot.create(:project) + unit = project.unit + task_definition = unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + tutor = unit.main_convenor_user + + task.update!(task_status: TaskStatus.ready_for_feedback) + task.add_status_comment(project.student, TaskStatus.ready_for_feedback) + + assert task.trigger_transition(trigger: 'complete', by_user: tutor) + assert_equal TaskStatus.complete, task.task_status + end + + def test_trigger_transition_requires_manual_feedback_before_assessment_outcomes_when_checking_feedback + project = FactoryBot.create(:project) + unit = project.unit + task_definition = unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + tutor = unit.main_convenor_user + + task.update!(task_status: TaskStatus.ready_for_feedback) + task.add_status_comment(project.student, TaskStatus.ready_for_feedback) + + assert_nil task.trigger_transition(trigger: 'complete', by_user: tutor, check_feedback: true) + assert_equal TaskStatus.ready_for_feedback, task.task_status + assert_includes task.errors.full_messages.to_sentence, 'until feedback has been given' + + task.reload + task.add_text_comment(project.student, 'Student follow-up') + + assert_nil task.trigger_transition(trigger: 'fix', by_user: tutor, check_feedback: true) + assert_equal TaskStatus.ready_for_feedback, task.task_status + + task.reload + task.add_text_comment(tutor, '**Automated Message:** Automated feedback is not enough') + + assert_nil task.trigger_transition(trigger: 'redo', by_user: tutor, check_feedback: true) + assert_equal TaskStatus.ready_for_feedback, task.task_status + + task.reload + task.add_text_comment(tutor, 'Manual tutor feedback') + + assert task.trigger_transition(trigger: 'complete', by_user: tutor, check_feedback: true) + assert_equal TaskStatus.complete, task.task_status + end + def test_days_awaiting_feedback_pauses_during_break travel_to Time.zone.parse('2026-04-10 00:00:00 UTC') do teaching_period = FactoryBot.create( From b2e4e64ac4b4ac13708302a184d32f6457a092f9 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 5 May 2026 11:18:03 +1000 Subject: [PATCH 042/199] chore(release): 10.0.0-107 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cd48b901a..aa70ffd742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-107](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-106...v10.0.0-107) (2026-05-05) + ## [10.0.0-106](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-105...v10.0.0-106) (2026-04-28) From 60082f00fa86686efcc8e506f28defd8c02b3d7f Mon Sep 17 00:00:00 2001 From: Prishnee Nuckcheddy Date: Thu, 7 May 2026 13:34:32 +1000 Subject: [PATCH 043/199] fix: mitigate URI credential leakage in relative URI merges Upgrade the uri gem and add a runtime guard that strips URI credentials when combining a credential-bearing base with relative paths to prevent CVE-2025-61594 style leakage in API runtime flows. --- Gemfile | 1 + Gemfile.lock | 3 ++- .../uri_credential_leak_mitigation.rb | 21 +++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 config/initializers/uri_credential_leak_mitigation.rb diff --git a/Gemfile b/Gemfile index 90b4021f63..5e198fe515 100644 --- a/Gemfile +++ b/Gemfile @@ -52,6 +52,7 @@ gem 'puma' gem 'bootsnap', require: false gem 'csv' +gem 'uri', '>= 1.1.1' # CVE-2025-61594 # Extend irb for better output gem 'hirb' diff --git a/Gemfile.lock b/Gemfile.lock index 6bb903f2a6..c7cec6e08b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -539,7 +539,7 @@ GEM unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) - uri (1.0.3) + uri (1.1.1) useragent (0.16.11) version_gem (1.1.6) warden (1.2.9) @@ -623,6 +623,7 @@ DEPENDENCIES sprockets-rails sys-filesystem tca_client + uri (>= 1.1.1) webmock RUBY VERSION diff --git a/config/initializers/uri_credential_leak_mitigation.rb b/config/initializers/uri_credential_leak_mitigation.rb new file mode 100644 index 0000000000..1096c5545a --- /dev/null +++ b/config/initializers/uri_credential_leak_mitigation.rb @@ -0,0 +1,21 @@ +require 'uri' + +# Mitigates credential leakage when combining a credential-bearing base URI +# with a relative URI via URI#+ by stripping userinfo from the merged result. +module UriCredentialLeakMitigation + def +(other) + combined = super + + return combined unless other.respond_to?(:absolute?) && !other.absolute? + return combined unless combined.respond_to?(:user=) || combined.respond_to?(:password=) + + sanitized = combined.dup + sanitized.user = nil if sanitized.respond_to?(:user=) + sanitized.password = nil if sanitized.respond_to?(:password=) + sanitized + rescue URI::InvalidComponentError + combined + end +end + +URI::Generic.prepend(UriCredentialLeakMitigation) From 11cc264b83d2c5e161b1926e12c496f2fd9add13 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Wed, 13 May 2026 14:19:22 +1000 Subject: [PATCH 044/199] feat: enable sentry error logging (#620) * feat: enable sentry error logging * chore: allow manual environment set --- Gemfile | 3 +++ Gemfile.lock | 9 +++++++++ app/api/api_root.rb | 1 + config/initializers/sentry.rb | 12 ++++++++++++ 4 files changed, 25 insertions(+) create mode 100644 config/initializers/sentry.rb diff --git a/Gemfile b/Gemfile index 90b4021f63..69ca22f0ba 100644 --- a/Gemfile +++ b/Gemfile @@ -120,3 +120,6 @@ gem 'pdf-reader' gem 'oauth2' gem "sys-filesystem" + +gem "sentry-rails" +gem "sentry-ruby" diff --git a/Gemfile.lock b/Gemfile.lock index 6bb903f2a6..8bc4f19b3a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -456,6 +456,13 @@ GEM ruby2_keywords (0.0.5) rubyzip (2.4.1) securerandom (0.4.1) + sentry-rails (6.5.0) + railties (>= 5.2.0) + sentry-ruby (~> 6.5.0) + sentry-ruby (6.5.0) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + logger set (1.1.1) sexp_processor (4.17.3) shellwords (0.2.2) @@ -613,6 +620,8 @@ DEPENDENCIES ruby-lsp ruby-saml rubyzip + sentry-rails + sentry-ruby shellwords sidekiq sidekiq-cron diff --git a/app/api/api_root.rb b/app/api/api_root.rb index e36e21226e..b30f0d8102 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -47,6 +47,7 @@ class ApiRoot < Grape::API message = "Sorry... something went wrong with your request." status = 500 end + Sentry.capture_exception(e) Rack::Response.new({ error: message }.to_json, status, { 'Content-type' => 'text/error' }) end diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb new file mode 100644 index 0000000000..1cb50372f7 --- /dev/null +++ b/config/initializers/sentry.rb @@ -0,0 +1,12 @@ +if ENV["SENTRY_DSN"].present? + Sentry.init do |config| + config.dsn = ENV.fetch("SENTRY_DSN", nil) + # get breadcrumbs from logs + config.breadcrumbs_logger = [:active_support_logger, :http_logger] + config.environment = ENV.fetch("SENTRY_ENVIRONMENT", Rails.env) + config.release = ENV["SENTRY_RELEASE"] if ENV["SENTRY_RELEASE"].present? + # Add data like request headers and IP for users, if applicable; + # see https://docs.sentry.io/platforms/ruby/data-management/data-collected/ for more info + config.send_default_pii = false + end +end From 7967ddf508b9c9b7f9e6d63635c9b701fd1583ff Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 13 May 2026 14:21:24 +1000 Subject: [PATCH 045/199] chore(release): 10.0.0-108 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa70ffd742..baaf2e9593 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-108](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-107...v10.0.0-108) (2026-05-13) + + +### Features + +* enable sentry error logging ([#620](https://github.com/b0ink/doubtfire-api/issues/620)) ([11cc264](https://github.com/b0ink/doubtfire-api/commit/11cc264b83d2c5e161b1926e12c496f2fd9add13)) + ## [10.0.0-107](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-106...v10.0.0-107) (2026-05-05) ## [10.0.0-106](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-105...v10.0.0-106) (2026-04-28) From a824b1b5fc4f47a12be33d410962af31fe810c89 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 13 May 2026 15:57:10 +1000 Subject: [PATCH 046/199] fix: ensure recent feedback has been left for fix --- app/models/task.rb | 24 ++++++++++++++++++++---- test/models/task_test.rb | 29 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/app/models/task.rb b/app/models/task.rb index 51af0a327c..492179b517 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -590,10 +590,17 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: return nil end - if check_feedback && [TaskStatus.complete, TaskStatus.fix_and_resubmit, TaskStatus.redo].include?(status) && - !has_manual_feedback_since_first_ready_for_feedback? - errors.add(:task_status, "cannot be moved to '#{status.name}' until feedback has been given") - return nil + if check_feedback + if status == TaskStatus.complete && !has_manual_feedback_since_first_ready_for_feedback? + errors.add(:task_status, "cannot be moved to '#{status.name}' until feedback has been given") + return nil + end + + if [TaskStatus.fix_and_resubmit, TaskStatus.redo].include?(status) && + !has_recent_manual_feedback_from_tutor?(by_user) + errors.add(:task_status, "cannot be moved to '#{status.name}' until feedback has been given") + return nil + end end if task_definition.assess_in_portfolio_only @@ -655,6 +662,15 @@ def has_manual_feedback_since_first_ready_for_feedback? feedback_comments.where.not("COALESCE(comment, '') LIKE ?", '**Automated Message:%').exists? end + def has_recent_manual_feedback_from_tutor?(tutor) + comments + .where(content_type: %w[text audio image pdf discussion]) + .where(user: tutor) + .where('created_at >= ?', 10.minutes.ago) + .where.not("COALESCE(comment, '') LIKE ?", '**Automated Message:%') + .exists? + end + def grade_desc grade_for(grade) end diff --git a/test/models/task_test.rb b/test/models/task_test.rb index dd8e72b3be..d7e7718fad 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -96,6 +96,35 @@ def test_trigger_transition_requires_manual_feedback_before_assessment_outcomes_ assert_equal TaskStatus.complete, task.task_status end + def test_trigger_transition_requires_recent_manual_tutor_feedback_for_fix_and_redo_when_checking_feedback + travel_to Time.zone.parse('2026-05-13 10:00:00 UTC') do + project = FactoryBot.create(:project) + unit = project.unit + task_definition = unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + tutor = unit.main_convenor_user + + task.update!(task_status: TaskStatus.ready_for_feedback) + task.add_status_comment(project.student, TaskStatus.ready_for_feedback) + task.add_text_comment(tutor, 'Older manual tutor feedback').update!(created_at: 11.minutes.ago) + + assert_nil task.trigger_transition(trigger: 'fix', by_user: tutor, check_feedback: true) + assert_equal TaskStatus.ready_for_feedback, task.task_status + + task.reload + task.add_text_comment(tutor, 'Recent manual tutor feedback') + + assert task.trigger_transition(trigger: 'fix', by_user: tutor, check_feedback: true) + assert_equal TaskStatus.fix_and_resubmit, task.task_status + + task.update!(task_status: TaskStatus.ready_for_feedback) + task.add_text_comment(tutor, 'Recent manual tutor feedback for redo') + + assert task.trigger_transition(trigger: 'redo', by_user: tutor, check_feedback: true) + assert_equal TaskStatus.redo, task.task_status + end + end + def test_days_awaiting_feedback_pauses_during_break travel_to Time.zone.parse('2026-04-10 00:00:00 UTC') do teaching_period = FactoryBot.create( From ed2add5d3b73447b4fd9a9136f1e9903ef963006 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 13 May 2026 16:28:35 +1000 Subject: [PATCH 047/199] chore(release): 10.0.0-109 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index baaf2e9593..10d4506ec0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-109](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-108...v10.0.0-109) (2026-05-13) + + +### Bug Fixes + +* ensure recent feedback has been left for fix ([a824b1b](https://github.com/b0ink/doubtfire-api/commit/a824b1b5fc4f47a12be33d410962af31fe810c89)) + ## [10.0.0-108](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-107...v10.0.0-108) (2026-05-13) From 7ea3d40df0cc47df89cd7a71c68ac1fdc5b1a1cd Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Wed, 3 Jun 2026 12:35:37 +1000 Subject: [PATCH 048/199] refactor: enable configuration of max shown comparisons in jplag report (#625) * refactor: enable configuration of max shown comparisons in jplag report * chore: split jplag command into multiline * chore: avoid redundant string --- .../similarity/unit_similarity_module.rb | 20 ++++++++++++++++++- config/application.rb | 10 ++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/app/models/similarity/unit_similarity_module.rb b/app/models/similarity/unit_similarity_module.rb index c003c36b56..1fd332d3ed 100644 --- a/app/models/similarity/unit_similarity_module.rb +++ b/app/models/similarity/unit_similarity_module.rb @@ -305,8 +305,26 @@ def run_jplag_on_done_files(task_definition, tasks_dir, tasks_with_files, unit_c skip_cluster_check = Doubtfire::Application.config.jplag_skip_cluster_check skip_cluster_string = skip_cluster_check ? '--cluster-skip' : '' + max_shown_comparisons = Doubtfire::Application.config.jplag_max_shown_comparisons + max_shown_comparisons = 2500 if max_shown_comparisons.nil? + # Run JPLAG on the extracted files. JPlag container should already be in the /jplag/ workdir. - docker_command = "docker exec -i jplag java -jar jplag-jar-with-dependencies.jar --skip-version-check #{tasks_dir_split}/submissions #{base_code_string} -l #{file_lang} --similarity-threshold=#{similarity_threshold} #{min_token_string} #{skip_cluster_string} -M RUN -r #{results_dir}/#{task_definition.abbreviation}-result --overwrite" + docker_command = [ + "docker exec -i jplag", + "java -jar jplag-jar-with-dependencies.jar", + "--skip-version-check", + "#{tasks_dir_split}/submissions", + base_code_string, + "-l #{file_lang}", + "--similarity-threshold=#{similarity_threshold}", + "--shown-comparisons=#{max_shown_comparisons}", + min_token_string, + skip_cluster_string, + "-M RUN", + "-r #{results_dir}/#{task_definition.abbreviation}-result", + "--overwrite" + ].join(" ") + logger.debug "Executing command: #{docker_command}" system(docker_command) diff --git a/config/application.rb b/config/application.rb index 23567a6463..8121ab0fbb 100644 --- a/config/application.rb +++ b/config/application.rb @@ -88,9 +88,19 @@ def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) # directory under root but is overridden using DF_JPLAG_REPORT_DIR environment # variable. config.jplag_report_dir = ENV['DF_JPLAG_REPORT_DIR'] || Rails.root.join('jplag/results').to_s + + # Tunes the comparison sensitivity by adjusting the minimum token required to be + # counted as a matching section. A smaller value increases the sensitivity + # but might lead to more false-positives config.jplag_min_tokens = ENV.fetch('DF_JPLAG_MIN_TOKENS', -1) + + # Skips the cluster calculation config.jplag_skip_cluster_check = ENV['DF_JPLAG_SKIP_CLUSTER_CHECK'].present? && (ENV['DF_JPLAG_SKIP_CLUSTER_CHECK'].to_s.downcase == "true" || ENV['DF_JPLAG_SKIP_CLUSTER_CHECK'].to_i == 1) + # The maximum number of comparisons that will be shown in the generated report + # if set to -1 all comparisons will be shown + config.jplag_max_shown_comparisons = ENV.fetch('DF_JPLAG_MAX_SHOWN_COMPARISONS', 2500) + # ==> File size limits # Sets the global file size limit per upload requirement # Defaults to 10MB (10,000,000 bytes) From 7a09ae5ab05c99a435879537b504ec821d9f40e0 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 3 Jun 2026 13:09:15 +1000 Subject: [PATCH 049/199] chore(release): 10.0.0-110 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10d4506ec0..e6db9dcfa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0-110](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-109...v10.0.0-110) (2026-06-03) + ## [10.0.0-109](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-108...v10.0.0-109) (2026-05-13) From d5d874214fe31f494b5a30f67f7286e711ba45b2 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:53:51 +1000 Subject: [PATCH 050/199] feat: support zip file submissions (#624) * feat: support zip file submissions * chore: fix rubocop * refactor: move uncompressed multiplier to env var and deny zip files containing zips * chore: update test --- app/api/submission/generate_helpers.rb | 2 +- app/helpers/file_helper.rb | 229 +++++++++++++++++- .../project_compile_portfolio_module.rb | 2 +- app/models/task.rb | 9 +- app/models/task_definition.rb | 6 +- app/views/shared/_file.pdf.erb | 6 + app/views/task/task_pdf.pdf.erb | 39 +++ config/application.rb | 13 + test/models/file_helper_test.rb | 124 ++++++++++ test/models/task_definition_test.rb | 28 +++ 10 files changed, 451 insertions(+), 7 deletions(-) diff --git a/app/api/submission/generate_helpers.rb b/app/api/submission/generate_helpers.rb index 1d74ac443e..c78564e4a2 100644 --- a/app/api/submission/generate_helpers.rb +++ b/app/api/submission/generate_helpers.rb @@ -18,7 +18,7 @@ def scoop_files(params, upload_reqs) files[key][:id] = files[key]['name'] files[key][:name] = detail['name'] - files[key][:type] = detail['type'] + files[key][:type] = detail['type'] == 'archive' ? 'zip' : detail['type'] end # File didn't get assigned an id above, then reject it since there was a mismatch diff --git a/app/helpers/file_helper.rb b/app/helpers/file_helper.rb index 1f589f08a8..41016e5000 100644 --- a/app/helpers/file_helper.rb +++ b/app/helpers/file_helper.rb @@ -6,14 +6,20 @@ require 'open3' require 'shellwords' require 'pdf-reader' +require 'zlib' +require 'rubygems/package' module FileHelper extend LogHelper extend TimeoutHelper extend MimeCheckHelpers + ZIP_NESTED_ARCHIVE_EXTENSIONS = %w[ + .7z .bz2 .ear .gz .jar .rar .tar .tar.bz2 .tar.gz .tar.xz .tbz .tbz2 .tgz .txz .war .xz .zip + ].freeze + def known_extension?(extn) - allow_extensions = %w(pdf ps csv xls xlsx pas cpp c cs csv h hpp java py js html coffee scss yaml yml xml json ts r rb rmd rnw rhtml rpres tex vb sql txt md jack hack asm hdl tst out cmp vm sh bat dat ipynb css png bmp tiff tif jpeg jpg gif zip gz tar wav ogg mp3 mp4 webm aac pcm aiff flac wma alac pml vue) + allow_extensions = %w(pdf ps csv xls xlsx pas cpp c cs csv h hpp java py js html coffee scss yaml yml xml json ts r rb rmd rnw rhtml rpres tex vb sql txt md jack hack asm hdl tst out cmp vm sh bat dat ipynb css png bmp tiff tif jpeg jpg gif zip gz tgz tar wav ogg mp3 mp4 webm aac pcm aiff flac wma alac pml vue) # Allow empty or nil extensions for blobs otherwise check that it matches the allowed list extn.blank? || allow_extensions.include?(extn) @@ -34,6 +40,16 @@ def accept_file(file, name, kind) 'application/tst', 'text/x-cmp', 'text/x-vm', 'application/x-sh', 'application/x-bat', 'application/dat', 'application/x-wine-extension-ini'] when 'document' mime_allow_list = [ 'application/pdf' ] + when 'zip', 'archive' + mime_allow_list = [ + 'application/zip', + 'application/x-zip', + 'application/x-zip-compressed', + 'multipart/x-zip', + 'application/x-tar', + 'application/gzip', + 'application/x-gzip' + ] when 'audio' mime_allow_list = ['audio/', 'video/webm', 'application/ogg', 'application/octet-stream'] when 'comment_attachment' @@ -87,6 +103,18 @@ def accept_file(file, name, kind) end end + if %w[zip archive].include?(kind) + zip_validation_result = validate_zip_upload(file['tempfile'].path, File.basename(file[:filename].to_s)) + + unless zip_validation_result[:valid] + logger.debug "Zip file is invalid: #{zip_validation_result[:msg]}" + return { + accepted: false, + msg: zip_validation_result[:msg] + } + end + end + logger.debug 'Uploaded file is accepted' # All checks are done @@ -383,6 +411,193 @@ def compress_pdf(path, max_size: 2_500_000, timeout_seconds: 30) FileUtils.rm_f tmp_file end + def zip_path_safe?(path) + return false if path.blank? + + clean_path = path.tr('\\', '/') + return false if clean_path.start_with?('/') || clean_path.include?("\0") + + clean_path.sub!(%r{\A\./+}, '') + clean_path.split('/').none? { |part| part.blank? || part == '.' || part == '..' } + end + + def zip_entry_limit + limit = Doubtfire::Application.config.zip_entry_limit.to_i + limit.positive? ? limit : 1_000 + end + + def zip_compression_ratio_limit + limit = Doubtfire::Application.config.zip_compression_ratio_limit.to_i + limit.positive? ? limit : 100 + end + + def zip_uncompressed_size_multiplier + multiplier = Doubtfire::Application.config.zip_uncompressed_size_multiplier.to_i + multiplier.positive? ? multiplier : 10 + end + + def zip_nested_archive?(path) + clean_path = path.to_s.downcase + ZIP_NESTED_ARCHIVE_EXTENSIONS.any? { |extension| clean_path.end_with?(extension) } + end + + def validate_zip_upload_entry!(name, size, zip_stats, _max_file_size, max_uncompressed_size) + raise 'Zip contains a file with an unsafe path.' unless zip_path_safe?(name) + raise 'Zip contains another archive file. Nested archives are not allowed.' if zip_nested_archive?(name) + + zip_stats[:entries] += 1 + zip_stats[:total_uncompressed_size] += size.to_i + + raise "Zip contains too many files. Limit is #{zip_entry_limit} files." if zip_stats[:entries] > zip_entry_limit + # raise "Zip contains a file larger than the #{max_file_size / 1_000_000}MB file limit." if size.to_i > max_file_size + if zip_stats[:total_uncompressed_size] > max_uncompressed_size + raise "Zip expands beyond the #{max_uncompressed_size / 1_000_000}MB uncompressed size limit." + end + end + + def validate_zip_file(path, max_file_size, max_uncompressed_size) + stats = { entries: 0, total_uncompressed_size: 0 } + + Zip::File.open(path) do |zip_file| + zip_file.each do |entry| + raise 'Encrypted zip entries are not supported.' if entry.respond_to?(:encrypted?) && entry.encrypted? + raise 'Zip contains an unsupported link entry.' if entry.respond_to?(:ftype) && entry.ftype == :symlink + next if entry.directory? + + validate_zip_upload_entry!(entry.name, entry.size, stats, max_file_size, max_uncompressed_size) + end + end + + stats + end + + def validate_tar_file(io, max_file_size, max_uncompressed_size) + stats = { entries: 0, total_uncompressed_size: 0 } + + Gem::Package::TarReader.new(io) do |tar| + tar.each do |entry| + next if entry.directory? + raise 'Zip contains an unsupported non-file entry.' unless entry.file? + + validate_zip_upload_entry!(entry.full_name, entry.header.size, stats, max_file_size, max_uncompressed_size) + end + end + + stats + end + + def validate_zip_upload(path, filename) + max_file_size = Doubtfire::Application.config.max_file_size.to_i + max_file_size = 10_000_000 if max_file_size <= 0 + max_uncompressed_size = max_file_size * zip_uncompressed_size_multiplier + return { valid: false, msg: "Zip exceeds the #{max_file_size / 1_000_000}MB file limit." } if File.size(path) > max_file_size + + begin + stats = + if filename.downcase.end_with?('.zip') + validate_zip_file(path, max_file_size, max_uncompressed_size) + elsif filename.downcase.end_with?('.tar') + File.open(path, 'rb') { |file| validate_tar_file(file, max_file_size, max_uncompressed_size) } + elsif filename.downcase.end_with?('.tar.gz', '.tgz') + Zlib::GzipReader.open(path) { |gzip| validate_tar_file(gzip, max_file_size, max_uncompressed_size) } + else + return { valid: false, msg: 'Unsupported zip format. Use .zip, .tar, .tar.gz, or .tgz.' } + end + + return { valid: false, msg: 'Zip must contain at least one file.' } if stats[:entries].zero? + + compressed_size = [File.size(path), 1].max + if stats[:total_uncompressed_size] / compressed_size > zip_compression_ratio_limit + return { valid: false, msg: "Zip compression ratio is too high. Limit is #{zip_compression_ratio_limit}:1." } + end + + { valid: true, msg: 'success' } + rescue Zip::Error, Zlib::Error, Gem::Package::TarInvalidError, EOFError + { valid: false, msg: 'Zip file is corrupted or not a supported zip.' } + rescue StandardError => e + { valid: false, msg: e.message } + end + end + + def zip_tree_add_path(tree, path) + clean_path = path.to_s.tr('\\', '/').sub(%r{\A\./+}, '').sub(%r{/+\z}, '') + return if clean_path.blank? + + parts = clean_path.split('/').reject(&:blank?) + node = tree + + parts.each_with_index do |part, index| + key = index == parts.length - 1 ? part : "#{part}/" + node[key] ||= {} + node = node[key] + end + end + + def zip_tree_walk(node, prefix = '', lines = []) + sorted_entries = node.keys.sort_by { |key| [key.end_with?('/') ? 0 : 1, key.downcase] } + + sorted_entries.each_with_index do |name, index| + last = index == sorted_entries.length - 1 + connector = '↳ ' + lines << "#{prefix}#{connector}#{name}" + zip_tree_walk(node[name], "#{prefix} ", lines) if node[name].any? + end + + lines + end + + def zip_file_tree(path, filename, display_limit: 200) + tree = {} + entries = 0 + + read_entry = lambda do |entry_name| + return unless zip_path_safe?(entry_name) + + entries += 1 + zip_tree_add_path(tree, entry_name) + end + + if filename.downcase.end_with?('.zip') + Zip::File.open(path) do |zip_file| + zip_file.each do |entry| + next if entry.directory? + + read_entry.call(entry.name) + end + end + elsif filename.downcase.end_with?('.tar') + File.open(path, 'rb') do |file| + Gem::Package::TarReader.new(file) do |tar| + tar.each do |entry| + next if entry.directory? + + read_entry.call(entry.full_name) + end + end + end + elsif filename.downcase.end_with?('.tar.gz', '.tgz', '.gz') + Zlib::GzipReader.open(path) do |gzip| + Gem::Package::TarReader.new(gzip) do |tar| + tar.each do |entry| + next if entry.directory? + + read_entry.call(entry.full_name) + end + end + end + end + + all_lines = zip_tree_walk(tree) + lines = all_lines.first(display_limit) + { lines: lines, entries: entries, tree_lines: all_lines.length, truncated: all_lines.length > display_limit } + rescue Zip::Error, Zlib::Error, Gem::Package::TarInvalidError, EOFError => e + logger.debug "Could not read zip file tree for #{filename}: #{e.message}" + { lines: [], entries: 0, truncated: false, error: true } + rescue StandardError => e + logger.debug "Could not read zip file tree for #{filename}: #{e.message}" + { lines: [], entries: 0, truncated: false, error: true } + end + def pages_in_pdf(path) exec = "qpdf --show-npages #{path}" @@ -790,6 +1005,18 @@ def line_wrap(path, width: 160) module_function :qpdf module_function :move_files module_function :validate_pdf + module_function :zip_path_safe? + module_function :zip_entry_limit + module_function :zip_compression_ratio_limit + module_function :zip_uncompressed_size_multiplier + module_function :zip_nested_archive? + module_function :validate_zip_upload_entry! + module_function :validate_zip_file + module_function :validate_tar_file + module_function :validate_zip_upload + module_function :zip_tree_add_path + module_function :zip_tree_walk + module_function :zip_file_tree module_function :copy_pdf module_function :read_file_to_str module_function :path_to_plagarism_html diff --git a/app/models/pdf_generation/project_compile_portfolio_module.rb b/app/models/pdf_generation/project_compile_portfolio_module.rb index 311bcaff57..8fb394340f 100644 --- a/app/models/pdf_generation/project_compile_portfolio_module.rb +++ b/app/models/pdf_generation/project_compile_portfolio_module.rb @@ -286,7 +286,7 @@ def portfolio_files(ensure_valid: false, force_ascii: false) result = [] Dir.chdir(portfolio_tmp_dir) - files = Dir.glob('*').select { |f| (f =~ /^\d{3}-(cover|document|code|image)/) == 0 } + files = Dir.glob('*').select { |f| (f =~ /^\d{3}-(cover|document|code|image|zip|archive)/) == 0 } files.each do |file| parts = file.split('-') idx = parts[0].to_i diff --git a/app/models/task.rb b/app/models/task.rb index 492179b517..5fdcfe7833 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -1133,7 +1133,7 @@ def compress_new_to_done(task_dir: student_work_dir(:new, false), zip_file_path: FileUtils.rm("#{task_dir}#{img}") unless dest_file == "#{task_dir}#{img}" end - input_files = Dir.entries(task_dir).select { |f| (f =~ /^\d{3}.(cover|document|code|image)/) == 0 } + input_files = Dir.entries(task_dir).select { |f| (f =~ /^\d{3}.(cover|document|code|image|zip|archive)/) == 0 } if input_files.length != task_definition.number_of_uploaded_files logger.error "Error processing task #{log_details} - missing files expected #{task_definition.number_of_uploaded_files} got #{input_files.length}" @@ -1321,6 +1321,7 @@ class TaskAppController < ApplicationController attr_accessor :base_path attr_accessor :image_path attr_accessor :include_pax + attr_accessor :submitted_files_url def init(task, is_retry) @task = task @@ -1331,6 +1332,10 @@ def init(task, is_retry) @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] @include_pax = !is_retry @work_id = "task-#{task.id}-#{Time.now.to_i}-#{Process.pid}-#{Thread.current.object_id}#{'-retry' if is_retry}" + host = Doubtfire::Application.config.institution[:host].to_s + host = "http://#{host}" unless host.match?(%r{\Ahttps?://}) + host = host.sub(%r{/*\z}, '') + @submitted_files_url = "#{host}/projects/#{task.project.id}/task_def_id/#{task.task_definition.id}/submission_files/download" end def make_pdf @@ -1481,7 +1486,7 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end end - raise LatexError.new(log_message), 'Failed to convert your submission to PDF. Check code files submitted for invalid characters, that documents are valid pdfs, and that images are valid.' + raise LatexError.new(log_message), 'Failed to convert your submission to PDF. Check code files submitted for invalid characters, that documents are valid pdfs, images are valid, and zip files are valid.' end end diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index 4e4b222c64..0a299f3985 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -340,6 +340,8 @@ def check_upload_requirements_format return end + req['type'] = 'zip' if req['type'] == 'archive' + # Check keys only contain key, type, name, tii_check, and tii_pct unless req.keys.excluding('key', 'type', 'name', 'tii_check', 'tii_pct').empty? errors.add(:upload_requirements, "has additional values for item #{i + 1} --> #{req.keys.join(' ')}.") @@ -350,8 +352,8 @@ def check_upload_requirements_format errors.add(:upload_requirements, "the name for item #{i + 1} does not seem to be a valid filename --> #{req['name']}.") end - # Check the type is either document or image or code - unless %w(document image code).include? req['type'] + # Check the type is either document, image, code, or zip + unless %w(document image code zip).include? req['type'] errors.add(:upload_requirements, "the type for item #{i + 1} is not valid --> #{req['type']}.") end diff --git a/app/views/shared/_file.pdf.erb b/app/views/shared/_file.pdf.erb index 036b4b2635..d97d945e7a 100644 --- a/app/views/shared/_file.pdf.erb +++ b/app/views/shared/_file.pdf.erb @@ -26,3 +26,9 @@ <% end # end for %> <% end %> + +<% if %w[zip archive].include?(file_type) %> +\begin{tcolorbox}[colback=blue!5!white,colframe=blue!75!black] + This zip file was uploaded with the portfolio. Zip contents are not expanded into this PDF. +\end{tcolorbox} +<% end %> diff --git a/app/views/task/task_pdf.pdf.erb b/app/views/task/task_pdf.pdf.erb index 44e8a83861..e279bff132 100644 --- a/app/views/task/task_pdf.pdf.erb +++ b/app/views/task/task_pdf.pdf.erb @@ -136,6 +136,45 @@ No Tutor % Supervisor's Name \includepdf[pages={<%= page_idx %>-<%= page_idx %>},fitpaper]{<%= file[:path] %>} <% end # end for %> <% end # end if %> + <% if %w[zip archive].include?(file[:type]) %> +<% archive_tree = FileHelper.zip_file_tree(file[:path], File.basename(file[:path])) %> +\begin{tcolorbox}[enhanced,breakable,colback=orange!8!white,colframe=orange!80!black,boxrule=0.9pt,left=4mm,right=4mm,top=3mm,bottom=3mm] +\begin{minipage}{0.12\textwidth} +\centering +{\Huge \faFileArchiveO} +\end{minipage} +\hfill +\begin{minipage}{0.82\textwidth} +\textbf{Zip submission file}\\[0.35em] +This zip file was uploaded with the submission. Zip contents are not expanded into this PDF.\\[0.65em] +\href{<%= @submitted_files_url %>}{\faDownload\ Download submitted files} {\footnotesize (Must be already authenticated in <%= lesc @doubtfire_product_name %>)} +\end{minipage} + +\tcblower +\textbf{Zip contents}\\[0.35em] +<% if archive_tree[:error] %> +The zip file tree could not be read while generating this PDF. Download the submitted files to inspect the contents. +<% elsif archive_tree[:lines].empty? %> +No readable file entries were found in this zip. +<% else %> +{\footnotesize\ttfamily +\begin{tabular}{@{}p{0.96\textwidth}@{}} +<% archive_tree[:lines].each do |line| %> +<% + indent = line[/\A */].length + content = line.lstrip + indent_width = indent * 0.45 +%> +\hspace*{<%= indent_width %>em}<%= raw lesc(content).to_s.gsub('↳', '\ensuremath{\hookrightarrow}') %>\\ +<% end %> +\end{tabular} +} +<% if archive_tree[:truncated] %> +\\[0.35em]\emph{Showing the first <%= archive_tree[:lines].length %> of <%= archive_tree[:tree_lines] %> tree lines. Download the submitted files to inspect everything.} +<% end %> +<% end %> +\end{tcolorbox} + <% end %> <% end # list of documents %> <% diff --git a/config/application.rb b/config/application.rb index 8121ab0fbb..9c1a67505e 100644 --- a/config/application.rb +++ b/config/application.rb @@ -106,6 +106,18 @@ def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) # Defaults to 10MB (10,000,000 bytes) config.max_file_size = ENV.fetch('DF_MAX_FILE_SIZE', 10_000_000) + # Max files inside an uploaded zip. If denied for "too many files", + # remove generated folders (e.g. node_modules/build) or raise this limit. + config.zip_entry_limit = ENV.fetch('DF_ZIP_ENTRY_LIMIT', 1_000) + + # Max zip compression ratio. If denied for "compression ratio is too high", + # check for repetitive/generated data before raising this zip-bomb guard. + config.zip_compression_ratio_limit = ENV.fetch('DF_ZIP_COMPRESSION_RATIO_LIMIT', 100) + + # Max expanded zip size as a multiple of DF_MAX_FILE_SIZE. + # Eg. If max file size is 10MB, the uncompressed size can be a maximum of 100MB + config.zip_uncompressed_size_multiplier = ENV.fetch('DF_ZIP_UNCOMPRESSED_SIZE_MULTIPLIER', 10) + # Prefer encrypted Rails credentials, while keeping env vars as a safe fallback. credentials.secret_key_base = Application.fetch_credential_or_env(:secret_key_base, env_key: 'DF_SECRET_KEY_BASE', default: Rails.env.production? ? nil : '9e010ee2f52af762916406fd2ac488c5694a6cc784777136e657511f8bbc7a73f96d59c0a9a778a0d7cf6406f8ecbf77efe4701dfbd63d8248fc7cc7f32dea97') credentials.secret_key_attr = Application.fetch_credential_or_env(:secret_key_attr, env_key: 'DF_SECRET_KEY_ATTR', default: Rails.env.production? ? nil : 'e69fc5960ca0e8700844a3a25fe80373b41c0a265d342eba06950113f3766fd983bad9ec51bf36eb615d9711bfe1dd90b8e35f01841b323f604ffee857e32055') @@ -140,6 +152,7 @@ def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) config.institution[:plagiarism] = ENV['DF_INSTITUTION_PLAGIARISM'] if ENV['DF_INSTITUTION_PLAGIARISM'] # Institution host becomes localhost in development config.institution[:host] ||= 'http://localhost:4200' if Rails.env.development? + config.institution[:settings] = ENV['DF_INSTITUTION_SETTINGS_RB'] if ENV['DF_INSTITUTION_SETTINGS_RB'] config.institution[:ffmpeg] = ENV['DF_FFMPEG_PATH'] || 'ffmpeg' diff --git a/test/models/file_helper_test.rb b/test/models/file_helper_test.rb index 5c3849b1d2..80eb68fd02 100644 --- a/test/models/file_helper_test.rb +++ b/test/models/file_helper_test.rb @@ -1,5 +1,6 @@ require "test_helper" require "open3" +require "zip" class FileHelperTest < ActiveSupport::TestCase def test_convert_use_with_gif @@ -27,6 +28,129 @@ def test_archive_paths assert_match %r{^#{FileHelper.student_work_root}/portfolio/}, original_portfolio_path end + def test_accept_zip_upload + Tempfile.create(['submission', '.zip']) do |zip_file| + Zip::File.open(zip_file.path, Zip::File::CREATE) do |zip| + zip.get_output_stream('src/main.rb') { |io| io.write("puts 'hello'\n") } + end + + result = FileHelper.accept_file( + { + filename: 'submission.zip', + 'tempfile' => zip_file + }, + 'Zip', + 'zip' + ) + + assert result[:accepted], result[:msg] + end + end + + def test_zip_upload_rejects_unsafe_paths + Tempfile.create(['submission', '.zip']) do |zip_file| + Zip::File.open(zip_file.path, Zip::File::CREATE) do |zip| + zip.get_output_stream('../escape.rb') { |io| io.write("puts 'bad'\n") } + end + + result = FileHelper.accept_file( + { + filename: 'submission.zip', + 'tempfile' => zip_file + }, + 'Zip', + 'zip' + ) + + refute result[:accepted] + assert_includes result[:msg], 'unsafe path' + end + end + + def test_zip_upload_rejects_nested_archives + Tempfile.create(['submission', '.zip']) do |zip_file| + Zip::File.open(zip_file.path, Zip::File::CREATE) do |zip| + zip.get_output_stream('lib/vendor.zip') { |io| io.write('nested archive') } + end + + result = FileHelper.accept_file( + { + filename: 'submission.zip', + 'tempfile' => zip_file + }, + 'Zip', + 'zip' + ) + + refute result[:accepted] + assert_includes result[:msg], 'Nested archives are not allowed' + end + end + + def test_zip_upload_accepts_entries_larger_than_file_limit + original_max_file_size = Doubtfire::Application.config.max_file_size + Doubtfire::Application.config.max_file_size = 1_000 + + Tempfile.create(['submission', '.zip']) do |zip_file| + Zip::File.open(zip_file.path, Zip::File::CREATE) do |zip| + zip.get_output_stream('large.txt') { |io| io.write('a' * 1_001) } + end + + result = FileHelper.accept_file( + { + filename: 'submission.zip', + 'tempfile' => zip_file + }, + 'Zip', + 'zip' + ) + + assert result[:accepted], result[:msg] + end + ensure + Doubtfire::Application.config.max_file_size = original_max_file_size + end + + def test_zip_upload_rejects_total_uncompressed_size_over_multiplier_limit + original_max_file_size = Doubtfire::Application.config.max_file_size + original_multiplier = Doubtfire::Application.config.zip_uncompressed_size_multiplier + Doubtfire::Application.config.max_file_size = 1_000 + Doubtfire::Application.config.zip_uncompressed_size_multiplier = 2 + + Tempfile.create(['submission', '.zip']) do |zip_file| + Zip::File.open(zip_file.path, Zip::File::CREATE) do |zip| + 3.times do |index| + zip.get_output_stream("file-#{index}.txt") { |io| io.write('a' * 900) } + end + end + + result = FileHelper.validate_zip_upload(zip_file.path, 'submission.zip') + + refute result[:valid] + assert_includes result[:msg], 'uncompressed size limit' + end + ensure + Doubtfire::Application.config.max_file_size = original_max_file_size + Doubtfire::Application.config.zip_uncompressed_size_multiplier = original_multiplier + end + + def test_zip_file_tree_lists_nested_paths + Tempfile.create(['submission', '.zip']) do |zip_file| + Zip::File.open(zip_file.path, Zip::File::CREATE) do |zip| + zip.get_output_stream('src/main.rb') { |io| io.write("puts 'hello'\n") } + zip.get_output_stream('README.md') { |io| io.write("# Read me\n") } + end + + tree = FileHelper.zip_file_tree(zip_file.path, 'submission.zip') + + assert_equal 2, tree[:entries] + assert_includes tree[:lines], '↳ src/' + assert_includes tree[:lines], ' ↳ main.rb' + assert_includes tree[:lines], '↳ README.md' + refute tree[:truncated] + end + end + def test_process_audio_converts_webm_audio Dir.mktmpdir("audio path ") do |dir| source_wav = File.join(dir, "source tone.wav") diff --git a/test/models/task_definition_test.rb b/test/models/task_definition_test.rb index ba62646455..d739028f55 100644 --- a/test/models/task_definition_test.rb +++ b/test/models/task_definition_test.rb @@ -113,6 +113,34 @@ def test_default_tii_settings td.destroy end + def test_upload_requirements_allow_zip + test_unit = Unit.first + td = TaskDefinition.new({ + unit_id: test_unit.id, + tutorial_stream: test_unit.tutorial_streams.first, + name: 'Test zip requirement', + description: 'test def', + weighting: 4, + target_grade: 0, + start_date: test_unit.start_date + 1.week, + target_date: test_unit.start_date + 2.weeks, + abbreviation: 'TestZipReq', + restrict_status_updates: false, + upload_requirements: [ + { + "key" => 'file0', + "name" => 'Source Zip', + "type" => 'zip' + } + ], + plagiarism_warn_pct: 0.8, + is_graded: false, + max_quality_pts: 5 + }) + + assert td.valid?, td.errors.full_messages.join(', ') + end + def test_group_tasks u = FactoryBot.create(:unit) activity_type = FactoryBot.create(:activity_type) From be3e59ed73da0283b7ee96b2753cb2b603f58e74 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:56:55 +1000 Subject: [PATCH 051/199] chore: update version --- app/api/api_root.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/api_root.rb b/app/api/api_root.rb index b30f0d8102..4e28e9c2b5 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -160,7 +160,7 @@ class ApiRoot < Grape::API add_swagger_documentation \ base_path: nil, - doc_version: 'v10.0.0', + doc_version: 'v11.0.0', hide_documentation_path: true, info: { title: 'Doubtfire API Documentation', From 53223c7a540ce3f08625a6e9612f5908b465fba4 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:01:02 +1000 Subject: [PATCH 052/199] chore(release): 10.0.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6db9dcfa3..eb0dda0510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [10.0.0](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-110...v10.0.0) (2026-06-03) + + +### Features + +* support zip file submissions ([#624](https://github.com/b0ink/doubtfire-deploy/issues/624)) ([d5d8742](https://github.com/b0ink/doubtfire-deploy/commit/d5d874214fe31f494b5a30f67f7286e711ba45b2)) + ## [10.0.0-110](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-109...v10.0.0-110) (2026-06-03) ## [10.0.0-109](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-108...v10.0.0-109) (2026-05-13) From ef5727b6bae31f0f6618dcbece58410534fd6075 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:01:17 +1000 Subject: [PATCH 053/199] chore(release): 11.0.0-1 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb0dda0510..60772f18e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-1](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-110...v11.0.0-1) (2026-06-03) + + +### Features + +* support zip file submissions ([#624](https://github.com/b0ink/doubtfire-deploy/issues/624)) ([d5d8742](https://github.com/b0ink/doubtfire-deploy/commit/d5d874214fe31f494b5a30f67f7286e711ba45b2)) + ## [10.0.0](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-110...v10.0.0) (2026-06-03) From bb453ce00f540bcf1d64d70e18c6d97b6f8bdc5a Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:08:02 +1000 Subject: [PATCH 054/199] feat: clear submission in process with no active sidekiq job (#626) * feat: clear submission in process with no active sidekiq job * chore: assign timeout variable --- lib/tasks/maintenance.rake | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/lib/tasks/maintenance.rake b/lib/tasks/maintenance.rake index f6ba8ebfcb..6a6e797214 100644 --- a/lib/tasks/maintenance.rake +++ b/lib/tasks/maintenance.rake @@ -1,6 +1,81 @@ require_all 'lib/helpers' +require 'sidekiq/api' namespace :maintenance do + def accept_submission_job_present?(task_id) + Sidekiq::Queue.new("default").each do |job| + return true if job.klass == 'AcceptSubmissionJob' && job.args[0] == task_id + end + + Sidekiq::Workers.new.each do |_process_id, _thread_id, work| + payload = JSON.parse(work['payload']) + + return true if payload['class'] == 'AcceptSubmissionJob' && payload['args'][0] == task_id + end + + false + end + + def notify_failed_submission(task, message) + if task.project.student.receive_task_notifications + begin + PortfolioEvidenceMailer.task_pdf_failed(task.project, [task]).deliver_now + rescue StandardError => e + Rails.logger.error "Failed to send task pdf failed email for project #{task.project.id}!\n#{e.message}" + end + end + + exception = StandardError.new(message) + Sentry.capture_exception(exception, extra: { task_id: task.id, project_id: task.project_id }) if defined?(Sentry) + + begin + mail = ErrorLogMailer.error_message('Accept Submission Cleanup', message, exception) + mail.deliver_now if mail.present? + rescue StandardError => e + Rails.logger.error "Failed to send error log to admin for task #{task.id}!\n#{e.message}" + end + end + + def mark_task_for_resubmission(task, _message) + tutor = task.project.tutor_for(task.task_definition) + + task.trigger_transition(trigger: 'fix', by_user: tutor) + task.add_text_comment(tutor, "**Automated Comment**: Something went wrong with compiling your submission. Please resubmit the task.") + rescue StandardError => e + Rails.logger.error "Failed to move task #{task.id} to fix and add automated comment!\n#{e.message}" + end + + def clear_abandoned_submissions! + in_process_path = FileHelper.student_work_dir(:in_process) + return unless Dir.exist?(in_process_path) + + abandoned_submission_timeout = 10.minutes + stale_before = abandoned_submission_timeout.ago + + Dir.foreach(in_process_path) do |entry| + next unless entry.match?(/^\d+$/) + + task_path = File.join(in_process_path, entry) + next unless File.directory?(task_path) + next unless File.mtime(task_path) < stale_before + + task = Task.includes(project: [:user, :unit]).find_by(id: entry.to_i) + next if task.nil? + + if accept_submission_job_present?(task.id) + Rails.logger.info "Skipping abandoned submission cleanup for task #{task.id} because AcceptSubmissionJob is still active" + next + end + + message = "Abandoned in-process submission detected for task #{task.log_details}. The stale in-process folder was older than #{abandoned_submission_timeout / 1.minute} minutes with no active AcceptSubmissionJob, has now been cleared, and the task requires resubmission." + Rails.logger.error message + + mark_task_for_resubmission(task, message) + task.clear_in_process + notify_failed_submission(task, message) + end + end + desc 'Cleanup temporary files' task cleanup: [:environment] do path = FileHelper.tmp_file_dir @@ -31,6 +106,12 @@ namespace :maintenance do .find_each(&:destroy!) AuthToken.destroy_old_tokens + clear_abandoned_submissions! + end + + desc 'Clear abandoned in-process submission folders and notify affected users' + task clear_abandoned_submissions: [:environment] do + clear_abandoned_submissions! end desc 'Remove PDFs from old submissions and archive units' From f59220292d023fe1579f1a4f484acfb969f1f3c4 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:08:23 +1000 Subject: [PATCH 055/199] chore(release): 11.0.0-2 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60772f18e9..16fd321523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-2](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-1...v11.0.0-2) (2026-06-04) + + +### Features + +* clear submission in process with no active sidekiq job ([#626](https://github.com/b0ink/doubtfire-deploy/issues/626)) ([bb453ce](https://github.com/b0ink/doubtfire-deploy/commit/bb453ce00f540bcf1d64d70e18c6d97b6f8bdc5a)) + ## [11.0.0-1](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-110...v11.0.0-1) (2026-06-03) From a85024cbc08a3ea759781d0823f3af3619ce7e8d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:36:11 +1000 Subject: [PATCH 056/199] chore: update task pdf failed subject line --- app/mailers/portfolio_evidence_mailer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mailers/portfolio_evidence_mailer.rb b/app/mailers/portfolio_evidence_mailer.rb index 7b2f56d817..b9fc7bd9ad 100644 --- a/app/mailers/portfolio_evidence_mailer.rb +++ b/app/mailers/portfolio_evidence_mailer.rb @@ -17,7 +17,7 @@ def task_pdf_failed(project, tasks) email_with_name = %("#{@student.name}" <#{@student.email}>) tutor_email = %("#{@tutor.name}" <#{@tutor.email}>) - subject = "#{project.unit.name}: Task PDFs ready to view" + subject = "#{project.unit.code} #{project.unit.name}: Task submission processing failed" mail(to: email_with_name, from: tutor_email, subject: subject) end From f5b383fdeda016dd7d391325f04700356cc58da5 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:41:32 +1000 Subject: [PATCH 057/199] fix: allow error log emails with no backtrace --- app/mailers/error_log_mailer.rb | 3 ++- test/mailers/error_log_mailer_test.rb | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/app/mailers/error_log_mailer.rb b/app/mailers/error_log_mailer.rb index e08607fd31..578a6f6fc3 100644 --- a/app/mailers/error_log_mailer.rb +++ b/app/mailers/error_log_mailer.rb @@ -8,7 +8,8 @@ def error_message(subject, message, exception) end @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] - @error_log = "#{message}\n\n#{exception.message}\n\n#{exception.backtrace.join("\n")}" + backtrace = exception.backtrace&.join("\n") || 'No backtrace available' + @error_log = "#{message}\n\n#{exception.message}\n\n#{backtrace}" mail(to: email, from: email, subject: "#{@doubtfire_product_name} Error Log - #{subject}") end diff --git a/test/mailers/error_log_mailer_test.rb b/test/mailers/error_log_mailer_test.rb index ed990597f9..843b3ef05d 100644 --- a/test/mailers/error_log_mailer_test.rb +++ b/test/mailers/error_log_mailer_test.rb @@ -30,4 +30,16 @@ def test_latex_error_logs_are_attached assert mail.attachments['log.txt'].present? assert mail.attachments['log.txt'].body.include? 'this is the content of the log' end + + def test_can_send_error_log_mail_without_backtrace + Doubtfire::Application.config.email_errors_to = 'test ' + exception = StandardError.new('test') + + mail = ErrorLogMailer.error_message('test', 'test message', exception) + + assert mail.present? + assert mail.to.include? 'test@test.com' + assert mail.body.include? exception.message + assert mail.body.include? 'No backtrace available' + end end From 6e2b1e27d4dccdb1c56e2dd98c969e27ab33c99a Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:41:51 +1000 Subject: [PATCH 058/199] chore: capture sentry alert instead of error --- lib/tasks/maintenance.rake | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/tasks/maintenance.rake b/lib/tasks/maintenance.rake index 6a6e797214..be664b13f6 100644 --- a/lib/tasks/maintenance.rake +++ b/lib/tasks/maintenance.rake @@ -25,10 +25,16 @@ namespace :maintenance do end end - exception = StandardError.new(message) - Sentry.capture_exception(exception, extra: { task_id: task.id, project_id: task.project_id }) if defined?(Sentry) + if defined?(Sentry) + Sentry.capture_message( + "Cleared abandoned in-process submission for task #{task.id}", + level: :info, + extra: { task_id: task.id, project_id: task.project_id, detail: message } + ) + end begin + exception = StandardError.new(message) mail = ErrorLogMailer.error_message('Accept Submission Cleanup', message, exception) mail.deliver_now if mail.present? rescue StandardError => e From 7fda505acfd3dc235d5eb66d300e8454dcb1bd40 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:45:45 +1000 Subject: [PATCH 059/199] chore: raise sentry alerts where error log mailer is used --- app/helpers/turn_it_in.rb | 1 + app/sidekiq/accept_submission_job.rb | 1 + app/sidekiq/archive_old_units_job.rb | 1 + lib/tasks/generate_pdfs.rake | 1 + 4 files changed, 4 insertions(+) diff --git a/app/helpers/turn_it_in.rb b/app/helpers/turn_it_in.rb index fdcafd788a..c2c0e765ef 100644 --- a/app/helpers/turn_it_in.rb +++ b/app/helpers/turn_it_in.rb @@ -84,6 +84,7 @@ def self.handle_tii_error(action, error) @@delay_call_until = DateTime.now + 1.minute when 403 # forbidden, issue with authentication... notify admin begin + Sentry.capture_exception(error) if defined?(Sentry) ErrorLogMailer.error_message('TII Credentials', "TII Error: #{error.message}", error).deliver rescue StandardError => e Rails.logger.error "Failed to send error email: #{e}" diff --git a/app/sidekiq/accept_submission_job.rb b/app/sidekiq/accept_submission_job.rb index 99dfdc04d9..00a063aeec 100644 --- a/app/sidekiq/accept_submission_job.rb +++ b/app/sidekiq/accept_submission_job.rb @@ -41,6 +41,7 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) begin # Notify system admin + Sentry.capture_exception(e) if defined?(Sentry) mail = ErrorLogMailer.error_message('Accept Submission', "Failed to convert submission to PDF for task #{task.log_details}", e) mail.deliver if mail.present? rescue StandardError => e diff --git a/app/sidekiq/archive_old_units_job.rb b/app/sidekiq/archive_old_units_job.rb index 7ebb68e666..db4ce2f11b 100644 --- a/app/sidekiq/archive_old_units_job.rb +++ b/app/sidekiq/archive_old_units_job.rb @@ -17,6 +17,7 @@ def perform rescue StandardError => e begin # Notify system admin + Sentry.capture_exception(e) if defined?(Sentry) mail = ErrorLogMailer.error_message('Archive Units', "Failed to move old units to archive", e) mail.deliver if mail.present? diff --git a/lib/tasks/generate_pdfs.rake b/lib/tasks/generate_pdfs.rake index f558bd2a1f..5db26d07e7 100644 --- a/lib/tasks/generate_pdfs.rake +++ b/lib/tasks/generate_pdfs.rake @@ -140,6 +140,7 @@ namespace :submission do success = false begin # Notify system admin + Sentry.capture_exception(e) if defined?(Sentry) mail = ErrorLogMailer.error_message("Failed portfolio job: #{project.log_details}", "Failed to create portfolio for project #{project.log_details}", e) mail.deliver if mail.present? rescue StandardError => e From e2afc12ec2f3bed9072b08e557e6b92419fd22b9 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:05:58 +1000 Subject: [PATCH 060/199] refactor: improve failed pdfgen sentry error --- app/sidekiq/accept_submission_job.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/sidekiq/accept_submission_job.rb b/app/sidekiq/accept_submission_job.rb index 00a063aeec..223e6ebe6a 100644 --- a/app/sidekiq/accept_submission_job.rb +++ b/app/sidekiq/accept_submission_job.rb @@ -41,7 +41,17 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) begin # Notify system admin - Sentry.capture_exception(e) if defined?(Sentry) + if defined?(Sentry) + Sentry.capture_exception( + e, + extra: { + task_id: task.id, + task_definition_abbreviation: task.task_definition.abbreviation, + username: task.project.user.username, + latex_log_message: e.respond_to?(:log_message) ? e.log_message.to_s.last(5000) : nil + } + ) + end mail = ErrorLogMailer.error_message('Accept Submission', "Failed to convert submission to PDF for task #{task.log_details}", e) mail.deliver if mail.present? rescue StandardError => e From 7f469a66f89c0ce2e0ad7755e1fd0fc46487a0ea Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:11:08 +1000 Subject: [PATCH 061/199] chore: improve stale in process submission sentry error --- lib/tasks/maintenance.rake | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/tasks/maintenance.rake b/lib/tasks/maintenance.rake index be664b13f6..2960881dbf 100644 --- a/lib/tasks/maintenance.rake +++ b/lib/tasks/maintenance.rake @@ -29,7 +29,12 @@ namespace :maintenance do Sentry.capture_message( "Cleared abandoned in-process submission for task #{task.id}", level: :info, - extra: { task_id: task.id, project_id: task.project_id, detail: message } + extra: { + task_id: task.id, + task_definition: task.task_definition.abbreviation, + username: task.project.user.username, + detail: message + } ) end From d139b7b7be800c83befdf5497fa4a22f3cf13d03 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:23:03 +1000 Subject: [PATCH 062/199] chore: capture additional error on failed accept submission job --- app/sidekiq/accept_submission_job.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/sidekiq/accept_submission_job.rb b/app/sidekiq/accept_submission_job.rb index 223e6ebe6a..388cf227d3 100644 --- a/app/sidekiq/accept_submission_job.rb +++ b/app/sidekiq/accept_submission_job.rb @@ -89,6 +89,16 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) end rescue StandardError => e # to raise error message to avoid unnecessary retry logger.error e + if defined?(Sentry) + Sentry.capture_exception( + e, + extra: { + task_id: task&.id, + task_definition_abbreviation: task&.task_definition&.abbreviation, + username: task&.project&.user&.username + } + ) + end task.clear_in_process end end From 14c3d45fe63c6d91392a8faa795ab59a5fef7d50 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:23:22 +1000 Subject: [PATCH 063/199] chore(release): 11.0.0-3 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16fd321523..bf020a9940 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-3](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-2...v11.0.0-3) (2026-06-04) + + +### Bug Fixes + +* allow error log emails with no backtrace ([f5b383f](https://github.com/b0ink/doubtfire-deploy/commit/f5b383fdeda016dd7d391325f04700356cc58da5)) + ## [11.0.0-2](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-1...v11.0.0-2) (2026-06-04) From 3790c9f3dc4899d9785f6996a77a5b550f1c5d87 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:44:43 +1000 Subject: [PATCH 064/199] fix: ensure we check for queued jobs --- lib/tasks/maintenance.rake | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/tasks/maintenance.rake b/lib/tasks/maintenance.rake index 2960881dbf..61e569a92f 100644 --- a/lib/tasks/maintenance.rake +++ b/lib/tasks/maintenance.rake @@ -2,15 +2,20 @@ require_all 'lib/helpers' require 'sidekiq/api' namespace :maintenance do - def accept_submission_job_present?(task_id) - Sidekiq::Queue.new("default").each do |job| - return true if job.klass == 'AcceptSubmissionJob' && job.args[0] == task_id - end + def accept_submission_job_matches_task?(job_class, job_args, task_id) + job_class == 'AcceptSubmissionJob' && job_args.first.to_i == task_id + end + def accept_submission_job_present?(task_id) Sidekiq::Workers.new.each do |_process_id, _thread_id, work| - payload = JSON.parse(work['payload']) + payload = work['payload'].is_a?(String) ? JSON.parse(work['payload']) : work['payload'] + + return true if accept_submission_job_matches_task?(payload['class'], payload['args'], task_id) + end - return true if payload['class'] == 'AcceptSubmissionJob' && payload['args'][0] == task_id + # TODO: We may need to iterate through each queue when we implement parallel sidekiq jobs + Sidekiq::Queue.new("default").each do |job| + return true if accept_submission_job_matches_task?(job.klass, job.args, task_id) end false @@ -74,11 +79,11 @@ namespace :maintenance do next if task.nil? if accept_submission_job_present?(task.id) - Rails.logger.info "Skipping abandoned submission cleanup for task #{task.id} because AcceptSubmissionJob is still active" + Rails.logger.info "Skipping abandoned submission cleanup for task #{task.id} because AcceptSubmissionJob is still running or queued" next end - message = "Abandoned in-process submission detected for task #{task.log_details}. The stale in-process folder was older than #{abandoned_submission_timeout / 1.minute} minutes with no active AcceptSubmissionJob, has now been cleared, and the task requires resubmission." + message = "Abandoned in-process submission detected for task #{task.log_details}. The stale in-process folder was older than #{abandoned_submission_timeout / 1.minute} minutes with no running or queued AcceptSubmissionJob, has now been cleared, and the task requires resubmission." Rails.logger.error message mark_task_for_resubmission(task, message) From 678f3866632d174713ab46bc7876c43dd2df56cd Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:01:04 +1000 Subject: [PATCH 065/199] feat: suspend any stale overseer assessments --- lib/tasks/maintenance.rake | 82 ++++++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 7 deletions(-) diff --git a/lib/tasks/maintenance.rake b/lib/tasks/maintenance.rake index 61e569a92f..f2bc661554 100644 --- a/lib/tasks/maintenance.rake +++ b/lib/tasks/maintenance.rake @@ -2,25 +2,41 @@ require_all 'lib/helpers' require 'sidekiq/api' namespace :maintenance do - def accept_submission_job_matches_task?(job_class, job_args, task_id) - job_class == 'AcceptSubmissionJob' && job_args.first.to_i == task_id - end - - def accept_submission_job_present?(task_id) + def sidekiq_job_present_in_workers_or_default_queue?(&matcher) Sidekiq::Workers.new.each do |_process_id, _thread_id, work| payload = work['payload'].is_a?(String) ? JSON.parse(work['payload']) : work['payload'] - return true if accept_submission_job_matches_task?(payload['class'], payload['args'], task_id) + return true if matcher.call(payload['class'], payload['args']) end # TODO: We may need to iterate through each queue when we implement parallel sidekiq jobs Sidekiq::Queue.new("default").each do |job| - return true if accept_submission_job_matches_task?(job.klass, job.args, task_id) + return true if matcher.call(job.klass, job.args) end false end + def accept_submission_job_matches_task?(job_class, job_args, task_id) + job_class == 'AcceptSubmissionJob' && job_args.first.to_i == task_id + end + + def accept_submission_job_present?(task_id) + sidekiq_job_present_in_workers_or_default_queue? do |job_class, job_args| + accept_submission_job_matches_task?(job_class, job_args, task_id) + end + end + + def accept_overseer_job_matches_assessment?(job_class, job_args, overseer_assessment_id) + job_class == 'AcceptOverseerJob' && job_args.last.to_i == overseer_assessment_id + end + + def accept_overseer_job_present?(overseer_assessment_id) + sidekiq_job_present_in_workers_or_default_queue? do |job_class, job_args| + accept_overseer_job_matches_assessment?(job_class, job_args, overseer_assessment_id) + end + end + def notify_failed_submission(task, message) if task.project.student.receive_task_notifications begin @@ -61,6 +77,15 @@ namespace :maintenance do Rails.logger.error "Failed to move task #{task.id} to fix and add automated comment!\n#{e.message}" end + def mark_task_for_overseer_resubmission(task) + tutor = task.project.tutor_for(task.task_definition) + + task.trigger_transition(trigger: 'fix', by_user: tutor) + task.add_text_comment(tutor, "**Automated Comment**: Something went wrong while running the automated tests for this submission. Please resubmit the task.") + rescue StandardError => e + Rails.logger.error "Failed to move task #{task.id} to fix and add Overseer automated comment!\n#{e.message}" + end + def clear_abandoned_submissions! in_process_path = FileHelper.student_work_dir(:in_process) return unless Dir.exist?(in_process_path) @@ -92,6 +117,43 @@ namespace :maintenance do end end + def clear_abandoned_overseer_assessments! + abandoned_assessment_timeout = 10.minutes + stale_before = abandoned_assessment_timeout.ago + + OverseerAssessment + .pre_queued + .includes(task: [project: :user]) + .where('created_at < ?', stale_before) + .find_each do |assessment| + if accept_overseer_job_present?(assessment.id) + Rails.logger.info "Skipping abandoned OverseerAssessment cleanup for assessment #{assessment.id} because AcceptOverseerJob is still running or queued" + next + end + + task = assessment.task + message = "Abandoned OverseerAssessment detected for task #{task.log_details}. Assessment #{assessment.id} remained pre_queued for more than #{abandoned_assessment_timeout / 1.minute} minutes with no running or queued AcceptOverseerJob and has been marked failed." + Rails.logger.error message + + assessment.update!(status: :failed) + mark_task_for_overseer_resubmission(task) + + if defined?(Sentry) + Sentry.capture_message( + "Marked stale OverseerAssessment failed for task #{task.id}", + level: :warning, + extra: { + task_id: task.id, + overseer_assessment_id: assessment.id, + task_definition: task.task_definition.abbreviation, + username: task.project.user.username, + detail: message + } + ) + end + end + end + desc 'Cleanup temporary files' task cleanup: [:environment] do path = FileHelper.tmp_file_dir @@ -123,6 +185,7 @@ namespace :maintenance do AuthToken.destroy_old_tokens clear_abandoned_submissions! + clear_abandoned_overseer_assessments! end desc 'Clear abandoned in-process submission folders and notify affected users' @@ -130,6 +193,11 @@ namespace :maintenance do clear_abandoned_submissions! end + desc 'Clear abandoned pre-queued Overseer assessments and request resubmission' + task clear_abandoned_overseer_assessments: [:environment] do + clear_abandoned_overseer_assessments! + end + desc 'Remove PDFs from old submissions and archive units' task archive_submissions: [:environment] do archive_period = Doubtfire::Application.config.unit_archive_after_period From c206553ee508fa5c0f5d895dd905735aed8baf97 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 12:15:22 +1000 Subject: [PATCH 066/199] chore(release): 11.0.0-4 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf020a9940..fe02982826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-4](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-3...v11.0.0-4) (2026-06-04) + + +### Features + +* suspend any stale overseer assessments ([678f386](https://github.com/b0ink/doubtfire-deploy/commit/678f3866632d174713ab46bc7876c43dd2df56cd)) + + +### Bug Fixes + +* ensure we check for queued jobs ([3790c9f](https://github.com/b0ink/doubtfire-deploy/commit/3790c9f3dc4899d9785f6996a77a5b550f1c5d87)) + ## [11.0.0-3](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-2...v11.0.0-3) (2026-06-04) From 1949b42258355b843d7d0e5563be040d4d859164 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:34:57 +1000 Subject: [PATCH 067/199] fix: correctly get task --- test/api/marking_sessions_api_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api/marking_sessions_api_test.rb b/test/api/marking_sessions_api_test.rb index a680de77cb..f5079fd36d 100644 --- a/test/api/marking_sessions_api_test.rb +++ b/test/api/marking_sessions_api_test.rb @@ -366,7 +366,7 @@ def test_marking_sessions_tutorial_split # unit = Unit.first project = unit.projects.first - task = project.tasks.first + task = project.task_for_task_definition(unit.task_definitions.first) while current_time <= end_time travel_to current_time do From 4280df3a6446920b4ae3c9108d28421ab9b3e6c5 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:36:32 +1000 Subject: [PATCH 068/199] feat: notify students when overseer tests fail (#627) * feat: notify students when overseer tests fail * chore: fix rubocop * chore: fix tests * refactor: move overseer notification grace period to env var --- .ci-setup/crontab | 1 + app/mailers/portfolio_evidence_mailer.rb | 16 ++++ app/models/overseer_assessment.rb | 56 +++++++++++++ .../overseer_assessment_failed.html.erb | 72 ++++++++++++++++ .../overseer_assessment_failed.text.erb | 17 ++++ config/application.rb | 3 + ...ent_notified_at_to_overseer_assessments.rb | 7 ++ db/schema.rb | 4 +- lib/shell/send_overseer_notifications.sh | 11 +++ lib/tasks/overseer_notifications.rake | 28 +++++++ test/mailers/unit_mail_test.rb | 19 +++++ test/models/overseer_assessment_test.rb | 82 +++++++++++++++++++ 12 files changed, 315 insertions(+), 1 deletion(-) create mode 100644 app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb create mode 100644 app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb create mode 100644 db/migrate/20260604031804_add_student_notified_at_to_overseer_assessments.rb create mode 100644 lib/shell/send_overseer_notifications.sh create mode 100644 lib/tasks/overseer_notifications.rake create mode 100644 test/models/overseer_assessment_test.rb diff --git a/.ci-setup/crontab b/.ci-setup/crontab index 87c9c955d8..b1298d5e39 100644 --- a/.ci-setup/crontab +++ b/.ci-setup/crontab @@ -3,6 +3,7 @@ BASH_ENV=/container.env PATH=/tmp/texlive/bin/x86_64-linux:/tmp/texlive/bin/aarch64-linux:/usr/local/bundle/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bundle/bin 10,15,20,25,30,35,40,45,50,55 * * * * /doubtfire/lib/shell/generate_pdfs.sh +0,10,20,30,40,50 * * * * /doubtfire/lib/shell/send_overseer_notifications.sh 0 5 * * * /doubtfire/lib/shell/check_plagiarism.sh 0 8 * * * /doubtfire/lib/shell/portfolio_autogen_check.sh 0 7 * * 1 /doubtfire/lib/shell/send_weekly_emails.sh diff --git a/app/mailers/portfolio_evidence_mailer.rb b/app/mailers/portfolio_evidence_mailer.rb index b9fc7bd9ad..743503c25b 100644 --- a/app/mailers/portfolio_evidence_mailer.rb +++ b/app/mailers/portfolio_evidence_mailer.rb @@ -54,6 +54,22 @@ def task_feedback_ready(project, tasks) mail(to: email_with_name, from: tutor_email, subject: subject) end + def overseer_assessment_failed(project, tasks) + return nil if project.nil? || tasks.nil? || tasks.empty? + + add_general + @student = project.student + @project = project + @tasks = tasks.sort_by { |t| t.task_definition.abbreviation } + @tutor = project.main_convenor_user + return nil if @tutor.nil? || @student.nil? + + email_with_name = %("#{@student.name}" <#{@student.email}>) + tutor_email = %("#{@tutor.name}" <#{@tutor.email}>) + subject = "#{project.unit.code} #{project.unit.name}: Automated feedback needs your attention" + mail(to: email_with_name, from: tutor_email, subject: subject) + end + def portfolio_ready(project) return nil if project.nil? diff --git a/app/models/overseer_assessment.rb b/app/models/overseer_assessment.rb index 84a2077dd0..00772c9bbd 100644 --- a/app/models/overseer_assessment.rb +++ b/app/models/overseer_assessment.rb @@ -16,6 +16,58 @@ class OverseerAssessment < ApplicationRecord after_destroy :delete_associated_files + + + def self.student_notification_grace_period + Doubtfire::Application.config.overseer_student_notification_grace_period + end + + scope :awaiting_student_failure_notification, lambda { |grace_period: student_notification_grace_period| + notification_cutoff = grace_period.ago + + joins(task: { project: :user }) + .joins(<<~SQL.squish) + INNER JOIN task_comments assessment_comments + ON assessment_comments.commentable_type = 'OverseerAssessment' + AND assessment_comments.commentable_id = overseer_assessments.id + AND assessment_comments.type = 'AssessmentComment' + SQL + .joins(<<~SQL.squish) + LEFT JOIN comments_read_receipts student_read_receipts + ON student_read_receipts.task_comment_id = assessment_comments.id + AND student_read_receipts.user_id = projects.user_id + SQL + .where(status: statuses[:failed], student_notified_at: nil) + .where(users: { receive_task_notifications: true }) + .where('overseer_assessments.updated_at <= ?', notification_cutoff) + .where('student_read_receipts.id IS NULL') + .where(<<~SQL.squish) + assessment_comments.id = ( + SELECT latest_comment.id + FROM task_comments latest_comment + WHERE latest_comment.commentable_type = 'OverseerAssessment' + AND latest_comment.commentable_id = overseer_assessments.id + AND latest_comment.type = 'AssessmentComment' + ORDER BY latest_comment.created_at DESC, latest_comment.id DESC + LIMIT 1 + ) + SQL + .where(<<~SQL.squish) + NOT EXISTS ( + SELECT 1 + FROM overseer_assessments newer_assessments + WHERE newer_assessments.task_id = overseer_assessments.task_id + AND ( + newer_assessments.created_at > overseer_assessments.created_at OR + ( + newer_assessments.created_at = overseer_assessments.created_at AND + newer_assessments.id > overseer_assessments.id + ) + ) + ) + SQL + } + # TODO: track how many tests ran, and how many tests total at the time # TODO: we might not have an overseerStepResult because a new test was added later @@ -86,6 +138,10 @@ def output_path FileHelper.task_submission_identifier_path_with_timestamp(:done, task, submission_timestamp) end + def latest_assessment_comment + assessment_comments.order(created_at: :desc, id: :desc).first + end + def add_assessment_comment(text = 'Automated Assessment Started') text.strip! return nil if text.blank? diff --git a/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb new file mode 100644 index 0000000000..23b0e89a9a --- /dev/null +++ b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb @@ -0,0 +1,72 @@ + + + + + + +
+

<%= @doubtfire_product_name %> Notification

+

Hi <%= @student.first_name %>,

+

+ Our automated assessment found issues and left feedback on the following tasks. If you have already reviewed the task in <%= @doubtfire_product_name %>, you can ignore this email. +

+

+ Please log in and review the latest automated feedback for: +

+

+

+ Cheers,
+ The <%= @doubtfire_product_name %> Team on behalf of <%= @tutor.name %> +

+
+
+ Unsubscribe | Generated with <%= @doubtfire_product_name %> +
+ + diff --git a/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb new file mode 100644 index 0000000000..9375ee8a30 --- /dev/null +++ b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb @@ -0,0 +1,17 @@ +Hi <%= @student.first_name %>, + +Our automated assessment found issues and left feedback on the following tasks. If you have already reviewed the task in <%= @doubtfire_product_name %>, you can ignore this email. + +Please log in and review the latest automated feedback for: +<% @tasks.each do |task| %> + * <%= task.task_definition.abbreviation %> - <%= task.task_definition.name %> +<% end %> + +Cheers, +The <%= @doubtfire_product_name %> Team on behalf of <%= @tutor.name %> + +--- + +Visit <%= @unsubscribe_url %> to unsubscribe from these notifications. + +Generated with <%= @doubtfire_product_name %> diff --git a/config/application.rb b/config/application.rb index 9c1a67505e..42dd1e6e27 100644 --- a/config/application.rb +++ b/config/application.rb @@ -54,6 +54,9 @@ class Application < Rails::Application # Period for which to keep units config.unit_archive_after_period = ENV.fetch('DF_UNIT_ARCHIVE_PERIOD', 2).to_f * 1.year + # Minimum time to wait before notifying a student about an unread failed overseer assessment + config.overseer_student_notification_grace_period = ENV.fetch('OVERSEER_STUDENT_NOTIFICATION_GRACE_PERIOD_MINUTES', 30).to_i.minutes + # Limit number of pdf generators to run at once config.pdfgen_max_processes = ENV['DF_MAX_PDF_GEN_PROCESSES'] || 2 diff --git a/db/migrate/20260604031804_add_student_notified_at_to_overseer_assessments.rb b/db/migrate/20260604031804_add_student_notified_at_to_overseer_assessments.rb new file mode 100644 index 0000000000..a0b9be1695 --- /dev/null +++ b/db/migrate/20260604031804_add_student_notified_at_to_overseer_assessments.rb @@ -0,0 +1,7 @@ +class AddStudentNotifiedAtToOverseerAssessments < ActiveRecord::Migration[8.0] + def change + add_column :overseer_assessments, :student_notified_at, :datetime + add_index :overseer_assessments, [:status, :student_notified_at, :updated_at], + name: 'index_overseer_assessments_on_status_notified_updated' + end +end diff --git a/db/schema.rb b/db/schema.rb index 1e49bb2022..a6b03f2f20 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_03_27_041457) do +ActiveRecord::Schema[8.0].define(version: 2026_06_04_031804) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -242,6 +242,8 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "total_steps" + t.datetime "student_notified_at" + t.index ["status", "student_notified_at", "updated_at"], name: "index_overseer_assessments_on_status_notified_updated" t.index ["task_id", "submission_timestamp"], name: "index_overseer_assessments_on_task_id_and_submission_timestamp", unique: true t.index ["task_id"], name: "index_overseer_assessments_on_task_id" end diff --git a/lib/shell/send_overseer_notifications.sh b/lib/shell/send_overseer_notifications.sh new file mode 100644 index 0000000000..e5be6b305e --- /dev/null +++ b/lib/shell/send_overseer_notifications.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +#Get path to script +APP_PATH=`echo $0 | awk '{split($0,patharr,"/"); idx=1; while(patharr[idx+1] != "") { if (patharr[idx] != "/") {printf("%s/", patharr[idx]); idx++ }} }'` +APP_PATH=`cd "$APP_PATH"; pwd` + +ROOT_PATH=`cd "$APP_PATH"/../..; pwd` + +cd "$ROOT_PATH" + +DF_LOG_TO_STDOUT=true rails overseer_notifications:send_failed_assessment_notifications diff --git a/lib/tasks/overseer_notifications.rake b/lib/tasks/overseer_notifications.rake new file mode 100644 index 0000000000..2b69ee3f9e --- /dev/null +++ b/lib/tasks/overseer_notifications.rake @@ -0,0 +1,28 @@ +namespace :overseer_notifications do + def notify_failed_overseer_assessments! + assessments = OverseerAssessment + .awaiting_student_failure_notification + .includes(task: [{ project: :unit }, :task_definition]) + + assessments.group_by(&:project).each do |project, project_assessments| + tasks = project_assessments.map(&:task).uniq + + begin + mail = PortfolioEvidenceMailer.overseer_assessment_failed(project, tasks) + next if mail.blank? + + mail.deliver_now + project_assessments.each do |assessment| + assessment.update!(student_notified_at: Time.current) + end + rescue StandardError => e + Rails.logger.error "Failed to send overseer assessment email for project #{project.id}!\n#{e.message}" + end + end + end + + desc 'Send overdue overseer assessment failure notifications to students' + task send_failed_assessment_notifications: [:environment] do + notify_failed_overseer_assessments! + end +end diff --git a/test/mailers/unit_mail_test.rb b/test/mailers/unit_mail_test.rb index 1acf5aa90a..c84b18a16d 100644 --- a/test/mailers/unit_mail_test.rb +++ b/test/mailers/unit_mail_test.rb @@ -54,4 +54,23 @@ def test_send_portfolio_fail_from_main_convenor unit.destroy! end + def test_send_overseer_assessment_failed_email + unit = FactoryBot.create :unit + convenor = FactoryBot.create :user, :convenor + + ur = unit.employ_staff convenor, Role.convenor + + unit.update main_convenor: ur + + project = unit.active_projects.first + task = project.task_for_task_definition(unit.task_definitions.first) + + mail = PortfolioEvidenceMailer.overseer_assessment_failed(project, [task]) + + assert_equal 1, mail.from.count + assert_equal convenor.email, mail.from.first + assert_equal project.student.email, mail.to.first + assert mail.html_part.body.include? "projects/#{project.id}/dashboard/#{task.task_definition.abbreviation}" + end + end diff --git a/test/models/overseer_assessment_test.rb b/test/models/overseer_assessment_test.rb new file mode 100644 index 0000000000..c95eb97637 --- /dev/null +++ b/test/models/overseer_assessment_test.rb @@ -0,0 +1,82 @@ +require 'test_helper' + +class OverseerAssessmentTest < ActiveSupport::TestCase + def test_selects_latest_failed_unread_assessment_once_grace_period_has_elapsed + assessment = create_failed_assessment + + pending_ids = OverseerAssessment.awaiting_student_failure_notification.pluck(:id) + + assert_includes pending_ids, assessment.id + end + + def test_excludes_assessment_when_student_has_read_comment + assessment = create_failed_assessment + assessment.latest_assessment_comment.mark_as_read(assessment.project.student) + + pending_ids = OverseerAssessment.awaiting_student_failure_notification.pluck(:id) + + assert_not_includes pending_ids, assessment.id + end + + def test_excludes_older_failed_assessment_when_a_newer_assessment_exists + older_failed = create_failed_assessment + create_assessment(task: older_failed.task, status: :passed, age: 20.minutes) + + pending_ids = OverseerAssessment.awaiting_student_failure_notification.pluck(:id) + + assert_not_includes pending_ids, older_failed.id + end + + def test_only_selects_latest_failed_assessment_for_a_task + older_failed = create_failed_assessment + newer_failed = create_failed_assessment(task: older_failed.task, age: 35.minutes) + + pending_ids = OverseerAssessment.awaiting_student_failure_notification.pluck(:id) + + assert_not_includes pending_ids, older_failed.id + assert_includes pending_ids, newer_failed.id + end + + def test_excludes_already_notified_assessment + assessment = create_failed_assessment + assessment.update_column(:student_notified_at, Time.current) + + pending_ids = OverseerAssessment.awaiting_student_failure_notification.pluck(:id) + + assert_not_includes pending_ids, assessment.id + end + + private + + def create_failed_assessment(task: nil, age: 40.minutes) + create_assessment(task: task, status: :failed, age: age, create_comment: true) + end + + def create_assessment(status:, age:, task: nil, create_comment: false) + unit = FactoryBot.create(:unit, task_count: 1) if task.nil? + task ||= begin + project = unit.active_projects.first + project.task_for_task_definition(unit.task_definitions.first) + end + + assessment = OverseerAssessment.create!( + task: task, + status: status, + submission_timestamp: "#{Time.current.to_i}-#{SecureRandom.hex(2)}" + ) + assessment.update_columns(created_at: age.ago, updated_at: age.ago) + + if create_comment + comment = AssessmentComment.create!( + task: task, + user: task.project.tutor_for(task.task_definition), + recipient: task.project.student, + comment: 'Automated tests failed', + commentable: assessment + ) + comment.update_columns(created_at: age.ago) + end + + assessment + end +end From b07d4f571156f23813274e15ef8537352312588a Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 16:01:07 +1000 Subject: [PATCH 069/199] chore(release): 11.0.0-5 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe02982826..2829957258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-5](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-4...v11.0.0-5) (2026-06-04) + + +### Features + +* notify students when overseer tests fail ([#627](https://github.com/b0ink/doubtfire-deploy/issues/627)) ([4280df3](https://github.com/b0ink/doubtfire-deploy/commit/4280df3a6446920b4ae3c9108d28421ab9b3e6c5)) + + +### Bug Fixes + +* correctly get task ([1949b42](https://github.com/b0ink/doubtfire-deploy/commit/1949b42258355b843d7d0e5563be040d4d859164)) + ## [11.0.0-4](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-3...v11.0.0-4) (2026-06-04) From c359c0e858215151cfc108a3a24fd95113e62cb8 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 4 Jun 2026 16:30:36 +1000 Subject: [PATCH 070/199] chore: ensure test submissions dont receive overseer notification --- app/sidekiq/accept_submission_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/sidekiq/accept_submission_job.rb b/app/sidekiq/accept_submission_job.rb index 388cf227d3..c785943344 100644 --- a/app/sidekiq/accept_submission_job.rb +++ b/app/sidekiq/accept_submission_job.rb @@ -81,6 +81,7 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) if overseer_assessment.present? logger.info "Launching Overseer assessment for task_def_id: #{task.task_definition.id} task_id: #{task.id}" + overseer_assessment.update!(student_notified_at: Time.current) if test_submission overseer_assessment.send_to_overseer(test_submission: test_submission) else From f46a6ec85c76597653d5377ed5da3d582529b11b Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 5 Jun 2026 08:21:46 +1000 Subject: [PATCH 071/199] chore: set executable permissions --- lib/shell/send_overseer_notifications.sh | 0 lib/tasks/simulate_marking_sessions.rake | 184 +++++++++++++++++++++++ lib/tasks/simulate_overseer_tasks.rake | 11 ++ 3 files changed, 195 insertions(+) mode change 100644 => 100755 lib/shell/send_overseer_notifications.sh create mode 100644 lib/tasks/simulate_marking_sessions.rake create mode 100644 lib/tasks/simulate_overseer_tasks.rake diff --git a/lib/shell/send_overseer_notifications.sh b/lib/shell/send_overseer_notifications.sh old mode 100644 new mode 100755 diff --git a/lib/tasks/simulate_marking_sessions.rake b/lib/tasks/simulate_marking_sessions.rake new file mode 100644 index 0000000000..fc445a2ecf --- /dev/null +++ b/lib/tasks/simulate_marking_sessions.rake @@ -0,0 +1,184 @@ +require 'active_support/testing/time_helpers' + +# lib/tasks/simulate_marking_sessions.rake +namespace :db do + desc 'Simulate marking sessions' + task simulate_marking_sessions: [:skip_prod, :environment] do + include ActiveSupport::Testing::TimeHelpers + + unit = Unit.first + user = unit.staff.first.user + project = unit.projects.first + task = project.tasks.first + unit_role = unit.employ_staff(user, Role.convenor) + + MarkingSession.delete_all + + activity_type = ActivityType.find_by(name: "Feedback") + activity_type = ActivityType.create(name: "Feedback", abbreviation: "Feedback") if activity_type.nil? + + tutorial_stream = TutorialStream.find_by(abbreviation: "feedback1") + if tutorial_stream.nil? + tutorial_stream = TutorialStream.create!({ + name: "feedback1", + abbreviation: "feedback1", + unit: unit, + activity_type_id: activity_type.id, + activity_type: activity_type + }) + end + + tutorial_stream.tutorials.delete_all + tutorial1 = Tutorial.create!({ + unit: unit, + meeting_day: "Wednesday", + meeting_time: "8:00", + meeting_location: "-", + code: "Tutorial1", + unit_role: unit_role, + abbreviation: "Tutorial1", + tutorial_stream: tutorial_stream + }) + + tutorial2 = Tutorial.create!({ + unit: unit, + meeting_day: "Wednesday", + meeting_time: "11:00", + meeting_location: "-", + code: "Tutorial2", + unit_role: unit_role, + abbreviation: "Tutorial2", + tutorial_stream: tutorial_stream + }) + + tutorial3 = Tutorial.create!({ + unit: unit, + meeting_day: "Wednesday", + meeting_time: "13:00", + meeting_location: "-", + code: "Tutorial3", + unit_role: unit_role, + abbreviation: "Tutorial3", + tutorial_stream: tutorial_stream + }) + + tutorial4 = Tutorial.create!({ + unit: unit, + meeting_day: "Thursday", + meeting_time: "12:00", + meeting_location: "-", + code: "Tutorial4", + unit_role: unit_role, + abbreviation: "Tutorial4", + tutorial_stream: tutorial_stream + }) + + # Find the most recent Wednesday + today = Time.zone.today + wednesday_offset = (today.wday - 3) % 7 + wednesday = today - wednesday_offset.days + + # start_time = wednesday.to_time.change(hour: 10, min: 0) # 10:00am + start_time = wednesday.in_time_zone.change(hour: 5, min: 4) # 11:05am + end_time = wednesday.in_time_zone.change(hour: 18, min: 0) # 3:00pm + + current_time = start_time + + while current_time <= end_time + travel_to current_time do + SessionTracker.record_assessment_activity( + action: 'get-submission-details', + user: user, + project: project, + ip_address: '127.0.0.1', + task: task + ) + end + + current_time += 5.minutes + end + + thursday = wednesday + 1.day + # thursday_offset = (today.wday - 4) % 7 + # thursday = today - thursday_offset.days + + start_time = thursday.in_time_zone.change(hour: 12, min: 4) # 12:05pm + end_time = thursday.in_time_zone.change(hour: 15, min: 0) # 3:00pm + + current_time = start_time + + while current_time <= end_time + travel_to current_time do + SessionTracker.record_assessment_activity( + action: 'get-submission-details', + user: user, + project: project, + ip_address: '127.0.0.1', + task: task + ) + end + + current_time += 5.minutes + end + + byebug + + start_time = thursday.in_time_zone.change(hour: 17, min: 0) # 12:05pm + end_time = thursday.in_time_zone.change(hour: 18, min: 0) # 3:00pm + + current_time = start_time + + while current_time <= end_time + travel_to current_time do + SessionTracker.record_assessment_activity( + action: 'get-submission-details', + user: user, + project: project, + ip_address: '127.0.0.1', + task: task + ) + end + + current_time += 5.minutes + end + end +end + +def aggregate_task_complete_stats + result = {} + + unit = Unit.first + + unit.task_definitions.each do |td| + result[td.abbreviation] = {} + end + + unit.active_projects.each do |project| + campus_name = project.campus.name + result[campus_name] ||= {} + + unit.task_definitions.each do |td| + result[campus_name][td.abbreviation] ||= {} + + unless project.has_task_for_task_definition?(td) + # Count not started + result[campus_name][td.abbreviation]['1'] ||= 0 + result[campus_name][td.abbreviation]['1'] += 1 + next + end + + task = project.task_for_task_definition(td) + next unless task + + status = task.task_status.id.to_s + result[campus_name][td.abbreviation][status] ||= 0 + result[campus_name][td.abbreviation][status] += 1 + end + end + + file_server = Doubtfire::Application.config.student_work_dir + analytics_dir = File.join(file_server, "analytics") + FileUtils.mkdir_p(analytics_dir) + + File.write("#{analytics_dir}/#{unit.code}-#{unit.id}-stats.json", result.to_json) +end diff --git a/lib/tasks/simulate_overseer_tasks.rake b/lib/tasks/simulate_overseer_tasks.rake new file mode 100644 index 0000000000..08e262561a --- /dev/null +++ b/lib/tasks/simulate_overseer_tasks.rake @@ -0,0 +1,11 @@ +# lib/tasks/simulate_overseer_tasks.rake +namespace :db do + desc 'Simulate Overseer Tasks' + task simulate_overseer_tasks: [:skip_prod, :environment] do + unit = Unit.first + user = unit.staff.first.user + project = unit.projects.first + task = project.tasks.first + unit_role = unit.employ_staff(user, Role.convenor) + end +end From 3dfffd970d45b99a9f22fb8f46ea7b8a98f5b8a6 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 5 Jun 2026 08:22:03 +1000 Subject: [PATCH 072/199] chore(release): 11.0.0-6 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2829957258..b3ad00d0c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-6](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-5...v11.0.0-6) (2026-06-04) + ## [11.0.0-5](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-4...v11.0.0-5) (2026-06-04) From 8a1c6e680dee5d2396dc8247367606e4a7a97602 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 5 Jun 2026 08:32:41 +1000 Subject: [PATCH 073/199] chore: remove files --- lib/tasks/simulate_marking_sessions.rake | 184 ----------------------- lib/tasks/simulate_overseer_tasks.rake | 11 -- 2 files changed, 195 deletions(-) delete mode 100644 lib/tasks/simulate_marking_sessions.rake delete mode 100644 lib/tasks/simulate_overseer_tasks.rake diff --git a/lib/tasks/simulate_marking_sessions.rake b/lib/tasks/simulate_marking_sessions.rake deleted file mode 100644 index fc445a2ecf..0000000000 --- a/lib/tasks/simulate_marking_sessions.rake +++ /dev/null @@ -1,184 +0,0 @@ -require 'active_support/testing/time_helpers' - -# lib/tasks/simulate_marking_sessions.rake -namespace :db do - desc 'Simulate marking sessions' - task simulate_marking_sessions: [:skip_prod, :environment] do - include ActiveSupport::Testing::TimeHelpers - - unit = Unit.first - user = unit.staff.first.user - project = unit.projects.first - task = project.tasks.first - unit_role = unit.employ_staff(user, Role.convenor) - - MarkingSession.delete_all - - activity_type = ActivityType.find_by(name: "Feedback") - activity_type = ActivityType.create(name: "Feedback", abbreviation: "Feedback") if activity_type.nil? - - tutorial_stream = TutorialStream.find_by(abbreviation: "feedback1") - if tutorial_stream.nil? - tutorial_stream = TutorialStream.create!({ - name: "feedback1", - abbreviation: "feedback1", - unit: unit, - activity_type_id: activity_type.id, - activity_type: activity_type - }) - end - - tutorial_stream.tutorials.delete_all - tutorial1 = Tutorial.create!({ - unit: unit, - meeting_day: "Wednesday", - meeting_time: "8:00", - meeting_location: "-", - code: "Tutorial1", - unit_role: unit_role, - abbreviation: "Tutorial1", - tutorial_stream: tutorial_stream - }) - - tutorial2 = Tutorial.create!({ - unit: unit, - meeting_day: "Wednesday", - meeting_time: "11:00", - meeting_location: "-", - code: "Tutorial2", - unit_role: unit_role, - abbreviation: "Tutorial2", - tutorial_stream: tutorial_stream - }) - - tutorial3 = Tutorial.create!({ - unit: unit, - meeting_day: "Wednesday", - meeting_time: "13:00", - meeting_location: "-", - code: "Tutorial3", - unit_role: unit_role, - abbreviation: "Tutorial3", - tutorial_stream: tutorial_stream - }) - - tutorial4 = Tutorial.create!({ - unit: unit, - meeting_day: "Thursday", - meeting_time: "12:00", - meeting_location: "-", - code: "Tutorial4", - unit_role: unit_role, - abbreviation: "Tutorial4", - tutorial_stream: tutorial_stream - }) - - # Find the most recent Wednesday - today = Time.zone.today - wednesday_offset = (today.wday - 3) % 7 - wednesday = today - wednesday_offset.days - - # start_time = wednesday.to_time.change(hour: 10, min: 0) # 10:00am - start_time = wednesday.in_time_zone.change(hour: 5, min: 4) # 11:05am - end_time = wednesday.in_time_zone.change(hour: 18, min: 0) # 3:00pm - - current_time = start_time - - while current_time <= end_time - travel_to current_time do - SessionTracker.record_assessment_activity( - action: 'get-submission-details', - user: user, - project: project, - ip_address: '127.0.0.1', - task: task - ) - end - - current_time += 5.minutes - end - - thursday = wednesday + 1.day - # thursday_offset = (today.wday - 4) % 7 - # thursday = today - thursday_offset.days - - start_time = thursday.in_time_zone.change(hour: 12, min: 4) # 12:05pm - end_time = thursday.in_time_zone.change(hour: 15, min: 0) # 3:00pm - - current_time = start_time - - while current_time <= end_time - travel_to current_time do - SessionTracker.record_assessment_activity( - action: 'get-submission-details', - user: user, - project: project, - ip_address: '127.0.0.1', - task: task - ) - end - - current_time += 5.minutes - end - - byebug - - start_time = thursday.in_time_zone.change(hour: 17, min: 0) # 12:05pm - end_time = thursday.in_time_zone.change(hour: 18, min: 0) # 3:00pm - - current_time = start_time - - while current_time <= end_time - travel_to current_time do - SessionTracker.record_assessment_activity( - action: 'get-submission-details', - user: user, - project: project, - ip_address: '127.0.0.1', - task: task - ) - end - - current_time += 5.minutes - end - end -end - -def aggregate_task_complete_stats - result = {} - - unit = Unit.first - - unit.task_definitions.each do |td| - result[td.abbreviation] = {} - end - - unit.active_projects.each do |project| - campus_name = project.campus.name - result[campus_name] ||= {} - - unit.task_definitions.each do |td| - result[campus_name][td.abbreviation] ||= {} - - unless project.has_task_for_task_definition?(td) - # Count not started - result[campus_name][td.abbreviation]['1'] ||= 0 - result[campus_name][td.abbreviation]['1'] += 1 - next - end - - task = project.task_for_task_definition(td) - next unless task - - status = task.task_status.id.to_s - result[campus_name][td.abbreviation][status] ||= 0 - result[campus_name][td.abbreviation][status] += 1 - end - end - - file_server = Doubtfire::Application.config.student_work_dir - analytics_dir = File.join(file_server, "analytics") - FileUtils.mkdir_p(analytics_dir) - - File.write("#{analytics_dir}/#{unit.code}-#{unit.id}-stats.json", result.to_json) -end diff --git a/lib/tasks/simulate_overseer_tasks.rake b/lib/tasks/simulate_overseer_tasks.rake deleted file mode 100644 index 08e262561a..0000000000 --- a/lib/tasks/simulate_overseer_tasks.rake +++ /dev/null @@ -1,11 +0,0 @@ -# lib/tasks/simulate_overseer_tasks.rake -namespace :db do - desc 'Simulate Overseer Tasks' - task simulate_overseer_tasks: [:skip_prod, :environment] do - unit = Unit.first - user = unit.staff.first.user - project = unit.projects.first - task = project.tasks.first - unit_role = unit.employ_staff(user, Role.convenor) - end -end From b33556b4d11d1e4d1bef18b8f1b27b4e1f0e1777 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 5 Jun 2026 09:39:15 +1000 Subject: [PATCH 074/199] refator: align wording closer to the automated comment --- .../overseer_assessment_failed.html.erb | 4 ++-- .../overseer_assessment_failed.text.erb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb index 23b0e89a9a..95fcda7a4e 100644 --- a/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb +++ b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb @@ -45,10 +45,10 @@

<%= @doubtfire_product_name %> Notification

Hi <%= @student.first_name %>,

- Our automated assessment found issues and left feedback on the following tasks. If you have already reviewed the task in <%= @doubtfire_product_name %>, you can ignore this email. + Automated assessment did not pass for the tasks below. Please review the Overseer report in <%= @doubtfire_product_name %>, verify your output, and resubmit.

- Please log in and review the latest automated feedback for: + Tasks to review:

    <% @tasks.each do |task| %>
  • diff --git a/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb index 9375ee8a30..37248debfa 100644 --- a/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb +++ b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb @@ -1,8 +1,8 @@ Hi <%= @student.first_name %>, -Our automated assessment found issues and left feedback on the following tasks. If you have already reviewed the task in <%= @doubtfire_product_name %>, you can ignore this email. +Automated assessment did not pass for the tasks below. Please review the Overseer report in <%= @doubtfire_product_name %>, verify your output, and resubmit. -Please log in and review the latest automated feedback for: +Tasks to review: <% @tasks.each do |task| %> * <%= task.task_definition.abbreviation %> - <%= task.task_definition.name %> <% end %> From 0717d9051181811c7d58dd453abf076b87cb0444 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 5 Jun 2026 12:03:52 +1000 Subject: [PATCH 075/199] fix: ensure start and target dates set correctly if unit start date is a monday --- app/models/teaching_period.rb | 5 ++++- test/models/teaching_period_test.rb | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/teaching_period.rb b/app/models/teaching_period.rb index eaf5d6b2c9..e8c0195ac0 100644 --- a/app/models/teaching_period.rb +++ b/app/models/teaching_period.rb @@ -115,7 +115,10 @@ def date_for_week_and_day(week, day) start_day_num = start_date.wday - result = week_start + (day_num - start_day_num).days + day_offset = day_num - start_day_num + day_offset += 7 if day_offset.negative? + + result = week_start + day_offset.days for a_break in breaks do if result >= a_break.start_date && result < a_break.end_date diff --git a/test/models/teaching_period_test.rb b/test/models/teaching_period_test.rb index 863507e2e6..e630bb35f9 100644 --- a/test/models/teaching_period_test.rb +++ b/test/models/teaching_period_test.rb @@ -119,7 +119,7 @@ def test_create_teaching_period_with_invalid_dates assert_equal tp.start_date + 3.day + 2.week, tp.date_for_week_and_day(3, 'Thu') assert_equal tp.start_date + 4.day + 2.week, tp.date_for_week_and_day(3, 'Fri') assert_equal tp.start_date + 5.day, tp.date_for_week_and_day(1, 'Sat') - assert_equal tp.start_date - 1.day, tp.date_for_week_and_day(1, 'Sun') + assert_equal tp.start_date + 6.days, tp.date_for_week_and_day(1, 'Sun') end test 'can map week and day to date after break' do From a01e4d7e7ca7ca255b2080f1f3e44da48db4c270 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 6 Jun 2026 14:41:42 +1000 Subject: [PATCH 076/199] fix: handle incomplete notebook metadata and markdown lists in pdfs --- app/views/layouts/application.pdf.erbtex | 6 ++++++ app/views/layouts/jupynotex.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/views/layouts/application.pdf.erbtex b/app/views/layouts/application.pdf.erbtex index b051661808..4c1c48e55f 100644 --- a/app/views/layouts/application.pdf.erbtex +++ b/app/views/layouts/application.pdf.erbtex @@ -27,6 +27,12 @@ } \usepackage[fencedCode,hashEnumerators,pipeTables,texMathDollars]{markdown} +\markdownSetup{rendererPrototypes={ + ulBeginTight={\begin{itemize}}, + ulEndTight={\end{itemize}}, + olBeginTight={\begin{enumerate}}, + olEndTight={\end{enumerate}} +}} \usepackage{luatextra} \defaultfontfeatures{Ligatures=TeX} diff --git a/app/views/layouts/jupynotex.py b/app/views/layouts/jupynotex.py index 525a424f48..bd339da29b 100644 --- a/app/views/layouts/jupynotex.py +++ b/app/views/layouts/jupynotex.py @@ -169,8 +169,8 @@ def __init__(self, path, config_options): with open(path, 'rt', encoding='utf8') as fh: nb_data = json.load(fh) - # get the languaje, to highlight - lang = nb_data['metadata']['language_info']['name'] + # get the language, when available, to highlight + lang = nb_data.get('metadata', {}).get('language_info', {}).get('name') self._highlight_delimiters = HIGHLIGHTERS.get(lang, HIGHLIGHTERS[None]) # get all cells From b950e55ad801fbdd6f3dafb3732aefc0b6cc6987 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 8 Jun 2026 12:38:34 +1000 Subject: [PATCH 077/199] fix: normalize notebook source lines - Jupyter may save cell source as a string; normalize it so PDF rendering does not split text character by character. --- app/views/layouts/jupynotex.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/views/layouts/jupynotex.py b/app/views/layouts/jupynotex.py index bd339da29b..7b3c49f2eb 100644 --- a/app/views/layouts/jupynotex.py +++ b/app/views/layouts/jupynotex.py @@ -60,6 +60,13 @@ def _validator_positive_int(value): return value +def _as_lines(value): + """Return notebook text stored as either a string or list of strings.""" + if isinstance(value, str): + return value.splitlines() + return value + + def _process_plain_text(lines, config_options=None): """Wrap a series of lines around a verbatim indication.""" if config_options is None: @@ -67,6 +74,7 @@ def _process_plain_text(lines, config_options=None): result = [] result.extend(VERBATIM_BEGIN) + lines = _as_lines(lines) for line in lines: line = line.strip() @@ -186,7 +194,7 @@ def _validate_config(self, config): def _proc_src(self, content): """Process the source of a cell.""" - source = content['source'] + source = _as_lines(content['source']) result = [] if content['cell_type'] == 'code': begin, end = self._highlight_delimiters From 280f3a0f6ce34acc4e30a85d6f7f4c58c944c93c Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:14:07 +1000 Subject: [PATCH 078/199] chore(release): 11.0.0-7 --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ad00d0c9..cc7f160a9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-7](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-6...v11.0.0-7) (2026-06-08) + + +### Bug Fixes + +* ensure start and target dates set correctly if unit start date is a monday ([0717d90](https://github.com/b0ink/doubtfire-deploy/commit/0717d9051181811c7d58dd453abf076b87cb0444)) +* handle incomplete notebook metadata and markdown lists in pdfs ([a01e4d7](https://github.com/b0ink/doubtfire-deploy/commit/a01e4d7e7ca7ca255b2080f1f3e44da48db4c270)) +* normalize notebook source lines ([b950e55](https://github.com/b0ink/doubtfire-deploy/commit/b950e55ad801fbdd6f3dafb3732aefc0b6cc6987)) + ## [11.0.0-6](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-5...v11.0.0-6) (2026-06-04) ## [11.0.0-5](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-4...v11.0.0-5) (2026-06-04) From 4a7141fd185e261321610c46108c3a397a4f64d8 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:26:27 +1000 Subject: [PATCH 079/199] chore: include task definition abbreviation in portfolio overall task status --- app/views/portfolio/portfolio_pdf.pdf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/portfolio/portfolio_pdf.pdf.erb b/app/views/portfolio/portfolio_pdf.pdf.erb index edf3f4fda7..6e47a8d719 100644 --- a/app/views/portfolio/portfolio_pdf.pdf.erb +++ b/app/views/portfolio/portfolio_pdf.pdf.erb @@ -130,7 +130,7 @@ No Tutor \textbf{Task} & \textbf{Status} & \textbf{Times Assessed} \\ \hline <% @task_defs.each do |td| %> - <%= lesc td.name %> & + <%= lesc "#{td.abbreviation} #{td.name}" %> & <% task = task_for_def(td) if task.nil? %> From 97249090eac3f7e214991f7c665f92768948b6db Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Tue, 9 Jun 2026 12:15:48 +1000 Subject: [PATCH 080/199] feat: communications system (#617) * feat: add communications system * fix: execute single set * feat: ability to edit conditions and actions * feat: enable communication set execution and mailing * refactor: fix rubocop * refactor: add action log email to convenors * fix: rubocop * feat: add scheduling system * fix: rubocop * fix: attempt nil tutorials fix * chore: fix test * feat: enable rollover of communication sets * refactor: ensure schedule next run is recalculated when updating teaching periods * chore: remove schema * feat: ensure communication sets only execute when the unit is active * feat: ensure communication schedules dont execute outside of unit start/end dates * chore: rollback migration * chore: bump migration * feat: add spec con days condition * feat: leave task comment action * chore: dont add automated comment text * chore: expose unit current week number * chore: ensure user is convenor to manage communications --- Gemfile | 1 + Gemfile.lock | 1 + app/api/api_root.rb | 2 + app/api/communication_rules_api.rb | 780 ++++++++++++++++++ .../entities/communication_action_entity.rb | 13 + .../communication_condition_entity.rb | 18 + app/api/entities/communication_rule_entity.rb | 17 + app/api/entities/communication_set_entity.rb | 17 + .../communication_set_schedule_entity.rb | 21 + app/api/entities/unit_entity.rb | 4 + app/mailers/communications_mailer.rb | 37 + app/models/break.rb | 9 + app/models/communication/campus_condition.rb | 4 + .../change_target_grade_action.rb | 3 + .../communication/communication_action.rb | 13 + .../communication/communication_condition.rb | 93 +++ .../communication/communication_rule.rb | 134 +++ app/models/communication/communication_set.rb | 113 +++ .../communication_set_schedule.rb | 175 ++++ .../communication/email_staff_action.rb | 15 + .../communication/email_student_action.rb | 4 + .../communication/login_status_condition.rb | 4 + .../communication/spec_con_condition.rb | 4 + .../communication/target_grade_condition.rb | 4 + .../communication/task_comment_action.rb | 4 + .../task_definition_status_condition.rb | 5 + .../task_status_count_condition.rb | 6 + .../tutorial_enrolment_condition.rb | 4 + .../tutorial_stream_enrolment_condition.rb | 4 + app/models/teaching_period.rb | 12 + app/models/unit.rb | 27 +- app/sidekiq/communication_rule_job.rb | 39 + app/sidekiq/execute_communication_set_job.rb | 601 ++++++++++++++ .../execute_communication_set_schedule_job.rb | 24 + .../poll_communication_set_schedules_job.rb | 19 + .../action_log_email.html.erb | 69 ++ .../action_log_email.text.erb | 18 + .../communication_email.html.erb | 53 ++ .../communication_email.text.erb | 12 + config/application.rb | 2 + config/initializers/sidekiq.rb | 5 + config/schedule.yml | 4 + .../20260604070032_add_communications_feat.rb | 123 +++ db/schema.rb | 88 +- test/factories/units_factory.rb | 4 +- test/models/communication_condition_test.rb | 40 + .../models/communication_set_schedule_test.rb | 74 ++ test/models/communication_set_test.rb | 69 ++ test/models/teaching_period_test.rb | 36 + test/models/unit_model_test.rb | 77 ++ .../communication_set_schedule_jobs_test.rb | 60 ++ .../execute_communication_set_job_test.rb | 57 ++ test/sidekiq/scheduled_job_test.rb | 3 +- 53 files changed, 3019 insertions(+), 6 deletions(-) create mode 100644 app/api/communication_rules_api.rb create mode 100644 app/api/entities/communication_action_entity.rb create mode 100644 app/api/entities/communication_condition_entity.rb create mode 100644 app/api/entities/communication_rule_entity.rb create mode 100644 app/api/entities/communication_set_entity.rb create mode 100644 app/api/entities/communication_set_schedule_entity.rb create mode 100644 app/mailers/communications_mailer.rb create mode 100644 app/models/communication/campus_condition.rb create mode 100644 app/models/communication/change_target_grade_action.rb create mode 100644 app/models/communication/communication_action.rb create mode 100644 app/models/communication/communication_condition.rb create mode 100644 app/models/communication/communication_rule.rb create mode 100644 app/models/communication/communication_set.rb create mode 100644 app/models/communication/communication_set_schedule.rb create mode 100644 app/models/communication/email_staff_action.rb create mode 100644 app/models/communication/email_student_action.rb create mode 100644 app/models/communication/login_status_condition.rb create mode 100644 app/models/communication/spec_con_condition.rb create mode 100644 app/models/communication/target_grade_condition.rb create mode 100644 app/models/communication/task_comment_action.rb create mode 100644 app/models/communication/task_definition_status_condition.rb create mode 100644 app/models/communication/task_status_count_condition.rb create mode 100644 app/models/communication/tutorial_enrolment_condition.rb create mode 100644 app/models/communication/tutorial_stream_enrolment_condition.rb create mode 100644 app/sidekiq/communication_rule_job.rb create mode 100644 app/sidekiq/execute_communication_set_job.rb create mode 100644 app/sidekiq/execute_communication_set_schedule_job.rb create mode 100644 app/sidekiq/poll_communication_set_schedules_job.rb create mode 100644 app/views/communications_mailer/action_log_email.html.erb create mode 100644 app/views/communications_mailer/action_log_email.text.erb create mode 100644 app/views/communications_mailer/communication_email.html.erb create mode 100644 app/views/communications_mailer/communication_email.text.erb create mode 100644 db/migrate/20260604070032_add_communications_feat.rb create mode 100644 test/models/communication_condition_test.rb create mode 100644 test/models/communication_set_schedule_test.rb create mode 100644 test/models/communication_set_test.rb create mode 100644 test/sidekiq/communication_set_schedule_jobs_test.rb create mode 100644 test/sidekiq/execute_communication_set_job_test.rb diff --git a/Gemfile b/Gemfile index 69ca22f0ba..b367b82225 100644 --- a/Gemfile +++ b/Gemfile @@ -102,6 +102,7 @@ gem 'net-smtp', require: false gem 'tca_client' # Async jobs +gem 'ice_cube' gem 'sidekiq' gem 'sidekiq-cron' gem 'sidekiq-status' diff --git a/Gemfile.lock b/Gemfile.lock index 8bc4f19b3a..9df7ab4c0a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -589,6 +589,7 @@ DEPENDENCIES grape-swagger-rails hirb icalendar + ice_cube json-jwt listen minitest diff --git a/app/api/api_root.rb b/app/api/api_root.rb index 4e28e9c2b5..c87cfc881e 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -80,6 +80,7 @@ class ApiRoot < Grape::API mount SidekiqApi mount LtiApi if Doubtfire::Application.config.lti_enabled mount TaskPrerequisitesApi + mount CommunicationRulesApi mount Tii::TurnItInApi mount Tii::TurnItInHooksApi @@ -135,6 +136,7 @@ class ApiRoot < Grape::API AuthenticationHelpers.add_auth_to SidekiqApi AuthenticationHelpers.add_auth_to LtiApi if Doubtfire::Application.config.lti_enabled AuthenticationHelpers.add_auth_to TaskPrerequisitesApi + AuthenticationHelpers.add_auth_to CommunicationRulesApi AuthenticationHelpers.add_auth_to Tii::TurnItInApi AuthenticationHelpers.add_auth_to Tii::TiiGroupAttachmentApi diff --git a/app/api/communication_rules_api.rb b/app/api/communication_rules_api.rb new file mode 100644 index 0000000000..af64417750 --- /dev/null +++ b/app/api/communication_rules_api.rb @@ -0,0 +1,780 @@ +require 'grape' +require 'entities/communication_set_entity' +require 'entities/communication_rule_entity' +require 'entities/communication_condition_entity' +require 'entities/communication_action_entity' +require 'entities/communication_set_schedule_entity' +require 'entities/sidekiq_job_entity' + +class CommunicationRulesApi < Grape::API + helpers AuthenticationHelpers + helpers AuthorisationHelpers + helpers SidekiqHelper + helpers do + def permitted_schedule_params(raw_schedule) + ActionController::Parameters.new(raw_schedule).permit( + :id, + :name, + :active, + :anchor_week, + :anchor_day, + :hour, + :minute, + :timezone, + :recurrence, + :interval, + :repeat_count, + :until_at + ) + end + + def schedule_params_from_request + communication_set_params = params[:communication_set] || params['communication_set'] || {} + communication_set_params[:schedules] || communication_set_params['schedules'] + end + + def sync_set_schedules!(communication_set, raw_schedules) + schedules = Array(raw_schedules).map { |schedule| permitted_schedule_params(schedule).to_h } + + keep_ids = schedules.filter_map { |schedule| schedule['id'] || schedule[:id] } + + communication_set.transaction do + communication_set.communication_set_schedules.where.not(id: keep_ids).destroy_all + + schedules.each do |schedule_attrs| + schedule_id = schedule_attrs.delete('id') || schedule_attrs.delete(:id) + + if schedule_id.present? + communication_set.communication_set_schedules.find(schedule_id).update!(schedule_attrs) + else + communication_set.communication_set_schedules.create!(schedule_attrs) + end + end + end + end + end + + before do + authenticated? + + unit = Unit.find(params[:unit_id]) + unless authorise? current_user, unit, :mannage_communications + error!({ error: 'Not authorised to manage unit communications' }, 403) + end + end + + desc 'Get communication sets for a unit' + params do + requires :unit_id, type: Integer + end + get '/units/:unit_id/communication_sets' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + present unit.communication_sets.includes(:communication_set_schedules, communication_rules: [:communication_conditions, :communication_actions]), + with: Entities::CommunicationSetEntity + end + + desc 'Get a communication set for a unit with preview data' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + get '/units/:unit_id/communication_sets/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_students + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + communication_set = unit.communication_sets + .includes(:communication_set_schedules, communication_rules: [:communication_conditions, :communication_actions]) + .find(params[:id]) + + previews = communication_set.preview_allocations_by_rule + + present( + id: communication_set.id, + unit_id: communication_set.unit_id, + name: communication_set.name, + active: communication_set.active, + schedules: Entities::CommunicationSetScheduleEntity.represent(communication_set.communication_set_schedules), + rules: Entities::CommunicationRuleEntity.represent(communication_set.communication_rules), + previews: communication_set.communication_rules.map do |rule| + { + target_rule_id: rule.id, + allocations: previews.fetch(rule.id, []).map do |allocation| + { + rule_id: allocation[:rule].id, + rule_name: allocation[:rule].name, + position: allocation[:rule].position, + students: allocation[:projects].map do |project| + { + first_name: project.user&.first_name, + last_name: project.user&.last_name, + preferred_name: project.user&.nickname, + username: project.user&.username, + student_id: project.user&.student_id, + full_name: [project.user&.first_name, project.user&.last_name].compact.join(' '), + target_grade: project.target_grade, + spec_con_days: project.spec_con_days, + last_sign_in_at: project.user&.last_sign_in_at, + campus: project.campus&.name + } + end + } + end + } + end + ) + end + + desc 'Create a communication set for a unit' + params do + requires :unit_id, type: Integer + requires :communication_set, type: Hash do + requires :name, type: String + optional :active, type: Boolean + optional :schedules, type: Array do + optional :name, type: String + optional :active, type: Boolean + optional :anchor_week, type: Integer + optional :anchor_day, type: String + optional :hour, type: Integer + optional :minute, type: Integer + optional :timezone, type: String + optional :recurrence, type: String + optional :interval, type: Integer + optional :repeat_count, type: Integer + optional :until_at, type: DateTime + end + end + end + post '/units/:unit_id/communication_sets' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + set_params = ActionController::Parameters.new(params) + .require(:communication_set) + .permit(:name, :active) + + communication_set = unit.communication_sets.create!(set_params) + sync_set_schedules!(communication_set, schedule_params_from_request) + communication_set.reload + present communication_set, with: Entities::CommunicationSetEntity + end + + desc 'Update a communication set' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + requires :communication_set, type: Hash do + optional :name, type: String + optional :active, type: Boolean + optional :schedules, type: Array do + optional :id, type: Integer + optional :name, type: String + optional :active, type: Boolean + optional :anchor_week, type: Integer + optional :anchor_day, type: String + optional :hour, type: Integer + optional :minute, type: Integer + optional :timezone, type: String + optional :recurrence, type: String + optional :interval, type: Integer + optional :repeat_count, type: Integer + optional :until_at, type: DateTime + end + end + end + put '/units/:unit_id/communication_sets/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + communication_set = unit.communication_sets.find(params[:id]) + set_params = ActionController::Parameters.new(params) + .require(:communication_set) + .permit(:name, :active) + + communication_set.update!(set_params) + sync_set_schedules!(communication_set, schedule_params_from_request) if schedule_params_from_request.present? || (params[:communication_set] || params['communication_set'] || {}).key?(:schedules) || (params[:communication_set] || params['communication_set'] || {}).key?('schedules') + communication_set.reload + present communication_set, with: Entities::CommunicationSetEntity + end + + desc 'Delete a communication set' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_sets/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + unit.communication_sets.find(params[:id]).destroy! + status 204 + end + + desc 'Execute a communication set' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + post '/units/:unit_id/communication_sets/:id/execute' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to execute unit communications' }, 403) + end + + communication_set = unit.communication_sets.find(params[:id]) + job_id = ExecuteCommunicationSetJob.perform_async(communication_set.id) + job = setup_job(job_id) + + present job, with: Entities::SidekiqJobEntity + end + + desc 'Get schedules for a communication set' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + end + get '/units/:unit_id/communication_sets/:communication_set_id/schedules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + present communication_set.communication_set_schedules.order(:id), + with: Entities::CommunicationSetScheduleEntity + end + + desc 'Get a communication schedule' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :id, type: Integer + end + get '/units/:unit_id/communication_sets/:communication_set_id/schedules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + schedule = communication_set.communication_set_schedules.find(params[:id]) + present schedule, with: Entities::CommunicationSetScheduleEntity + end + + desc 'Create a communication schedule' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :communication_set_schedule, type: Hash do + requires :name, type: String + optional :active, type: Boolean + requires :anchor_week, type: Integer + requires :anchor_day, type: String + optional :hour, type: Integer + optional :minute, type: Integer + optional :timezone, type: String + optional :recurrence, type: String + optional :interval, type: Integer + optional :repeat_count, type: Integer + optional :until_at, type: DateTime + end + end + post '/units/:unit_id/communication_sets/:communication_set_id/schedules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + schedule_params = permitted_schedule_params(params[:communication_set_schedule]).to_h + schedule = communication_set.communication_set_schedules.create!(schedule_params) + schedule.reload + present schedule, with: Entities::CommunicationSetScheduleEntity + end + + desc 'Update a communication schedule' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :id, type: Integer + requires :communication_set_schedule, type: Hash do + optional :name, type: String + optional :active, type: Boolean + optional :anchor_week, type: Integer + optional :anchor_day, type: String + optional :hour, type: Integer + optional :minute, type: Integer + optional :timezone, type: String + optional :recurrence, type: String + optional :interval, type: Integer + optional :repeat_count, type: Integer + optional :until_at, type: DateTime + end + end + put '/units/:unit_id/communication_sets/:communication_set_id/schedules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + schedule = communication_set.communication_set_schedules.find(params[:id]) + schedule_params = permitted_schedule_params(params[:communication_set_schedule]).to_h + schedule.update!(schedule_params) + schedule.reload + present schedule, with: Entities::CommunicationSetScheduleEntity + end + + desc 'Delete a communication schedule' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_sets/:communication_set_id/schedules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + communication_set.communication_set_schedules.find(params[:id]).destroy! + status 204 + end + + desc 'Get communication rules for a unit' + params do + requires :unit_id, type: Integer + end + get '/units/:unit_id/communication_rules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + present unit.communication_rules.includes(:communication_conditions, :communication_actions), + with: Entities::CommunicationRuleEntity + end + + desc 'Get communication rules for a set' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + end + get '/units/:unit_id/communication_sets/:communication_set_id/rules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + + present communication_set.communication_rules.includes(:communication_conditions, :communication_actions), + with: Entities::CommunicationRuleEntity + end + + desc 'Create a communication rule for a communication set' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :communication_rule, type: Hash do + requires :name, type: String + requires :operator, type: String + optional :position, type: Integer + optional :active, type: Boolean + optional :send_log_to_convenors, type: Boolean + end + end + post '/units/:unit_id/communication_sets/:communication_set_id/rules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + rule_params = ActionController::Parameters.new(params) + .require(:communication_rule) + .permit(:name, :operator, :position, :active, :send_log_to_convenors) + + rule_params[:position] = communication_set.communication_rules.count if rule_params[:position].nil? + rule = communication_set.communication_rules.create!(rule_params) + present rule, with: Entities::CommunicationRuleEntity + end + + desc 'Update a communication rule' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + requires :communication_rule, type: Hash do + optional :name, type: String + optional :operator, type: String + optional :position, type: Integer + optional :active, type: Boolean + optional :send_log_to_convenors, type: Boolean + end + end + put '/units/:unit_id/communication_rules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:id]) + rule_params = ActionController::Parameters.new(params) + .require(:communication_rule) + .permit(:name, :operator, :position, :active, :send_log_to_convenors) + + rule.update!(rule_params) + present rule, with: Entities::CommunicationRuleEntity + end + + desc 'Execute a communication rule within its communication set' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + post '/units/:unit_id/communication_rules/:id/execute' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to execute unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:id]) + job_id = ExecuteCommunicationSetJob.perform_async(rule.communication_set_id, rule.id) + job = setup_job(job_id) + + present job, with: Entities::SidekiqJobEntity + end + + desc 'Preview projects matched by a communication rule' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + post '/units/:unit_id/communication_rules/:id/preview' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_students + error!({ error: 'Not authorised to preview unit communications' }, 403) + end + + # rule = unit.communication_rules.find(params[:id]) + # job_id = CommunicationRuleJob.perform_async(rule.id) + # job = setup_job(job_id) + + # present job, with: Entities::SidekiqJobEntity + # rule = unit.communication_rules.find(params[:id]) + + rule = unit.communication_rules.find(params[:id]) + allocations = rule.communication_set.preview_allocations_for_rule(rule) + + present( + target_rule_id: rule.id, + allocations: allocations.map do |allocation| + { + rule_id: allocation[:rule].id, + rule_name: allocation[:rule].name, + position: allocation[:rule].position, + students: allocation[:projects].map do |project| + { + first_name: project.user&.first_name, + last_name: project.user&.last_name, + preferred_name: project.user&.nickname, + username: project.user&.username, + student_id: project.user&.student_id, + full_name: [project.user&.first_name, project.user&.last_name].compact.join(' '), + target_grade: project.target_grade, + last_sign_in_at: project.user&.last_sign_in_at, + campus: project.campus&.name + } + end + } + end + ) + end + + desc 'Get communication conditions for a rule' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + end + get '/units/:unit_id/communication_rules/:communication_rule_id/conditions' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + + present rule.communication_conditions, with: Entities::CommunicationConditionEntity + end + + desc 'Delete a communication rule' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_rules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:id]) + rule.destroy! + status 204 + end + + desc 'Create a communication condition' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :communication_condition, type: Hash do + requires :type, type: String + requires :operator, type: String + optional :target_grade, type: Integer + optional :task_definition_id, type: Integer + optional :task_statuses, type: Array[String] + optional :task_status_count, type: Integer + optional :task_target_grade, type: Integer + optional :last_sign_in_at, type: DateTime + optional :spec_con_days, type: Integer + optional :tutorial_id, type: Integer + optional :tutorial_stream_id, type: Integer + optional :campus_id, type: Integer + end + end + post '/units/:unit_id/communication_rules/:communication_rule_id/conditions' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + raw_condition_params = params[:communication_condition] + condition_params = { + type: raw_condition_params[:type], + operator: raw_condition_params[:operator], + target_grade: raw_condition_params[:target_grade], + task_definition_id: raw_condition_params[:task_definition_id], + task_status_count: raw_condition_params[:task_status_count], + task_target_grade: raw_condition_params[:task_target_grade], + last_sign_in_at: raw_condition_params[:last_sign_in_at], + spec_con_days: raw_condition_params[:spec_con_days], + tutorial_id: raw_condition_params[:tutorial_id], + tutorial_stream_id: raw_condition_params[:tutorial_stream_id], + campus_id: raw_condition_params[:campus_id] + }.compact + + task_statuses = raw_condition_params[:task_statuses] + condition_params[:task_statuses] = Array(task_statuses) unless task_statuses.nil? + + condition = rule.communication_conditions.create!(condition_params) + present condition, with: Entities::CommunicationConditionEntity + end + + desc 'Update a communication condition' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :id, type: Integer + requires :communication_condition, type: Hash do + optional :type, type: String + optional :operator, type: String + optional :target_grade, type: Integer + optional :task_definition_id, type: Integer + optional :task_statuses, type: Array[String] + optional :task_status_count, type: Integer + optional :task_target_grade, type: Integer + optional :last_sign_in_at, type: DateTime + optional :spec_con_days, type: Integer + optional :tutorial_id, type: Integer + optional :tutorial_stream_id, type: Integer + optional :campus_id, type: Integer + end + end + put '/units/:unit_id/communication_rules/:communication_rule_id/conditions/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + condition = rule.communication_conditions.find(params[:id]) + raw_condition_params = params[:communication_condition] + condition_params = { + type: raw_condition_params[:type], + operator: raw_condition_params[:operator], + target_grade: raw_condition_params[:target_grade], + task_definition_id: raw_condition_params[:task_definition_id], + task_status_count: raw_condition_params[:task_status_count], + task_target_grade: raw_condition_params[:task_target_grade], + last_sign_in_at: raw_condition_params[:last_sign_in_at], + spec_con_days: raw_condition_params[:spec_con_days], + tutorial_id: raw_condition_params[:tutorial_id], + tutorial_stream_id: raw_condition_params[:tutorial_stream_id], + campus_id: raw_condition_params[:campus_id] + }.compact + + task_statuses = raw_condition_params[:task_statuses] + condition_params[:task_statuses] = Array(task_statuses) unless task_statuses.nil? + + condition.update!(condition_params) + present condition, with: Entities::CommunicationConditionEntity + end + + desc 'Get communication actions for a rule' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + end + get '/units/:unit_id/communication_rules/:communication_rule_id/actions' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + + present rule.communication_actions, with: Entities::CommunicationActionEntity + end + + desc 'Delete a communication condition' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_rules/:communication_rule_id/conditions/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + rule.communication_conditions.find(params[:id]).destroy! + status 204 + end + + desc 'Create a communication action' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :communication_action, type: Hash do + requires :type, type: String + optional :subject, type: String + optional :body, type: String + optional :email_tutors, type: Boolean + optional :email_convenors, type: Boolean + optional :target_grade, type: Integer + optional :task_definition_id, type: Integer + end + end + post '/units/:unit_id/communication_rules/:communication_rule_id/actions' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + action_params = ActionController::Parameters.new(params) + .require(:communication_action) + .permit( + :type, + :subject, + :body, + :email_tutors, + :email_convenors, + :target_grade, + :task_definition_id + ) + + action = rule.communication_actions.create!(action_params) + present action, with: Entities::CommunicationActionEntity + end + + desc 'Update a communication action' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :id, type: Integer + requires :communication_action, type: Hash do + optional :type, type: String + optional :subject, type: String + optional :body, type: String + optional :email_tutors, type: Boolean + optional :email_convenors, type: Boolean + optional :target_grade, type: Integer + optional :task_definition_id, type: Integer + end + end + put '/units/:unit_id/communication_rules/:communication_rule_id/actions/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + action = rule.communication_actions.find(params[:id]) + action_params = ActionController::Parameters.new(params) + .require(:communication_action) + .permit(:type, :subject, :body, :email_tutors, :email_convenors, :target_grade, :task_definition_id) + + action.update!(action_params) + present action, with: Entities::CommunicationActionEntity + end + + desc 'Delete a communication action' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_rules/:communication_rule_id/actions/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + rule.communication_actions.find(params[:id]).destroy! + status 204 + end +end diff --git a/app/api/entities/communication_action_entity.rb b/app/api/entities/communication_action_entity.rb new file mode 100644 index 0000000000..f44ecc2cc9 --- /dev/null +++ b/app/api/entities/communication_action_entity.rb @@ -0,0 +1,13 @@ +module Entities + class CommunicationActionEntity < Grape::Entity + expose :id + expose :type + expose :communication_rule_id + expose :subject + expose :body + expose :email_tutors + expose :email_convenors + expose :target_grade + expose :task_definition_id + end +end diff --git a/app/api/entities/communication_condition_entity.rb b/app/api/entities/communication_condition_entity.rb new file mode 100644 index 0000000000..045b58d96b --- /dev/null +++ b/app/api/entities/communication_condition_entity.rb @@ -0,0 +1,18 @@ +module Entities + class CommunicationConditionEntity < Grape::Entity + expose :id + expose :type + expose :communication_id, as: :communication_rule_id + expose :operator + expose :target_grade + expose :task_definition_id + expose :task_statuses + expose :task_status_count + expose :task_target_grade + expose :last_sign_in_at + expose :spec_con_days + expose :tutorial_id + expose :tutorial_stream_id + expose :campus_id + end +end diff --git a/app/api/entities/communication_rule_entity.rb b/app/api/entities/communication_rule_entity.rb new file mode 100644 index 0000000000..97f56da5a6 --- /dev/null +++ b/app/api/entities/communication_rule_entity.rb @@ -0,0 +1,17 @@ +module Entities + class CommunicationRuleEntity < Grape::Entity + expose :id + expose :communication_set_id + expose :name + expose :operator + expose :position + expose :active + expose :send_log_to_convenors + expose :communication_conditions, + as: :conditions, + using: Entities::CommunicationConditionEntity + expose :communication_actions, + as: :actions, + using: Entities::CommunicationActionEntity + end +end diff --git a/app/api/entities/communication_set_entity.rb b/app/api/entities/communication_set_entity.rb new file mode 100644 index 0000000000..a9b6029b9d --- /dev/null +++ b/app/api/entities/communication_set_entity.rb @@ -0,0 +1,17 @@ +require 'entities/communication_rule_entity' +require 'entities/communication_set_schedule_entity' + +module Entities + class CommunicationSetEntity < Grape::Entity + expose :id + expose :unit_id + expose :name + expose :active + expose :communication_set_schedules, + as: :schedules, + using: Entities::CommunicationSetScheduleEntity + expose :communication_rules, + as: :rules, + using: Entities::CommunicationRuleEntity + end +end diff --git a/app/api/entities/communication_set_schedule_entity.rb b/app/api/entities/communication_set_schedule_entity.rb new file mode 100644 index 0000000000..1c1573b024 --- /dev/null +++ b/app/api/entities/communication_set_schedule_entity.rb @@ -0,0 +1,21 @@ +module Entities + class CommunicationSetScheduleEntity < Grape::Entity + expose :id + expose :communication_set_id + expose :name + expose :active + expose :anchor_week + expose :anchor_day + expose :hour + expose :minute + expose :timezone + expose :recurrence + expose :interval + expose :repeat_count + expose :until_at + expose :ice_cube_schedule + expose :next_run_at + expose :last_run_at + expose :last_enqueued_at + end +end diff --git a/app/api/entities/unit_entity.rb b/app/api/entities/unit_entity.rb index 6a9e0aa8b9..af6a823ea1 100644 --- a/app/api/entities/unit_entity.rb +++ b/app/api/entities/unit_entity.rb @@ -34,6 +34,10 @@ def can_read_unit_config?(my_role) expose :portfolio_auto_generation_date, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) }, expose_nil: false end + expose :current_unit_week do |unit| + unit.week_number(Time.current) + end + expose :active expose :overseer_image_id, unless: :summary_only, if: lambda { |unit, options| can_read_unit_config?(options[:my_role]) } diff --git a/app/mailers/communications_mailer.rb b/app/mailers/communications_mailer.rb new file mode 100644 index 0000000000..95034342d9 --- /dev/null +++ b/app/mailers/communications_mailer.rb @@ -0,0 +1,37 @@ +class CommunicationsMailer < ApplicationMailer + def communication_email(to:, from:, subject:, body:, recipient:, sender:, unit:, rule:) + @recipient = recipient + @sender = sender + @unit = unit + @rule = rule + @body = body.to_s + @body_paragraphs = @body.split(/\r?\n/).map(&:strip).reject(&:blank?) + + @doubtfire_host = Doubtfire::Application.config.institution[:host] + @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] + @unsubscribe_url = "#{@doubtfire_host}/edit_profile" + + mail(to: to, from: from, subject: subject) + end + + def action_log_email(payload) + @recipient = payload[:recipient] + @sender = payload[:sender] + @unit = payload[:unit] + @rule = payload[:rule] + @body = payload[:body].to_s + @body_paragraphs = @body.split(/\r?\n/).map(&:strip).reject(&:blank?) + @affected_students_count = payload[:affected_students_count] + + @doubtfire_host = Doubtfire::Application.config.institution[:host] + @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] + @unsubscribe_url = "#{@doubtfire_host}/edit_profile" + + attachments[payload[:csv_filename]] = { + mime_type: 'text/csv', + content: payload[:csv_content] + } + + mail(to: payload[:to], from: payload[:from], subject: payload[:subject]) + end +end diff --git a/app/models/break.rb b/app/models/break.rb index f857fd12ba..ed1c76b1ad 100644 --- a/app/models/break.rb +++ b/app/models/break.rb @@ -1,6 +1,9 @@ class Break < ApplicationRecord belongs_to :teaching_period, optional: false + after_save :refresh_teaching_period_communication_schedule_caches + after_destroy :refresh_teaching_period_communication_schedule_caches + validates :start_date, presence: true validates :number_of_weeks, presence: true validates :teaching_period_id, presence: true @@ -46,4 +49,10 @@ def monday_after_break def end_date start_date + duration end + + private + + def refresh_teaching_period_communication_schedule_caches + teaching_period.refresh_communication_schedule_caches + end end diff --git a/app/models/communication/campus_condition.rb b/app/models/communication/campus_condition.rb new file mode 100644 index 0000000000..07e2bc0096 --- /dev/null +++ b/app/models/communication/campus_condition.rb @@ -0,0 +1,4 @@ +class CampusCondition < CommunicationCondition + validates :campus, presence: true + validates :operator, inclusion: { in: ENROLMENT_OPERATORS } +end diff --git a/app/models/communication/change_target_grade_action.rb b/app/models/communication/change_target_grade_action.rb new file mode 100644 index 0000000000..07913b2d03 --- /dev/null +++ b/app/models/communication/change_target_grade_action.rb @@ -0,0 +1,3 @@ +class ChangeTargetGradeAction < CommunicationAction + validates :target_grade, presence: true +end diff --git a/app/models/communication/communication_action.rb b/app/models/communication/communication_action.rb new file mode 100644 index 0000000000..af287b6fa2 --- /dev/null +++ b/app/models/communication/communication_action.rb @@ -0,0 +1,13 @@ +class CommunicationAction < ApplicationRecord + VALID_TYPES = %w[ + EmailStudentAction + EmailStaffAction + ChangeTargetGradeAction + TaskCommentAction + ].freeze + + belongs_to :communication_rule, class_name: 'CommunicationRule' + belongs_to :task_definition, optional: true + + validates :type, presence: true, inclusion: { in: VALID_TYPES } +end diff --git a/app/models/communication/communication_condition.rb b/app/models/communication/communication_condition.rb new file mode 100644 index 0000000000..108667ce41 --- /dev/null +++ b/app/models/communication/communication_condition.rb @@ -0,0 +1,93 @@ +class CommunicationCondition < ApplicationRecord + VALID_TYPES = %w[ + TargetGradeCondition + TaskDefinitionStatusCondition + TaskStatusCountCondition + LoginStatusCondition + SpecConCondition + TutorialEnrolmentCondition + TutorialStreamEnrolmentCondition + CampusCondition + ].freeze + + GRADE_OPERATORS = %w[ + greater_than + greater_than_or_equal_to + less_than + less_than_or_equal_to + equal_to + not_equal_to + ].freeze + + EQUALITY_OPERATORS = %w[equal_to not_equal_to].freeze + DATE_OPERATORS = %w[before after].freeze + ENROLMENT_OPERATORS = %w[enrolled_in not_enrolled_in].freeze + TASK_STATUS_KEYS = %w[ + not_started + complete + need_help + working_on_it + fix_and_resubmit + feedback_exceeded + redo + discuss + ready_for_feedback + demonstrate + fail + time_exceeded + assess_in_portfolio + attention_required + ].freeze + + belongs_to :communication, + class_name: 'CommunicationRule', + inverse_of: :communication_conditions + + belongs_to :task_definition, optional: true + belongs_to :tutorial, optional: true + belongs_to :tutorial_stream, optional: true + belongs_to :campus, optional: true + + attribute :task_statuses, :json, default: -> { [] } + + validates :type, presence: true, inclusion: { in: VALID_TYPES } + validates :operator, presence: true + before_validation :normalize_task_statuses + + def task_statuses_must_be_present + unless task_statuses.is_a?(Array) && task_statuses.any?(&:present?) + errors.add(:task_statuses, 'must include at least one task status') + return + end + + invalid_statuses = task_statuses.reject { |status| TASK_STATUS_KEYS.include?(status) } + return if invalid_statuses.empty? + + errors.add(:task_statuses, "contains invalid task statuses: #{invalid_statuses.join(', ')}") + end + + private + + def normalize_task_statuses + parsed_statuses = + case task_statuses + when nil + nil + when String + begin + JSON.parse(task_statuses) + rescue JSON::ParserError + [task_statuses] + end + when Array + task_statuses + else + Array(task_statuses) + end + + self.task_statuses = + parsed_statuses&.filter_map do |status| + status.is_a?(String) ? status.strip.presence : status.presence + end + end +end diff --git a/app/models/communication/communication_rule.rb b/app/models/communication/communication_rule.rb new file mode 100644 index 0000000000..205192560e --- /dev/null +++ b/app/models/communication/communication_rule.rb @@ -0,0 +1,134 @@ +class CommunicationRule < ApplicationRecord + LOGICAL_OPERATORS = %w[and or].freeze + + belongs_to :communication_set, class_name: 'CommunicationSet' + delegate :unit, to: :communication_set + + has_many :communication_conditions, + class_name: 'CommunicationCondition', + foreign_key: :communication_id, + inverse_of: :communication, + dependent: :destroy + has_many :communication_actions, class_name: 'CommunicationAction', dependent: :destroy + + validates :name, presence: true + validates :operator, presence: true, inclusion: { in: LOGICAL_OPERATORS } + validates :position, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + def matching_projects(projects = nil) + projects ||= communication_set.eligible_projects + return projects if communication_conditions.empty? + + projects.select do |project| + matches = communication_conditions.map { |condition| condition_match?(project, condition) } + + operator == 'or' ? matches.any? : matches.all? + end + end + + private + + def condition_match?(project, condition) + case condition.type + when 'TargetGradeCondition' + target_grade_condition_match?(project, condition) + when 'TaskDefinitionStatusCondition' + task_definition_status_condition_match?(project, condition) + when 'TaskStatusCountCondition' + task_status_count_condition_match?(project, condition) + when 'LoginStatusCondition' + login_status_condition_match?(project, condition) + when 'SpecConCondition' + spec_con_condition_match?(project, condition) + when 'TutorialEnrolmentCondition' + tutorial_enrolment_condition_match?(project, condition) + when 'TutorialStreamEnrolmentCondition' + tutorial_stream_enrolment_condition_match?(project, condition) + when 'CampusCondition' + campus_condition_match?(project, condition) + else + false + end + end + + def target_grade_condition_match?(project, condition) + project_target_grade = project.target_grade + return false if project_target_grade.nil? + + compare_value(project_target_grade, condition.target_grade, condition.operator) + end + + def task_definition_status_condition_match?(project, condition) + task = project.tasks.find { |t| t.task_definition_id == condition.task_definition_id } + status = task&.task_status&.status_key&.to_s || 'not_started' + statuses = condition.task_statuses || [] + + case condition.operator + when 'equal_to' then statuses.include?(status) + when 'not_equal_to' then !statuses.include?(status) + else false + end + end + + def task_status_count_condition_match?(project, condition) + statuses = condition.task_statuses || [] + relevant_task_definitions = project.unit.task_definitions.select do |task_definition| + task_definition.target_grade == condition.task_target_grade + end + + count = relevant_task_definitions.count do |task_definition| + task = project.tasks.find { |project_task| project_task.task_definition_id == task_definition.id } + task_status = task&.task_status&.status_key&.to_s || 'not_started' + + statuses.include?(task_status) + end + + compare_value(count, condition.task_status_count, condition.operator) + end + + def login_status_condition_match?(project, condition) + last_sign_in_at = project.user&.last_sign_in_at + + case condition.operator + when 'before' then last_sign_in_at.present? && last_sign_in_at < condition.last_sign_in_at + when 'after' then last_sign_in_at.present? && last_sign_in_at > condition.last_sign_in_at + else false + end + end + + def spec_con_condition_match?(project, condition) + compare_value(project.spec_con_days, condition.spec_con_days, condition.operator) + end + + def tutorial_enrolment_condition_match?(project, condition) + enrolled = project.tutorial_enrolments.any? { |enrolment| enrolment.tutorial_id == condition.tutorial_id } + + condition.operator == 'not_enrolled_in' ? !enrolled : enrolled + end + + def tutorial_stream_enrolment_condition_match?(project, condition) + enrolled = project.tutorial_enrolments.any? do |enrolment| + enrolment.tutorial&.tutorial_stream_id == condition.tutorial_stream_id + end + + condition.operator == 'not_enrolled_in' ? !enrolled : enrolled + end + + def campus_condition_match?(project, condition) + enrolled = project.campus_id == condition.campus_id + + condition.operator == 'not_enrolled_in' ? !enrolled : enrolled + end + + def compare_value(left, right, operator) + case operator + when 'greater_than' then left > right + when 'greater_than_or_equal_to' then left >= right + when 'less_than' then left < right + when 'less_than_or_equal_to' then left <= right + when 'equal_to' then left == right + when 'not_equal_to' then left != right + else false + end + end +end diff --git a/app/models/communication/communication_set.rb b/app/models/communication/communication_set.rb new file mode 100644 index 0000000000..9d9a818996 --- /dev/null +++ b/app/models/communication/communication_set.rb @@ -0,0 +1,113 @@ +class CommunicationSet < ApplicationRecord + belongs_to :unit + + has_many :communication_set_schedules, + class_name: 'CommunicationSetSchedule', + inverse_of: :communication_set, + dependent: :destroy + + has_many :communication_rules, + -> { order(:position) }, + class_name: 'CommunicationRule', + inverse_of: :communication_set, + dependent: :destroy + + validates :name, presence: true + + def eligible_projects + unit.projects + .where(enrolled: true) + .includes(:user, :campus, { tasks: [:task_status, :task_definition] }, { tutorial_enrolments: :tutorial }) + .to_a + end + + def preview_projects_for_rule(target_rule) + preview_allocations_for_rule(target_rule) + .find { |allocation| allocation[:rule].id == target_rule.id } + &.fetch(:projects, []) || [] + end + + def preview_allocations_by_rule + communication_rules.each_with_object({}) do |rule, allocations_by_rule| + allocations_by_rule[rule.id] = preview_allocations_for_rule(rule) + end + end + + def preview_allocations_for_rule(target_rule) + remaining_projects = eligible_projects + allocations = [] + + communication_rules.each do |rule| + matched_projects = rule.matching_projects(remaining_projects) + allocations << { rule: rule, projects: matched_projects } + return allocations if rule.id == target_rule.id + + remaining_projects -= matched_projects + end + + allocations + end + + def copy_to(other_unit) + new_set = dup + new_set.unit = other_unit + new_set.save! + + communication_set_schedules.each do |schedule| + new_schedule = schedule.dup + new_schedule.communication_set = new_set + new_schedule.ice_cube_schedule = nil + new_schedule.next_run_at = nil + new_schedule.last_run_at = nil + new_schedule.last_enqueued_at = nil + new_schedule.save! + end + + communication_rules.each do |rule| + new_rule = rule.dup + new_rule.communication_set = new_set + new_rule.save! + + rule.communication_conditions.each do |condition| + new_condition = condition.dup + new_condition.communication = new_rule + new_condition.task_definition = matching_task_definition(other_unit, condition) + new_condition.tutorial_stream = matching_tutorial_stream(other_unit, condition) + new_condition.tutorial = matching_tutorial(other_unit, condition) + new_condition.save! + end + + rule.communication_actions.each do |action| + new_action = action.dup + new_action.communication_rule = new_rule + new_action.task_definition = matching_task_definition(other_unit, action) + new_action.save! + end + end + + new_set + end + + private + + def matching_task_definition(unit, condition) + return nil if condition.task_definition.blank? + + unit.task_definitions.find_by(abbreviation: condition.task_definition.abbreviation) + end + + def matching_tutorial_stream(unit, condition) + return nil if condition.tutorial_stream.blank? + + unit.tutorial_streams.find_by(abbreviation: condition.tutorial_stream.abbreviation) + end + + def matching_tutorial(unit, condition) + return nil if condition.tutorial.blank? + + unit.tutorials.find_by( + abbreviation: condition.tutorial.abbreviation, + campus_id: condition.tutorial.campus_id + ) + end +end diff --git a/app/models/communication/communication_set_schedule.rb b/app/models/communication/communication_set_schedule.rb new file mode 100644 index 0000000000..26f0a41628 --- /dev/null +++ b/app/models/communication/communication_set_schedule.rb @@ -0,0 +1,175 @@ +class CommunicationSetSchedule < ApplicationRecord + RECURRENCES = %w[none daily weekly monthly].freeze + VALID_DAYS = Date::DAYNAMES.freeze + DAY_ABBREVIATIONS = Date::ABBR_DAYNAMES.freeze + + belongs_to :communication_set, class_name: 'CommunicationSet', inverse_of: :communication_set_schedules + delegate :unit, to: :communication_set + + validates :name, presence: true + validates :anchor_week, presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :anchor_day, presence: true + validates :hour, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than: 24 } + validates :minute, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than: 60 } + validates :interval, presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :recurrence, presence: true, inclusion: { in: RECURRENCES } + validates :repeat_count, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true + validates :active, inclusion: { in: [true, false] } + validate :anchor_day_supported + validate :timezone_supported + validate :anchor_date_resolves + + before_validation :normalize_anchor_day + before_validation :default_timezone + before_validation :sync_schedule_cache + + scope :active, -> { where(active: true) } + scope :due, ->(time = Time.zone.now) { where('next_run_at IS NOT NULL AND next_run_at <= ?', time) } + scope :with_active_unit, -> { joins(communication_set: :unit).where(units: { active: true }) } + + def resolved_anchor_date + return nil if unit.blank? || anchor_week.blank? || anchor_day.blank? + + unit.date_for_week_and_day(anchor_week, anchor_day_abbreviation) + end + + def resolved_start_at + date = resolved_anchor_date + return nil if date.nil? + + timezone_object.local(date.year, date.month, date.day, hour, minute) + end + + def build_ice_cube_schedule + start_at = resolved_start_at + return nil if start_at.nil? + + schedule = IceCube::Schedule.new(start_at) + rule = recurrence_rule + schedule.add_recurrence_rule(rule) if rule + schedule + end + + def next_occurrence_after(time = Time.zone.now) + schedule = build_ice_cube_schedule + return nil if schedule.nil? + + time_for_schedule = time.in_time_zone(timezone_object) + occurrence = if recurrence == 'none' + occurrence = schedule.start_time + occurrence >= time_for_schedule ? occurrence : nil + else + schedule.next_occurrence(time_for_schedule) + end + + return nil if occurrence.blank? + + occurrence_within_unit_dates?(occurrence) ? occurrence : nil + end + + def due?(time = Time.zone.now) + active_for_scheduling? && next_run_at.present? && next_run_at <= time + end + + def refresh_next_run_at!(from_time = Time.zone.now) + update!(next_run_at: active_for_scheduling? ? next_occurrence_after(from_time) : nil) + end + + def anchor_day_abbreviation + return nil if anchor_day.blank? + + day = anchor_day.to_s.strip + return day if DAY_ABBREVIATIONS.include?(day) + + index = VALID_DAYS.index(day.titlecase) + index.nil? ? nil : DAY_ABBREVIATIONS[index] + end + + private + + def recurrence_rule + case recurrence + when 'daily' + IceCube::Rule.daily(interval) + when 'weekly' + IceCube::Rule.weekly(interval) + when 'monthly' + IceCube::Rule.monthly(interval) + end&.tap do |rule| + rule.count(repeat_count) if repeat_count.present? + rule.until(until_at.in_time_zone(timezone_object)) if until_at.present? + end + end + + def timezone_object + ActiveSupport::TimeZone[timezone.presence || Time.zone.name] || Time.zone + end + + def normalize_anchor_day + return if anchor_day.blank? + + full_day = + if VALID_DAYS.include?(anchor_day.to_s.titlecase) + anchor_day.to_s.titlecase + else + day_index = DAY_ABBREVIATIONS.index(anchor_day.to_s.titlecase) + day_index.nil? ? anchor_day : VALID_DAYS[day_index] + end + + self.anchor_day = full_day + end + + def default_timezone + self.timezone = timezone.presence || Time.zone.name + end + + def sync_schedule_cache + schedule = build_ice_cube_schedule + self.ice_cube_schedule = schedule.present? ? JSON.generate(schedule.to_hash) : nil + self.next_run_at = next_occurrence_after(Time.zone.now) if active_for_scheduling? && should_refresh_next_run_at? + self.next_run_at = nil unless active_for_scheduling? + end + + def active_for_scheduling? + active? && unit&.active? + end + + def occurrence_within_unit_dates?(occurrence) + return true if unit&.end_date.blank? + + occurrence.to_date <= unit.end_date + end + + def should_refresh_next_run_at? + will_save_change_to_anchor_week? || + will_save_change_to_anchor_day? || + will_save_change_to_hour? || + will_save_change_to_minute? || + will_save_change_to_timezone? || + will_save_change_to_recurrence? || + will_save_change_to_interval? || + will_save_change_to_repeat_count? || + will_save_change_to_until_at? || + will_save_change_to_active? || + next_run_at.blank? + end + + def anchor_day_supported + return if anchor_day.blank? || VALID_DAYS.include?(anchor_day.to_s.titlecase) + + errors.add(:anchor_day, 'must be a valid day name') + end + + def timezone_supported + return if timezone.blank? || ActiveSupport::TimeZone[timezone].present? + + errors.add(:timezone, 'must be a valid timezone') + end + + def anchor_date_resolves + return if unit.blank? || anchor_week.blank? || anchor_day.blank? + return if resolved_anchor_date.present? + + errors.add(:base, 'schedule anchor could not be resolved for this unit') + end +end diff --git a/app/models/communication/email_staff_action.rb b/app/models/communication/email_staff_action.rb new file mode 100644 index 0000000000..fc1c2d36f3 --- /dev/null +++ b/app/models/communication/email_staff_action.rb @@ -0,0 +1,15 @@ +class EmailStaffAction < CommunicationAction + validates :subject, presence: true + validates :body, presence: true + validates :email_tutors, inclusion: { in: [true, false] } + validates :email_convenors, inclusion: { in: [true, false] } + validate :staff_recipient? + + private + + def staff_recipient? + return if email_tutors || email_convenors + + errors.add(:base, 'must email tutors or convenors') + end +end diff --git a/app/models/communication/email_student_action.rb b/app/models/communication/email_student_action.rb new file mode 100644 index 0000000000..fdd0ecb17d --- /dev/null +++ b/app/models/communication/email_student_action.rb @@ -0,0 +1,4 @@ +class EmailStudentAction < CommunicationAction + validates :subject, presence: true + validates :body, presence: true +end diff --git a/app/models/communication/login_status_condition.rb b/app/models/communication/login_status_condition.rb new file mode 100644 index 0000000000..c972c76fdc --- /dev/null +++ b/app/models/communication/login_status_condition.rb @@ -0,0 +1,4 @@ +class LoginStatusCondition < CommunicationCondition + validates :last_sign_in_at, presence: true + validates :operator, inclusion: { in: DATE_OPERATORS } +end diff --git a/app/models/communication/spec_con_condition.rb b/app/models/communication/spec_con_condition.rb new file mode 100644 index 0000000000..cebeb8565f --- /dev/null +++ b/app/models/communication/spec_con_condition.rb @@ -0,0 +1,4 @@ +class SpecConCondition < CommunicationCondition + validates :spec_con_days, presence: true, numericality: { only_integer: true } + validates :operator, inclusion: { in: GRADE_OPERATORS } +end diff --git a/app/models/communication/target_grade_condition.rb b/app/models/communication/target_grade_condition.rb new file mode 100644 index 0000000000..7f0c30f560 --- /dev/null +++ b/app/models/communication/target_grade_condition.rb @@ -0,0 +1,4 @@ +class TargetGradeCondition < CommunicationCondition + validates :target_grade, presence: true + validates :operator, inclusion: { in: GRADE_OPERATORS } +end diff --git a/app/models/communication/task_comment_action.rb b/app/models/communication/task_comment_action.rb new file mode 100644 index 0000000000..d42165b51e --- /dev/null +++ b/app/models/communication/task_comment_action.rb @@ -0,0 +1,4 @@ +class TaskCommentAction < CommunicationAction + validates :task_definition, presence: true + validates :body, presence: true +end diff --git a/app/models/communication/task_definition_status_condition.rb b/app/models/communication/task_definition_status_condition.rb new file mode 100644 index 0000000000..83ab40bdec --- /dev/null +++ b/app/models/communication/task_definition_status_condition.rb @@ -0,0 +1,5 @@ +class TaskDefinitionStatusCondition < CommunicationCondition + validates :task_definition, presence: true + validates :operator, inclusion: { in: EQUALITY_OPERATORS } + validate :task_statuses_must_be_present +end diff --git a/app/models/communication/task_status_count_condition.rb b/app/models/communication/task_status_count_condition.rb new file mode 100644 index 0000000000..13d24aaacb --- /dev/null +++ b/app/models/communication/task_status_count_condition.rb @@ -0,0 +1,6 @@ +class TaskStatusCountCondition < CommunicationCondition + validates :task_status_count, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :task_target_grade, presence: true, inclusion: { in: GradeHelper::RANGE } + validates :operator, inclusion: { in: GRADE_OPERATORS } + validate :task_statuses_must_be_present +end diff --git a/app/models/communication/tutorial_enrolment_condition.rb b/app/models/communication/tutorial_enrolment_condition.rb new file mode 100644 index 0000000000..024ed79ce1 --- /dev/null +++ b/app/models/communication/tutorial_enrolment_condition.rb @@ -0,0 +1,4 @@ +class TutorialEnrolmentCondition < CommunicationCondition + validates :tutorial, presence: true + validates :operator, inclusion: { in: ENROLMENT_OPERATORS } +end diff --git a/app/models/communication/tutorial_stream_enrolment_condition.rb b/app/models/communication/tutorial_stream_enrolment_condition.rb new file mode 100644 index 0000000000..e2153037fa --- /dev/null +++ b/app/models/communication/tutorial_stream_enrolment_condition.rb @@ -0,0 +1,4 @@ +class TutorialStreamEnrolmentCondition < CommunicationCondition + validates :tutorial_stream, presence: true + validates :operator, inclusion: { in: ENROLMENT_OPERATORS } +end diff --git a/app/models/teaching_period.rb b/app/models/teaching_period.rb index e8c0195ac0..6d55ea0e48 100644 --- a/app/models/teaching_period.rb +++ b/app/models/teaching_period.rb @@ -18,6 +18,7 @@ class TeachingPeriod < ApplicationRecord validate :validate_end_date_after_start_date, :validate_active_until_after_end_date after_update :propogate_date_changes + after_update :refresh_communication_schedule_caches, if: :saved_change_to_teaching_dates? # Public methods @@ -135,6 +136,13 @@ def future_teaching_periods TeachingPeriod.where("start_date > :end_date", end_date: end_date) end + def refresh_communication_schedule_caches + CommunicationSetSchedule + .joins(communication_set: :unit) + .where(units: { teaching_period_id: id }) + .find_each(&:refresh_next_run_at!) + end + private def can_destroy? @@ -163,4 +171,8 @@ def propogate_date_changes u.update(start_date: self.start_date, end_date: self.end_date) end end + + def saved_change_to_teaching_dates? + saved_change_to_start_date? || saved_change_to_end_date? + end end diff --git a/app/models/unit.rb b/app/models/unit.rb index 7647c50915..7f59a86b4a 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -67,7 +67,8 @@ def self.permissions :get_tutor_times_summary, :get_marking_sessions, :upload_grades_csv, - :get_staff_notes + :get_staff_notes, + :mannage_communications ] # What can admin do with units? @@ -146,6 +147,7 @@ def role_for(user) after_update :move_files_on_code_change, if: :saved_change_to_code? after_update :propogate_date_changes_to_tasks, if: :saved_change_to_start_date? after_update :update_overdue_tasks_aip, if: :saved_change_to_mark_late_submissions_as_assess_in_portfolio? + after_update :refresh_communication_schedule_caches, if: :saved_change_to_communication_schedule_inputs? # Model associations. # When a Unit is destroyed, any TaskDefinitions, Tutorials, and ProjectConvenor instances will also be destroyed. @@ -157,6 +159,9 @@ def role_for(user) has_many :unit_roles, dependent: :destroy, inverse_of: :unit has_many :learning_outcomes, as: :context, dependent: :destroy # inverse_of: :unit has_many :marking_sessions, dependent: :destroy + has_many :communication_sets, class_name: 'CommunicationSet', dependent: :destroy + has_many :communication_rules, through: :communication_sets, class_name: 'CommunicationRule' + has_many :communication_set_schedules, through: :communication_sets, class_name: 'CommunicationSetSchedule' has_many :comments, through: :projects has_many :tasks, through: :projects @@ -238,6 +243,14 @@ def active_projects projects.where(enrolled: true) end + def refresh_communication_schedule_caches + communication_set_schedules.find_each(&:refresh_next_run_at!) + end + + def saved_change_to_communication_schedule_inputs? + saved_change_to_active? || saved_change_to_start_date? || saved_change_to_end_date? + end + def ordered_task_definitions task_definitions.order('start_date ASC, abbreviation ASC') end @@ -435,6 +448,10 @@ def rollover(teaching_period, start_date, end_date, new_code) end end + communication_sets.each do |communication_set| + communication_set.copy_to(new_unit) + end + # Now duplicate all feedback chips chip_mapping = {} @@ -1550,15 +1567,19 @@ def date_for_week_and_day(week, day) start_day_num = start_date.wday - start_date + week.weeks + (day_num - start_day_num).days + start_date + (week - 1).weeks + (day_num - start_day_num).days end end def week_number(date) + return nil if date.nil? || start_date.nil? + if teaching_period.present? teaching_period.week_number(date) else - ((date - start_date) / 1.week).floor + 1 + target_date = date.to_date + unit_start_date = start_date.to_date + ((target_date - unit_start_date).to_i / 7).floor + 1 end end diff --git a/app/sidekiq/communication_rule_job.rb b/app/sidekiq/communication_rule_job.rb new file mode 100644 index 0000000000..3d1b02c6ce --- /dev/null +++ b/app/sidekiq/communication_rule_job.rb @@ -0,0 +1,39 @@ +class CommunicationRuleJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + include ApplicationHelper + include FileHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first, args.last, 'communication-rule'] }, + on_conflict: :reject, + retry: 1 + + def perform(rule_id) + logger.info "Starting communication rule job..." + + at(0) + total(1) + + rule = CommunicationRule.find(rule_id) + + projects = rule.communication_set.preview_projects_for_rule(rule) + store( + result: projects.map do |project| + { + username: project.user&.username, + student_id: project.user&.student_id, + target_grade: project.target_grade, + last_sign_in_at: project.user&.last_sign_in_at + } + end + ) + + logger.info "Completed communication job" + rescue StandardError => e + logger.error e + raise e + end + +end diff --git a/app/sidekiq/execute_communication_set_job.rb b/app/sidekiq/execute_communication_set_job.rb new file mode 100644 index 0000000000..3c772dff31 --- /dev/null +++ b/app/sidekiq/execute_communication_set_job.rb @@ -0,0 +1,601 @@ +require 'csv' + +class ExecuteCommunicationSetJob + include Sidekiq::Job + include Sidekiq::Status::Worker + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args[0], args[1], 'communication-set'] }, + on_conflict: :reject, + retry: 1 + + def perform(communication_set_id, target_rule_id = nil) + communication_set = CommunicationSet.find(communication_set_id) + rules = communication_set.communication_rules.to_a + target_rule_id = target_rule_id&.to_i + + if target_rule_id.present? && rules.none? { |rule| rule.id == target_rule_id } + raise ActiveRecord::RecordNotFound, "CommunicationRule #{target_rule_id} not found in set #{communication_set_id}" + end + + eligible_projects = communication_set.eligible_projects + remaining_projects = eligible_projects.dup + executed_rules = [] + action_results = [] + + rules_to_process = + if target_rule_id.present? + cutoff_index = rules.index { |rule| rule.id == target_rule_id } + rules.first(cutoff_index + 1) + else + rules + end + + at(0) + total(rules_to_process.length.nonzero? || 1) + + rules_to_process.each_with_index do |rule, index| + matched_projects = rule.matching_projects(remaining_projects) + + executed_rules << { + rule_id: rule.id, + rule_name: rule.name, + matched_project_ids: matched_projects.map(&:id) + } + + should_execute_actions = target_rule_id.present? ? rule.id == target_rule_id : true + + if should_execute_actions + rule_action_results = rule.communication_actions.flat_map do |action| + execute_action(action, matched_projects, communication_set.unit, rule) + end + + action_results.concat(rule_action_results) + if rule.send_log_to_convenors? + action_results.concat( + send_action_log_to_convenors( + matched_projects, + communication_set.unit, + rule, + rule_action_results + ) + ) + end + end + + remaining_projects -= matched_projects + at(index + 1) + end + + store( + result: { + communication_set_id: communication_set.id, + target_rule_id: target_rule_id, + executed_rule_ids: executed_rules.map { |item| item[:rule_id] }, + remaining_project_ids: remaining_projects.map(&:id), + rules: executed_rules, + actions: action_results + } + ) + rescue StandardError => e + logger.error("ExecuteCommunicationSetJob failed: #{e.class} #{e.message}") + raise e + end + + private + + def execute_action(action, projects, unit, rule) + case action.type + when 'ChangeTargetGradeAction' + execute_change_target_grade_action(action, projects) + when 'EmailStudentAction' + execute_email_student_action(action, projects, unit, rule) + when 'EmailStaffAction' + execute_email_staff_action(action, projects, unit, rule) + when 'TaskCommentAction' + execute_task_comment_action(action, projects, unit, rule) + else + [{ + action_id: action.id, + action_type: action.type, + status: 'skipped', + reason: 'unsupported action type' + }] + end + end + + def execute_change_target_grade_action(action, projects) + projects.map do |project| + previous_target_grade = project.target_grade + + project.update!(target_grade: action.target_grade) + + { + action_id: action.id, + action_type: action.type, + status: 'updated', + project_id: project.id, + username: project.user&.username, + previous_target_grade: previous_target_grade, + target_grade: action.target_grade + } + end + end + + def execute_email_student_action(action, projects, unit, rule) + projects.filter_map do |project| + recipient = project.user + sender = sender_for(unit) + + if recipient&.email.blank? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + reason: 'student email missing' + } + end + + if sender.blank? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + reason: 'sender email missing' + } + end + + subject = render_template(action.subject, project, unit, rule, projects.length) + body = render_template(action.body, project, unit, rule, projects.length) + + CommunicationsMailer.communication_email( + to: formatted_email(recipient), + from: sender, + subject: subject, + body: body, + recipient: recipient, + sender: sender_user_for(unit), + unit: unit, + rule: rule + ).deliver_now + + { + action_id: action.id, + action_type: action.type, + status: 'sent', + project_id: project.id, + username: recipient.username, + recipient_email: recipient.email + } + end + end + + def execute_email_staff_action(action, projects, unit, rule) + projects.flat_map do |project| + sender = sender_for(unit) + + if sender.blank? + next [{ + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + username: project.user&.username, + reason: 'sender email missing' + }] + end + + staff_recipients_for(project, unit, action).map do |recipient| + subject = render_template(action.subject, project, unit, rule, projects.length) + body = render_template(action.body, project, unit, rule, projects.length) + + CommunicationsMailer.communication_email( + to: formatted_email(recipient), + from: sender, + subject: subject, + body: body, + recipient: recipient, + sender: sender_user_for(unit), + unit: unit, + rule: rule + ).deliver_now + + { + action_id: action.id, + action_type: action.type, + status: 'sent', + project_id: project.id, + username: project.user&.username, + recipient_email: recipient.email, + recipient_username: recipient.username + } + end + end + end + + def execute_task_comment_action(action, projects, unit, rule) + comment_author = sender_user_for(unit) + + if comment_author.blank? + return [{ + action_id: action.id, + action_type: action.type, + status: 'skipped', + reason: 'comment author missing' + }] + end + + comment_text_template = action.body.to_s.strip + + projects.map do |project| + task_definition = action.task_definition || unit.task_definitions.find_by(id: action.task_definition_id) + if task_definition.blank? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + username: project.user&.username, + reason: 'task definition missing' + } + end + + task = project.task_for_task_definition(task_definition) + rendered_comment = render_template(comment_text_template, project, unit, rule, projects.length) + + if rendered_comment.blank? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + username: project.user&.username, + reason: 'comment text missing' + } + end + + comment = task.add_text_comment(comment_author, rendered_comment) + + if comment.nil? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + username: project.user&.username, + reason: 'duplicate comment' + } + end + + { + action_id: action.id, + action_type: action.type, + status: 'commented', + project_id: project.id, + username: project.user&.username, + task_definition_id: task_definition.id, + task_definition_name: task_definition.name, + comment_id: comment.id + } + end + end + + def send_action_log_to_convenors(projects, unit, rule, prior_action_results) + sender = sender_for(unit) + + if sender.blank? + return [{ + action_id: nil, + action_type: 'SendLogToConvenors', + status: 'skipped', + reason: 'sender email missing' + }] + end + + recipients = unit.convenors.includes(:user).map(&:user).select { |user| user&.email.present? }.uniq(&:id) + + if recipients.empty? + return [{ + action_id: nil, + action_type: 'SendLogToConvenors', + status: 'skipped', + reason: 'convenor email missing' + }] + end + + csv_content = build_action_log_csv(rule, projects, prior_action_results) + csv_filename = "communication-rule-#{rule.id}-action-log.csv" + body = action_log_email_body(rule, prior_action_results) + subject = action_log_email_subject(unit, rule) + + recipients.map do |recipient| + CommunicationsMailer.action_log_email( + to: formatted_email(recipient), + from: sender, + subject: subject, + body: body, + recipient: recipient, + sender: sender_user_for(unit), + unit: unit, + rule: rule, + csv_content: csv_content, + csv_filename: csv_filename, + affected_students_count: projects.length + ).deliver_now + + { + action_id: nil, + action_type: 'SendLogToConvenors', + status: 'sent', + recipient_email: recipient.email, + recipient_username: recipient.username, + attachment_filename: csv_filename, + affected_students_count: projects.length + } + end + end + + def staff_recipients_for(project, unit, action) + recipients = [] + + if action.email_tutors + recipients.concat( + project.tutorial_enrolments.filter_map do |tutorial_enrolment| + tutorial_enrolment.tutorial&.unit_role&.user + end + ) + end + + if action.email_convenors + recipients.concat(unit.convenors.includes(:user).map(&:user)) + end + + recipients.select { |recipient| recipient&.email.present? }.uniq(&:id) + end + + def render_template(template, project, unit, rule, affected_students_count, target_grade_override = nil, action_results = []) + return '' if template.blank? + + student = project&.user + target_grade_value = target_grade_override.nil? ? project&.target_grade : target_grade_override + + replacements = { + '{{student.first_name}}' => student&.first_name.to_s, + '{{student.last_name}}' => student&.last_name.to_s, + '{{student.preferred_name}}' => (student&.nickname.presence || student&.first_name).to_s, + '{{student.full_name}}' => [student&.first_name, student&.last_name].compact.join(' '), + '{{student.username}}' => student&.username.to_s, + '{{student.student_id}}' => student&.student_id.to_s, + '{{affected_students_count}}' => affected_students_count.to_s, + '{{unit.code}}' => unit.code.to_s, + '{{unit.name}}' => unit.name.to_s, + '{{rule.name}}' => rule.name.to_s, + '{{target_grade}}' => target_grade_name(target_grade_value), + '{{conditions_summary}}' => conditions_summary(rule), + '{{actions_summary}}' => actions_summary(rule, action_results) + } + + replacements.reduce(template.dup) do |rendered, (token, value)| + rendered.gsub(token, value) + end + end + + def target_grade_name(value) + GradeHelper.grade_for(value).to_s + end + + def formatted_email(user) + return nil if user&.email.blank? + + %("#{user.name}" <#{user.email}>) + end + + def sender_for(unit) + formatted_email(sender_user_for(unit)) + end + + def sender_user_for(unit) + unit.main_convenor_user || unit.convenors.includes(:user).first&.user + end + + def conditions_summary(rule) + rule.communication_conditions.map do |condition| + "- #{human_condition_summary(condition)}" + end.join("\n") + end + + def actions_summary(rule, action_results) + return rule.communication_actions.map { |action| "- #{human_action_summary(action)}" }.join("\n") if action_results.blank? + + rule.communication_actions.map { |action| "- #{human_action_summary(action)}" }.join("\n") + end + + def build_action_log_csv(rule, projects, action_results) + action_order = rule.communication_actions.each_with_index.to_h { |action, index| [action.id, index] } + ordered_results = action_results.sort_by do |result| + project = projects.find { |item| item.id == result[:project_id] } + student = project&.user + + [ + student&.username.to_s, + action_order.fetch(result[:action_id], Float::INFINITY), + result[:recipient_email].to_s + ] + end + + CSV.generate(headers: true) do |csv| + csv << [ + 'student_username', + 'student_id', + 'student_name', + 'rule_name', + 'action_type', + 'status', + 'details', + # 'previous_target_grade', + # 'new_target_grade', + 'recipient_email', + 'executed_at' + ] + + ordered_results.each do |result| + project = projects.find { |item| item.id == result[:project_id] } + student = project&.user + details = if result[:status] == 'updated' + "Changed target grade from #{target_grade_name(result[:previous_target_grade])} to #{target_grade_name(result[:target_grade])}" + elsif result[:status] == 'commented' + task_definition = TaskDefinition.find_by(id: result[:task_definition_id]) + "Added comment to #{task_definition_label(task_definition)}" + elsif result[:recipient_email].present? + "Sent email to #{result[:recipient_email]}" + else + result[:reason].to_s + end + + csv << [ + student&.username, + student&.student_id, + student&.name, + rule.name, + result[:action_type], + result[:status], + details, + # target_grade_name(result[:previous_target_grade]), + # target_grade_name(result[:target_grade]), + result[:recipient_email], + Time.current.iso8601 + ] + end + end + end + + def action_log_email_subject(unit, rule) + "#{unit.code} #{rule.name} action log" + end + + def action_log_email_body(rule, action_results) + [ + action_log_conditions_intro(rule), + conditions_summary(rule), + 'The following actions have been applied to these students:', + actions_summary(rule, action_results) + ].join("\n") + end + + def action_log_conditions_intro(rule) + if rule.operator == 'or' + 'A scheduled rule has been run for students that match any of the following conditions:' + else + 'A scheduled rule has been run for students that match all of the following conditions:' + end + end + + def human_condition_summary(condition) + case condition.type + when 'TaskDefinitionStatusCondition' + predicate = condition.operator == 'not_equal_to' ? 'Not In' : 'In' + task = TaskDefinition.find_by(id: condition.task_definition_id) + task_label = if task + "Task #{task.abbreviation} #{task.name}" + else + "Task #{condition.task_definition_id}" + end + "Students that have #{task_label} #{predicate} [#{Array(condition.task_statuses).map { |status| status.to_s.titleize }.join(', ')}]" + when 'TargetGradeCondition' + "Students with a Target Grade #{operator_label(condition.operator)} #{target_grade_name(condition.target_grade)}" + when 'TaskStatusCountCondition' + grade_label = target_grade_name(condition.task_target_grade) + statuses = Array(condition.task_statuses).map { |status| status.to_s.titleize }.join(', ') + "Students that have #{operator_label(condition.operator)} #{condition.task_status_count} #{grade_label} tasks in [#{statuses}]" + when 'LoginStatusCondition' + "Students whose last sign in is #{condition.operator.to_s.humanize.downcase} #{condition.last_sign_in_at}" + when 'SpecConCondition' + "Students with Special Consideration Days #{operator_label(condition.operator)} #{condition.spec_con_days}" + when 'TutorialEnrolmentCondition' + tutorial = Tutorial.find_by(id: condition.tutorial_id) + tutorial_label = + if tutorial + [tutorial.abbreviation, tutorial.name].compact.join(' ') + else + "Tutorial #{condition.tutorial_id}" + end + "Students #{enrolment_label(condition.operator).downcase} #{tutorial_label}" + when 'TutorialStreamEnrolmentCondition' + tutorial_stream = TutorialStream.find_by(id: condition.tutorial_stream_id) + stream_label = + if tutorial_stream + [tutorial_stream.abbreviation, tutorial_stream.name].compact.join(' ') + else + "Tutorial Stream #{condition.tutorial_stream_id}" + end + "Students #{enrolment_label(condition.operator).downcase} #{stream_label}" + when 'CampusCondition' + campus = Campus.find_by(id: condition.campus_id) + campus_label = campus&.name || "Campus #{condition.campus_id}" + "Students #{enrolment_label(condition.operator).downcase} #{campus_label}" + else + "#{condition.type.to_s.underscore.humanize} #{condition.operator.to_s.humanize}" + end + end + + def human_action_summary(action) + case action.type + when 'EmailStudentAction' + 'Send email' + when 'EmailStaffAction' + 'Send staff email' + when 'ChangeTargetGradeAction' + "Change Target Grade to #{target_grade_name(action.target_grade)}" + when 'TaskCommentAction' + "Add comment to #{task_definition_label(action.task_definition)}" + else + human_action_type_name(action.type) + end + end + + def human_action_type_name(type) + case type + when 'EmailStudentAction' + 'Send email' + when 'EmailStaffAction' + 'Send staff email' + when 'ChangeTargetGradeAction' + 'Change target grade' + when 'TaskCommentAction' + 'Add task comment' + else + type.to_s.underscore.humanize + end + end + + def operator_label(operator) + case operator.to_s + when 'greater_than' + 'Greater Than' + when 'greater_than_or_equal_to' + 'Greater Than Or Equal To' + when 'less_than' + 'Less Than' + when 'less_than_or_equal_to' + 'Less Than Or Equal To' + when 'equal_to' + 'Equal To' + when 'not_equal_to' + 'Not Equal To' + else + operator.to_s.humanize + end + end + + def enrolment_label(operator) + operator.to_s == 'not_enrolled_in' ? 'Not Enrolled In' : 'Enrolled In' + end + + def task_definition_label(task_definition) + return 'Task' if task_definition.blank? + + "Task #{task_definition.abbreviation} #{task_definition.name}" + end +end diff --git a/app/sidekiq/execute_communication_set_schedule_job.rb b/app/sidekiq/execute_communication_set_schedule_job.rb new file mode 100644 index 0000000000..6960a96dd4 --- /dev/null +++ b/app/sidekiq/execute_communication_set_schedule_job.rb @@ -0,0 +1,24 @@ +class ExecuteCommunicationSetScheduleJob + include Sidekiq::Job + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first, 'communication-set-schedule'] }, + on_conflict: :reject, + retry: 1 + + def perform(schedule_id) + schedule = CommunicationSetSchedule.find(schedule_id) + return unless schedule.active? + return unless schedule.unit.active? + + now = Time.zone.now + return unless schedule.due?(now) + + ExecuteCommunicationSetJob.perform_async(schedule.communication_set_id) + schedule.update!( + last_enqueued_at: now, + last_run_at: now, + next_run_at: schedule.next_occurrence_after(now + 1.second) + ) + end +end diff --git a/app/sidekiq/poll_communication_set_schedules_job.rb b/app/sidekiq/poll_communication_set_schedules_job.rb new file mode 100644 index 0000000000..b5f0f803f2 --- /dev/null +++ b/app/sidekiq/poll_communication_set_schedules_job.rb @@ -0,0 +1,19 @@ +class PollCommunicationSetSchedulesJob + include Sidekiq::Job + + sidekiq_options lock: :until_executed, + lock_args_method: ->(_args) { ['poll-communication-set-schedules'] }, + on_conflict: :reject, + retry: 1 + + def perform + CommunicationSetSchedule + .includes(:communication_set) + .active + .with_active_unit + .due(Time.zone.now) + .find_each do |schedule| + ExecuteCommunicationSetScheduleJob.perform_async(schedule.id) + end + end +end diff --git a/app/views/communications_mailer/action_log_email.html.erb b/app/views/communications_mailer/action_log_email.html.erb new file mode 100644 index 0000000000..7df5658bf1 --- /dev/null +++ b/app/views/communications_mailer/action_log_email.html.erb @@ -0,0 +1,69 @@ + + + + + + +
    +

    <%= @doubtfire_product_name %> Notification

    + +

    Hi <%= @recipient&.nickname.presence || @recipient&.first_name || @recipient&.name %>,

    + + <% @body_paragraphs.each do |paragraph| %> +

    <%= paragraph %>

    + <% end %> + +
    + +

    + <%= @affected_students_count %> + students matched this rule and were affected by the configured actions. +

    + +

    A full log of actions has been attached as a CSV.

    + +

    + Cheers,
    + The <%= @doubtfire_product_name %> Team +

    + +
    + + diff --git a/app/views/communications_mailer/action_log_email.text.erb b/app/views/communications_mailer/action_log_email.text.erb new file mode 100644 index 0000000000..f980a817f3 --- /dev/null +++ b/app/views/communications_mailer/action_log_email.text.erb @@ -0,0 +1,18 @@ +Hi <%= @recipient&.nickname.presence || @recipient&.first_name || @recipient&.name %>, + +<% @body_paragraphs.each do |paragraph| %> +<%= paragraph %> + +<% end %> +--- + +<%= @affected_students_count %> students matched this rule and were affected by the configured actions. + +A full log of actions has been attached as a CSV. + +Cheers, +The <%= @doubtfire_product_name %> Team + +--- + +Generated with <%= @doubtfire_product_name %> diff --git a/app/views/communications_mailer/communication_email.html.erb b/app/views/communications_mailer/communication_email.html.erb new file mode 100644 index 0000000000..7933f82475 --- /dev/null +++ b/app/views/communications_mailer/communication_email.html.erb @@ -0,0 +1,53 @@ + + + + + + +
    +

    <%= @doubtfire_product_name %> Notification

    + + <% @body_paragraphs.each do |paragraph| %> +

    <%= paragraph %>

    + <% end %> + +
    + + diff --git a/app/views/communications_mailer/communication_email.text.erb b/app/views/communications_mailer/communication_email.text.erb new file mode 100644 index 0000000000..5ad89c15fb --- /dev/null +++ b/app/views/communications_mailer/communication_email.text.erb @@ -0,0 +1,12 @@ +<%# Hi <%= @recipient.nickname.presence || @recipient.first_name %> %> + +<% @body_paragraphs.each do |paragraph| %> +<%= paragraph %> + +<% end %> +<%# Cheers, +The <%= @doubtfire_product_name %> Team on behalf of <%= @sender.name %> %> + +--- + +Generated with <%= @doubtfire_product_name %> diff --git a/config/application.rb b/config/application.rb index 42dd1e6e27..21afadb5da 100644 --- a/config/application.rb +++ b/config/application.rb @@ -271,6 +271,7 @@ def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) config.autoload_paths << Rails.root.join('app') << Rails.root.join('app/models/comments') << + Rails.root.join('app/models/communication') << Rails.root.join('app/models/turn_it_in') << Rails.root.join('app/models/similarity') << Rails.root.join('app/models/d2l') @@ -278,6 +279,7 @@ def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) config.eager_load_paths << Rails.root.join('app') << Rails.root.join('app/models/comments') << + Rails.root.join('app/models/communication') << Rails.root.join('app/models/turn_it_in') << Rails.root.join('app/models/similarity') << Rails.root.join('app/models/d2l') diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index dc00e55825..942c043b5d 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -14,6 +14,11 @@ Sidekiq::Status.configure_server_middleware(config, expiration: 30.minutes.to_i) Sidekiq::Status.configure_client_middleware(config, expiration: 30.minutes.to_i) + + config.on(:startup) do + schedule_file = Rails.root.join('config/schedule.yml') + Sidekiq::Cron::Job.load_from_hash!(YAML.load_file(schedule_file)) if File.exist?(schedule_file) + end end Sidekiq.configure_client do |config| diff --git a/config/schedule.yml b/config/schedule.yml index b0bd370f02..b26a1309d2 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -16,6 +16,10 @@ refresh_moderation_feedback_timestamps: cron: "every 60 minutes" class: "RefreshModerationFeedbackTimestampsJob" +poll_communication_set_schedules: + cron: "every 5 minutes" + class: "PollCommunicationSetSchedulesJob" + # archive_old_units: # cron: "every 6 months" # class: "ArchiveOldUnitsJob" diff --git a/db/migrate/20260604070032_add_communications_feat.rb b/db/migrate/20260604070032_add_communications_feat.rb new file mode 100644 index 0000000000..b28e520cc4 --- /dev/null +++ b/db/migrate/20260604070032_add_communications_feat.rb @@ -0,0 +1,123 @@ +class AddCommunicationsFeat < ActiveRecord::Migration[8.0] + def change + create_table :communication_sets do |t| + t.references :unit, null: false + t.string :name + t.boolean :active, null: false, default: true + t.timestamps + end + + create_table :communication_rules do |t| + t.references :communication_set, null: false + + t.integer :position, null: false, default: 0 + t.string :name + # AND | OR + t.string :operator # AND | OR + + t.boolean :send_log_to_convenors, null: false, default: false + + t.boolean :active, null: false, default: true + + t.timestamps + end + + create_table :communication_conditions do |t| + t.string :type, null: false + t.references :communication, null: false + + # TargetGradeCondition + t.integer :target_grade + # t.string :target_grade_comparison # gt|gte|lt|lte|notequal|equal + + # TaskDefinitionStatusCondition + t.references :task_definition + t.json :task_statuses + # t.string :task_status_comparison # equal|notequal + + # LoginStatusCondition + t.datetime :last_sign_in_at + # t.string :last_sign_in_comparison # before|after + + # TutorialEnrolmentCondition + t.references :tutorial + # t.string :tutorial_comparison # enrolled|notenrolled + + # TutorialStreamEnrolmentCondition + t.references :tutorial_stream + # t.string :tutorial_stream_comparison # enrolled|notenrolled + + # CampusCondition + t.references :campus + # t.string :campus_comparison # enrolled|notenrolled + + # TaskStatusCountCondition + t.integer :task_status_count + t.integer :task_target_grade + + # SpecConDaysCondition + t.integer :spec_con_days + + t.string :operator, null: false + + t.timestamps + end + + create_table :communication_actions do |t| + t.string :type, null: false + t.references :communication_rule, null: false + + # EmailStudentAction / EmailStaffAction + t.string :subject + t.text :body + + # EmailStaffAction + t.boolean :email_tutors, null: false, default: false + t.boolean :email_convenors, null: false, default: false + + # ChangeTargetGradeAction + t.integer :target_grade + + # TaskCommentAction + t.references :task_definition, null: true + + t.timestamps + end + + create_table :communication_set_schedules do |t| + t.references :communication_set, null: false + t.string :name + t.boolean :active, null: false, default: true + + # Anchor the schedule to the unit's teaching calendar. The actual start + # datetime is resolved through Unit#date_for_week_and_day. + t.integer :anchor_week, null: false + t.string :anchor_day, null: false + t.integer :hour, null: false, default: 8 + t.integer :minute, null: false, default: 0 + t.string :timezone, null: false, default: "UTC" + + # Canonical recurrence settings to hydrate IceCube rules from. + t.string :recurrence, null: false, default: "none" + t.integer :interval, null: false, default: 1 + + # Optional limits for recurring schedules. + t.integer :repeat_count + t.datetime :until_at + + # Serialized payload to rebuild an IceCube schedule without guessing. + t.json :ice_cube_schedule + + # Derived state for the worker that enqueues due communication runs. + t.datetime :next_run_at + t.datetime :last_run_at + t.datetime :last_enqueued_at + + t.timestamps + end + + add_index :communication_actions, :type + add_index :communication_conditions, :type + add_index :communication_set_schedules, [:active, :next_run_at] + end +end diff --git a/db/schema.rb b/db/schema.rb index a6b03f2f20..add38ef602 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_06_04_031804) do +ActiveRecord::Schema[8.0].define(version: 2026_06_04_070032) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -73,6 +73,92 @@ t.index ["user_id"], name: "index_comments_read_receipts_on_user_id" end + create_table "communication_actions", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.string "type", null: false + t.bigint "communication_rule_id", null: false + t.string "subject" + t.text "body" + t.boolean "email_tutors", default: false, null: false + t.boolean "email_convenors", default: false, null: false + t.integer "target_grade" + t.bigint "task_definition_id" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["communication_rule_id"], name: "index_communication_actions_on_communication_rule_id" + t.index ["task_definition_id"], name: "index_communication_actions_on_task_definition_id" + t.index ["type"], name: "index_communication_actions_on_type" + end + + create_table "communication_conditions", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.string "type", null: false + t.bigint "communication_id", null: false + t.integer "target_grade" + t.bigint "task_definition_id" + t.text "task_statuses", size: :long, collation: "utf8mb4_bin" + t.datetime "last_sign_in_at" + t.bigint "tutorial_id" + t.bigint "tutorial_stream_id" + t.bigint "campus_id" + t.integer "task_status_count" + t.integer "task_target_grade" + t.integer "spec_con_days" + t.string "operator", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["campus_id"], name: "index_communication_conditions_on_campus_id" + t.index ["communication_id"], name: "index_communication_conditions_on_communication_id" + t.index ["task_definition_id"], name: "index_communication_conditions_on_task_definition_id" + t.index ["tutorial_id"], name: "index_communication_conditions_on_tutorial_id" + t.index ["tutorial_stream_id"], name: "index_communication_conditions_on_tutorial_stream_id" + t.index ["type"], name: "index_communication_conditions_on_type" + t.check_constraint "json_valid(`task_statuses`)", name: "task_statuses" + end + + create_table "communication_rules", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "communication_set_id", null: false + t.integer "position", default: 0, null: false + t.string "name" + t.string "operator" + t.boolean "send_log_to_convenors", default: false, null: false + t.boolean "active", default: true, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["communication_set_id"], name: "index_communication_rules_on_communication_set_id" + end + + create_table "communication_set_schedules", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "communication_set_id", null: false + t.string "name" + t.boolean "active", default: true, null: false + t.integer "anchor_week", null: false + t.string "anchor_day", null: false + t.integer "hour", default: 8, null: false + t.integer "minute", default: 0, null: false + t.string "timezone", default: "UTC", null: false + t.string "recurrence", default: "none", null: false + t.integer "interval", default: 1, null: false + t.integer "repeat_count" + t.datetime "until_at" + t.text "ice_cube_schedule", size: :long, collation: "utf8mb4_bin" + t.datetime "next_run_at" + t.datetime "last_run_at" + t.datetime "last_enqueued_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active", "next_run_at"], name: "index_communication_set_schedules_on_active_and_next_run_at" + t.index ["communication_set_id"], name: "index_communication_set_schedules_on_communication_set_id" + t.check_constraint "json_valid(`ice_cube_schedule`)", name: "ice_cube_schedule" + end + + create_table "communication_sets", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.string "name" + t.boolean "active", default: true, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["unit_id"], name: "index_communication_sets_on_unit_id" + end + create_table "d2l_assessment_mappings", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "unit_id", null: false t.string "org_unit_id" diff --git a/test/factories/units_factory.rb b/test/factories/units_factory.rb index 7cc6de1090..a25b6102a0 100644 --- a/test/factories/units_factory.rb +++ b/test/factories/units_factory.rb @@ -143,6 +143,8 @@ next end + next if campus_tutorials.empty? + if campus_tutorials.first.tutorial_stream.present? tutorial_streams.each_with_index do |ts, i| p.enrol_in ts.tutorials.where(campus_id: c.id).sample @@ -153,7 +155,7 @@ end eval.part_enrolled_student_count.times do - unit.tutorial_enrolments.joins(:project).where('projects.campus_id = :campus_id', campus_id: c.id).sample.destroy + unit.tutorial_enrolments.joins(:project).where('projects.campus_id = :campus_id', campus_id: c.id).sample&.destroy end end diff --git a/test/models/communication_condition_test.rb b/test/models/communication_condition_test.rb new file mode 100644 index 0000000000..428aaa0410 --- /dev/null +++ b/test/models/communication_condition_test.rb @@ -0,0 +1,40 @@ +require 'test_helper' + +class CommunicationConditionTest < ActiveSupport::TestCase + def test_task_definition_status_condition_accepts_multiple_task_statuses + unit = FactoryBot.create(:unit, with_students: false, task_count: 1, stream_count: 0, tutorials: 0) + communication_set = unit.communication_sets.create!(name: 'Test Set', active: true) + communication_rule = communication_set.communication_rules.create!(name: 'Test Rule', operator: 'and', position: 0) + + condition = TaskDefinitionStatusCondition.new( + communication: communication_rule, + operator: 'equal_to', + task_definition: unit.task_definitions.first, + task_statuses: %w[not_started working_on_it fix_and_resubmit] + ) + + assert condition.valid?, condition.errors.full_messages + condition.save! + + condition.reload + assert_equal %w[not_started working_on_it fix_and_resubmit], condition.task_statuses + end + + def test_spec_con_condition_accepts_integer_spec_con_days + unit = FactoryBot.create(:unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0) + communication_set = unit.communication_sets.create!(name: 'Test Set', active: true) + communication_rule = communication_set.communication_rules.create!(name: 'Test Rule', operator: 'and', position: 0) + + condition = SpecConCondition.new( + communication: communication_rule, + operator: 'greater_than_or_equal_to', + spec_con_days: 4 + ) + + assert condition.valid?, condition.errors.full_messages + condition.save! + + condition.reload + assert_equal 4, condition.spec_con_days + end +end diff --git a/test/models/communication_set_schedule_test.rb b/test/models/communication_set_schedule_test.rb new file mode 100644 index 0000000000..4351859954 --- /dev/null +++ b/test/models/communication_set_schedule_test.rb @@ -0,0 +1,74 @@ +require 'test_helper' + +class CommunicationSetScheduleTest < ActiveSupport::TestCase + def test_next_run_at_is_nil_when_next_occurrence_is_after_unit_end_date + travel_to Time.zone.local(2026, 1, 1, 9, 0, 0) do + unit = create_unit(start_date: Date.parse('2026-02-02'), end_date: Date.parse('2026-02-02')) + schedule = create_schedule(unit, anchor_day: 'Tuesday') + + assert_nil schedule.next_run_at + end + end + + def test_next_run_at_refreshes_when_unit_dates_change + travel_to Time.zone.local(2026, 1, 1, 9, 0, 0) do + unit = create_unit(start_date: Date.parse('2026-02-02'), end_date: Date.parse('2026-02-10')) + schedule = create_schedule(unit, anchor_day: 'Tuesday') + + assert_equal Time.zone.local(2026, 2, 3, 9, 30, 0), schedule.next_run_at + + unit.update!(end_date: Date.parse('2026-02-02')) + assert_nil schedule.reload.next_run_at + + unit.update!(start_date: Date.parse('2026-02-09'), end_date: Date.parse('2026-02-17')) + assert_equal Time.zone.local(2026, 2, 10, 9, 30, 0), schedule.reload.next_run_at + end + end + + def test_next_run_at_refreshes_when_unit_active_status_changes + travel_to Time.zone.local(2026, 1, 1, 9, 0, 0) do + unit = create_unit(start_date: Date.parse('2026-02-02'), end_date: Date.parse('2026-05-11')) + schedule = create_schedule(unit, anchor_day: 'Monday') + + assert_equal Time.zone.local(2026, 2, 2, 9, 30, 0), schedule.next_run_at + + unit.update!(active: false) + assert_nil schedule.reload.next_run_at + + unit.update!(active: true) + assert_equal Time.zone.local(2026, 2, 2, 9, 30, 0), schedule.reload.next_run_at + end + end + + private + + def create_unit(start_date:, end_date:) + FactoryBot.create( + :unit, + active: true, + with_students: false, + task_count: 0, + tutorials: 0, + outcome_count: 0, + staff_count: 0, + campus_count: 0, + start_date: start_date, + end_date: end_date + ) + end + + def create_schedule(unit, anchor_day:) + communication_set = unit.communication_sets.create!(name: 'Scheduled communications', active: true) + communication_set.communication_set_schedules.create!( + name: 'Week 1 schedule', + active: true, + anchor_week: 1, + anchor_day: anchor_day, + hour: 9, + minute: 30, + timezone: 'UTC', + recurrence: 'none', + interval: 1 + ) + end +end diff --git a/test/models/communication_set_test.rb b/test/models/communication_set_test.rb new file mode 100644 index 0000000000..931c201762 --- /dev/null +++ b/test/models/communication_set_test.rb @@ -0,0 +1,69 @@ +require 'test_helper' + +class CommunicationSetTest < ActiveSupport::TestCase + def test_preview_projects_for_rule_excludes_students_claimed_by_earlier_rules + unit = FactoryBot.create( + :unit, + student_count: 2, + unenrolled_student_count: 0, + part_enrolled_student_count: 0, + inactive_student_count: 0, + task_count: 1, + stream_count: 0, + tutorials: 0 + ) + communication_set = unit.communication_sets.create!(name: 'Test Set', active: true) + + first_rule = communication_set.communication_rules.create!(name: 'First Rule', operator: 'and', position: 0) + second_rule = communication_set.communication_rules.create!(name: 'Second Rule', operator: 'and', position: 1) + + first_rule.communication_conditions.create!( + type: 'TaskDefinitionStatusCondition', + operator: 'equal_to', + task_definition: unit.task_definitions.first, + task_statuses: ['not_started'] + ) + + second_rule.communication_conditions.create!( + type: 'TaskDefinitionStatusCondition', + operator: 'equal_to', + task_definition: unit.task_definitions.first, + task_statuses: ['not_started'] + ) + + first_rule_matches = communication_set.preview_projects_for_rule(first_rule) + second_rule_matches = communication_set.preview_projects_for_rule(second_rule) + + assert_equal 2, first_rule_matches.length + assert_empty second_rule_matches + end + + def test_preview_projects_for_rule_matches_spec_con_days + unit = FactoryBot.create( + :unit, + with_students: false, + task_count: 0, + stream_count: 0, + tutorials: 0, + outcome_count: 0, + staff_count: 0, + campus_count: 1 + ) + + matching_project = FactoryBot.create(:project, unit: unit, spec_con_days: 4) + FactoryBot.create(:project, unit: unit, spec_con_days: 1) + + communication_set = unit.communication_sets.create!(name: 'Test Set', active: true) + communication_rule = communication_set.communication_rules.create!(name: 'Spec Con Rule', operator: 'and', position: 0) + + communication_rule.communication_conditions.create!( + type: 'SpecConCondition', + operator: 'greater_than_or_equal_to', + spec_con_days: 3 + ) + + matched_projects = communication_set.preview_projects_for_rule(communication_rule) + + assert_equal [matching_project.id], matched_projects.map(&:id) + end +end diff --git a/test/models/teaching_period_test.rb b/test/models/teaching_period_test.rb index e630bb35f9..c18fbd5d85 100644 --- a/test/models/teaching_period_test.rb +++ b/test/models/teaching_period_test.rb @@ -220,4 +220,40 @@ def test_create_teaching_period_with_invalid_dates tp.destroy assert tp.destroyed? end + + test 'communication schedule next run refreshes when breaks change' do + travel_to Time.zone.local(2026, 1, 1, 9, 0, 0) do + tp = TeachingPeriod.create!( + year: 2026, + period: 'T1', + start_date: Date.parse('2026-02-02'), + end_date: Date.parse('2026-05-11'), + active_until: Date.parse('2026-05-18') + ) + unit = FactoryBot.create(:unit, teaching_period: tp, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 0) + communication_set = unit.communication_sets.create!(name: 'Weekly check in', active: true) + schedule = communication_set.communication_set_schedules.create!( + name: 'Week 3 Monday', + active: true, + anchor_week: 3, + anchor_day: 'Monday', + hour: 9, + minute: 30, + timezone: 'UTC', + recurrence: 'none', + interval: 1 + ) + + assert_equal Time.zone.local(2026, 2, 16, 9, 30, 0), schedule.next_run_at + + teaching_break = tp.add_break(Date.parse('2026-02-09'), 1) + assert_equal Time.zone.local(2026, 2, 23, 9, 30, 0), schedule.reload.next_run_at + + tp.update_break(teaching_break.id, Date.parse('2026-02-23'), 1) + assert_equal Time.zone.local(2026, 2, 16, 9, 30, 0), schedule.reload.next_run_at + + teaching_break.destroy + assert_equal Time.zone.local(2026, 2, 16, 9, 30, 0), schedule.reload.next_run_at + end + end end diff --git a/test/models/unit_model_test.rb b/test/models/unit_model_test.rb index 39144c51d7..eca67c6755 100644 --- a/test/models/unit_model_test.rb +++ b/test/models/unit_model_test.rb @@ -237,6 +237,83 @@ def test_rollover_of_group_tasks unit2.destroy end + def test_rollover_of_communication_sets + task_definition = FactoryBot.create(:task_definition, unit: @unit, tutorial_stream: @unit.tutorial_streams.first) + communication_set = @unit.communication_sets.create!(name: 'At Risk Follow Up', active: true) + communication_set.communication_set_schedules.create!( + name: 'Weekly follow up', + active: true, + anchor_week: 1, + anchor_day: 'Monday', + hour: 9, + minute: 30, + timezone: 'UTC', + recurrence: 'weekly', + interval: 1, + last_run_at: Time.zone.now, + last_enqueued_at: Time.zone.now + ) + communication_rule = communication_set.communication_rules.create!( + name: 'Not started', + operator: 'and', + position: 0, + send_log_to_convenors: true, + active: true + ) + communication_rule.communication_conditions.create!( + type: 'TaskDefinitionStatusCondition', + operator: 'equal_to', + task_definition: task_definition, + task_statuses: ['not_started'] + ) + communication_rule.communication_conditions.create!( + type: 'TutorialStreamEnrolmentCondition', + operator: 'enrolled_in', + tutorial_stream: @unit.tutorial_streams.first + ) + communication_rule.communication_actions.create!( + type: 'EmailStudentAction', + subject: 'Please start {{unit.code}}', + body: 'Hello {{student.first_name}}' + ) + + unit2 = @unit.rollover TeachingPeriod.find(2), nil, nil, nil + + assert_equal 1, unit2.communication_sets.count + new_set = unit2.communication_sets.first + assert_not_equal communication_set.id, new_set.id + assert_equal 'At Risk Follow Up', new_set.name + assert_equal true, new_set.active + + assert_equal 1, new_set.communication_set_schedules.count + new_schedule = new_set.communication_set_schedules.first + assert_not_equal communication_set.communication_set_schedules.first.id, new_schedule.id + assert_equal 'Weekly follow up', new_schedule.name + assert_equal 'weekly', new_schedule.recurrence + assert_nil new_schedule.last_run_at + assert_nil new_schedule.last_enqueued_at + + assert_equal 1, new_set.communication_rules.count + new_rule = new_set.communication_rules.first + assert_not_equal communication_rule.id, new_rule.id + assert_equal 'Not started', new_rule.name + assert_equal true, new_rule.send_log_to_convenors + + new_task_definition = unit2.task_definitions.find_by!(abbreviation: task_definition.abbreviation) + task_condition = new_rule.communication_conditions.find_by!(type: 'TaskDefinitionStatusCondition') + assert_equal new_task_definition, task_condition.task_definition + assert_equal ['not_started'], task_condition.task_statuses + + new_tutorial_stream = unit2.tutorial_streams.find_by!(abbreviation: @unit.tutorial_streams.first.abbreviation) + stream_condition = new_rule.communication_conditions.find_by!(type: 'TutorialStreamEnrolmentCondition') + assert_equal new_tutorial_stream, stream_condition.tutorial_stream + + assert_equal 1, new_rule.communication_actions.count + assert_equal 'Please start {{unit.code}}', new_rule.communication_actions.first.subject + + unit2.destroy + end + def test_rollover_of_tasks_have_same_start_week_and_day @unit.import_tasks_from_csv File.open(Rails.root.join('test_files',"#{@unit.code}-Tasks.csv")) diff --git a/test/sidekiq/communication_set_schedule_jobs_test.rb b/test/sidekiq/communication_set_schedule_jobs_test.rb new file mode 100644 index 0000000000..fe7af97194 --- /dev/null +++ b/test/sidekiq/communication_set_schedule_jobs_test.rb @@ -0,0 +1,60 @@ +require 'test_helper' + +class CommunicationSetScheduleJobsTest < ActiveSupport::TestCase + def test_poll_does_not_enqueue_schedules_for_inactive_units + travel_to Time.zone.local(2026, 2, 2, 10, 0, 0) do + active_schedule = create_due_schedule(unit_active: true) + inactive_schedule = create_due_schedule(unit_active: false) + + PollCommunicationSetSchedulesJob.new.perform + + enqueued_schedule_ids = ExecuteCommunicationSetScheduleJob.jobs.map { |job| job['args'].first } + assert_includes enqueued_schedule_ids, active_schedule.id + assert_not_includes enqueued_schedule_ids, inactive_schedule.id + end + end + + def test_schedule_execution_does_not_enqueue_communication_set_for_inactive_unit + travel_to Time.zone.local(2026, 2, 2, 10, 0, 0) do + schedule = create_due_schedule(unit_active: false) + + ExecuteCommunicationSetScheduleJob.new.perform(schedule.id) + + assert_empty ExecuteCommunicationSetJob.jobs + assert_nil schedule.reload.last_enqueued_at + assert_nil schedule.last_run_at + end + end + + private + + def create_due_schedule(unit_active:) + unit = FactoryBot.create( + :unit, + active: unit_active, + with_students: false, + task_count: 0, + tutorials: 0, + outcome_count: 0, + staff_count: 0, + campus_count: 0, + start_date: Date.parse('2026-02-02'), + end_date: Date.parse('2026-05-11') + ) + communication_set = unit.communication_sets.create!(name: 'Scheduled communications', active: true) + + schedule = communication_set.communication_set_schedules.create!( + name: 'Due schedule', + active: true, + anchor_week: 1, + anchor_day: 'Monday', + hour: 9, + minute: 0, + timezone: 'UTC', + recurrence: 'none', + interval: 1 + ) + schedule.update_column(:next_run_at, Time.zone.local(2026, 2, 2, 9, 0, 0)) + schedule + end +end diff --git a/test/sidekiq/execute_communication_set_job_test.rb b/test/sidekiq/execute_communication_set_job_test.rb new file mode 100644 index 0000000000..8ec1afb8b3 --- /dev/null +++ b/test/sidekiq/execute_communication_set_job_test.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'test_helper' + +class ExecuteCommunicationSetJobTest < ActiveSupport::TestCase + def test_task_comment_action_adds_a_comment_to_each_selected_students_task + unit = FactoryBot.create( + :unit, + with_students: false, + task_count: 1, + stream_count: 0, + tutorials: 0, + outcome_count: 0, + staff_count: 1 + ) + + task_definition = unit.task_definitions.first + campus = Campus.first + comment_author = unit.main_convenor_user + + student_one = FactoryBot.create(:user, :student) + student_one.update!(first_name: 'Ada', last_name: 'Lovelace') + project_one = unit.enrol_student(student_one, campus) + + student_two = FactoryBot.create(:user, :student) + student_two.update!(first_name: 'Grace', last_name: 'Hopper') + project_two = unit.enrol_student(student_two, campus) + + communication_set = unit.communication_sets.create!(name: 'Test Set', active: true) + communication_rule = communication_set.communication_rules.create!( + name: 'Comment Rule', + operator: 'and', + position: 0 + ) + communication_rule.communication_actions.create!( + type: 'TaskCommentAction', + task_definition: task_definition, + body: 'Please review {{student.first_name}} for {{unit.code}}' + ) + + ExecuteCommunicationSetJob.new.perform(communication_set.id) + + task_one = project_one.task_for_task_definition(task_definition) + task_two = project_two.task_for_task_definition(task_definition) + + assert_equal 1, TaskComment.where(task: task_one).count + assert_equal 1, TaskComment.where(task: task_two).count + + comment_one = task_one.comments.last + comment_two = task_two.comments.last + + assert_equal comment_author, comment_one.user + assert_equal comment_author, comment_two.user + assert_equal 'Please review Ada for ' + unit.code, comment_one.comment + assert_equal 'Please review Grace for ' + unit.code, comment_two.comment + end +end diff --git a/test/sidekiq/scheduled_job_test.rb b/test/sidekiq/scheduled_job_test.rb index ba1b1b7e00..e56b1b7766 100644 --- a/test/sidekiq/scheduled_job_test.rb +++ b/test/sidekiq/scheduled_job_test.rb @@ -6,13 +6,14 @@ class TiiCheckProgressJobTest < ActiveSupport::TestCase def test_jobs_are_scheduled Sidekiq::Cron::Job.destroy_all! Sidekiq::Cron::Job.load_from_hash!(YAML.load_file(Rails.root.join('config/schedule.yml'))) - assert_equal 4, Sidekiq::Cron::Job.all.count, Sidekiq::Cron::Job.all.map(&:name) + assert_equal 5, Sidekiq::Cron::Job.all.count, Sidekiq::Cron::Job.all.map(&:name) Sidekiq::Cron::Job.all.each(&:enqueue!) assert_equal 1, TiiRegisterWebHookJob.jobs.count assert_equal 1, TiiCheckProgressJob.jobs.count assert_equal 1, ClearAccessTokensJob.jobs.count assert_equal 1, RefreshModerationFeedbackTimestampsJob.jobs.count + assert_equal 1, PollCommunicationSetSchedulesJob.jobs.count # assert_equal 1, ArchiveOldUnitsJob.jobs.count end From a417f0e6cabb15f33e286068d31c90b08171e15c Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 9 Jun 2026 12:16:43 +1000 Subject: [PATCH 081/199] chore(release): 11.0.0-8 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc7f160a9f..3505a27c96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-8](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-7...v11.0.0-8) (2026-06-09) + + +### Features + +* communications system ([#617](https://github.com/b0ink/doubtfire-deploy/issues/617)) ([9724909](https://github.com/b0ink/doubtfire-deploy/commit/97249090eac3f7e214991f7c665f92768948b6db)) + ## [11.0.0-7](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-6...v11.0.0-7) (2026-06-08) From 93599d27d736ebfaf7b997dfd82d764a45195881 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:15:26 +1000 Subject: [PATCH 082/199] fix: ensure more reliable test --- test/api/tasks_api_test.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/api/tasks_api_test.rb b/test/api/tasks_api_test.rb index 75a6fb005f..cfdeae172b 100644 --- a/test/api/tasks_api_test.rb +++ b/test/api/tasks_api_test.rb @@ -914,7 +914,10 @@ def test_resubmission_doesnt_change_submission_date target_grade: 0, outcome_count: 0 ) - td.update!(due_date: Time.zone.today + 1.week) + td.update!( + target_date: Time.zone.today + 1.week, + due_date: Time.zone.today + 1.week + ) student1 = FactoryBot.create(:user, :student) student2 = FactoryBot.create(:user, :student) @@ -966,9 +969,9 @@ def test_resubmission_doesnt_change_submission_date task1.reload assert task2.submission_date > task1.submission_date assert_equal original_submission_date, task1.submission_date - assert TaskStatus.ready_for_feedback, task1.task_status + assert_equal TaskStatus.ready_for_feedback, task1.task_status - # Submit the task again after the duedate, ensure the submission_date hasn't changed (student1) + # Submit the task again later, ensure the submission_date hasn't changed (student1) travel 2.days task1.submit(student1) @@ -981,7 +984,7 @@ def test_resubmission_doesnt_change_submission_date task1.reload assert task2.submission_date > task1.submission_date assert_equal original_submission_date, task1.submission_date - assert TaskStatus.ready_for_feedback, task1.task_status + assert_equal TaskStatus.ready_for_feedback, task1.task_status task1.update(task_status_id: TaskStatus.fix_and_resubmit.id) @@ -990,7 +993,7 @@ def test_resubmission_doesnt_change_submission_date task1.submit(student1) task1.reload - assert TaskStatus.ready_for_feedback, task1.task_status + assert_equal TaskStatus.ready_for_feedback, task1.task_status tasks = unit.tasks_for_task_inbox(tutor, false) From 8ff358a78918c459bd23185b6b7585804806a1e7 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:14:43 +1000 Subject: [PATCH 083/199] refactor: improve latex work directory naming (#632) * refactor: improve latex work directory naming * chore: restore retry suffix --- app/models/pdf_generation/project_compile_portfolio_module.rb | 4 +++- app/models/task.rb | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/pdf_generation/project_compile_portfolio_module.rb b/app/models/pdf_generation/project_compile_portfolio_module.rb index 8fb394340f..93694a54d9 100644 --- a/app/models/pdf_generation/project_compile_portfolio_module.rb +++ b/app/models/pdf_generation/project_compile_portfolio_module.rb @@ -71,7 +71,9 @@ def init(project, is_retry) @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] @is_retry = is_retry @include_pax = !is_retry - @work_id = "portfolio-#{project.id}-#{Time.now.to_i}-#{Process.pid}-#{Thread.current.object_id}#{'-retry' if is_retry}" + @work_id = FileHelper.sanitized_path( + "portfolio-#{Time.current.strftime('%Y%m%d-%H%M')}-#{project.student.username}-#{project.id}-#{Process.pid}#{'-retry' if is_retry}" + ) end def make_pdf diff --git a/app/models/task.rb b/app/models/task.rb index 5fdcfe7833..cfd9b6b8ba 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -1331,7 +1331,9 @@ def init(task, is_retry) @institution_name = Doubtfire::Application.config.institution[:name] @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] @include_pax = !is_retry - @work_id = "task-#{task.id}-#{Time.now.to_i}-#{Process.pid}-#{Thread.current.object_id}#{'-retry' if is_retry}" + @work_id = FileHelper.sanitized_path( + "task-#{Time.current.strftime('%Y%m%d-%H%M')}-#{task.project.student.username}-#{task.task_definition.abbreviation}-#{task.id}-#{Process.pid}#{'-retry' if is_retry}" + ) host = Doubtfire::Application.config.institution[:host].to_s host = "http://#{host}" unless host.match?(%r{\Ahttps?://}) host = host.sub(%r{/*\z}, '') From 0bedbfcd1f983a97098b5c1867d92bf851adbdc1 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:20:47 +1000 Subject: [PATCH 084/199] refactor: reset portfolio compilation after completion (#630) * refactor: reset portfolio compilation after completion * chore: add log --- .../project_compile_portfolio_module.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/models/pdf_generation/project_compile_portfolio_module.rb b/app/models/pdf_generation/project_compile_portfolio_module.rb index 93694a54d9..3ad5d1eb55 100644 --- a/app/models/pdf_generation/project_compile_portfolio_module.rb +++ b/app/models/pdf_generation/project_compile_portfolio_module.rb @@ -115,9 +115,7 @@ def initialize(log_message) # Create the portfolio for this project def create_portfolio - self.compile_portfolio = false - save! - + logger.info "Creating portfolio for #{user.username} in #{unit.code}" begin pac = ProjectAppController.new pac.init(self, false) @@ -144,8 +142,13 @@ def create_portfolio logger.info "Created portfolio at #{portfolio_path} - #{log_details}" self.portfolio_production_date = Time.zone.now - save + self.compile_portfolio = false + save! + true rescue StandardError => e + self.compile_portfolio = false + save! + logger.error "Failed to convert portfolio to PDF - #{log_details} -\nError: #{e.message}" log_file = e.message.scan(%r{/.*\.log}).first From 5d3407e2c897fa05f6e5c6ee41d8e0379fb6a367 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:15:28 +1000 Subject: [PATCH 085/199] feat: expose route to fetch tasks waiting for pdf gen --- app/api/projects_api.rb | 13 +++++++++++++ .../project_compile_portfolio_module.rb | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/app/api/projects_api.rb b/app/api/projects_api.rb index fb88a683a1..c2954bdcba 100644 --- a/app/api/projects_api.rb +++ b/app/api/projects_api.rb @@ -215,4 +215,17 @@ class ProjectsApi < Grape::API present portfolio_tasks.map(&:id) end + desc 'Get IDs of tasks that are still processing a PDF ' + get '/projects/:id/tasks_processing' do + project = Project.find(params[:id]) + + unless authorise? current_user, project, :get + error!({ error: "Couldn't find Project with id=#{params[:id]}" }, 403) + end + + portfolio_tasks = project.tasks_processing_pdf + + present portfolio_tasks.map(&:id) + end + end diff --git a/app/models/pdf_generation/project_compile_portfolio_module.rb b/app/models/pdf_generation/project_compile_portfolio_module.rb index 3ad5d1eb55..eab5a5292b 100644 --- a/app/models/pdf_generation/project_compile_portfolio_module.rb +++ b/app/models/pdf_generation/project_compile_portfolio_module.rb @@ -205,6 +205,19 @@ def portfolio_tasks end end + # Return the tasks that are currently being processed + def tasks_processing_pdf + # Get assigned tasks that should be included in the portfolio + tasks = self.tasks.joins(:task_definition).order('task_definitions.target_date, task_definitions.abbreviation') + + # Select tasks that should have a PDF submission, but is currently being processed + tasks.select do |task| + !task.has_pdf && + task.processing_pdf? && + task.task_definition.upload_requirements.present? + end + end + # # Return the path to the student's learning summary report. # This returns nil if there is no learning summary report. From 5a5dea779fb85a8c2331363faaccc056982a9697 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:19:38 +1000 Subject: [PATCH 086/199] test: cover portfolio task pdf processing state --- test/models/project_model_test.rb | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/models/project_model_test.rb b/test/models/project_model_test.rb index b237689de3..ea53a4a333 100644 --- a/test/models/project_model_test.rb +++ b/test/models/project_model_test.rb @@ -131,6 +131,49 @@ def test_create_empty_portfolio unit.destroy! end + def test_portfolio_tasks_excludes_tasks_until_pdf_processing_finishes + unit = FactoryBot.create(:unit, with_students: false, task_count: 0) + project = FactoryBot.create(:project, unit: unit) + no_upload_task_definition = FactoryBot.create(:task_definition, unit: unit, upload_requirements: []) + finished_task_definition = FactoryBot.create(:task_definition, unit: unit) + processing_task_definition = FactoryBot.create(:task_definition, unit: unit) + + no_upload_task = FactoryBot.create( + :task, + project: project, + task_definition: no_upload_task_definition, + task_status: TaskStatus.ready_for_feedback + ) + finished_task = FactoryBot.create( + :task, + project: project, + task_definition: finished_task_definition, + task_status: TaskStatus.ready_for_feedback + ) + processing_task = FactoryBot.create( + :task, + project: project, + task_definition: processing_task_definition, + task_status: TaskStatus.ready_for_feedback + ) + + FileUtils.touch(finished_task.final_pdf_path) + FileUtils.touch(processing_task.final_pdf_path) + processing_dir = FileHelper.student_work_dir(:new, processing_task, true) + + assert_includes project.portfolio_tasks, no_upload_task + assert_includes project.portfolio_tasks, finished_task + assert_not_includes project.portfolio_tasks, processing_task + assert_includes project.tasks_processing_pdf, processing_task + + FileUtils.rm_r(processing_dir) + + assert_includes project.portfolio_tasks, processing_task + assert_not_includes project.tasks_processing_pdf, processing_task + + unit.destroy! + end + def test_create_portfolio_with_lsr project = FactoryBot.create(:project) unit = project.unit From 065180f248db7e80dd86f0491822e2051a9fa220 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:58:39 +1000 Subject: [PATCH 087/199] feat: overflow task claim logs (#614) * feat: overflow task claim logs * chore: change name * chore: reset schema * chore: bump migration * fix: standardise timestamp * refactor: track days awaiting feedback in log * chore: fix test * chore: dont format time * chore: fix test --- app/api/tasks_api.rb | 29 ++++++---- app/api/units_api.rb | 12 +++++ app/models/overflow_task_claim.rb | 1 + app/models/overflow_task_claim_log.rb | 8 +++ app/models/unit.rb | 37 +++++++++++++ .../download_overflow_task_claims_csv_job.rb | 33 ++++++++++++ ...0065531_create_overflow_task_claim_logs.rb | 17 ++++++ db/schema.rb | 22 +++++++- test/api/csv_test.rb | 54 +++++++++++++++++++ test/api/tasks_api_test.rb | 47 ++++++++++++++++ test/factories/overflow_task_claim_logs.rb | 14 +++++ test/factories/overflow_task_claims.rb | 3 +- 12 files changed, 266 insertions(+), 11 deletions(-) create mode 100644 app/models/overflow_task_claim_log.rb create mode 100644 app/sidekiq/download_overflow_task_claims_csv_job.rb create mode 100644 db/migrate/20260610065531_create_overflow_task_claim_logs.rb create mode 100644 test/factories/overflow_task_claim_logs.rb diff --git a/app/api/tasks_api.rb b/app/api/tasks_api.rb index d08c43ca8b..afd9851fe3 100644 --- a/app/api/tasks_api.rb +++ b/app/api/tasks_api.rb @@ -516,19 +516,30 @@ class TasksApi < Grape::API error!({ error: "This task has already been claimed by another tutor" }, 409) end - inactive_claim = task.overflow_task_claim - inactive_claim&.destroy! + claimed_at = Time.zone.now + original_tutor = task.tutor - task_claim = OverflowTaskClaim.create!({ - task: task, - claimed_by_unit_role_id: my_unit_role.id - }) + ActiveRecord::Base.transaction do + task.overflow_task_claim&.destroy! - unless task_claim.valid? - error!({ error: "Failed to claim task" }, 400) + OverflowTaskClaim.create!( + task: task, + claimed_by_unit_role: my_unit_role + ) + + OverflowTaskClaimLog.create!( + unit: unit, + task: task, + claimed_by_unit_role: my_unit_role, + claimed_by_user: current_user, + original_tutor_user: original_tutor, + student_user: project.student, + days_awaiting_feedback: task.days_awaiting_feedback(claimed_at), + claimed_at: claimed_at + ) end - logger.info "Overflow task claim: {\"user_id\": #{current_user.id},\"task_id\": #{task.id}, \"timestamp\": \"#{Time.zone.now}\", \"original_tutor_user_id\": #{task.tutor ? task.tutor.id : -1}}" + logger.info "Overflow task claim: {\"user_id\": #{current_user.id},\"task_id\": #{task.id}, \"timestamp\": \"#{claimed_at}\", \"original_tutor_user_id\": #{original_tutor ? original_tutor.id : -1}}" true end diff --git a/app/api/units_api.rb b/app/api/units_api.rb index 8741ffb32e..59160e06e9 100644 --- a/app/api/units_api.rb +++ b/app/api/units_api.rb @@ -536,6 +536,18 @@ class UnitsApi < Grape::API present job, with: Entities::SidekiqJobEntity end + desc 'Download CSV of overflow task claims in this unit' + get '/csv/units/:id/overflow_task_claims' do + unit = Unit.find(params[:id]) + unless authorise? current_user, unit, :download_stats + error!({ error: "Not authorised to download overflow task claim stats for #{unit.code}" }, 403) + end + + job_id = DownloadOverflowTaskClaimsCsvJob.perform_async(unit.id) + job = setup_job(job_id) + present job, with: Entities::SidekiqJobEntity + end + desc 'Download CSV of all student tasks in this unit' get '/csv/units/:id/task_completion' do unit = Unit.find(params[:id]) diff --git a/app/models/overflow_task_claim.rb b/app/models/overflow_task_claim.rb index 2559ce5964..986d055b6d 100644 --- a/app/models/overflow_task_claim.rb +++ b/app/models/overflow_task_claim.rb @@ -1,4 +1,5 @@ class OverflowTaskClaim < ApplicationRecord belongs_to :task + belongs_to :claimed_by_unit_role, class_name: 'UnitRole' end diff --git a/app/models/overflow_task_claim_log.rb b/app/models/overflow_task_claim_log.rb new file mode 100644 index 0000000000..9b4b606115 --- /dev/null +++ b/app/models/overflow_task_claim_log.rb @@ -0,0 +1,8 @@ +class OverflowTaskClaimLog < ApplicationRecord + belongs_to :unit + belongs_to :task, optional: true + belongs_to :claimed_by_unit_role, class_name: 'UnitRole', optional: true + belongs_to :claimed_by_user, class_name: 'User', optional: true + belongs_to :original_tutor_user, class_name: 'User', optional: true + belongs_to :student_user, class_name: 'User', optional: true +end diff --git a/app/models/unit.rb b/app/models/unit.rb index 7f59a86b4a..634942e47f 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -2534,6 +2534,43 @@ def tutor_assessment_csv end end + def overflow_task_claims_csv + CSV.generate do |csv| + csv << [ + 'Tutor who claimed', + 'Claiming Unit Role ID', + 'Original Tutor', + 'Student Username', + 'Student ID', + 'Task ID', + 'Task Definition', + 'Days Awaiting Feedback', + 'Timestamp' + ] + + OverflowTaskClaimLog + .where(unit_id: id) + .includes(:claimed_by_user, :original_tutor_user, :student_user, task: :task_definition) + .order(:claimed_at) + .each do |claim| + task = claim.task + student = claim.student_user + + csv << [ + claim.claimed_by_user&.name, + claim.claimed_by_unit_role_id, + claim.original_tutor_user&.name, + student&.username, + student&.student_id, + claim.task_id, + task&.task_definition&.abbreviation, + claim.days_awaiting_feedback, + claim.claimed_at, + ] + end + end + end + #---------------------------------------------------------------------------- # Task updates from offline download/upload #---------------------------------------------------------------------------- diff --git a/app/sidekiq/download_overflow_task_claims_csv_job.rb b/app/sidekiq/download_overflow_task_claims_csv_job.rb new file mode 100644 index 0000000000..5fd7429b66 --- /dev/null +++ b/app/sidekiq/download_overflow_task_claims_csv_job.rb @@ -0,0 +1,33 @@ +require 'csv' + +class DownloadOverflowTaskClaimsCsvJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + include ApplicationHelper + include FileHelper + include MimeCheckHelpers + include CsvHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first] }, + on_conflict: :reject, + retry: false + + def perform(unit_id) + logger.info "Starting overflow task claims csv download..." + + at(0) + total(1) + + unit = Unit.find(unit_id) + csv = unit.overflow_task_claims_csv + + store(result: csv) + + logger.info "Completed overflow task claims csv download!" + rescue StandardError => e + logger.error e + raise e + end +end diff --git a/db/migrate/20260610065531_create_overflow_task_claim_logs.rb b/db/migrate/20260610065531_create_overflow_task_claim_logs.rb new file mode 100644 index 0000000000..d02ea8762a --- /dev/null +++ b/db/migrate/20260610065531_create_overflow_task_claim_logs.rb @@ -0,0 +1,17 @@ +class CreateOverflowTaskClaimLogs < ActiveRecord::Migration[8.0] + def change + create_table :overflow_task_claim_logs do |t| + t.references :unit, null: false + t.references :task, null: false + t.references :claimed_by_unit_role, null: false + t.references :claimed_by_user, null: false + t.references :original_tutor_user + t.references :student_user, null: false + t.integer :days_awaiting_feedback, null: false + t.datetime :claimed_at, null: false + t.timestamps + end + + add_index :overflow_task_claim_logs, [:unit_id, :claimed_at] + end +end diff --git a/db/schema.rb b/db/schema.rb index add38ef602..dd1dbdd7fd 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_06_04_070032) do +ActiveRecord::Schema[8.0].define(version: 2026_06_10_065531) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -312,6 +312,26 @@ t.index ["task_id"], name: "index_moderated_tasks_on_task_id" end + create_table "overflow_task_claim_logs", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.bigint "task_id", null: false + t.bigint "claimed_by_unit_role_id", null: false + t.bigint "claimed_by_user_id", null: false + t.bigint "original_tutor_user_id" + t.bigint "student_user_id", null: false + t.integer "days_awaiting_feedback", null: false + t.datetime "claimed_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["claimed_by_unit_role_id"], name: "index_overflow_task_claim_logs_on_claimed_by_unit_role_id" + t.index ["claimed_by_user_id"], name: "index_overflow_task_claim_logs_on_claimed_by_user_id" + t.index ["original_tutor_user_id"], name: "index_overflow_task_claim_logs_on_original_tutor_user_id" + t.index ["student_user_id"], name: "index_overflow_task_claim_logs_on_student_user_id" + t.index ["task_id"], name: "index_overflow_task_claim_logs_on_task_id" + t.index ["unit_id", "claimed_at"], name: "index_overflow_task_claim_logs_on_unit_id_and_claimed_at" + t.index ["unit_id"], name: "index_overflow_task_claim_logs_on_unit_id" + end + create_table "overflow_task_claims", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "task_id", null: false t.bigint "claimed_by_unit_role_id", null: false diff --git a/test/api/csv_test.rb b/test/api/csv_test.rb index 0b45e7d761..42a9f8ebdd 100644 --- a/test/api/csv_test.rb +++ b/test/api/csv_test.rb @@ -989,6 +989,60 @@ def test_download_csv_stats_tutor_assessed end end + # GET /api/csv/units/{id}/overflow_task_claims + def test_download_csv_overflow_task_claims + Sidekiq::Testing.inline! do + unit = create(:unit, student_count: 1, task_count: 1, stream_count: 0, tutorials: 1) + convenor = unit.main_convenor.user + tutor_user = create(:user, :tutor, first_name: 'ATutor', last_name: 'Tutor') + tutor_role = unit.employ_staff(tutor_user, Role.tutor) + tutorial = unit.tutorials.first + tutorial.update!(unit_role: tutor_role) + + project = unit.active_projects.first + student = project.student + student.update!(username: 'student-one', student_id: 's1234567') + + task_definition = unit.task_definitions.first + task_definition.update!(abbreviation: 'T1') + + task = project.task_for_task_definition(task_definition) + claimed_at = Time.zone.parse('2026-04-01 10:30:00 UTC') + create( + :overflow_task_claim_log, + unit_id: unit.id, + task_id: task.id, + claimed_by_unit_role_id: tutor_role.id, + claimed_by_user_id: tutor_user.id, + original_tutor_user_id: tutor_user.id, + student_user_id: student.id, + days_awaiting_feedback: 12, + claimed_at: claimed_at + ) + + add_auth_header_for(user: convenor) + + get "/api/csv/units/#{unit.id}/overflow_task_claims" + + assert_equal 200, last_response.status + assert_equal unit.overflow_task_claims_csv, last_response_body['result'] + + rows = CSV.parse(last_response_body['result'], headers: true) + assert_equal 1, rows.length + assert_equal 'ATutor Tutor', rows[0]['Tutor who claimed'] + assert_equal tutor_role.id.to_s, rows[0]['Claiming Unit Role ID'] + assert_equal 'ATutor Tutor', rows[0]['Original Tutor'] + assert_equal 'student-one', rows[0]['Student Username'] + assert_equal 's1234567', rows[0]['Student ID'] + assert_equal task.id.to_s, rows[0]['Task ID'] + assert_equal 'T1', rows[0]['Task Definition'] + assert_equal '12', rows[0]['Days Awaiting Feedback'] + assert_equal claimed_at.to_s, rows[0]['Timestamp'] + assert_equal claimed_at, Time.zone.parse(rows[0]['Timestamp']) + Sidekiq::Testing.fake! + end + end + #47: Testing for unit ID error with empty user ID #GET /api/csv/units/{id}/tutor_assessments def test_download_csv_stats_tutor_assessed_with_empty_unit_id diff --git a/test/api/tasks_api_test.rb b/test/api/tasks_api_test.rb index cfdeae172b..016f789e7f 100644 --- a/test/api/tasks_api_test.rb +++ b/test/api/tasks_api_test.rb @@ -1068,4 +1068,51 @@ def test_task_target_date_permissions unit.update!(allow_flexible_dates: false) end + def test_claim_overflow_task_creates_analytics_log + travel_to Time.zone.parse('2026-04-15 10:30:00 UTC') do + unit = create( + :unit, + student_count: 1, + unenrolled_student_count: 0, + part_enrolled_student_count: 0, + inactive_student_count: 0, + task_count: 1, + stream_count: 0, + tutorials: 1 + ) + original_tutor = create(:user, :tutor) + original_tutor_role = unit.employ_staff(original_tutor, Role.tutor) + unit.tutorials.first.update!(unit_role: original_tutor_role) + + claiming_tutor = create(:user, :tutor) + claiming_role = unit.employ_staff(claiming_tutor, Role.tutor) + claiming_role.update!(can_mark_overflow_tasks: true) + + project = unit.active_projects.first + task_definition = unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + task.update!(submission_date: 12.days.ago) + + add_auth_header_for(user: claiming_tutor) + + assert_difference('OverflowTaskClaim.count', 1) do + assert_difference('OverflowTaskClaimLog.count', 1) do + post "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/claim_overflow_task" + end + end + + assert_equal 201, last_response.status + + claim_log = OverflowTaskClaimLog.order(:id).last + assert_equal unit, claim_log.unit + assert_equal task, claim_log.task + assert_equal claiming_role, claim_log.claimed_by_unit_role + assert_equal claiming_tutor, claim_log.claimed_by_user + assert_equal original_tutor, claim_log.original_tutor_user + assert_equal project.student, claim_log.student_user + assert_equal 12, claim_log.days_awaiting_feedback + assert_equal Time.zone.now, claim_log.claimed_at + end + end + end diff --git a/test/factories/overflow_task_claim_logs.rb b/test/factories/overflow_task_claim_logs.rb new file mode 100644 index 0000000000..a853348015 --- /dev/null +++ b/test/factories/overflow_task_claim_logs.rb @@ -0,0 +1,14 @@ +FactoryBot.define do + factory :overflow_task_claim_log do + task { create(:task) } + unit { task.project.unit } + claimed_by_unit_role do + create(:unit_role, unit: unit, user: create(:user, :tutor), role: Role.tutor) + end + claimed_by_user { claimed_by_unit_role.user } + original_tutor_user_id { task.tutor&.id } + student_user_id { task.project.student.id } + days_awaiting_feedback { task.days_awaiting_feedback } + claimed_at { Time.zone.now } + end +end diff --git a/test/factories/overflow_task_claims.rb b/test/factories/overflow_task_claims.rb index b4def1a12c..1bb3abb700 100644 --- a/test/factories/overflow_task_claims.rb +++ b/test/factories/overflow_task_claims.rb @@ -1,5 +1,6 @@ FactoryBot.define do factory :overflow_task_claim do - + task + claimed_by_unit_role { create(:unit_role, unit: task.project.unit, user: create(:user, :tutor), role: Role.tutor) } end end From eb5d872aafd6c475a7c8344ffb110feff22a67a9 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:07:26 +1000 Subject: [PATCH 088/199] chore(release): 11.0.0-9 --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3505a27c96..e78b935362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-9](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-8...v11.0.0-9) (2026-06-11) + + +### Features + +* expose route to fetch tasks waiting for pdf gen ([5d3407e](https://github.com/b0ink/doubtfire-deploy/commit/5d3407e2c897fa05f6e5c6ee41d8e0379fb6a367)) +* overflow task claim logs ([#614](https://github.com/b0ink/doubtfire-deploy/issues/614)) ([065180f](https://github.com/b0ink/doubtfire-deploy/commit/065180f248db7e80dd86f0491822e2051a9fa220)) + + +### Bug Fixes + +* ensure more reliable test ([93599d2](https://github.com/b0ink/doubtfire-deploy/commit/93599d27d736ebfaf7b997dfd82d764a45195881)) + ## [11.0.0-8](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-7...v11.0.0-8) (2026-06-09) From f7b877c9759089ce1bb4c4e99490a039c0761181 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:31:57 +1000 Subject: [PATCH 089/199] feat: add column for days awaiting feedback including breaks --- app/models/task.rb | 7 +++++++ app/models/unit.rb | 6 ++++-- test/models/task_test.rb | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/models/task.rb b/app/models/task.rb index cfd9b6b8ba..965bdd311a 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -421,6 +421,13 @@ def days_awaiting_feedback(now_time = Time.zone.now) ([0, current_time - submission_time - paused_seconds].max / 1.day).floor end + # Excludes any breaks that would otherwise "pause" feedback + def calendar_days_awaiting_feedback(now_time = Time.zone.now) + return 0 if submission_date.blank? + + [0, (now_time.to_date - submission_date.to_date).to_i].max + end + def complete? status == :complete end diff --git a/app/models/unit.rb b/app/models/unit.rb index 634942e47f..2a61b7dbe3 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -1751,7 +1751,8 @@ def days_awaiting_feedback_by_tutorial_csv 'Project ID', 'Task Definition', 'Task ID', - 'Days Awaiting Feedback' + 'Days Awaiting Feedback', + 'Days Awaiting Feedback (Incl. Breaks)' ] # Add data @@ -1791,7 +1792,8 @@ def days_awaiting_feedback_by_tutorial_csv row['project_id'], row['task_abbr'], row['task_id'], - row.days_awaiting_feedback + row.days_awaiting_feedback, + row.calendar_days_awaiting_feedback ] end end diff --git a/test/models/task_test.rb b/test/models/task_test.rb index d7e7718fad..abb30ec69c 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -140,6 +140,7 @@ def test_days_awaiting_feedback_pauses_during_break task.update!(submission_date: Time.zone.parse('2026-03-29 00:00:00 UTC')) assert_equal 3.0, task.days_awaiting_feedback + assert_equal 12.0, task.calendar_days_awaiting_feedback end travel_back end From 90cf453d17ba2fa4d656c2d087f6a32a86cfd5eb Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 11 Jun 2026 15:18:48 +1000 Subject: [PATCH 090/199] chore: refresh feedback in active units only --- app/sidekiq/refresh_moderation_feedback_timestamps_job.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/sidekiq/refresh_moderation_feedback_timestamps_job.rb b/app/sidekiq/refresh_moderation_feedback_timestamps_job.rb index de38dc9a80..ad3d8abf77 100644 --- a/app/sidekiq/refresh_moderation_feedback_timestamps_job.rb +++ b/app/sidekiq/refresh_moderation_feedback_timestamps_job.rb @@ -6,6 +6,8 @@ class RefreshModerationFeedbackTimestampsJob def perform ModeratedTask .where(state: %i[open waiting_for_new_feedback]) + .joins(task: { project: :unit }) + .where(units: { active: true }) .includes(task: :comments) .find_each do |moderated_task| task = moderated_task.task From ba7ca98cdda21cf2988c12aa6c97c56352661033 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 17 Jun 2026 10:41:25 +1000 Subject: [PATCH 091/199] chore: enforce utf8mb4 encoding in development --- config/database.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/database.yml b/config/database.yml index eaaf83f653..7b132ba964 100644 --- a/config/database.yml +++ b/config/database.yml @@ -5,6 +5,8 @@ development: password: <%= Rails.application.credentials.dig(:database, :development, :password) || ENV['DF_DEV_DB_PASSWORD'] %> host: <%= Rails.application.credentials.dig(:database, :development, :host) || ENV['DF_DEV_DB_HOST'] %> min_messages: warning + encoding: utf8mb4 + collation: utf8mb4_general_ci test: adapter: <%= Rails.application.credentials.dig(:database, :test, :adapter) || ENV['DF_TEST_DB_ADAPTER'] %> From 433e60d0e31783fff4c4f7adb29f7992c6603a24 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 17 Jun 2026 10:44:48 +1000 Subject: [PATCH 092/199] chore: enforce utf8mb4 encoding in tests --- config/database.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/database.yml b/config/database.yml index 7b132ba964..81ebb08640 100644 --- a/config/database.yml +++ b/config/database.yml @@ -15,7 +15,9 @@ test: password: <%= Rails.application.credentials.dig(:database, :test, :password) || ENV['DF_TEST_DB_PASSWORD'] %> host: <%= Rails.application.credentials.dig(:database, :test, :host) || ENV['DF_TEST_DB_HOST'] %> min_messages: warning - + encoding: utf8mb4 + collation: utf8mb4_general_ci + staging: adapter: <%= Rails.application.credentials.dig(:database, :staging, :adapter) || ENV['DF_STAGING_DB_ADAPTER'] %> host: <%= Rails.application.credentials.dig(:database, :staging, :host) || ENV['DF_STAGING_DB_HOST'] %> From b0bf4d3b905c09620af12f2aef8ad916ed966707 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 17 Jun 2026 10:45:16 +1000 Subject: [PATCH 093/199] test: ensure native uft8 emojis can be added --- test/models/task_test.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/models/task_test.rb b/test/models/task_test.rb index abb30ec69c..ab80938026 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -49,6 +49,21 @@ def test_comments_for_user end end + def test_add_text_comment_with_raw_utf8_emoji_bytes + project = FactoryBot.create(:project) + unit = project.unit + convenor = unit.main_convenor_user + task_definition = unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + comment_text = "\xF0\x9F\x98\x82".force_encoding(Encoding::UTF_8) + + comment = task.add_text_comment(convenor, comment_text) + + assert comment.persisted? + assert_equal comment_text, comment.comment + assert_equal comment_text, TaskComment.find(comment.id).read_attribute(:comment) + end + def test_trigger_transition_allows_assessment_outcomes_without_feedback_check_by_default project = FactoryBot.create(:project) unit = project.unit From 4f95d61c8d897183ce83ca6d96b06570e51e4def Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Wed, 17 Jun 2026 11:35:56 +1000 Subject: [PATCH 094/199] feat: submission history (#636) * feat: submission history * chore: combine migrations * chore: only enable submission history if overseer was originally enabled * refactor: store submission history in zip * test: fix submission history test stubbing * refactor: dont store submissions in zip within the history archive --- .../entities/overseer_assessment_entity.rb | 1 + app/api/entities/submission_history_entity.rb | 13 + app/api/submission/portfolio_evidence_api.rb | 55 +++- app/models/overseer_assessment.rb | 65 ++--- app/models/submission_history.rb | 172 +++++++++++++ app/models/task.rb | 5 + app/models/task_definition.rb | 16 +- app/sidekiq/accept_overseer_job.rb | 39 +-- app/sidekiq/accept_submission_job.rb | 17 +- app/sidekiq/create_submission_history_job.rb | 27 ++ ...60611053643_create_submission_histories.rb | 243 ++++++++++++++++++ db/schema.rb | 13 +- lib/tasks/maintenance.rake | 29 +++ test/factories/overseer_assessments.rb | 5 +- test/factories/submission_histories.rb | 6 + test/models/overseer_assessment_test.rb | 7 +- test/models/submission_history_test.rb | 119 +++++++++ test/models/task_definition_test.rb | 27 ++ 18 files changed, 773 insertions(+), 86 deletions(-) create mode 100644 app/api/entities/submission_history_entity.rb create mode 100644 app/models/submission_history.rb create mode 100644 app/sidekiq/create_submission_history_job.rb create mode 100644 db/migrate/20260611053643_create_submission_histories.rb create mode 100644 test/factories/submission_histories.rb create mode 100644 test/models/submission_history_test.rb diff --git a/app/api/entities/overseer_assessment_entity.rb b/app/api/entities/overseer_assessment_entity.rb index e44d84d4ee..4e893a5d0b 100644 --- a/app/api/entities/overseer_assessment_entity.rb +++ b/app/api/entities/overseer_assessment_entity.rb @@ -2,6 +2,7 @@ module Entities class OverseerAssessmentEntity < Grape::Entity expose :id expose :task_id + expose :submission_history_id expose :submission_timestamp expose :result_task_status expose :status diff --git a/app/api/entities/submission_history_entity.rb b/app/api/entities/submission_history_entity.rb new file mode 100644 index 0000000000..d33eee7193 --- /dev/null +++ b/app/api/entities/submission_history_entity.rb @@ -0,0 +1,13 @@ +module Entities + class SubmissionHistoryEntity < Grape::Entity + expose :id + expose :task_id + expose :submission_timestamp + expose :created_at + expose :has_submission_files?, as: :has_submission_files + + expose :overseer_assessment_id do |history| + history.overseer_assessment&.id + end + end +end diff --git a/app/api/submission/portfolio_evidence_api.rb b/app/api/submission/portfolio_evidence_api.rb index d17d340a27..82f87088cf 100644 --- a/app/api/submission/portfolio_evidence_api.rb +++ b/app/api/submission/portfolio_evidence_api.rb @@ -182,6 +182,48 @@ def self.logger present result, with: Entities::OverseerAssessmentEntity end + desc 'Get all retained submission histories for a task' + get '/projects/:id/task_def_id/:task_definition_id/submission_histories' do + project = Project.find(params[:id]) + task_definition = project.unit.task_definitions.find(params[:task_definition_id]) + + unless authorise? current_user, project.unit, :provide_feedback + error!({ error: "Not authorised to get submission history for task '#{task_definition.name}'" }, 401) + end + + task = project.task_for_task_definition(task_definition) + unless task + error!({ error: 'A submission for this task definition has never been created' }, 404) + end + + present task.submission_histories.order(submission_timestamp: :desc), + with: Entities::SubmissionHistoryEntity + end + + desc 'Download a retained submission history archive' + get '/projects/:id/task_def_id/:task_definition_id/submission_histories/:history_id/files' do + project = Project.find(params[:id]) + task_definition = project.unit.task_definitions.find(params[:task_definition_id]) + + unless authorise? current_user, project.unit, :provide_feedback + error!({ error: "Not authorised to get submission history for task '#{task_definition.name}'" }, 401) + end + + task = project.task_for_task_definition(task_definition) + history = task&.submission_histories&.find_by(id: params[:history_id]) + error!({ error: 'Submission history was not found' }, 404) unless history + error!({ error: 'Submission history files are not available' }, 404) unless history.has_submission_files? + + filename = "#{project.student.username}-#{task_definition.abbreviation}-#{history.submission_timestamp}.zip" + + content_type 'application/octet-stream' + header['Content-Disposition'] = "attachment; filename=#{filename}" + submission_zip_data = history.submission_zip_data + header['Content-Length'] = submission_zip_data.bytesize.to_s + env['api.format'] = :binary + body submission_zip_data + end + desc 'Trigger an overseer assessment to run again' put '/projects/:id/task_def_id/:task_definition_id/overseer_assessment/:oa_id/trigger' do project = Project.find(params[:id]) @@ -280,12 +322,12 @@ def self.logger error!({ error: 'A submission for this task definition have never been created' }, 401) end - oa = task.overseer_assessments.find_by(submission_timestamp: params[:timestamp]) - unless oa - error!({ error: "No overseer assessment found for timestamp '#{params[:timestamp]}'" }, 404) + history = task.submission_histories.find_by(submission_timestamp: params[:timestamp]) + unless history + error!({ error: "No submission history found for timestamp '#{params[:timestamp]}'" }, 404) end - unless oa.has_submission_files? + unless history.has_submission_files? error!({ error: "No submission files are available for timestamp '#{params[:timestamp]}'" }, 404) end @@ -294,7 +336,10 @@ def self.logger content_type 'application/octet-stream' header['Content-Disposition'] = "attachment; filename=#{filename}" - stream_file oa.submission_zip_file_name + submission_zip_data = history.submission_zip_data + header['Content-Length'] = submission_zip_data.bytesize.to_s + env['api.format'] = :binary + body submission_zip_data end desc 'Get the result of the submission of a task made last' diff --git a/app/models/overseer_assessment.rb b/app/models/overseer_assessment.rb index 00772c9bbd..d01a3210ed 100644 --- a/app/models/overseer_assessment.rb +++ b/app/models/overseer_assessment.rb @@ -1,6 +1,7 @@ # rubocop:disable Rails/Output class OverseerAssessment < ApplicationRecord belongs_to :task, optional: false + belongs_to :submission_history, optional: false has_one :project, through: :task has_many :assessment_comments, as: :commentable, dependent: :destroy @@ -11,12 +12,16 @@ class OverseerAssessment < ApplicationRecord validates :submission_timestamp, presence: true validates :submission_timestamp, uniqueness: { scope: :task_id } + validates :submission_history_id, uniqueness: true + validate :submission_history_matches_task enum :status, { pre_queued: 0, passed: 1, failed: 2 } - after_destroy :delete_associated_files - + def submission_history_matches_task + return if submission_history.nil? || task.nil? || submission_history.task_id == task_id + errors.add(:submission_history, 'must belong to the same task') + end def self.student_notification_grace_period Doubtfire::Application.config.overseer_student_notification_grace_period @@ -72,13 +77,14 @@ def self.student_notification_grace_period # TODO: we might not have an overseerStepResult because a new test was added later # Creates an OverseerAssessment object for a new submission - def self.create_for(task, test_submission) + def self.create_for(submission_history, test_submission) # Create only if: # unit's assessment is enabled && # task's assessment is enabled && # task definition has an assessment resources zip file && # task has a student submission + task = submission_history.task task_definition = task.task_definition unit = task_definition.unit @@ -92,51 +98,18 @@ def self.create_for(task, test_submission) return nil if docker_image_name_tag.nil? || docker_image_name_tag.strip.empty? - result = OverseerAssessment.create!( + OverseerAssessment.create!( task: task, + submission_history: submission_history, status: :pre_queued, - submission_timestamp: Time.now.utc.to_i + submission_timestamp: submission_history.submission_timestamp ) - - # Create the submission folder and give access - FileUtils.mkdir_p result.output_path - result.grant_access_to_submission - - result.copy_latest_files_to_submission - - result - end - - def has_submission_files? - File.exist? submission_zip_file_name - end - - def submission_zip_file_name - "#{output_path}/submission.zip" - end - - def grant_access_to_submission - # TODO: Use FACL instead in future. - `chmod o+w #{output_path}` - end - - def copy_latest_files_to_submission - zip_file_path = submission_zip_file_name - - if task.has_new_files? - puts "Copying new files to submission at: #{zip_file_path}" - # Generate a zip file for this particular submission with timestamp value and put it here - task.compress_new_to_done zip_file_path: zip_file_path, rm_task_dir: false, rename_files: true - else - puts "Copying done file to submission at: #{zip_file_path}" - task.copy_done_to zip_file_path - end end - # Path to where the submission and output are stored - includes the submission when it is to be processed - def output_path - FileHelper.task_submission_identifier_path_with_timestamp(:done, task, submission_timestamp) - end + delegate :has_submission_files?, + :submission_zip_file_name, + :output_path, + to: :submission_history def latest_assessment_comment assessment_comments.order(created_at: :desc, id: :desc).first @@ -212,7 +185,7 @@ def send_to_overseer(test_submission: false) return { error: "This assessment is no longer setup for automated feedback. Automated feedback is turned off at either the unit or task level, or the task does not have the scripts needed to automate assessment." } end - unless File.exist? submission_zip_file_name + unless has_submission_files? puts "ERROR: Student submission history zip file doesn't exist #{submission_zip_file_name}. Unable to send - OverseerAssessment #{id}" return { error: "We no longer have the files associated with this submission. Please test a later submission, or upload your work again." } end @@ -315,10 +288,6 @@ def update_from_output(work_dir_path) self.save! end - def delete_associated_files - FileUtils.rm_rf output_path - end - def base64?(value) value.is_a?(String) && Base64.strict_encode64(Base64.decode64(value)) == value end diff --git a/app/models/submission_history.rb b/app/models/submission_history.rb new file mode 100644 index 0000000000..745925ac99 --- /dev/null +++ b/app/models/submission_history.rb @@ -0,0 +1,172 @@ +require 'zip' +require 'stringio' + +class SubmissionHistory < ApplicationRecord + belongs_to :task, optional: false + has_one :overseer_assessment, dependent: :destroy + + validates :submission_timestamp, presence: true, uniqueness: { scope: :task_id } + + after_destroy :delete_associated_files + + def self.enabled_requirements(task) + task.upload_requirements.each_index.select do |index| + task.upload_requirements[index]['submission_history'] == true + end + end + + def self.create_archive!(task, submission_timestamp) + if exists?(task: task, submission_timestamp: submission_timestamp.to_s) + raise ActiveRecord::RecordNotUnique, 'Submission history already exists for this task and timestamp' + end + + enabled_indexes = enabled_requirements(task) + raise 'No upload requirements are enabled for submission history' if enabled_indexes.empty? + + source_path = FileHelper.zip_file_path_for_done_task(task) + raise "Submission file not found: #{source_path}" unless File.exist?(source_path) + + history = new(task: task, submission_timestamp: submission_timestamp.to_s) + FileUtils.mkdir_p(history.output_path) + + temporary_history = "#{history.archive_file_name}.tmp-#{SecureRandom.hex(8)}" + copied_files = 0 + archive_updated = false + begin + Zip::File.open(temporary_history, create: true) do |destination| + copy_archive_entries(history.archive_file_name, destination) if File.exist?(history.archive_file_name) + + Zip::File.open(source_path) do |source| + source.each do |entry| + next if entry.name_is_directory? + + file_name = entry.name.split('/').last + next unless file_name&.match?(/^\d{3}-(?:document|code|image|zip|archive)/) + next unless enabled_indexes.include?(file_name.to_i) + + destination.get_output_stream(File.join(history.entry_prefix, entry.name)) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + copied_files += 1 + end + end + end + + raise 'No selected submission files were found in the completed submission' if copied_files.zero? + + FileUtils.mv(temporary_history, history.archive_file_name) + archive_updated = true + system('chmod', 'o+w', history.output_path) + history.save! + history + ensure + FileUtils.rm_f(temporary_history) + history.delete_associated_files if archive_updated && !history.persisted? + if !history.persisted? && Dir.exist?(history.output_path) && Dir.empty?(history.output_path) + FileUtils.rm_rf(history.output_path) + end + end + end + + def output_path + FileHelper.task_submission_identifier_path(:done, task) + end + + def archive_file_name + File.join(output_path, 'history.zip') + end + + def entry_prefix + "#{FileHelper.sanitized_path(submission_timestamp.to_s)}/" + end + + def submission_entry_prefix + File.join(entry_prefix, task.id.to_s, '/') + end + + # Kept for Overseer compatibility; submissions are entries within this archive. + def submission_zip_file_name + archive_file_name + end + + def submission_zip_data + buffer = Zip::OutputStream.write_buffer do |output| + Zip::File.open(archive_file_name) do |archive| + submission_entries(archive).each do |entry| + output.put_next_entry(entry.name.delete_prefix(entry_prefix)) + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + end + + buffer.string + end + + def has_submission_files? # rubocop:disable Naming/PredicateName + return false unless File.exist?(archive_file_name) + + Zip::File.open(archive_file_name) { |archive| submission_entries(archive).any? } + rescue Zip::Error + false + end + + def delete_associated_files + return unless File.exist?(archive_file_name) + + temporary_history = "#{archive_file_name}.tmp-#{SecureRandom.hex(8)}" + + Zip::File.open(temporary_history, create: true) do |destination| + self.class.copy_archive_entries(archive_file_name, destination, excluding_prefix: entry_prefix) + end + + if Zip::File.open(temporary_history) { |archive| archive.entries.empty? } + FileUtils.rm_f(archive_file_name) + FileUtils.rm_f(temporary_history) + FileUtils.rm_rf(output_path) if Dir.empty?(output_path) + else + FileUtils.mv(temporary_history, archive_file_name) + end + ensure + FileUtils.rm_f(temporary_history) if temporary_history + end + + def submission_entries(archive) + archive.entries.reject(&:name_is_directory?).select do |entry| + entry.name.start_with?(submission_entry_prefix) + end + end + + def self.copy_archive_entries(source_path, destination, excluding_prefix: nil) + Zip::File.open(source_path) do |source| + source.each do |entry| + next if excluding_prefix && entry.name.start_with?(excluding_prefix) + + if entry.name_is_directory? + destination.mkdir(entry.name) unless destination.find_entry(entry.name) + else + destination.get_output_stream(entry.name) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + end + end + end + + def self.pending_marker_path(task) + File.join(FileHelper.task_submission_identifier_path(:pending, task), 'submission-history') + end + + def self.mark_pending(task) + marker_path = pending_marker_path(task) + FileUtils.mkdir_p(File.dirname(marker_path)) + FileUtils.touch(marker_path) + end + + def self.clear_pending(task) + FileUtils.rm_f(pending_marker_path(task)) + end + + def self.pending?(task) + File.exist?(pending_marker_path(task)) + end +end diff --git a/app/models/task.rb b/app/models/task.rb index 965bdd311a..ff7fa130ed 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -134,6 +134,7 @@ def specific_permission_hash(role, perm_hash, _other) has_many :reverse_moss_similarities, class_name: 'MossTaskSimilarity', dependent: :destroy, inverse_of: :other_task, foreign_key: 'other_task_id' has_many :task_engagements, dependent: :destroy has_many :task_submissions, dependent: :destroy + has_many :submission_histories, dependent: :destroy has_many :overseer_assessments, dependent: :destroy has_many :tii_submissions, dependent: :destroy has_many :test_attempts, dependent: :destroy @@ -1577,6 +1578,10 @@ def accept_submission(current_user, files, ui, contributions, trigger, alignment ui.error!({ 'error' => 'A submission is already being processed. Please wait for the current submission process to complete.' }, 403) end + if SubmissionHistory.pending?(self) + ui.error!({ 'error' => 'Submission history is still being created. Please wait before submitting again.' }, 403) + end + if !test_submission && (overseer_enabled? || task_definition.assessment_enabled) && overseer_assessments.where(status: OverseerAssessment.statuses[:pre_queued]).exists? ui.error!({ 'error' => 'A submission is already waiting for automated feedback. Please wait for the current Overseer job to complete before submitting again.' }, 403) diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index 0a299f3985..e9f1c1444b 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -97,6 +97,7 @@ def self.permissions validates :max_quality_pts, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 100, message: 'must be between 0 and 100' } validate :upload_requirements, :check_upload_requirements_format + validate :submission_history_required_for_overseer validates :description, length: { maximum: 4095, allow_blank: true } @@ -342,8 +343,8 @@ def check_upload_requirements_format req['type'] = 'zip' if req['type'] == 'archive' - # Check keys only contain key, type, name, tii_check, and tii_pct - unless req.keys.excluding('key', 'type', 'name', 'tii_check', 'tii_pct').empty? + # Check keys only contain supported upload requirement settings + unless req.keys.excluding('key', 'type', 'name', 'tii_check', 'tii_pct', 'submission_history').empty? errors.add(:upload_requirements, "has additional values for item #{i + 1} --> #{req.keys.join(' ')}.") end @@ -367,10 +368,21 @@ def check_upload_requirements_format errors.add(:upload_requirements, "the tii_pct for item #{i + 1} is not a non-negative number --> #{req['tii_pct']}.") end + unless req['submission_history'].blank? || [true, false].include?(req['submission_history']) + errors.add(:upload_requirements, "the submission_history for item #{i + 1} is not a boolean --> #{req['submission_history']}.") + end + i += 1 end end + def submission_history_required_for_overseer + return unless assessment_enabled? + return if upload_requirements&.any? { |requirement| requirement['submission_history'] == true } + + errors.add(:upload_requirements, 'must include at least one file in submission history when Overseer is enabled') + end + def number_of_uploaded_files upload_requirements.length end diff --git a/app/sidekiq/accept_overseer_job.rb b/app/sidekiq/accept_overseer_job.rb index 7d00f9017c..de54c078a5 100644 --- a/app/sidekiq/accept_overseer_job.rb +++ b/app/sidekiq/accept_overseer_job.rb @@ -40,7 +40,7 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment work_dir = Rails.root.join("tmp", "overseer", work_dir_name) FileUtils.mkdir_p(work_dir) - extract_student_submission_files(task, submission, work_dir) + extract_student_submission_files(task, submission, work_dir, timestamp) extract_overseer_resource_files(assessment, work_dir) success_status = nil @@ -240,25 +240,32 @@ def run_overseer_step(step:, work_dir:, work_dir_name:, task_id:, timestamp:, do ) end - def extract_student_submission_files(task, submission, work_dir) - # Extract submission files, removing any parent folders - Zip::File.open(submission) do |zip_file| - zip_file.each do |entry| - next if entry.name_is_directory? + def extract_student_submission_files(task, submission, work_dir, timestamp) + # Submission files are stored directly under their timestamp in the task archive. + Zip::File.open(submission) do |history_zip| + prefix = "#{FileHelper.sanitized_path(timestamp.to_s)}/" + entries = history_zip.entries.reject(&:name_is_directory?).select { |entry| entry.name.start_with?(prefix) } + raise "Submission history entries not found for timestamp: #{timestamp}" if entries.empty? - parts = entry.name.split('/')[1..] - next unless parts.length >= 1 + extract_submission_entries(task, entries, work_dir, prefix) + end + end - file_name = parts.first - index = file_name.to_i + def extract_submission_entries(task, entries, work_dir, prefix) + # Extract submission files, removing any parent folders. + entries.each do |entry| + parts = entry.name.delete_prefix(prefix).split('/') + next unless parts.first == task.id.to_s && parts.length >= 2 - file = task.upload_requirements[index] - final_name = file['name'] + file_name = parts.second + index = file_name.to_i - dest_path = File.join(work_dir, final_name) - FileUtils.mkdir_p(File.dirname(dest_path)) - zip_file.extract(entry, dest_path) { true } - end + file = task.upload_requirements[index] + final_name = file['name'] + + dest_path = File.join(work_dir, final_name) + FileUtils.mkdir_p(File.dirname(dest_path)) + entry.extract(dest_path) { true } end end diff --git a/app/sidekiq/accept_submission_job.rb b/app/sidekiq/accept_submission_job.rb index c785943344..3010ea1ac1 100644 --- a/app/sidekiq/accept_submission_job.rb +++ b/app/sidekiq/accept_submission_job.rb @@ -76,17 +76,12 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) task.send_documents_to_tii(user, accepted_tii_eula: accepted_tii_eula) end - if task.overseer_enabled? || test_submission - overseer_assessment = OverseerAssessment.create_for(task, test_submission) - if overseer_assessment.present? - logger.info "Launching Overseer assessment for task_def_id: #{task.task_definition.id} task_id: #{task.id}" - - overseer_assessment.update!(student_notified_at: Time.current) if test_submission - overseer_assessment.send_to_overseer(test_submission: test_submission) - - else - logger.info "Overseer assessment for task_def_id: #{task.task_definition.id} task_id: #{task.id} was not performed #{overseer_assessment.inspect}" - end + if SubmissionHistory.enabled_requirements(task).any? + submission_timestamp = Time.now.utc.to_i + SubmissionHistory.mark_pending(task) + CreateSubmissionHistoryJob.perform_async(task.id, submission_timestamp, test_submission) + elsif task.overseer_enabled? || test_submission + logger.error "Overseer assessment was not performed because task definition #{task.task_definition.id} has no submission history files configured" end rescue StandardError => e # to raise error message to avoid unnecessary retry logger.error e diff --git a/app/sidekiq/create_submission_history_job.rb b/app/sidekiq/create_submission_history_job.rb new file mode 100644 index 0000000000..9efedb25ff --- /dev/null +++ b/app/sidekiq/create_submission_history_job.rb @@ -0,0 +1,27 @@ +class CreateSubmissionHistoryJob + include Sidekiq::Job + include LogHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first, 'submission-history'] }, + on_conflict: :reject, + retry: false + + def perform(task_id, submission_timestamp, test_submission) + task = Task.find(task_id) + history = SubmissionHistory.create_archive!(task, submission_timestamp) + + return unless task.overseer_enabled? || test_submission + + assessment = OverseerAssessment.create_for(history, test_submission) + return if assessment.nil? + + assessment.update!(student_notified_at: Time.current) if test_submission + assessment.send_to_overseer(test_submission: test_submission) + rescue StandardError => e + logger.error e + Sentry.capture_exception(e, extra: { task_id: task_id }) if defined?(Sentry) + ensure + SubmissionHistory.clear_pending(task) if task + end +end diff --git a/db/migrate/20260611053643_create_submission_histories.rb b/db/migrate/20260611053643_create_submission_histories.rb new file mode 100644 index 0000000000..6661f139c1 --- /dev/null +++ b/db/migrate/20260611053643_create_submission_histories.rb @@ -0,0 +1,243 @@ +# rubocop:disable Rails/SkipsModelValidations +require 'zip' + +class CreateSubmissionHistories < ActiveRecord::Migration[8.0] + # Migration-local models provide direct access to the tables as they exist at + # this migration point, without using future application callbacks or validations. + class MigrationTaskDefinition < ApplicationRecord + self.table_name = 'task_definitions' + end + + class MigrationOverseerAssessment < ApplicationRecord + self.table_name = 'overseer_assessments' + end + + class MigrationSubmissionHistory < ApplicationRecord + self.table_name = 'submission_histories' + end + + class MigrationTask < ApplicationRecord + self.table_name = 'tasks' + belongs_to :project, class_name: 'CreateSubmissionHistories::MigrationProject' + end + + class MigrationProject < ApplicationRecord + self.table_name = 'projects' + belongs_to :unit, class_name: 'CreateSubmissionHistories::MigrationUnit' + belongs_to :user, class_name: 'CreateSubmissionHistories::MigrationUser' + + def student + user + end + end + + class MigrationUnit < ApplicationRecord + self.table_name = 'units' + end + + class MigrationUser < ApplicationRecord + self.table_name = 'users' + end + + # `up` applies the migration when moving the database to this version. + def up + # Store one completed submission-history archive per task and timestamp. + create_table :submission_histories do |t| + t.references :task, null: false + t.string :submission_timestamp, null: false + + t.timestamps + end + + add_index :submission_histories, + [:task_id, :submission_timestamp], + unique: true, + name: 'index_submission_histories_on_task_and_timestamp' + + # Enable submission history for existing Overseer task definitions. Preserve + # an explicit value if this key has already been added to a requirement. + MigrationTaskDefinition.reset_column_information + MigrationTaskDefinition.find_each do |task_definition| + requirements = JSON.parse(task_definition.upload_requirements.presence || '[]') + next unless requirements.is_a?(Array) + + requirements.each do |requirement| + next unless requirement.is_a?(Hash) + next if requirement.key?('submission_history') + + requirement['submission_history'] = task_definition.assessment_enabled? + end + + task_definition.update_columns(upload_requirements: requirements.to_json) + rescue JSON::ParserError + next + end + + # Add the association as nullable first because existing Overseer assessments + # do not have a submission-history row yet. + add_reference :overseer_assessments, :submission_history, index: false + + # Refresh Active Record's cached columns after creating the table and column. + MigrationOverseerAssessment.reset_column_information + MigrationSubmissionHistory.reset_column_information + + # Preserve existing Overseer archives by creating a history with the same + # task and timestamp, then linking the assessment to that history. + MigrationOverseerAssessment.find_each do |assessment| + history = MigrationSubmissionHistory.find_or_create_by!( + task_id: assessment.task_id, + submission_timestamp: assessment.submission_timestamp + ) + assessment.update_columns(submission_history_id: history.id) + end + + # Replace each task's timestamp directories with one history.zip. Every + # directory and filename is preserved inside the archive. + MigrationSubmissionHistory.distinct.pluck(:task_id).each do |task_id| + task = MigrationTask.find_by(id: task_id) + archive_existing_history_directories(task) if task + end + + # Once all existing rows are linked, make the association required and + # ensure a submission history cannot belong to multiple Overseer assessments. + change_column_null :overseer_assessments, :submission_history_id, false + add_index :overseer_assessments, :submission_history_id, unique: true + end + + # `down` reverses `up` when rolling the database back from this version. + def down + # Restore timestamp directories before removing the records that identify + # which task archives need to be unpacked. + MigrationSubmissionHistory.distinct.pluck(:task_id).each do |task_id| + task = MigrationTask.find_by(id: task_id) + restore_history_directories(task) if task + end + + # Remove the dependent association before removing its referenced table. + remove_reference :overseer_assessments, :submission_history + + # Restore upload requirements to their shape before this migration. + MigrationTaskDefinition.find_each do |task_definition| + requirements = JSON.parse(task_definition.upload_requirements.presence || '[]') + next unless requirements.is_a?(Array) + + requirements.each do |requirement| + requirement.delete('submission_history') if requirement.is_a?(Hash) + end + + task_definition.update_columns(upload_requirements: requirements.to_json) + rescue JSON::ParserError + next + end + + # Submission-history rows are no longer needed after the association and + # upload-requirement configuration have been removed. + drop_table :submission_histories + end + + private + + def archive_existing_history_directories(task) + task_path = FileHelper.task_submission_identifier_path(:done, task) + return unless Dir.exist?(task_path) + + directories = Dir.children(task_path).filter_map do |name| + path = File.join(task_path, name) + [name, path] if File.directory?(path) + end + return if directories.empty? + + archive_path = File.join(task_path, 'history.zip') + temporary_path = "#{archive_path}.tmp-#{SecureRandom.hex(8)}" + + Zip::File.open(temporary_path, create: true) do |archive| + copy_zip_entries(archive_path, archive) if File.exist?(archive_path) + directories.each { |name, path| add_directory_to_zip(archive, name, path) } + end + + FileUtils.mv(temporary_path, archive_path) + directories.each { |directory| FileUtils.rm_rf(directory.last) } + ensure + FileUtils.rm_f(temporary_path) if temporary_path + end + + def restore_history_directories(task) + task_path = FileHelper.task_submission_identifier_path(:done, task) + archive_path = File.join(task_path, 'history.zip') + return unless File.exist?(archive_path) + + submission_entries = Hash.new { |entries, timestamp| entries[timestamp] = [] } + Zip::File.open(archive_path) do |archive| + archive.each do |entry| + next if entry.name_is_directory? + + timestamp, relative_path = entry.name.split('/', 2) + next unless relative_path + + if relative_path.start_with?("#{task.id}/") + submission_entries[timestamp] << [relative_path, entry.get_input_stream.read] + else + destination = File.join(task_path, timestamp, relative_path) + FileUtils.mkdir_p(File.dirname(destination)) + entry.extract(destination) { true } + end + end + end + + submission_entries.each do |timestamp, entries| + timestamp_path = File.join(task_path, timestamp) + FileUtils.mkdir_p(timestamp_path) + Zip::File.open(File.join(timestamp_path, 'submission.zip'), create: true) do |submission_zip| + entries.each do |relative_path, contents| + submission_zip.get_output_stream(relative_path) { |output| output.write(contents) } + end + end + end + + FileUtils.rm_f(archive_path) + end + + def add_directory_to_zip(archive, root_name, path) + archive.mkdir("#{root_name}/") unless archive.find_entry("#{root_name}/") + + Dir.glob(File.join(path, '**', '*'), File::FNM_DOTMATCH).sort.each do |source| + next if ['.', '..'].include?(File.basename(source)) + + relative_path = File.join(root_name, source.delete_prefix("#{path}/")) + if File.directory?(source) + archive.mkdir("#{relative_path}/") unless archive.find_entry("#{relative_path}/") + elsif File.basename(source) == 'submission.zip' + add_submission_zip_to_history(archive, root_name, source) + else + archive.add(relative_path, source) + end + end + end + + def add_submission_zip_to_history(archive, root_name, submission_path) + Zip::File.open(submission_path) do |submission_zip| + submission_zip.each do |entry| + next if entry.name_is_directory? + + archive.get_output_stream(File.join(root_name, entry.name)) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + end + end + + def copy_zip_entries(source_path, destination) + Zip::File.open(source_path) do |source| + source.each do |entry| + if entry.name_is_directory? + destination.mkdir(entry.name) unless destination.find_entry(entry.name) + else + destination.get_output_stream(entry.name) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + end + end + end +end +# rubocop:enable Rails/SkipsModelValidations diff --git a/db/schema.rb b/db/schema.rb index dd1dbdd7fd..a77cfc3d97 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_06_10_065531) do +ActiveRecord::Schema[8.0].define(version: 2026_06_11_053643) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -349,7 +349,9 @@ t.datetime "updated_at", null: false t.integer "total_steps" t.datetime "student_notified_at" + t.bigint "submission_history_id", null: false t.index ["status", "student_notified_at", "updated_at"], name: "index_overseer_assessments_on_status_notified_updated" + t.index ["submission_history_id"], name: "index_overseer_assessments_on_submission_history_id", unique: true t.index ["task_id", "submission_timestamp"], name: "index_overseer_assessments_on_task_id_and_submission_timestamp", unique: true t.index ["task_id"], name: "index_overseer_assessments_on_task_id" end @@ -481,6 +483,15 @@ t.index ["user_id"], name: "index_staff_notes_on_user_id" end + create_table "submission_histories", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "task_id", null: false + t.string "submission_timestamp", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["task_id", "submission_timestamp"], name: "index_submission_histories_on_task_and_timestamp", unique: true + t.index ["task_id"], name: "index_submission_histories_on_task_id" + end + create_table "task_comments", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "task_id", null: false t.bigint "user_id", null: false diff --git a/lib/tasks/maintenance.rake b/lib/tasks/maintenance.rake index f2bc661554..d7053b6e2b 100644 --- a/lib/tasks/maintenance.rake +++ b/lib/tasks/maintenance.rake @@ -1,6 +1,7 @@ require_all 'lib/helpers' require 'sidekiq/api' +# rubocop:disable Metrics/BlockLength namespace :maintenance do def sidekiq_job_present_in_workers_or_default_queue?(&matcher) Sidekiq::Workers.new.each do |_process_id, _thread_id, work| @@ -37,6 +38,12 @@ namespace :maintenance do end end + def create_submission_history_job_present?(task_id) + sidekiq_job_present_in_workers_or_default_queue? do |job_class, job_args| + job_class == 'CreateSubmissionHistoryJob' && job_args.first.to_i == task_id + end + end + def notify_failed_submission(task, message) if task.project.student.receive_task_notifications begin @@ -154,6 +161,21 @@ namespace :maintenance do end end + def clear_abandoned_submission_history_markers! + stale_before = 10.minutes.ago + marker_pattern = File.join(FileHelper.root_submission_history_dir, '**', 'pending', '*', 'submission-history') + + Dir.glob(marker_pattern).each do |marker_path| + next unless File.mtime(marker_path) < stale_before + + task_id = File.basename(File.dirname(marker_path)).to_i + next if create_submission_history_job_present?(task_id) + + Rails.logger.error "Clearing abandoned submission history marker for task #{task_id}" + FileUtils.rm_f(marker_path) + end + end + desc 'Cleanup temporary files' task cleanup: [:environment] do path = FileHelper.tmp_file_dir @@ -185,6 +207,7 @@ namespace :maintenance do AuthToken.destroy_old_tokens clear_abandoned_submissions! + clear_abandoned_submission_history_markers! clear_abandoned_overseer_assessments! end @@ -198,6 +221,11 @@ namespace :maintenance do clear_abandoned_overseer_assessments! end + desc 'Clear stale submission history markers with no queued or running job' + task clear_abandoned_submission_history_markers: [:environment] do + clear_abandoned_submission_history_markers! + end + desc 'Remove PDFs from old submissions and archive units' task archive_submissions: [:environment] do archive_period = Doubtfire::Application.config.unit_archive_after_period @@ -239,3 +267,4 @@ namespace :maintenance do `find #{FileHelper.root_portfolio_dir} -name "*pdf.old" -exec rm {} \;` end end +# rubocop:enable Metrics/BlockLength diff --git a/test/factories/overseer_assessments.rb b/test/factories/overseer_assessments.rb index cd1020a653..d858906fb6 100644 --- a/test/factories/overseer_assessments.rb +++ b/test/factories/overseer_assessments.rb @@ -1,7 +1,8 @@ FactoryBot.define do factory :overseer_assessment do - task nil - submission_timestamp { "MyString" } + submission_history + task { submission_history.task } + submission_timestamp { submission_history.submission_timestamp } result_task_status { "MyString" } end end diff --git a/test/factories/submission_histories.rb b/test/factories/submission_histories.rb new file mode 100644 index 0000000000..974aa49390 --- /dev/null +++ b/test/factories/submission_histories.rb @@ -0,0 +1,6 @@ +FactoryBot.define do + factory :submission_history do + task + sequence(:submission_timestamp) { |n| "#{Time.current.to_i}-#{n}" } + end +end diff --git a/test/models/overseer_assessment_test.rb b/test/models/overseer_assessment_test.rb index c95eb97637..2bcd58f3ba 100644 --- a/test/models/overseer_assessment_test.rb +++ b/test/models/overseer_assessment_test.rb @@ -59,10 +59,15 @@ def create_assessment(status:, age:, task: nil, create_comment: false) project.task_for_task_definition(unit.task_definitions.first) end + submission_timestamp = "#{Time.current.to_i}-#{SecureRandom.hex(2)}" assessment = OverseerAssessment.create!( task: task, + submission_history: SubmissionHistory.create!( + task: task, + submission_timestamp: submission_timestamp + ), status: status, - submission_timestamp: "#{Time.current.to_i}-#{SecureRandom.hex(2)}" + submission_timestamp: submission_timestamp ) assessment.update_columns(created_at: age.ago, updated_at: age.ago) diff --git a/test/models/submission_history_test.rb b/test/models/submission_history_test.rb new file mode 100644 index 0000000000..266d987309 --- /dev/null +++ b/test/models/submission_history_test.rb @@ -0,0 +1,119 @@ +require 'test_helper' +require 'tmpdir' +require 'zip' + +class SubmissionHistoryTest < ActiveSupport::TestCase + def test_creates_archive_with_only_selected_upload_requirements + unit = FactoryBot.create(:unit, task_count: 1) + task = unit.active_projects.first.task_for_task_definition(unit.task_definitions.first) + task.task_definition.update!( + assessment_enabled: false, + upload_requirements: [ + { 'key' => 'file0', 'name' => 'main.rb', 'type' => 'code', 'submission_history' => true }, + { 'key' => 'file1', 'name' => 'report.pdf', 'type' => 'document', 'submission_history' => false } + ] + ) + + Dir.mktmpdir do |dir| + source_path = File.join(dir, 'done.zip') + output_path = File.join(dir, 'history') + create_source_archive(source_path, task.id) + + with_file_helper_methods( + zip_file_path_for_done_task: source_path, + task_submission_identifier_path: output_path + ) do + history = SubmissionHistory.create_archive!(task, '12345') + + assert history.persisted? + Zip::File.open(history.archive_file_name) do |archive| + assert archive.find_entry("12345/#{task.id}/000-code.rb") + assert_nil archive.find_entry("12345/#{task.id}/001-document.pdf") + end + + Zip::File.open_buffer(StringIO.new(history.submission_zip_data)) do |download| + assert download.find_entry("#{task.id}/000-code.rb") + assert_nil download.find_entry("#{task.id}/001-document.pdf") + end + end + end + end + + def test_does_not_create_record_when_archive_copy_fails + unit = FactoryBot.create(:unit, task_count: 1) + task = unit.active_projects.first.task_for_task_definition(unit.task_definitions.first) + task.task_definition.update!( + assessment_enabled: false, + upload_requirements: [ + { 'key' => 'file0', 'name' => 'main.rb', 'type' => 'code', 'submission_history' => true } + ] + ) + + assert_no_difference('SubmissionHistory.count') do + assert_raises(RuntimeError) do + with_file_helper_methods(zip_file_path_for_done_task: '/missing/submission.zip') do + SubmissionHistory.create_archive!(task, '12345') + end + end + end + end + + def test_keeps_multiple_timestamps_in_one_task_archive + unit = FactoryBot.create(:unit, task_count: 1) + task = unit.active_projects.first.task_for_task_definition(unit.task_definitions.first) + task.task_definition.update!( + assessment_enabled: false, + upload_requirements: [ + { 'key' => 'file0', 'name' => 'main.rb', 'type' => 'code', 'submission_history' => true } + ] + ) + + Dir.mktmpdir do |dir| + source_path = File.join(dir, 'done.zip') + output_path = File.join(dir, 'history') + create_source_archive(source_path, task.id) + + with_file_helper_methods( + zip_file_path_for_done_task: source_path, + task_submission_identifier_path: output_path + ) do + first = SubmissionHistory.create_archive!(task, '12345') + SubmissionHistory.create_archive!(task, '67890') + + Zip::File.open(first.archive_file_name) do |archive| + assert archive.find_entry("12345/#{task.id}/000-code.rb") + assert archive.find_entry("67890/#{task.id}/000-code.rb") + end + + first.destroy! + + Zip::File.open(File.join(output_path, 'history.zip')) do |archive| + assert_nil archive.find_entry("12345/#{task.id}/000-code.rb") + assert archive.find_entry("67890/#{task.id}/000-code.rb") + end + end + end + end + + private + + def create_source_archive(path, task_id) + Zip::File.open(path, create: true) do |zip| + zip.get_output_stream("#{task_id}/000-code.rb") { |file| file.write('puts "hello"') } + zip.get_output_stream("#{task_id}/001-document.pdf") { |file| file.write('%PDF') } + end + end + + def with_file_helper_methods(replacements) + originals = replacements.to_h { |name, _value| [name, FileHelper.method(name)] } + replacements.each do |name, value| + FileHelper.define_singleton_method(name) { |*_args| value } + end + + yield + ensure + originals&.each do |name, implementation| + FileHelper.define_singleton_method(name, implementation) + end + end +end diff --git a/test/models/task_definition_test.rb b/test/models/task_definition_test.rb index d739028f55..8d3500e7cc 100644 --- a/test/models/task_definition_test.rb +++ b/test/models/task_definition_test.rb @@ -8,6 +8,33 @@ def app Rails.application end + def test_overseer_requires_a_submission_history_upload + task_definition = FactoryBot.build( + :task_definition, + assessment_enabled: true, + upload_requirements: [ + { 'key' => 'file0', 'name' => 'main.rb', 'type' => 'code', 'submission_history' => false } + ] + ) + + assert_not task_definition.valid? + assert_includes task_definition.errors[:upload_requirements], + 'must include at least one file in submission history when Overseer is enabled' + end + + def test_overseer_accepts_a_submission_history_upload + task_definition = FactoryBot.build( + :task_definition, + assessment_enabled: true, + upload_requirements: [ + { 'key' => 'file0', 'name' => 'main.rb', 'type' => 'code', 'submission_history' => true } + ] + ) + + task_definition.validate + assert_empty task_definition.errors[:upload_requirements] + end + def test_default_quality_points test_unit = Unit.first td = TaskDefinition.new({ From 91d0b19fdd8261dec6ff484f3bc7349b9b1251f8 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 17 Jun 2026 12:16:18 +1000 Subject: [PATCH 095/199] chore(release): 11.0.0-10 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e78b935362..93240005d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-10](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-9...v11.0.0-10) (2026-06-17) + + +### Features + +* add column for days awaiting feedback including breaks ([f7b877c](https://github.com/b0ink/doubtfire-deploy/commit/f7b877c9759089ce1bb4c4e99490a039c0761181)) +* submission history ([#636](https://github.com/b0ink/doubtfire-deploy/issues/636)) ([4f95d61](https://github.com/b0ink/doubtfire-deploy/commit/4f95d61c8d897183ce83ca6d96b06570e51e4def)) + ## [11.0.0-9](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-8...v11.0.0-9) (2026-06-11) From 6836a20e582473eff4edb41e2fd1f6c4a5a9751d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 17 Jun 2026 14:46:57 +1000 Subject: [PATCH 096/199] fix: serve public web cal as raw text --- app/api/webcal_public_api.rb | 5 +++-- test/api/webcal_api_test.rb | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/api/webcal_public_api.rb b/app/api/webcal_public_api.rb index a2bc091e9f..4c18aebc1f 100644 --- a/app/api/webcal_public_api.rb +++ b/app/api/webcal_public_api.rb @@ -14,9 +14,10 @@ class WebcalPublicApi < Grape::API webcal = Webcal.find_by!(guid: params[:guid]) # Serve the iCalendar with the correct MIME type. + env['api.format'] = :txt content_type 'text/calendar' - # Seve ical. - present webcal.to_ical.to_ical + # Serve ical. + webcal.to_ical.to_ical end end diff --git a/test/api/webcal_api_test.rb b/test/api/webcal_api_test.rb index ae7f6c37ac..735b3cae6b 100644 --- a/test/api/webcal_api_test.rb +++ b/test/api/webcal_api_test.rb @@ -77,7 +77,7 @@ def app assert_equal current_guid, last_response_body['guid'] end - test 'Ical endpoint is public and serves webcal with corect content type' do + test 'Ical endpoint is public and serves webcal with correct content type' do add_auth_header_for user: @student # Enable webcal, get GUID put_json '/api/webcal', { webcal: { enabled: true } } @@ -89,6 +89,9 @@ def app # Ensure correct content type assert_equal 200, last_response.status assert_equal 'text/calendar', last_response['Content-Type'] + assert last_response.body.start_with?("BEGIN:VCALENDAR\r\n") + assert_includes last_response.body, "\r\nEND:VCALENDAR\r\n" + assert_not last_response.body.start_with?('"') end test 'Reminder must be specified with both time & unit' do From c782911a1140ce1f5bfe3041ff47425b60efa1bd Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:23:19 +1000 Subject: [PATCH 097/199] refactor: ensure webcal uses students local start/target dates --- app/models/task.rb | 32 +++++++++++++++++++ app/models/webcal.rb | 64 ++++++++++++++++++++++++++++++++------ test/models/webcal_test.rb | 55 +++++++++++++++++++++++++++++--- 3 files changed, 138 insertions(+), 13 deletions(-) diff --git a/app/models/task.rb b/app/models/task.rb index ff7fa130ed..30f148f5d8 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -409,6 +409,38 @@ def due_date return extension_date end + def local_due_date + if unit.allow_flexible_dates + return target_due_date if target_due_date.present? + + grade_target_date = case project.target_grade + when 1 then task_definition.c_target_date + when 2 then task_definition.d_target_date + when 3 then task_definition.hd_target_date + end + return grade_target_date if grade_target_date.present? + end + + due_date + end + + def local_start_date + if unit.allow_flexible_dates + return target_start_date if target_start_date.present? + + grade_start_date = case project.target_grade + when 1 then task_definition.c_start_date + when 2 then task_definition.d_start_date + when 3 then task_definition.hd_start_date + end + return grade_start_date if grade_start_date.present? + end + + return task_definition.start_date + extensions.weeks if extensions.negative? + + task_definition.start_date + end + def days_awaiting_feedback(now_time = Time.zone.now) return 0 if submission_date.blank? diff --git a/app/models/webcal.rb b/app/models/webcal.rb index 06ce4f3f61..4278eff6b5 100644 --- a/app/models/webcal.rb +++ b/app/models/webcal.rb @@ -13,6 +13,20 @@ def self.valid_time_units %w(W D H M) end + def reminder_trigger + return nil unless reminder? + + trigger = + case reminder_unit + when 'W', 'D' + "-P#{reminder_time}#{reminder_unit}" + when 'H', 'M' + "-PT#{reminder_time}#{reminder_unit}" + end + + Icalendar::Values::Duration.new(trigger, 'RELATED' => 'START') + end + # # Represents the presence of `reminder_time` and `reminder_unit`. # @@ -28,7 +42,7 @@ def reminder? def task_definitions TaskDefinition .joins(:unit, unit: :projects) - .includes(:unit, unit: :projects) + .includes(:grade_due_dates, unit: :projects) .where( projects: { user_id: user_id, enrolled: true }, units: { active: true } @@ -71,8 +85,15 @@ def to_ical(task_defs = task_definitions) ical.publish ical.prodid = Doubtfire::Application.config.institution[:product_name] - # load all of the tasks... uses the preloaded project - tasks = Task.where(task_definition: task_defs, project: task_defs.map { |t| t.unit.projects.first }.uniq) + projects = Project + .includes(:unit) + .where(user_id: user_id, enrolled: true, unit_id: task_defs.map(&:unit_id).uniq) + .index_by(&:unit_id) + + tasks = Task + .includes(:project, task_definition: :grade_due_dates) + .where(task_definition: task_defs, project: projects.values) + .index_by(&:task_definition_id) # Add iCalendar events for the specified definition. task_defs.each do |td| @@ -83,7 +104,7 @@ def to_ical(task_defs = task_definitions) ev_date_format = '%Y%m%d' ev_reminders = reminder? - ev_reminder_trigger = "-PT#{reminder_time}#{reminder_unit}" + ev_reminder_trigger = reminder_trigger # Add event for start date, if the user opted in. if include_start_dates @@ -91,7 +112,8 @@ def to_ical(task_defs = task_definitions) ev.uid = "S-#{td.id}" ev.summary = event_name_for_task_definition(td, 'start') ev.status = 'CONFIRMED' - ev.dtstart = ev.dtend = Icalendar::Values::Date.new(td.start_date.strftime(ev_date_format)) + start_date = Webcal.start_date_for_task_definition(td, tasks[td.id], projects[td.unit_id]) + ev.dtstart = ev.dtend = Icalendar::Values::Date.new(start_date.strftime(ev_date_format)) Webcal.add_metadata_to_ical_event(ev, td) @@ -110,7 +132,8 @@ def to_ical(task_defs = task_definitions) ev.uid = "E-#{td.id}" ev.summary = event_name_for_task_definition(td, 'end') ev.status = 'CONFIRMED' - ev.dtstart = ev.dtend = Icalendar::Values::Date.new(Webcal.end_date_for_task_definition(td, tasks).strftime(ev_date_format)) + end_date = Webcal.end_date_for_task_definition(td, tasks[td.id], projects[td.unit_id]) + ev.dtstart = ev.dtend = Icalendar::Values::Date.new(end_date.strftime(ev_date_format)) Webcal.add_metadata_to_ical_event(ev, td) @@ -137,9 +160,32 @@ def to_ical(task_defs = task_definitions) # # Returns the target/extended date for the specified task definition. # - def self.end_date_for_task_definition(task_def, tasks) - task = tasks.select { |t| t.task_definition_id == task_def.id }.first - task.present? ? task.due_date : task_def.target_date + def self.end_date_for_task_definition(task_def, task = nil, project = nil) + return task.local_due_date if task.present? + + flexible_grade_date_for_task_definition(task_def, project, :target_date) || task_def.target_date + end + + # + # Returns the start date for the specified task definition. + # + def self.start_date_for_task_definition(task_def, task = nil, project = nil) + return task.local_start_date if task.present? + + flexible_grade_date_for_task_definition(task_def, project, :start_date) || task_def.start_date + end + + def self.flexible_grade_date_for_task_definition(task_def, project, date_type) + return nil unless project&.unit&.allow_flexible_dates + + case [project.target_grade, date_type] + when [1, :target_date] then task_def.c_target_date + when [2, :target_date] then task_def.d_target_date + when [3, :target_date] then task_def.hd_target_date + when [1, :start_date] then task_def.c_start_date + when [2, :start_date] then task_def.d_start_date + when [3, :start_date] then task_def.hd_start_date + end end # diff --git a/test/models/webcal_test.rb b/test/models/webcal_test.rb index b8de99efe3..a04c787eae 100644 --- a/test/models/webcal_test.rb +++ b/test/models/webcal_test.rb @@ -134,6 +134,52 @@ class WebcalTest < ActiveSupport::TestCase task.update(extensions: 0) end + test 'Includes events with flexible planned task dates if available' do + @webcal.update(include_start_dates: true) + @current_unit1.update!(allow_flexible_dates: true) + + td = @current_unit1.task_definitions.first + task = @current_project1.task_for_task_definition(td) + task.update!( + target_start_date: td.start_date + 2.days, + target_due_date: td.target_date + 3.days + ) + + cal = @webcal.to_ical + td_start_event = cal.events.detect { |e| e.summary == @webcal.event_name_for_task_definition(td, 'start') } + td_end_event = cal.events.detect { |e| e.summary == @webcal.event_name_for_task_definition(td, 'end') } + + assert_equal task.target_start_date.to_date, td_start_event.dtstart.to_date + assert_equal task.target_start_date.to_date, td_start_event.dtend.to_date + assert_equal task.target_due_date.to_date, td_end_event.dtstart.to_date + assert_equal task.target_due_date.to_date, td_end_event.dtend.to_date + end + + test 'Includes events with flexible grade guideline dates if no planned task dates exist' do + @webcal.update(include_start_dates: true) + @current_unit2.update!(allow_flexible_dates: true) + + td = @current_unit2.task_definitions.first + Task.where(project: @current_project2, task_definition: td).destroy_all + + grade_start_date = td.start_date + 2.days + grade_due_date = td.target_date + 3.days + td.grade_due_dates.create!( + target_grade: @current_project2.target_grade, + start_date: grade_start_date, + target_due_date: grade_due_date + ) + + cal = @webcal.to_ical + td_start_event = cal.events.detect { |e| e.summary == @webcal.event_name_for_task_definition(td, 'start') } + td_end_event = cal.events.detect { |e| e.summary == @webcal.event_name_for_task_definition(td, 'end') } + + assert_equal grade_start_date.to_date, td_start_event.dtstart.to_date + assert_equal grade_start_date.to_date, td_start_event.dtend.to_date + assert_equal grade_due_date.to_date, td_end_event.dtstart.to_date + assert_equal grade_due_date.to_date, td_end_event.dtend.to_date + end + test 'Includes webcal reminders correctly' do cal = @webcal.to_ical all_task_defs = @current_unit1.task_definitions + @current_unit2.task_definitions @@ -155,15 +201,16 @@ class WebcalTest < ActiveSupport::TestCase time = 2 checks = [ - { unit: 'W', trigger_symbol: :weeks }, - { unit: 'D', trigger_symbol: :days }, - { unit: 'H', trigger_symbol: :hours }, - { unit: 'M', trigger_symbol: :minutes }, + { unit: 'W', trigger_symbol: :weeks, expected_trigger: 'TRIGGER;RELATED=START:-P2W' }, + { unit: 'D', trigger_symbol: :days, expected_trigger: 'TRIGGER;RELATED=START:-P2D' }, + { unit: 'H', trigger_symbol: :hours, expected_trigger: 'TRIGGER;RELATED=START:-PT2H' }, + { unit: 'M', trigger_symbol: :minutes, expected_trigger: 'TRIGGER;RELATED=START:-PT2M' }, ] checks.each do |check| @webcal.update(reminder_time: time, reminder_unit: check[:unit]) cal = @webcal.to_ical + assert_includes cal.to_ical, check[:expected_trigger] per_task_def.call do |td, ev| From d32145ef62c7ad014524e5e3610844b7bda20b76 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 18 Jun 2026 11:39:38 +1000 Subject: [PATCH 098/199] feat: engagement passport (#631) * feat: engagement passport * feat: allow replies * fix: auto orient images * chore: reset schema * chore: bump migration * chore: reset schema * chore: bump migration --- app/api/api_root.rb | 2 + app/api/engagements_api.rb | 223 +++++++++++++ app/api/entities/engagement_comment_entity.rb | 10 + app/api/entities/engagement_detail_entity.rb | 7 + app/api/entities/engagement_entity.rb | 21 ++ app/helpers/file_helper.rb | 9 +- app/models/engagement.rb | 112 +++++++ app/models/engagement_comment.rb | 27 ++ app/models/project.rb | 17 +- app/models/unit.rb | 3 +- app/models/user.rb | 2 + .../20260618001447_create_engagements.rb | 27 ++ db/schema.rb | 30 +- test/api/engagements_api_test.rb | 302 ++++++++++++++++++ test/factories/engagements_factory.rb | 15 + test/models/engagement_test.rb | 96 ++++++ 16 files changed, 896 insertions(+), 7 deletions(-) create mode 100644 app/api/engagements_api.rb create mode 100644 app/api/entities/engagement_comment_entity.rb create mode 100644 app/api/entities/engagement_detail_entity.rb create mode 100644 app/api/entities/engagement_entity.rb create mode 100644 app/models/engagement.rb create mode 100644 app/models/engagement_comment.rb create mode 100644 db/migrate/20260618001447_create_engagements.rb create mode 100644 test/api/engagements_api_test.rb create mode 100644 test/factories/engagements_factory.rb create mode 100644 test/models/engagement_test.rb diff --git a/app/api/api_root.rb b/app/api/api_root.rb index c87cfc881e..3dbc682297 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -60,6 +60,7 @@ class ApiRoot < Grape::API mount AuthenticationApi mount BreaksApi mount DiscussionCommentApi + mount EngagementsApi mount ExtensionCommentsApi mount ScormExtensionCommentsApi mount GroupSetsApi @@ -118,6 +119,7 @@ class ApiRoot < Grape::API AuthenticationHelpers.add_auth_to ActivityTypesAuthenticatedApi AuthenticationHelpers.add_auth_to BreaksApi AuthenticationHelpers.add_auth_to DiscussionCommentApi + AuthenticationHelpers.add_auth_to EngagementsApi AuthenticationHelpers.add_auth_to ExtensionCommentsApi AuthenticationHelpers.add_auth_to ScormExtensionCommentsApi AuthenticationHelpers.add_auth_to GroupSetsApi diff --git a/app/api/engagements_api.rb b/app/api/engagements_api.rb new file mode 100644 index 0000000000..b26d687ead --- /dev/null +++ b/app/api/engagements_api.rb @@ -0,0 +1,223 @@ +require 'grape' + +class EngagementsApi < Grape::API + helpers AuthenticationHelpers + helpers AuthorisationHelpers + helpers FileStreamHelper + + before do + authenticated? + end + + helpers do + def engagement_for(project) + project.engagements.find(params[:id]) + end + + def validate_evidence!(attachment, evidence_url, remove_evidence: false) + if remove_evidence && (attachment.present? || evidence_url.present?) + error!({ error: 'Cannot remove evidence and provide replacement evidence together.' }, 400) + end + + if attachment.present? && evidence_url.present? + error!({ error: 'Provide either an attachment or an evidence URL, not both.' }, 400) + end + + return nil if attachment.blank? + + error!({ error: 'Attachment is empty.' }, 400) if File.size?(attachment['tempfile'].path).blank? + if File.size?(attachment['tempfile'].path) >= 30_000_000 + error!({ error: 'Attachment exceeds the maximum attachment size of 30MB.' }, 400) + end + + image_result = FileHelper.accept_file(attachment, 'engagement evidence image', 'image') + return 'image' if image_result[:accepted] + + pdf_result = FileHelper.accept_file(attachment, 'engagement evidence PDF', 'document') + return 'pdf' if pdf_result[:accepted] + + error!({ error: "File is not an acceptable image or PDF: #{pdf_result[:msg]}" }, 400) + end + end + + desc 'Get engagements for a project' + get '/projects/:project_id/engagements' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to view these engagements.' }, 403) unless authorise?(current_user, project, :get_engagements) + + engagements = project.engagements + .includes(:user, :engagement_comments) + .order(:occurred_at, :created_at) + present engagements, with: Entities::EngagementEntity + end + + desc 'Get an engagement for a project' + get '/projects/:project_id/engagements/:id' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to view this engagement.' }, 403) unless authorise?(current_user, project, :get_engagements) + + engagement = project.engagements + .includes(:user, engagement_comments: :user) + .find(params[:id]) + present engagement, with: Entities::EngagementDetailEntity + end + + desc 'Create an engagement for a project' + params do + requires :engagement_type, type: String + requires :note, type: String + requires :occurred_at, type: DateTime + optional :evidence_url, type: String + optional :attachment, type: File + end + post '/projects/:project_id/engagements' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to create an engagement.' }, 403) unless authorise?(current_user, project, :create_engagement) + + attachment = params[:attachment] + attachment_type = validate_evidence!(attachment, params[:evidence_url]) + + engagement = project.engagements.create!( + user: current_user, + engagement_type: params[:engagement_type], + note: params[:note], + occurred_at: params[:occurred_at], + evidence_url: attachment.present? ? nil : params[:evidence_url] + ) + + engagement.replace_attachment(attachment, attachment_type) if attachment.present? + present engagement, with: Entities::EngagementDetailEntity + rescue StandardError + engagement&.destroy + raise + end + + desc 'Update an engagement for a project' + params do + optional :engagement_type, type: String + optional :note, type: String + optional :occurred_at, type: DateTime + optional :evidence_url, type: String + optional :attachment, type: File + optional :remove_evidence, type: Boolean, default: false + end + put '/projects/:project_id/engagements/:id' do + project = Project.find(params[:project_id]) + engagement = engagement_for(project) + + can_edit = authorise?(current_user, project, :edit_engagement) && engagement.user_id == current_user.id + error!({ error: 'You do not have permission to edit this engagement.' }, 403) unless can_edit + + attachment = params[:attachment] + evidence_url = params.key?(:evidence_url) ? params[:evidence_url] : nil + attachment_type = validate_evidence!( + attachment, + evidence_url, + remove_evidence: params[:remove_evidence] + ) + + attributes = {} + attributes[:engagement_type] = params[:engagement_type] if params.key?(:engagement_type) + attributes[:note] = params[:note] if params.key?(:note) + attributes[:occurred_at] = params[:occurred_at] if params.key?(:occurred_at) + + if params[:remove_evidence] + engagement.remove_attachment + engagement.evidence_url = nil + elsif attachment.present? + engagement.assign_attributes(attributes) + engagement.replace_attachment(attachment, attachment_type) + attributes = {} + elsif params.key?(:evidence_url) + engagement.remove_attachment + engagement.evidence_url = evidence_url + end + + engagement.update!(attributes) + present engagement.reload, with: Entities::EngagementDetailEntity + end + + desc 'Delete an engagement for a project' + delete '/projects/:project_id/engagements/:id' do + project = Project.find(params[:project_id]) + engagement = engagement_for(project) + error!({ error: 'You do not have permission to delete this engagement.' }, 403) unless authorise?(current_user, project.unit, :delete_engagement) + + engagement.destroy! + present engagement.destroyed?, with: Grape::Presenters::Presenter + end + + desc 'Get evidence attached to an engagement' + params do + optional :as_attachment, type: Boolean, default: false + end + get '/projects/:project_id/engagements/:id/attachment' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to view this evidence.' }, 403) unless authorise?(current_user, project, :get_engagements) + + engagement = engagement_for(project) + error!({ error: 'No attachment for this engagement.' }, 404) unless engagement.attachment? + error!({ error: 'File missing.' }, 404) unless File.exist?(engagement.attachment_path) + + content_type engagement.attachment_mime_type + env['api.format'] = :binary + if params[:as_attachment] + header['Content-Disposition'] = "attachment; filename=#{engagement.attachment_file_name}" + end + + stream_file engagement.attachment_path + end + + desc 'Add a comment to an engagement' + params do + requires :comment, type: String + optional :reply_to_id, type: Integer + end + post '/projects/:project_id/engagements/:id/comments' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to comment on this engagement.' }, 403) unless authorise?(current_user, project, :comment_engagement) + + engagement = engagement_for(project) + reply_to = engagement.engagement_comments.find(params[:reply_to_id]) if params[:reply_to_id].present? + comment = engagement.engagement_comments.create!( + user: current_user, + comment: params[:comment], + reply_to: reply_to + ) + present comment, with: Entities::EngagementCommentEntity + end + + desc 'Update an engagement comment' + params do + requires :comment, type: String + end + put '/projects/:project_id/engagements/:id/comments/:comment_id' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to comment on this engagement.' }, 403) unless authorise?(current_user, project, :comment_engagement) + + engagement = engagement_for(project) + comment = engagement.engagement_comments.find(params[:comment_id]) + error!({ error: 'You can only edit your own comments.' }, 403) unless comment.user_id == current_user.id + if comment.created_at < 10.minutes.ago + error!({ error: 'Comments can only be edited within 10 minutes of being created.' }, 403) + end + + comment.update!(comment: params[:comment]) + present comment, with: Entities::EngagementCommentEntity + end + + desc 'Delete an engagement comment' + delete '/projects/:project_id/engagements/:id/comments/:comment_id' do + project = Project.find(params[:project_id]) + engagement = engagement_for(project) + comment = engagement.engagement_comments.find(params[:comment_id]) + + can_delete_own = comment.user_id == current_user.id && + authorise?(current_user, project, :comment_engagement) + can_delete_any = authorise?(current_user, project.unit, :delete_engagement) + error!({ error: 'You do not have permission to delete this comment.' }, 403) unless can_delete_own || can_delete_any + + comment.destroy! + present comment.destroyed?, with: Grape::Presenters::Presenter + end +end diff --git a/app/api/entities/engagement_comment_entity.rb b/app/api/entities/engagement_comment_entity.rb new file mode 100644 index 0000000000..4abf5deb8a --- /dev/null +++ b/app/api/entities/engagement_comment_entity.rb @@ -0,0 +1,10 @@ +module Entities + class EngagementCommentEntity < Grape::Entity + expose :id + expose :comment + expose :reply_to_id + expose :user, using: Entities::Minimal::MinimalUserEntity + expose :created_at + expose :updated_at + end +end diff --git a/app/api/entities/engagement_detail_entity.rb b/app/api/entities/engagement_detail_entity.rb new file mode 100644 index 0000000000..d0bf9fc591 --- /dev/null +++ b/app/api/entities/engagement_detail_entity.rb @@ -0,0 +1,7 @@ +module Entities + class EngagementDetailEntity < EngagementEntity + expose :engagement_comments, + as: :comments, + using: Entities::EngagementCommentEntity + end +end diff --git a/app/api/entities/engagement_entity.rb b/app/api/entities/engagement_entity.rb new file mode 100644 index 0000000000..984fb74015 --- /dev/null +++ b/app/api/entities/engagement_entity.rb @@ -0,0 +1,21 @@ +module Entities + class EngagementEntity < Grape::Entity + expose :id + expose :project_id + expose :engagement_type + expose :note + expose :occurred_at + expose :evidence_url + expose :content_type + expose :has_attachment do |engagement, _options| + engagement.attachment? + end + expose :attachment_file_name, if: ->(engagement, _) { engagement.attachment? } + expose :user, using: Entities::Minimal::MinimalUserEntity + expose :comment_count do |engagement| + engagement.engagement_comments.size + end + expose :created_at + expose :updated_at + end +end diff --git a/app/helpers/file_helper.rb b/app/helpers/file_helper.rb index 41016e5000..2f63d6b504 100644 --- a/app/helpers/file_helper.rb +++ b/app/helpers/file_helper.rb @@ -334,6 +334,12 @@ def comment_attachment_path(task_comment, attachment_extension) "#{File.join(student_work_dir(:comment, task_comment.task), "#{task_comment.id.to_s}#{attachment_extension}")}" end + def engagement_attachment_path(engagement, attachment_extension) + dir = File.join(project_work_root(engagement.project), 'engagement') + FileUtils.mkdir_p(dir) + File.join(dir, "#{engagement.id}#{attachment_extension}") + end + def comment_prompt_path(task_comment, attachment_extension, count) "#{File.join(student_work_dir(:discussion, task_comment.task), "#{task_comment.id.to_s}_#{count.to_s}#{attachment_extension}")}" end @@ -345,7 +351,7 @@ def comment_reply_prompt_path(discussion_comment, attachment_extension) def compress_image_to_dest(source, dest, delete_frames = false) exec = "convert -quiet \ \"#{source}\" \ - #{delete_frames ? '-delete 1--1' : ''} -strip -density 72 -quality 85% -resize 2048x2048\\> -resize 48x48\\< \ + #{delete_frames ? '-delete 1--1' : ''} -auto-orient -strip -density 72 -quality 85% -resize 2048x2048\\> -resize 48x48\\< \ \"#{dest}\" >>/dev/null 2>>/dev/null" system_try_within 40, 'compressing image using convert', exec @@ -998,6 +1004,7 @@ def line_wrap(path, width: 160) module_function :student_portfolio_dir module_function :student_portfolio_path module_function :comment_attachment_path + module_function :engagement_attachment_path module_function :comment_prompt_path module_function :comment_reply_prompt_path module_function :compress_image_to_dest diff --git a/app/models/engagement.rb b/app/models/engagement.rb new file mode 100644 index 0000000000..5d7cc78fe1 --- /dev/null +++ b/app/models/engagement.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require 'uri' + +class Engagement < ApplicationRecord + include FileHelper + include MimeCheckHelpers + + belongs_to :project, optional: false, inverse_of: :engagements + belongs_to :user, optional: false, inverse_of: :engagements + + has_many :engagement_comments, + -> { order(:created_at) }, + dependent: :destroy, + inverse_of: :engagement + + validates :engagement_type, presence: true, length: { maximum: 255 } + validates :note, presence: true, length: { maximum: 4095 } + validates :occurred_at, presence: true + validates :evidence_url, length: { maximum: 2048, allow_blank: true } + validates :content_type, inclusion: { in: %w[image pdf], allow_nil: true } + validate :valid_evidence_url + validate :single_evidence_source + validate :consistent_attachment_metadata + + before_validation :normalise_text + before_destroy :delete_attachment + + def attachment? + content_type.present? && attachment_extension.present? + end + + def attachment_path + FileHelper.engagement_attachment_path(self, attachment_extension) + end + + def attachment_file_name + "engagement-#{id}#{attachment_extension}" + end + + def attachment_mime_type + mime_type(attachment_path) + end + + def replace_attachment(file_upload, attachment_type) + delete_attachment + self.evidence_url = nil + self.content_type = attachment_type + + if attachment_type == 'image' + self.attachment_extension = + if mime_type(file_upload['tempfile'].path).starts_with?('image/gif') + '.gif' + else + '.jpg' + end + save! + image_saved = FileHelper.compress_image_to_dest(file_upload['tempfile'].path, attachment_path) + raise 'Failed to save engagement image attachment' unless image_saved && File.exist?(attachment_path) + else + self.attachment_extension = '.pdf' + save! + FileHelper.compress_pdf(file_upload['tempfile'].path) + FileUtils.mv(file_upload['tempfile'].path, attachment_path) + raise 'Failed to save engagement PDF attachment' unless File.exist?(attachment_path) + end + + file_upload['tempfile'].unlink if File.exist?(file_upload['tempfile'].path) + true + end + + def remove_attachment + delete_attachment + self.content_type = nil + self.attachment_extension = nil + end + + private + + def normalise_text + self.engagement_type = engagement_type&.strip + self.note = note&.strip + self.evidence_url = evidence_url&.strip.presence + end + + def valid_evidence_url + return if evidence_url.blank? + + uri = URI.parse(evidence_url) + return if uri.is_a?(URI::HTTP) && uri.host.present? + + errors.add(:evidence_url, 'must be a valid HTTP or HTTPS URL') + rescue URI::InvalidURIError + errors.add(:evidence_url, 'must be a valid HTTP or HTTPS URL') + end + + def single_evidence_source + return unless evidence_url.present? && (content_type.present? || attachment_extension.present?) + + errors.add(:base, 'An engagement can have either an evidence URL or an attachment, not both') + end + + def consistent_attachment_metadata + return if content_type.present? == attachment_extension.present? + + errors.add(:base, 'Attachment content type and extension must both be present') + end + + def delete_attachment + FileUtils.rm_f(attachment_path) if attachment_extension.present? + end +end diff --git a/app/models/engagement_comment.rb b/app/models/engagement_comment.rb new file mode 100644 index 0000000000..d02b4fca89 --- /dev/null +++ b/app/models/engagement_comment.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class EngagementComment < ApplicationRecord + belongs_to :engagement, optional: false, inverse_of: :engagement_comments + belongs_to :user, optional: false, inverse_of: :engagement_comments + belongs_to :reply_to, class_name: 'EngagementComment', optional: true + has_many :replies, + class_name: 'EngagementComment', + foreign_key: :reply_to_id, + dependent: :nullify, + inverse_of: :reply_to + + validates :comment, presence: true, length: { maximum: 4095 } + validate :reply_belongs_to_engagement + + before_validation do + self.comment = comment&.strip + end + + private + + def reply_belongs_to_engagement + return if reply_to.nil? || reply_to.engagement_id == engagement_id + + errors.add(:reply_to, 'must belong to the same engagement') + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 1444730f9b..dbe0405ad7 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -33,6 +33,7 @@ class Project < ApplicationRecord has_many :session_activities, dependent: :destroy has_many :staff_notes, dependent: :destroy + has_many :engagements, dependent: :destroy, inverse_of: :project # Callbacks - methods called are private before_destroy :can_destroy? @@ -59,7 +60,9 @@ def self.permissions :make_submission, :get_submission, :change, - :reprocess_submission + :reprocess_submission, + :get_engagements, + :comment_engagement ] # What can tutors do with projects? tutor_role_permissions = [ @@ -74,7 +77,11 @@ def self.permissions :get_staff_note, :create_staff_note, :reprocess_submission, - :get_discussion_prompt + :get_discussion_prompt, + :get_engagements, + :create_engagement, + :edit_engagement, + :comment_engagement ] # What can admins do with projects? @@ -82,7 +89,8 @@ def self.permissions :get, :get_submission, :reprocess_submission, - :get_discussion_prompt + :get_discussion_prompt, + :get_engagements ] # What can auditors do with projects? @@ -91,7 +99,8 @@ def self.permissions :get_submission, :get_staff_note, :reprocess_submission, - :get_discussion_prompt + :get_discussion_prompt, + :get_engagements ] # What can nil users do with projects? diff --git a/app/models/unit.rb b/app/models/unit.rb index 2a61b7dbe3..5c3d72bbbe 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -68,7 +68,8 @@ def self.permissions :get_marking_sessions, :upload_grades_csv, :get_staff_notes, - :mannage_communications + :mannage_communications, + :delete_engagement ] # What can admin do with units? diff --git a/app/models/user.rb b/app/models/user.rb index c360c140ff..fa86462bb4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -143,6 +143,8 @@ def token_for_text?(a_token, token_type) belongs_to :role, optional: false # Foreign Key has_many :unit_roles, dependent: :destroy, inverse_of: :user has_many :projects, dependent: :restrict_with_exception, inverse_of: :user + has_many :engagements, dependent: :restrict_with_exception, inverse_of: :user + has_many :engagement_comments, dependent: :restrict_with_exception, inverse_of: :user has_many :auth_tokens, dependent: :destroy, inverse_of: :user has_many :user_oauth_tokens, dependent: :destroy, inverse_of: :user has_many :user_oauth_states, dependent: :destroy, inverse_of: :user diff --git a/db/migrate/20260618001447_create_engagements.rb b/db/migrate/20260618001447_create_engagements.rb new file mode 100644 index 0000000000..c66c8a759f --- /dev/null +++ b/db/migrate/20260618001447_create_engagements.rb @@ -0,0 +1,27 @@ +class CreateEngagements < ActiveRecord::Migration[8.0] + def change + create_table :engagements do |t| + t.references :project, null: false, index: true + t.references :user, null: false, index: true + t.string :engagement_type, null: false + t.text :note, null: false + t.datetime :occurred_at, null: false + t.text :evidence_url + t.string :content_type + t.string :attachment_extension + + t.timestamps + end + + create_table :engagement_comments do |t| + t.references :engagement, null: false, index: true + t.references :user, null: false, index: true + t.references :reply_to, null: true, index: true + t.text :comment, null: false + + t.timestamps + end + + add_index :engagements, [:project_id, :occurred_at] + end +end diff --git a/db/schema.rb b/db/schema.rb index a77cfc3d97..3e5727b917 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_06_11_053643) do +ActiveRecord::Schema[8.0].define(version: 2026_06_18_001447) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -185,6 +185,34 @@ t.index ["task_definition_id"], name: "index_discussion_prompts_on_task_definition_id" end + create_table "engagement_comments", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "engagement_id", null: false + t.bigint "user_id", null: false + t.bigint "reply_to_id" + t.text "comment", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["engagement_id"], name: "index_engagement_comments_on_engagement_id" + t.index ["reply_to_id"], name: "index_engagement_comments_on_reply_to_id" + t.index ["user_id"], name: "index_engagement_comments_on_user_id" + end + + create_table "engagements", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "project_id", null: false + t.bigint "user_id", null: false + t.string "engagement_type", null: false + t.text "note", null: false + t.datetime "occurred_at", null: false + t.text "evidence_url" + t.string "content_type" + t.string "attachment_extension" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["project_id", "occurred_at"], name: "index_engagements_on_project_id_and_occurred_at" + t.index ["project_id"], name: "index_engagements_on_project_id" + t.index ["user_id"], name: "index_engagements_on_user_id" + end + create_table "feedback_chips", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "type" t.text "chip_text" diff --git a/test/api/engagements_api_test.rb b/test/api/engagements_api_test.rb new file mode 100644 index 0000000000..a0688ed46f --- /dev/null +++ b/test/api/engagements_api_test.rb @@ -0,0 +1,302 @@ +require 'test_helper' + +class EngagementsApiTest < ActiveSupport::TestCase + include Rack::Test::Methods + include TestHelpers::AuthHelper + include TestHelpers::JsonHelper + include TestHelpers::TestFileHelper + + def app + Rails.application + end + + def setup + @unit = FactoryBot.create(:unit, with_students: false) + @student = FactoryBot.create(:user, :student) + @project = @unit.enrol_student(@student, nil) + @tutor = FactoryBot.create(:user, :tutor) + @unit.employ_staff(@tutor, Role.tutor) + @convenor = @unit.main_convenor_user + end + + def engagement_params(overrides = {}) + { + engagement_type: 'attendance', + note: 'Attended the weekly discussion.', + occurred_at: Time.zone.now.iso8601 + }.merge(overrides) + end + + def create_engagement(user: @tutor, overrides: {}) + add_auth_header_for(user: user) + post_json "/api/projects/#{@project.id}/engagements", engagement_params(overrides) + assert_equal 201, last_response.status, last_response.body + Engagement.find(last_response_body['id']) + end + + def test_tutor_can_create_and_student_can_read_engagements + later = create_engagement(overrides: { engagement_type: 'forum', occurred_at: 1.day.from_now.iso8601 }) + earlier = create_engagement(overrides: { engagement_type: 'negative', occurred_at: 1.day.ago.iso8601 }) + + add_auth_header_for(user: @student) + get "/api/projects/#{@project.id}/engagements" + + assert_equal 200, last_response.status + assert_equal [earlier.id, later.id], (last_response_body.map { |engagement| engagement['id'] }) + assert_equal 'negative', last_response_body.first['engagement_type'] + assert_equal @tutor.id, last_response_body.first.dig('user', 'id') + end + + def test_student_and_unrelated_tutor_cannot_create_engagements + add_auth_header_for(user: @student) + post_json "/api/projects/#{@project.id}/engagements", engagement_params + assert_equal 403, last_response.status + + unrelated_tutor = FactoryBot.create(:user, :tutor) + add_auth_header_for(user: unrelated_tutor) + post_json "/api/projects/#{@project.id}/engagements", engagement_params + assert_equal 403, last_response.status + end + + def test_only_author_with_current_teaching_access_can_edit + engagement = create_engagement + other_tutor = FactoryBot.create(:user, :tutor) + @unit.employ_staff(other_tutor, Role.tutor) + + add_auth_header_for(user: other_tutor) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}", + { note: 'Changed by another tutor.' } + ) + assert_equal 403, last_response.status + + add_auth_header_for(user: @tutor) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}", + { note: 'Corrected note.', occurred_at: 2.days.ago.iso8601 } + ) + assert_equal 200, last_response.status + assert_equal 'Corrected note.', engagement.reload.note + + @unit.unit_role_for(@tutor).destroy! + assert Engagement.exists?(engagement.id) + + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}", + { note: 'Changed after role removal.' } + ) + assert_equal 403, last_response.status + assert_equal 'Corrected note.', engagement.reload.note + end + + def test_only_convenor_can_delete + engagement = create_engagement + + add_auth_header_for(user: @tutor) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}" + assert_equal 403, last_response.status + + add_auth_header_for(user: @convenor) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}" + assert_equal 200, last_response.status + assert_nil Engagement.find_by(id: engagement.id) + end + + def test_student_and_teaching_staff_can_comment + engagement = create_engagement + + add_auth_header_for(user: @student) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'I have added the supporting context.' } + ) + assert_equal 201, last_response.status + assert_equal @student.id, last_response_body.dig('user', 'id') + + add_auth_header_for(user: @tutor) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'Thanks, this clarifies the evidence.' } + ) + assert_equal 201, last_response.status + + unrelated_student = FactoryBot.create(:user, :student) + add_auth_header_for(user: unrelated_student) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'Unrelated comment.' } + ) + assert_equal 403, last_response.status + + add_auth_header_for(user: @student) + get "/api/projects/#{@project.id}/engagements/#{engagement.id}" + assert_equal 200, last_response.status + assert_equal 2, last_response_body['comments'].length + assert_equal( + ['I have added the supporting context.', 'Thanks, this clarifies the evidence.'], + last_response_body['comments'].map { |comment| comment['comment'] } + ) + end + + def test_comment_can_reply_to_comment_in_same_engagement + engagement = create_engagement + + add_auth_header_for(user: @student) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'Original comment.' } + ) + assert_equal 201, last_response.status + original_comment_id = last_response_body['id'] + + add_auth_header_for(user: @tutor) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'Reply comment.', reply_to_id: original_comment_id } + ) + assert_equal 201, last_response.status + assert_equal original_comment_id, last_response_body['reply_to_id'] + + get "/api/projects/#{@project.id}/engagements/#{engagement.id}" + assert_equal original_comment_id, last_response_body['comments'].last['reply_to_id'] + end + + def test_comment_cannot_reply_to_comment_in_another_engagement + engagement = create_engagement + other_engagement = create_engagement(overrides: { note: 'Another engagement.' }) + original_comment = engagement.engagement_comments.create!( + user: @student, + comment: 'Comment on the first engagement.' + ) + + add_auth_header_for(user: @tutor) + post_json( + "/api/projects/#{@project.id}/engagements/#{other_engagement.id}/comments", + { comment: 'Invalid reply.', reply_to_id: original_comment.id } + ) + assert_equal 404, last_response.status + end + + def test_comment_author_can_edit_within_ten_minutes + engagement = create_engagement + comment = engagement.engagement_comments.create!(user: @student, comment: 'Original comment.') + + add_auth_header_for(user: @student) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{comment.id}", + { comment: 'Updated comment.' } + ) + + assert_equal 200, last_response.status + assert_equal 'Updated comment.', comment.reload.comment + end + + def test_comment_cannot_be_edited_after_ten_minutes_or_by_another_user + engagement = create_engagement + comment = engagement.engagement_comments.create!(user: @student, comment: 'Original comment.') + + add_auth_header_for(user: @tutor) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{comment.id}", + { comment: 'Tutor edit.' } + ) + assert_equal 403, last_response.status + + comment.update_column(:created_at, 11.minutes.ago) + add_auth_header_for(user: @student) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{comment.id}", + { comment: 'Late edit.' } + ) + assert_equal 403, last_response.status + assert_equal 'Original comment.', comment.reload.comment + end + + def test_comment_author_and_convenor_can_delete + engagement = create_engagement + student_comment = engagement.engagement_comments.create!( + user: @student, + comment: 'Student comment.' + ) + + add_auth_header_for(user: @tutor) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{student_comment.id}" + assert_equal 403, last_response.status + + add_auth_header_for(user: @student) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{student_comment.id}" + assert_equal 200, last_response.status + assert_not EngagementComment.exists?(student_comment.id) + + tutor_comment = engagement.engagement_comments.create!(user: @tutor, comment: 'Tutor comment.') + add_auth_header_for(user: @convenor) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{tutor_comment.id}" + assert_equal 200, last_response.status + assert_not EngagementComment.exists?(tutor_comment.id) + end + + def test_rejects_file_and_url_together + add_auth_header_for(user: @tutor) + data = engagement_params( + evidence_url: 'https://example.com/evidence', + attachment: upload_file('test_files/submissions/boo.png', 'image/png') + ) + + post "/api/projects/#{@project.id}/engagements", data + + assert_equal 400, last_response.status + assert_equal 0, @project.engagements.count + end + + def test_image_attachment_can_be_retrieved_and_replaced_with_url + add_auth_header_for(user: @tutor) + data = engagement_params( + attachment: upload_file('test_files/submissions/Deakin_Logo.jpeg', 'image/jpeg') + ) + + post "/api/projects/#{@project.id}/engagements", data + assert_equal 201, last_response.status, last_response.body + + engagement = Engagement.find(last_response_body['id']) + assert engagement.attachment? + assert File.exist?(engagement.attachment_path) + + add_auth_header_for(user: @student) + get "/api/projects/#{@project.id}/engagements/#{engagement.id}/attachment" + assert_equal 200, last_response.status + + add_auth_header_for(user: @tutor) + old_attachment_path = engagement.attachment_path + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}", + { evidence_url: 'https://example.com/replacement' } + ) + assert_equal 200, last_response.status + + engagement.reload + assert_not engagement.attachment? + assert_not File.exist?(old_attachment_path) + assert_equal 'https://example.com/replacement', engagement.evidence_url + end + + def test_pdf_attachment_is_accepted_and_removed_with_project + add_auth_header_for(user: @tutor) + data = engagement_params( + attachment: upload_file('test_files/submissions/00_question.pdf', 'application/pdf') + ) + + post "/api/projects/#{@project.id}/engagements", data + assert_equal 201, last_response.status, last_response.body + + engagement = Engagement.find(last_response_body['id']) + assert_equal 'pdf', engagement.content_type + attachment_path = engagement.attachment_path + assert File.exist?(attachment_path) + + @project.tutorial_enrolments.destroy_all + @project.destroy! + + assert_nil Engagement.find_by(id: engagement.id) + assert_not File.exist?(attachment_path) + end +end diff --git a/test/factories/engagements_factory.rb b/test/factories/engagements_factory.rb new file mode 100644 index 0000000000..0f21d6fd17 --- /dev/null +++ b/test/factories/engagements_factory.rb @@ -0,0 +1,15 @@ +FactoryBot.define do + factory :engagement do + project + user + engagement_type { 'attendance' } + note { 'Attended the weekly discussion.' } + occurred_at { Time.zone.now } + end + + factory :engagement_comment do + engagement + user + comment { 'Thanks for recording this.' } + end +end diff --git a/test/models/engagement_test.rb b/test/models/engagement_test.rb new file mode 100644 index 0000000000..8b984db3c6 --- /dev/null +++ b/test/models/engagement_test.rb @@ -0,0 +1,96 @@ +require 'test_helper' + +class EngagementTest < ActiveSupport::TestCase + def test_requires_core_attributes + engagement = Engagement.new + + assert_not engagement.valid? + assert_includes engagement.errors[:project], 'must exist' + assert_includes engagement.errors[:user], 'must exist' + assert_includes engagement.errors[:engagement_type], "can't be blank" + assert_includes engagement.errors[:note], "can't be blank" + assert_includes engagement.errors[:occurred_at], "can't be blank" + end + + def test_accepts_free_form_engagement_types_and_trims_text + engagement = FactoryBot.build( + :engagement, + engagement_type: ' negative ', + note: ' Missed an agreed check-in. ' + ) + + assert engagement.valid? + assert_equal 'negative', engagement.engagement_type + assert_equal 'Missed an agreed check-in.', engagement.note + end + + def test_validates_evidence_url + engagement = FactoryBot.build(:engagement, evidence_url: 'ftp://example.com/evidence') + + assert_not engagement.valid? + assert_includes engagement.errors[:evidence_url], 'must be a valid HTTP or HTTPS URL' + + engagement.evidence_url = 'https://example.com/evidence' + assert engagement.valid? + end + + def test_rejects_attachment_and_url_together + engagement = FactoryBot.build( + :engagement, + evidence_url: 'https://example.com/evidence', + content_type: 'image', + attachment_extension: '.jpg' + ) + + assert_not engagement.valid? + assert_includes engagement.errors[:base], 'An engagement can have either an evidence URL or an attachment, not both' + end + + def test_destroy_removes_comments_and_attachment + engagement = FactoryBot.create( + :engagement, + content_type: 'image', + attachment_extension: '.jpg' + ) + comment = FactoryBot.create(:engagement_comment, engagement: engagement) + FileUtils.touch(engagement.attachment_path) + + attachment_path = engagement.attachment_path + engagement.destroy! + + assert_not File.exist?(attachment_path) + assert_nil EngagementComment.find_by(id: comment.id) + end + + def test_project_destroy_removes_engagement + engagement = FactoryBot.create(:engagement) + + engagement.project.tutorial_enrolments.destroy_all + engagement.project.destroy! + + assert_nil Engagement.find_by(id: engagement.id) + end + + def test_comment_reply_must_belong_to_same_engagement + engagement = FactoryBot.create(:engagement) + other_engagement = FactoryBot.create(:engagement) + original_comment = FactoryBot.create(:engagement_comment, engagement: engagement) + reply = FactoryBot.build( + :engagement_comment, + engagement: other_engagement, + reply_to: original_comment + ) + + assert_not reply.valid? + assert_includes reply.errors[:reply_to], 'must belong to the same engagement' + end + + def test_author_cannot_be_deleted_while_engagement_exists + engagement = FactoryBot.create(:engagement) + + assert_raises(ActiveRecord::DeleteRestrictionError) do + engagement.user.destroy! + end + assert Engagement.exists?(engagement.id) + end +end From c16f7558e9a6a471a72a13756e4dc5fa87fd4ad7 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 18 Jun 2026 12:19:25 +1000 Subject: [PATCH 099/199] refactor: make access and refresh token expiry configurable (#637) * refactor: make access and refresh token expiry configurable * chore: expose auth token expiry * refactor: scale auth token reuse window with configured expiry --- app/api/authentication_api.rb | 6 +++++- app/helpers/authentication_helpers.rb | 2 +- app/models/user.rb | 15 +++++++++++++-- config/application.rb | 2 ++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/api/authentication_api.rb b/app/api/authentication_api.rb index f2b525af3a..e81de94d1f 100644 --- a/app/api/authentication_api.rb +++ b/app/api/authentication_api.rb @@ -81,6 +81,7 @@ class AuthenticationApi < Grape::API # Return user details present :user, user, with: Entities::UserEntity present :auth_token, token.authentication_token + present :auth_token_expiry, token.auth_token_expiry set_refresh_cookie_in_response(remember) end end @@ -376,6 +377,7 @@ class AuthenticationApi < Grape::API # Respond user details with new auth token present :user, user, with: Entities::UserEntity present :auth_token, token.authentication_token + present :auth_token_expiry, token.auth_token_expiry set_refresh_cookie_in_response(params[:remember]) end end @@ -499,8 +501,10 @@ class AuthenticationApi < Grape::API end end # Return user details + token = current_user.generate_authentication_token!(token_type: :general, force_new: false) present :user, current_user, with: Entities::UserEntity - present :auth_token, current_user.generate_authentication_token!(token_type: :general, force_new: false).authentication_token + present :auth_token, token.authentication_token + present :auth_token_expiry, token.auth_token_expiry else present nil end diff --git a/app/helpers/authentication_helpers.rb b/app/helpers/authentication_helpers.rb index d43c635c09..1f81dbe848 100644 --- a/app/helpers/authentication_helpers.rb +++ b/app/helpers/authentication_helpers.rb @@ -221,7 +221,7 @@ def set_refresh_cookie_in_response(remember) # Generate a new token when the old one is absent or getting close to expiring if token.nil? || token.auth_token_expiry <= Time.zone.now - 12.hours - token = current_user.generate_authentication_token!(token_type: :refresh_token, expiry: Time.zone.now + 1.week) + token = current_user.generate_authentication_token!(token_type: :refresh_token) end domain = Doubtfire::Application.config.institution[:cookie_domain] diff --git a/app/models/user.rb b/app/models/user.rb index fa86462bb4..159b9aab7f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -94,12 +94,23 @@ def authenticate?(data) # Force-generates a new authentication token, regardless of whether or not # it is actually expired # - def generate_authentication_token!(remember: false, expiry: Time.zone.now + 2.hours, token_type: :general, force_new: true) + def generate_authentication_token!(remember: false, expiry: nil, token_type: :general, force_new: true) # Ensure this user is saved... so it has an id self.save unless self.persisted? + expiry_duration = + if token_type.to_sym == :refresh_token + Doubtfire::Application.config.refresh_token_expiry + else + Doubtfire::Application.config.access_token_expiry + end + expiry ||= Time.zone.now + expiry_duration + + # Reuse tokens for up to 75% of their configured lifetime, then rotate early. + token_reuse_duration = expiry_duration * 0.75 + # Get a recent token, or create a new one token = self.auth_tokens.where(token_type: token_type).last unless force_new - if token.nil? || token.created_at <= Time.zone.now - 90.minutes + if token.nil? || token.auth_token_expiry <= Time.zone.now || token.created_at <= Time.zone.now - token_reuse_duration token = AuthToken.generate(self, remember, expiry, token_type) end diff --git a/config/application.rb b/config/application.rb index 21afadb5da..2466620f08 100644 --- a/config/application.rb +++ b/config/application.rb @@ -36,6 +36,8 @@ class Application < Rails::Application # are: database, ldap, aaf, or saml. It can be overridden using the DF_AUTH_METHOD # environment variable. config.auth_method = (ENV['DF_AUTH_METHOD'] || :database).to_sym + config.access_token_expiry = ENV.fetch('DF_ACCESS_TOKEN_EXPIRY_SECONDS', 2.hours.to_i).to_i.seconds + config.refresh_token_expiry = ENV.fetch('DF_REFRESH_TOKEN_EXPIRY_SECONDS', 1.week.to_i).to_i.seconds # ==> Student work directory # File server location for storing student's work. Defaults to `student_work` From ea1bd13323a03b056b6a7d9347f558a37c111f97 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 18 Jun 2026 12:20:09 +1000 Subject: [PATCH 100/199] chore(release): 11.0.0-11 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93240005d4..77674715fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-11](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-10...v11.0.0-11) (2026-06-18) + + +### Features + +* engagement passport ([#631](https://github.com/b0ink/doubtfire-deploy/issues/631)) ([d32145e](https://github.com/b0ink/doubtfire-deploy/commit/d32145ef62c7ad014524e5e3610844b7bda20b76)) + + +### Bug Fixes + +* serve public web cal as raw text ([6836a20](https://github.com/b0ink/doubtfire-deploy/commit/6836a20e582473eff4edb41e2fd1f6c4a5a9751d)) + ## [11.0.0-10](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-9...v11.0.0-10) (2026-06-17) From dabd06d268fbc981ba6fb7854bf6cb443259bcf5 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 11:43:16 +1000 Subject: [PATCH 101/199] chore: ignore commit in git blame --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..53de1f9cbd --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Jan 11 2022: Repository wide format (support zeitwerk mode loader) +f4050e826af83166e45b0b4f2af14179b40305dd From 57cb9e250b7fbfe9eb477021c9077d86d4b696ab Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:13:11 +1000 Subject: [PATCH 102/199] feat: customisable grades (#638) * feat: customisable grades * refactor: allow custom grade labels * chore: fix rubocop * chore: fix test * chore: reset schema * chore: bump migration * refactor: use dynamic grades * chore: fix rubocop --- .../entities/minimal/minimal_unit_entity.rb | 2 + app/api/entities/task_definition_entity.rb | 10 +- app/api/entities/unit_entity.rb | 2 + app/api/projects_api.rb | 8 +- app/api/task_definitions_api.rb | 68 ++++----- app/api/units_api.rb | 18 ++- .../change_target_grade_action.rb | 9 ++ .../communication/target_grade_condition.rb | 9 ++ .../task_status_count_condition.rb | 11 +- app/models/project.rb | 26 +--- app/models/task.rb | 33 ++--- app/models/task_definition.rb | 47 +++---- app/models/unit.rb | 133 +++++++++++++++++- app/models/webcal.rb | 11 +- app/sidekiq/execute_communication_set_job.rb | 14 +- ...0260618033139_add_grade_values_to_units.rb | 8 ++ db/schema.rb | 4 +- 17 files changed, 281 insertions(+), 132 deletions(-) create mode 100644 db/migrate/20260618033139_add_grade_values_to_units.rb diff --git a/app/api/entities/minimal/minimal_unit_entity.rb b/app/api/entities/minimal/minimal_unit_entity.rb index bbead2cd81..06f9659f2a 100644 --- a/app/api/entities/minimal/minimal_unit_entity.rb +++ b/app/api/entities/minimal/minimal_unit_entity.rb @@ -20,6 +20,8 @@ class MinimalUnitEntity < Grape::Entity end expose :active + expose :grade_values + expose :grade_definitions end end end diff --git a/app/api/entities/task_definition_entity.rb b/app/api/entities/task_definition_entity.rb index 04c15e74d0..6027226c3d 100644 --- a/app/api/entities/task_definition_entity.rb +++ b/app/api/entities/task_definition_entity.rb @@ -19,16 +19,10 @@ def staff?(my_role) expose :target_date expose :due_date expose :start_date - # expose :p_target_date, expose_nil: false - expose :c_target_date, expose_nil: false - expose :d_target_date, expose_nil: false - expose :hd_target_date, expose_nil: false - - expose :c_start_date, expose_nil: false - expose :d_start_date, expose_nil: false - expose :hd_start_date, expose_nil: false end + expose :grade_due_date_overrides, as: :grade_due_dates, expose_nil: false + expose :upload_requirements, expose_nil: false do |task_definition, options| if staff?(options[:my_role]) task_definition.upload_requirements diff --git a/app/api/entities/unit_entity.rb b/app/api/entities/unit_entity.rb index af6a823ea1..46f26976be 100644 --- a/app/api/entities/unit_entity.rb +++ b/app/api/entities/unit_entity.rb @@ -39,6 +39,8 @@ def can_read_unit_config?(my_role) end expose :active + expose :grade_values + expose :grade_definitions expose :overseer_image_id, unless: :summary_only, if: lambda { |unit, options| can_read_unit_config?(options[:my_role]) } expose :assessment_enabled, unless: :summary_only diff --git a/app/api/projects_api.rb b/app/api/projects_api.rb index c2954bdcba..a895007ff3 100644 --- a/app/api/projects_api.rb +++ b/app/api/projects_api.rb @@ -98,6 +98,10 @@ class ProjectsApi < Grape::API error!({ error: "You do not have permissions to change this student" }, 403) end + unless project.unit.grade_value?(params[:target_grade]) + error!({ error: 'Target grade is not enabled for this unit' }, 422) + end + project.target_grade = params[:target_grade] project.save elsif !params[:submitted_grade].nil? @@ -107,6 +111,9 @@ class ProjectsApi < Grape::API if project.portfolio_exists? error!({ error: "You cannot change your submitted grade after portfolio submission" }, 403) end + unless project.unit.grade_value?(params[:submitted_grade]) + error!({ error: 'Submitted grade is not enabled for this unit' }, 422) + end project.submitted_grade = params[:submitted_grade] project.save @@ -126,7 +133,6 @@ class ProjectsApi < Grape::API if params[:old_grade] != project.grade error!({ error: 'Existing project grade does not match current grade. Refresh project and try again.' }, 403) end - for_student = false project.grade = params[:grade] diff --git a/app/api/task_definitions_api.rb b/app/api/task_definitions_api.rb index 814561628b..5e0be83ee7 100644 --- a/app/api/task_definitions_api.rb +++ b/app/api/task_definitions_api.rb @@ -86,6 +86,10 @@ class TaskDefinitionsApi < Grape::API task_params[:unit_id] = unit.id task_params[:upload_requirements] = params[:task_def][:upload_requirements].present? ? JSON.parse(params[:task_def][:upload_requirements]) : [] + unless unit.grade_value?(task_params[:target_grade]) + error!({ error: 'Target grade is not enabled for this unit' }, 422) + end + task_def = TaskDefinition.new(task_params) # Set the tutorial stream @@ -139,13 +143,11 @@ class TaskDefinitionsApi < Grape::API optional :requires_discussion, type: Boolean, desc: 'Whether task must be discussed in class before it can be signed off as complete' optional :use_resources_for_jplag_base_code, type: Boolean, desc: 'Include the common base code from task resources for JPlag comparisons' optional :lock_assessments_to_tutorial_stream, type: Boolean, desc: 'Only allow tutors in this tutorial stream to assess this task' - # optional :p_target_date, type: Date, desc: 'Pass due date override' - optional :c_target_date, type: Date, desc: 'Credit due date override' - optional :d_target_date, type: Date, desc: 'Distinction due date override' - optional :hd_target_date, type: Date, desc: 'High Distinction due date override' - optional :c_start_date, type: Date, desc: 'Credit start date override' - optional :d_start_date, type: Date, desc: 'Distinction start date override' - optional :hd_start_date, type: Date, desc: 'High Distinction start date override' + optional :grade_due_dates, type: Array do + requires :target_grade, type: Integer + optional :target_due_date, type: Date + optional :start_date, type: Date + end end end put '/units/:unit_id/task_definitions/:id' do @@ -157,14 +159,7 @@ class TaskDefinitionsApi < Grape::API end # strip these out so TaskDefinition#update! never sees them - grade_due_overrides = params[:task_def].slice( - 'p_target_date', 'c_target_date', 'd_target_date', 'hd_target_date', - 'p_start_date', 'c_start_date', 'd_start_date', 'hd_start_date' - ) - params[:task_def].except!( - 'p_target_date', 'c_target_date', 'd_target_date', 'hd_target_date', - 'p_start_date', 'c_start_date', 'd_start_date', 'hd_start_date' - ) + grade_due_date_rows = params[:task_def].delete('grade_due_dates') task_params = ActionController::Parameters.new(params) .require(:task_def) @@ -212,6 +207,10 @@ class TaskDefinitionsApi < Grape::API end end + if task_params.key?(:target_grade) && !unit.grade_value?(task_params[:target_grade]) + error!({ error: 'Target grade is not enabled for this unit' }, 422) + end + # Bulk update task definition with permitted parameters task_def.update!(task_params) @@ -239,29 +238,32 @@ class TaskDefinitionsApi < Grape::API end end - grade_number = { 'c' => 1, 'd' => 2, 'hd' => 3 } - field_map = { 'target_date' => :target_due_date, 'start_date' => :start_date } - - grade_due_overrides.each do |key, date| - next if date.blank? - - # if task_def.start_date > date - # error!({ error: 'Target date cannot be earlier than start date' }, 400) - # end - + if grade_due_date_rows.present? unless unit.allow_flexible_dates error!({ error: 'This unit must have Allow Flexible Dates enabled to modify target dates per grade' }, 403) end - grade_key, kind = key.to_s.split('_', 2) # e.g. "c", "target_date" - next unless grade_number.key?(grade_key) - next unless field_map.key?(kind) + grade_due_date_rows.each do |row_params| + target_grade = row_params[:target_grade] || row_params['target_grade'] + next if target_grade.to_i.zero? - row = TaskDefinitionGradeDueDate.find_or_initialize_by( - task_definition: task_def, - target_grade: grade_number[grade_key] - ) - row.update!(field_map[kind] => date) + unless unit.grade_value?(target_grade) + error!({ error: "Target grade #{target_grade} is not enabled for this unit" }, 422) + end + + target_due_date = row_params[:target_due_date] || row_params['target_due_date'] + start_date = row_params[:start_date] || row_params['start_date'] + row = TaskDefinitionGradeDueDate.find_or_initialize_by( + task_definition: task_def, + target_grade: target_grade + ) + + if target_due_date.blank? && start_date.blank? + row.destroy if row.persisted? + else + row.update!(target_due_date: target_due_date, start_date: start_date) + end + end end present task_def, with: Entities::TaskDefinitionEntity, my_role: unit.role_for(current_user) diff --git a/app/api/units_api.rb b/app/api/units_api.rb index 59160e06e9..4d1e80b971 100644 --- a/app/api/units_api.rb +++ b/app/api/units_api.rb @@ -93,6 +93,12 @@ class UnitsApi < Grape::API optional :feedback_warning_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its highlighted in the tutors inbox' optional :feedback_overflow_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its added to overflow marking' optional :enforce_feedback_before_discussed_in_class, type: Boolean, desc: 'Require feedback to be completed before tasks can be marked discussed in class' + optional :grade_definitions, type: Array do + requires :id, type: String + requires :value, type: Integer + requires :label, type: String + requires :abbreviation, type: String + end mutually_exclusive :teaching_period_id, :start_date mutually_exclusive :teaching_period_id, :end_date @@ -128,7 +134,8 @@ class UnitsApi < Grape::API :assessment_enabled, :feedback_warning_threshold_days, :feedback_overflow_threshold_days, - :enforce_feedback_before_discussed_in_class + :enforce_feedback_before_discussed_in_class, + grade_definitions: [:id, :value, :label, :abbreviation] ) if unit.teaching_period_id.present? && (unit_parameters.key?(:start_date) || unit_parameters['teaching_period_id'] == -1) @@ -177,6 +184,12 @@ class UnitsApi < Grape::API optional :feedback_warning_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its highlighted in the tutors inbox' optional :feedback_overflow_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its added to overflow marking' optional :enforce_feedback_before_discussed_in_class, type: Boolean, desc: 'Require feedback to be completed before tasks can be marked discussed in class', default: false + optional :grade_definitions, type: Array do + requires :id, type: String + requires :value, type: Integer + requires :label, type: String + requires :abbreviation, type: String + end mutually_exclusive :teaching_period_id, :start_date mutually_exclusive :teaching_period_id, :end_date @@ -209,7 +222,8 @@ class UnitsApi < Grape::API :allow_student_change_tutorial, :feedback_warning_threshold_days, :feedback_overflow_threshold_days, - :enforce_feedback_before_discussed_in_class + :enforce_feedback_before_discussed_in_class, + grade_definitions: [:id, :value, :label, :abbreviation] ) # Ensure the user is authorised to convene units diff --git a/app/models/communication/change_target_grade_action.rb b/app/models/communication/change_target_grade_action.rb index 07913b2d03..efcfc2cb8e 100644 --- a/app/models/communication/change_target_grade_action.rb +++ b/app/models/communication/change_target_grade_action.rb @@ -1,3 +1,12 @@ class ChangeTargetGradeAction < CommunicationAction validates :target_grade, presence: true + validate :target_grade_enabled_for_unit + + private + + def target_grade_enabled_for_unit + return if target_grade.nil? || communication_rule&.unit&.grade_value?(target_grade) + + errors.add(:target_grade, 'is not enabled for this unit') + end end diff --git a/app/models/communication/target_grade_condition.rb b/app/models/communication/target_grade_condition.rb index 7f0c30f560..77f195a473 100644 --- a/app/models/communication/target_grade_condition.rb +++ b/app/models/communication/target_grade_condition.rb @@ -1,4 +1,13 @@ class TargetGradeCondition < CommunicationCondition validates :target_grade, presence: true validates :operator, inclusion: { in: GRADE_OPERATORS } + validate :target_grade_enabled_for_unit + + private + + def target_grade_enabled_for_unit + return if target_grade.nil? || communication&.unit&.grade_value?(target_grade) + + errors.add(:target_grade, 'is not enabled for this unit') + end end diff --git a/app/models/communication/task_status_count_condition.rb b/app/models/communication/task_status_count_condition.rb index 13d24aaacb..42659a930e 100644 --- a/app/models/communication/task_status_count_condition.rb +++ b/app/models/communication/task_status_count_condition.rb @@ -1,6 +1,15 @@ class TaskStatusCountCondition < CommunicationCondition validates :task_status_count, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } - validates :task_target_grade, presence: true, inclusion: { in: GradeHelper::RANGE } + validates :task_target_grade, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :operator, inclusion: { in: GRADE_OPERATORS } validate :task_statuses_must_be_present + validate :task_target_grade_enabled_for_unit + + private + + def task_target_grade_enabled_for_unit + return if task_target_grade.nil? || communication&.unit&.grade_value?(task_target_grade) + + errors.add(:task_target_grade, 'is not enabled for this unit') + end end diff --git a/app/models/project.rb b/app/models/project.rb index dbe0405ad7..9ea9e9d2db 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -270,16 +270,7 @@ def active? # Get a string representation of the Target Grade # def target_grade_desc - case target_grade - when 1 - 'Credit' - when 2 - 'Distinction' - when 3 - 'High Distinction' - else - 'Pass' - end + unit.grade_label(target_grade) end def reference_date @@ -370,7 +361,7 @@ def top_tasks # overdue_tasks = task_states.select { |ts| to_target.call(ts) < Time.zone.today } - for i in GradeHelper::RANGE + for i in unit.grade_values graded_tasks = overdue_tasks.select { |ts| ts[:task_definition].target_grade == i } graded_tasks.each do |ts| @@ -386,7 +377,7 @@ def top_tasks # soon_tasks = task_states.select { |ts| to_target.call(ts) >= Time.zone.today && to_target.call(ts) < Time.zone.today + 7.days } - for i in GradeHelper::RANGE + for i in unit.grade_values graded_tasks = soon_tasks.select { |ts| ts[:task_definition].target_grade == i } graded_tasks.each do |ts| @@ -401,7 +392,7 @@ def top_tasks # ahead_tasks = task_states.select { |ts| to_target.call(ts) >= Time.zone.today + 7.days } - for i in GradeHelper::RANGE + for i in unit.grade_values graded_tasks = ahead_tasks.select { |ts| ts[:task_definition].target_grade == i } graded_tasks.each do |ts| @@ -554,19 +545,14 @@ def revert_overdue_tasks # task_stats field def update_task_stats # generate SQL for columns that count the number of tasks per grade - count_by_grade = (GradeHelper::RANGE).map { |grade_id| "SUM(CASE WHEN target_grade <= #{grade_id} THEN 1 ELSE 0 END) AS count_#{grade_id}" } + count_by_grade = unit.grade_values.map { |grade_id| "SUM(CASE WHEN target_grade <= #{grade_id} THEN 1 ELSE 0 END) AS count_#{grade_id}" } # Get the count of the total number of tasks less than each target grade task_count = unit .task_definitions .select(*count_by_grade) # create columns for each grade .map do |r| # map to array - [ - r['count_0'].to_f || 0.0, - r['count_1'].to_f || 0.0, - r['count_2'].to_f || 0.0, - r['count_3'].to_f || 0.0 - ] + unit.grade_values.index_with { |grade_id| r["count_#{grade_id}"].to_f || 0.0 } end .first # there is only one row returned... diff --git a/app/models/task.rb b/app/models/task.rb index 30f148f5d8..9533bda8f7 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -413,11 +413,7 @@ def local_due_date if unit.allow_flexible_dates return target_due_date if target_due_date.present? - grade_target_date = case project.target_grade - when 1 then task_definition.c_target_date - when 2 then task_definition.d_target_date - when 3 then task_definition.hd_target_date - end + grade_target_date = task_definition.grade_target_date(project.target_grade) return grade_target_date if grade_target_date.present? end @@ -428,11 +424,7 @@ def local_start_date if unit.allow_flexible_dates return target_start_date if target_start_date.present? - grade_start_date = case project.target_grade - when 1 then task_definition.c_start_date - when 2 then task_definition.d_start_date - when 3 then task_definition.hd_start_date - end + grade_start_date = task_definition.grade_start_date(project.target_grade) return grade_start_date if grade_start_date.present? end @@ -724,13 +716,9 @@ def grade_task(new_grade, ui = nil, grading_group = false) raise message end - grade_map = { - 'f' => -1, - 'p' => 0, - 'c' => 1, - 'd' => 2, - 'hd' => 3 - } + grade_map = unit.grade_definitions.to_h do |definition| + [definition['abbreviation'].downcase, definition['value']] + end if task_definition.is_graded if new_grade.nil? raise_error.call("No grade was supplied for a graded task (task id #{id})") @@ -742,13 +730,16 @@ def grade_task(new_grade, ui = nil, grading_group = false) if new_grade.is_a?(String) if grade_map.keys.include?(new_grade.downcase) # convert string representation to integer representation - new_grade = grade_map[new_grade] + new_grade = grade_map[new_grade.downcase] else - raise_error.call("New grade supplied to task is not a valid string - expects one of {f|p|c|d|hd} (task id #{id})") + raise_error.call("New grade supplied to task is not a valid abbreviation (task id #{id})") end end - unless new_grade.is_a?(Integer) && grade_map.values.include?(new_grade.to_i) - raise_error.call("New grade supplied to task is not a valid integer - expects one of {-1|0|1|2|3} (task id #{id})") + unless new_grade.is_a?(Integer) + raise_error.call("New grade supplied to task is not a valid integer (task id #{id})") + end + unless unit.assessment_grade_value?(new_grade) + raise_error.call("Grade is not enabled for this unit (task id #{id})") end # propagate new grade to all OTHER group members if group_task? && !grading_group diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index e9f1c1444b..7ef377811f 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -93,7 +93,8 @@ def self.permissions validates :name, uniqueness: { scope: :unit_id } # task definition names within a unit must be unique validates :abbreviation, uniqueness: { scope: :unit_id } # task definition names within a unit must be unique - validates :target_grade, inclusion: { in: GradeHelper::RANGE, message: '%{value} is not a valid target grade' } + validates :target_grade, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validate :target_grade_enabled_for_unit validates :max_quality_pts, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 100, message: 'must be between 0 and 100' } validate :upload_requirements, :check_upload_requirements_format @@ -114,36 +115,22 @@ def self.permissions include TaskDefinitionTiiModule include TaskDefinitionSimilarityModule - # def p_target_date - # due_date - # end - - # Per-grade target date overrides - - def c_target_date - grade_due_dates.find { |g| g.target_grade == 1 }&.target_due_date - end - - def d_target_date - grade_due_dates.find { |g| g.target_grade == 2 }&.target_due_date - end - - def hd_target_date - grade_due_dates.find { |g| g.target_grade == 3 }&.target_due_date - end - - # Per-grade start date overrides - - def c_start_date - grade_due_dates.find { |g| g.target_grade == 1 }&.start_date + def grade_due_date_overrides + grade_due_dates.map do |override| + { + target_grade: override.target_grade, + target_due_date: override.target_due_date, + start_date: override.start_date + } + end end - def d_start_date - grade_due_dates.find { |g| g.target_grade == 2 }&.start_date + def grade_target_date(target_grade) + grade_due_dates.find { |g| g.target_grade == target_grade.to_i }&.target_due_date end - def hd_start_date - grade_due_dates.find { |g| g.target_grade == 3 }&.start_date + def grade_start_date(target_grade) + grade_due_dates.find { |g| g.target_grade == target_grade.to_i }&.start_date end def unit_must_be_same @@ -940,6 +927,12 @@ def read_file_from_resources(filename) private + def target_grade_enabled_for_unit + return if unit.nil? || target_grade.nil? || unit.grade_value?(target_grade) + + errors.add(:target_grade, 'is not enabled for this unit') + end + def delete_associated_files() remove_task_sheet() remove_task_resources() diff --git a/app/models/unit.rb b/app/models/unit.rb index 5c3d72bbbe..9fa3b4465b 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -6,8 +6,19 @@ require 'moss_ruby' require 'csv_helper' require 'grade_helper' +require 'securerandom' class Unit < ApplicationRecord + DEFAULT_GRADE_DEFINITIONS = [ + { 'id' => 'fail', 'value' => -1, 'label' => 'Fail', 'abbreviation' => 'F' }, + { 'id' => 'pass', 'value' => 0, 'label' => 'Pass', 'abbreviation' => 'P' }, + { 'id' => 'credit', 'value' => 1, 'label' => 'Credit', 'abbreviation' => 'C' }, + { 'id' => 'distinction', 'value' => 2, 'label' => 'Distinction', 'abbreviation' => 'D' }, + { 'id' => 'high-distinction', 'value' => 3, 'label' => 'High Distinction', 'abbreviation' => 'HD' } + ].freeze + + serialize :grade_values, coder: JSON + include ApplicationHelper include FileHelper include MimeCheckHelpers @@ -213,6 +224,8 @@ def role_for(user) validate :autogen_date_within_unit_active_period, if: -> { start_date_changed? || end_date_changed? || teaching_period_id_changed? || portfolio_auto_generation_date_changed? } validate :cant_disable_aip_only_if_aip_tasks_exist + validate :grade_definitions_are_valid + validate :configured_grades_preserve_used_values, if: :will_save_change_to_grade_values? scope :current, -> { current_for_date(Time.zone.now) } scope :current_for_date, ->(date) { where('start_date <= ? AND end_date >= ?', date, date) } @@ -310,6 +323,46 @@ def has_teaching_period? self.teaching_period.present? end + def grade_values + grade_definitions.filter_map { |definition| definition['value'] unless definition['value'] == -1 } + end + + def grade_definitions + normalize_grade_definitions(self[:grade_values]) + end + + def grade_value?(value) + grade_values.include?(value.to_i) + end + + def assessment_grade_value?(value) + grade_definitions.any? { |definition| definition['value'] == value.to_i } + end + + def grade_definition(value) + return nil if value.nil? + + grade_definitions.find { |definition| definition['value'] == value.to_i } + end + + def grade_label(value) + grade_definition(value)&.fetch('label', nil) || GradeHelper.grade_for(value) + end + + def grade_abbreviation(value) + grade_definition(value)&.fetch('abbreviation', nil) || GradeHelper.short_grade_for(value) + end + + def grade_definitions=(definitions) + normalized = normalize_grade_definitions(definitions, sort: false) + fail_definition = normalized.find { |definition| definition['value'] == -1 } + target_definitions = normalized.reject { |definition| definition['value'] == -1 } + + fail_definition['value'] = -1 + target_definitions.each_with_index { |definition, index| definition['value'] = index } + self[:grade_values] = [fail_definition, *target_definitions] + end + def ensure_teaching_period_dates_match if self[:start_date] != teaching_period.start_date errors.add(:start_date, "should match teaching period date") @@ -336,6 +389,80 @@ def cant_disable_aip_only_if_aip_tasks_exist end end + def grade_definitions_are_valid + definitions = grade_definitions + values = definitions.map { |definition| definition['value'] } + target_values = values.reject { |value| value == -1 } + + errors.add(:grade_definitions, 'must include a failure grade and at least one target grade') if definitions.length < 2 + errors.add(:grade_definitions, 'must include index -1 exactly once') unless values.count(-1) == 1 + errors.add(:grade_definitions, 'target grade indexes must be unique non-negative integers') unless target_values.all? { |value| value >= 0 } && target_values.uniq.length == target_values.length + errors.add(:grade_definitions, 'must use unique identifiers') unless definitions.map { |definition| definition['id'] }.uniq.length == definitions.length + errors.add(:grade_definitions, 'must use unique labels') unless definitions.map { |definition| definition['label'].downcase }.uniq.length == definitions.length + errors.add(:grade_definitions, 'must use unique abbreviations') unless definitions.map { |definition| definition['abbreviation'].downcase }.uniq.length == definitions.length + + definitions.each do |definition| + errors.add(:grade_definitions, 'labels must be present and no longer than 50 characters') unless definition['label'].present? && definition['label'].length <= 50 + errors.add(:grade_definitions, 'abbreviations must be present and no longer than 10 characters') unless definition['abbreviation'].present? && definition['abbreviation'].length <= 10 + end + end + + def configured_grades_preserve_used_values + used_values = task_definitions.distinct.pluck(:target_grade) + used_values |= projects.distinct.pluck(:target_grade, :submitted_grade).flatten.compact + used_values |= tasks.distinct.pluck(:grade).compact + used_values |= communication_rules.joins(:communication_conditions) + .pluck('communication_conditions.target_grade', 'communication_conditions.task_target_grade') + .flatten + .compact + used_values |= communication_rules.joins(:communication_actions) + .pluck('communication_actions.target_grade') + .compact + previous_definitions = normalize_grade_definitions(attribute_in_database('grade_values')) + current_by_id = grade_definitions.index_by { |definition| definition['id'] } + + changed_values = used_values.select do |value| + previous_definition = previous_definitions.find { |definition| definition['value'] == value } + current_definition = current_by_id[previous_definition&.fetch('id', nil)] + current_definition.nil? || current_definition['value'] != value + end + + errors.add(:grade_definitions, "cannot remove or reorder grades currently in use at indexes: #{changed_values.uniq.sort.join(', ')}") if changed_values.any? + end + + def normalize_grade_definitions(raw_definitions, sort: true) + raw_definitions = JSON.parse(raw_definitions) if raw_definitions.is_a?(String) + raw_definitions = DEFAULT_GRADE_DEFINITIONS if raw_definitions.blank? + + definitions = Array(raw_definitions).map do |definition| + definition = definition.to_h if definition.respond_to?(:to_h) + if definition.is_a?(Hash) + definition = definition.stringify_keys + { + 'id' => definition['id'].presence || SecureRandom.uuid, + 'value' => definition['value'].to_i, + 'label' => definition['label'].to_s.strip, + 'abbreviation' => definition['abbreviation'].to_s.strip.upcase + } + else + value = definition.to_i + default = DEFAULT_GRADE_DEFINITIONS.find { |item| item['value'] == value } + default&.dup || { + 'id' => "grade-#{value}", + 'value' => value, + 'label' => "Grade #{value}", + 'abbreviation' => "G#{value}" + } + end + end + + unless definitions.any? { |definition| definition['value'] == -1 } + definitions.unshift(DEFAULT_GRADE_DEFINITIONS.first.dup) + end + + sort ? definitions.sort_by { |definition| definition['value'] } : definitions + end + def validate_end_date_after_start_date if end_date.present? && start_date.present? && end_date < start_date @@ -1868,7 +1995,7 @@ def task_completion_csv row['student_id'], row['username'], "#{row['first_name']} #{row['last_name']}", - GradeHelper.grade_for(row['target_grade']), + grade_label(row['target_grade']), row['email'], row['portfolio_production_date'].present? && !row['compile_portfolio'] && File.exist?(FileHelper.student_portfolio_path(self, row['username'], create: true)), row['grade'] > 0 ? row['grade'] : nil, @@ -1884,7 +2011,7 @@ def task_completion_csv row["grp_#{gs.id}"] end + task_def_by_grade.map do |td| result = [row["status_#{td.id}"].nil? ? TaskStatus.not_started.name : row["status_#{td.id}"]] - result << GradeHelper.short_grade_for(row["grade_#{td.id}"]) if td.is_graded? + result << grade_abbreviation(row["grade_#{td.id}"]) if td.is_graded? result << row["stars_#{td.id}"] if td.has_stars? result << row["people_#{td.id}"] if td.is_group_task? result @@ -2502,7 +2629,7 @@ def student_task_completion_stats result[:tutorial][t.id] = _calculate_task_completion_stats(data.select { |r| r[:tutorial_id] == t.id }) end - for i in GradeHelper::RANGE do + for i in grade_values do result[:grade][i] = _calculate_task_completion_stats(data.select { |r| r[:grade] == i }) end diff --git a/app/models/webcal.rb b/app/models/webcal.rb index 4278eff6b5..d0ca3c3e3b 100644 --- a/app/models/webcal.rb +++ b/app/models/webcal.rb @@ -178,14 +178,9 @@ def self.start_date_for_task_definition(task_def, task = nil, project = nil) def self.flexible_grade_date_for_task_definition(task_def, project, date_type) return nil unless project&.unit&.allow_flexible_dates - case [project.target_grade, date_type] - when [1, :target_date] then task_def.c_target_date - when [2, :target_date] then task_def.d_target_date - when [3, :target_date] then task_def.hd_target_date - when [1, :start_date] then task_def.c_start_date - when [2, :start_date] then task_def.d_start_date - when [3, :start_date] then task_def.hd_start_date - end + return task_def.grade_target_date(project.target_grade) if date_type == :target_date + + task_def.grade_start_date(project.target_grade) end # diff --git a/app/sidekiq/execute_communication_set_job.rb b/app/sidekiq/execute_communication_set_job.rb index 3c772dff31..fa279e1124 100644 --- a/app/sidekiq/execute_communication_set_job.rb +++ b/app/sidekiq/execute_communication_set_job.rb @@ -372,7 +372,7 @@ def render_template(template, project, unit, rule, affected_students_count, targ '{{unit.code}}' => unit.code.to_s, '{{unit.name}}' => unit.name.to_s, '{{rule.name}}' => rule.name.to_s, - '{{target_grade}}' => target_grade_name(target_grade_value), + '{{target_grade}}' => target_grade_name(target_grade_value, unit), '{{conditions_summary}}' => conditions_summary(rule), '{{actions_summary}}' => actions_summary(rule, action_results) } @@ -382,8 +382,8 @@ def render_template(template, project, unit, rule, affected_students_count, targ end end - def target_grade_name(value) - GradeHelper.grade_for(value).to_s + def target_grade_name(value, unit = nil) + (unit&.grade_label(value) || GradeHelper.grade_for(value)).to_s end def formatted_email(user) @@ -444,7 +444,7 @@ def build_action_log_csv(rule, projects, action_results) project = projects.find { |item| item.id == result[:project_id] } student = project&.user details = if result[:status] == 'updated' - "Changed target grade from #{target_grade_name(result[:previous_target_grade])} to #{target_grade_name(result[:target_grade])}" + "Changed target grade from #{target_grade_name(result[:previous_target_grade], rule.unit)} to #{target_grade_name(result[:target_grade], rule.unit)}" elsif result[:status] == 'commented' task_definition = TaskDefinition.find_by(id: result[:task_definition_id]) "Added comment to #{task_definition_label(task_definition)}" @@ -504,9 +504,9 @@ def human_condition_summary(condition) end "Students that have #{task_label} #{predicate} [#{Array(condition.task_statuses).map { |status| status.to_s.titleize }.join(', ')}]" when 'TargetGradeCondition' - "Students with a Target Grade #{operator_label(condition.operator)} #{target_grade_name(condition.target_grade)}" + "Students with a Target Grade #{operator_label(condition.operator)} #{target_grade_name(condition.target_grade, condition.communication.unit)}" when 'TaskStatusCountCondition' - grade_label = target_grade_name(condition.task_target_grade) + grade_label = target_grade_name(condition.task_target_grade, condition.communication.unit) statuses = Array(condition.task_statuses).map { |status| status.to_s.titleize }.join(', ') "Students that have #{operator_label(condition.operator)} #{condition.task_status_count} #{grade_label} tasks in [#{statuses}]" when 'LoginStatusCondition' @@ -547,7 +547,7 @@ def human_action_summary(action) when 'EmailStaffAction' 'Send staff email' when 'ChangeTargetGradeAction' - "Change Target Grade to #{target_grade_name(action.target_grade)}" + "Change Target Grade to #{target_grade_name(action.target_grade, action.communication_rule.unit)}" when 'TaskCommentAction' "Add comment to #{task_definition_label(action.task_definition)}" else diff --git a/db/migrate/20260618033139_add_grade_values_to_units.rb b/db/migrate/20260618033139_add_grade_values_to_units.rb new file mode 100644 index 0000000000..6f82693d02 --- /dev/null +++ b/db/migrate/20260618033139_add_grade_values_to_units.rb @@ -0,0 +1,8 @@ +class AddGradeValuesToUnits < ActiveRecord::Migration[8.0] + def change + add_column :units, :grade_values, :json, null: false, default: [0, 1, 2, 3] + + change_column_null :task_definition_grade_due_dates, :target_due_date, true + add_column :task_definition_grade_due_dates, :start_date, :datetime, null: true unless column_exists?(:task_definition_grade_due_dates, :start_date) + end +end diff --git a/db/schema.rb b/db/schema.rb index 3e5727b917..54788bc2a6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_06_18_001447) do +ActiveRecord::Schema[8.0].define(version: 2026_06_18_033139) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -891,10 +891,12 @@ t.integer "feedback_warning_threshold_days", default: 5 t.integer "feedback_overflow_threshold_days", default: 7 t.boolean "enforce_feedback_before_discussed_in_class", default: false, null: false + t.text "grade_values", size: :long, default: "[0,1,2,3]", null: false, collation: "utf8mb4_bin" t.index ["draft_task_definition_id"], name: "index_units_on_draft_task_definition_id" t.index ["main_convenor_id"], name: "index_units_on_main_convenor_id" t.index ["overseer_image_id"], name: "index_units_on_overseer_image_id" t.index ["teaching_period_id"], name: "index_units_on_teaching_period_id" + t.check_constraint "json_valid(`grade_values`)", name: "grade_values" end create_table "user_oauth_states", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| From 9c3f8b174d384d6c36b5590a763c79bd0f27ec68 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:19:46 +1000 Subject: [PATCH 103/199] chore(release): 11.0.0-12 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77674715fe..66cc9fb9d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-12](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-11...v11.0.0-12) (2026-06-24) + + +### Features + +* customisable grades ([#638](https://github.com/b0ink/doubtfire-deploy/issues/638)) ([57cb9e2](https://github.com/b0ink/doubtfire-deploy/commit/57cb9e250b7fbfe9eb477021c9077d86d4b696ab)) + ## [11.0.0-11](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-10...v11.0.0-11) (2026-06-18) From 3a0c879398a62469524f6e49410cccf5da0a2a33 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:32:55 +1000 Subject: [PATCH 104/199] fix: avoid json default --- .../20260618033139_add_grade_values_to_units.rb | 14 ++++++++++++-- db/schema.rb | 3 +-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/db/migrate/20260618033139_add_grade_values_to_units.rb b/db/migrate/20260618033139_add_grade_values_to_units.rb index 6f82693d02..5f94a693a1 100644 --- a/db/migrate/20260618033139_add_grade_values_to_units.rb +++ b/db/migrate/20260618033139_add_grade_values_to_units.rb @@ -1,8 +1,18 @@ class AddGradeValuesToUnits < ActiveRecord::Migration[8.0] - def change - add_column :units, :grade_values, :json, null: false, default: [0, 1, 2, 3] + DEFAULT_GRADE_VALUES = [0, 1, 2, 3].to_json + + def up + add_column :units, :grade_values, :json + execute "UPDATE units SET grade_values = #{connection.quote(DEFAULT_GRADE_VALUES)}" + change_column_null :units, :grade_values, false change_column_null :task_definition_grade_due_dates, :target_due_date, true add_column :task_definition_grade_due_dates, :start_date, :datetime, null: true unless column_exists?(:task_definition_grade_due_dates, :start_date) end + + def down + remove_column :task_definition_grade_due_dates, :start_date if column_exists?(:task_definition_grade_due_dates, :start_date) + change_column_null :task_definition_grade_due_dates, :target_due_date, false + remove_column :units, :grade_values + end end diff --git a/db/schema.rb b/db/schema.rb index 54788bc2a6..ed0b4ace5f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -891,12 +891,11 @@ t.integer "feedback_warning_threshold_days", default: 5 t.integer "feedback_overflow_threshold_days", default: 7 t.boolean "enforce_feedback_before_discussed_in_class", default: false, null: false - t.text "grade_values", size: :long, default: "[0,1,2,3]", null: false, collation: "utf8mb4_bin" + t.text "grade_values", size: :long, null: false, collation: "utf8mb4_bin" t.index ["draft_task_definition_id"], name: "index_units_on_draft_task_definition_id" t.index ["main_convenor_id"], name: "index_units_on_main_convenor_id" t.index ["overseer_image_id"], name: "index_units_on_overseer_image_id" t.index ["teaching_period_id"], name: "index_units_on_teaching_period_id" - t.check_constraint "json_valid(`grade_values`)", name: "grade_values" end create_table "user_oauth_states", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| From 8b586f9b76c9cc82f71a33bcace5532e4ad9c4ec Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:39:03 +1000 Subject: [PATCH 105/199] fix: allow nullable grade values --- db/migrate/20260618033139_add_grade_values_to_units.rb | 1 - db/schema.rb | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrate/20260618033139_add_grade_values_to_units.rb b/db/migrate/20260618033139_add_grade_values_to_units.rb index 5f94a693a1..1cc8ca1b4c 100644 --- a/db/migrate/20260618033139_add_grade_values_to_units.rb +++ b/db/migrate/20260618033139_add_grade_values_to_units.rb @@ -4,7 +4,6 @@ class AddGradeValuesToUnits < ActiveRecord::Migration[8.0] def up add_column :units, :grade_values, :json execute "UPDATE units SET grade_values = #{connection.quote(DEFAULT_GRADE_VALUES)}" - change_column_null :units, :grade_values, false change_column_null :task_definition_grade_due_dates, :target_due_date, true add_column :task_definition_grade_due_dates, :start_date, :datetime, null: true unless column_exists?(:task_definition_grade_due_dates, :start_date) diff --git a/db/schema.rb b/db/schema.rb index ed0b4ace5f..8e5a22b10e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -891,11 +891,12 @@ t.integer "feedback_warning_threshold_days", default: 5 t.integer "feedback_overflow_threshold_days", default: 7 t.boolean "enforce_feedback_before_discussed_in_class", default: false, null: false - t.text "grade_values", size: :long, null: false, collation: "utf8mb4_bin" + t.text "grade_values", size: :long, collation: "utf8mb4_bin" t.index ["draft_task_definition_id"], name: "index_units_on_draft_task_definition_id" t.index ["main_convenor_id"], name: "index_units_on_main_convenor_id" t.index ["overseer_image_id"], name: "index_units_on_overseer_image_id" t.index ["teaching_period_id"], name: "index_units_on_teaching_period_id" + t.check_constraint "json_valid(`grade_values`)", name: "grade_values" end create_table "user_oauth_states", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| From 5c3c0dc221002ce4f42036fc8736999af88f7a5a Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:44:47 +1000 Subject: [PATCH 106/199] chore(release): 11.0.0-13 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66cc9fb9d8..b55d13968a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-13](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-12...v11.0.0-13) (2026-06-24) + + +### Bug Fixes + +* allow nullable grade values ([8b586f9](https://github.com/b0ink/doubtfire-deploy/commit/8b586f9b76c9cc82f71a33bcace5532e4ad9c4ec)) +* avoid json default ([3a0c879](https://github.com/b0ink/doubtfire-deploy/commit/3a0c879398a62469524f6e49410cccf5da0a2a33)) + ## [11.0.0-12](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-11...v11.0.0-12) (2026-06-24) From 1a475640e211d3dec7e3b5034b3f0efddaa6e29d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:00:16 +1000 Subject: [PATCH 107/199] fix: avoid serialisation --- app/models/unit.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index 9fa3b4465b..57e664e874 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -17,8 +17,6 @@ class Unit < ApplicationRecord { 'id' => 'high-distinction', 'value' => 3, 'label' => 'High Distinction', 'abbreviation' => 'HD' } ].freeze - serialize :grade_values, coder: JSON - include ApplicationHelper include FileHelper include MimeCheckHelpers From ea7c5db44e3f1751cf6d93028a07f0f82cca3bec Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:08:41 +1000 Subject: [PATCH 108/199] fix: check grade value is valid json --- app/models/unit.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index 57e664e874..e514d9a7ac 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -358,7 +358,15 @@ def grade_definitions=(definitions) fail_definition['value'] = -1 target_definitions.each_with_index { |definition, index| definition['value'] = index } - self[:grade_values] = [fail_definition, *target_definitions] + self[:grade_values] = grade_values_for_column([fail_definition, *target_definitions]) + end + + def grade_values_for_column(definitions) + if self.class.type_for_attribute('grade_values').is_a?(ActiveRecord::Type::Json) + definitions + else + definitions.to_json + end end def ensure_teaching_period_dates_match From b6648a35938106bc938be64853466512de4c3a30 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:08:45 +1000 Subject: [PATCH 109/199] chore(release): 11.0.0-14 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b55d13968a..7bda61a545 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-14](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-13...v11.0.0-14) (2026-06-24) + + +### Bug Fixes + +* avoid serialisation ([1a47564](https://github.com/b0ink/doubtfire-deploy/commit/1a475640e211d3dec7e3b5034b3f0efddaa6e29d)) +* check grade value is valid json ([ea7c5db](https://github.com/b0ink/doubtfire-deploy/commit/ea7c5db44e3f1751cf6d93028a07f0f82cca3bec)) + ## [11.0.0-13](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-12...v11.0.0-13) (2026-06-24) From 3dd6106bb65eb739c81f8eab5332684a915d0c9f Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:10:54 +1000 Subject: [PATCH 110/199] fix: check grade value is valid json --- app/models/unit.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index e514d9a7ac..d1980a6abb 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -437,8 +437,8 @@ def configured_grades_preserve_used_values end def normalize_grade_definitions(raw_definitions, sort: true) - raw_definitions = JSON.parse(raw_definitions) if raw_definitions.is_a?(String) raw_definitions = DEFAULT_GRADE_DEFINITIONS if raw_definitions.blank? + raw_definitions = JSON.parse(raw_definitions) if raw_definitions.is_a?(String) definitions = Array(raw_definitions).map do |definition| definition = definition.to_h if definition.respond_to?(:to_h) @@ -469,7 +469,6 @@ def normalize_grade_definitions(raw_definitions, sort: true) sort ? definitions.sort_by { |definition| definition['value'] } : definitions end - def validate_end_date_after_start_date if end_date.present? && start_date.present? && end_date < start_date errors.add(:end_date, "should be after the Start date") From 9fff32829f3aca2b923d67b4bba9e433b72e1dc7 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:10:59 +1000 Subject: [PATCH 111/199] chore(release): 11.0.0-15 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bda61a545..4beb2f83c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-15](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-14...v11.0.0-15) (2026-06-24) + + +### Bug Fixes + +* check grade value is valid json ([3dd6106](https://github.com/b0ink/doubtfire-deploy/commit/3dd6106bb65eb739c81f8eab5332684a915d0c9f)) + ## [11.0.0-14](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-13...v11.0.0-14) (2026-06-24) From 3eecd2f8e7733e79307830e173e6687fc4d2632f Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:53:23 +1000 Subject: [PATCH 112/199] feat: add sentry tunnel support --- app/api/api_root.rb | 1 + app/api/sentry_tunnel_api.rb | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 app/api/sentry_tunnel_api.rb diff --git a/app/api/api_root.rb b/app/api/api_root.rb index 3dbc682297..6f6b6f1669 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -66,6 +66,7 @@ class ApiRoot < Grape::API mount GroupSetsApi mount LearningOutcomesApi mount ProjectsApi + mount SentryTunnelApi mount SettingsApi mount StudentsApi mount Submission::PortfolioApi diff --git a/app/api/sentry_tunnel_api.rb b/app/api/sentry_tunnel_api.rb new file mode 100644 index 0000000000..d2d6392cb2 --- /dev/null +++ b/app/api/sentry_tunnel_api.rb @@ -0,0 +1,45 @@ +require 'grape' +require 'rest-client' +require 'uri' + +class SentryTunnelApi < Grape::API + helpers do + def sentry_envelope_url + dsn = ENV.fetch('SENTRY_DSN', nil) + return nil if dsn.blank? + + uri = URI.parse(dsn) + project_id = uri.path.delete_prefix('/') + public_key = uri.user + return nil if project_id.blank? || public_key.blank? + + "#{uri.scheme}://#{uri.host}/api/#{project_id}/envelope/?sentry_key=#{URI.encode_www_form_component(public_key)}" + rescue URI::InvalidURIError + nil + end + end + + desc 'Forward browser Sentry envelopes to Sentry' + post '/sentry/tunnel' do + envelope_url = sentry_envelope_url + status 204 + return if envelope_url.blank? + + body = request.body.read + return if body.blank? + + RestClient::Request.execute( + method: :post, + url: envelope_url, + payload: body, + headers: { content_type: request.content_type || 'application/x-sentry-envelope' }, + timeout: 5, + open_timeout: 2 + ) + + nil + rescue RestClient::Exception, SocketError, Timeout::Error => e + logger.warn "Unable to forward Sentry envelope: #{e.class}" + nil + end +end From 5edfe6c4e2bb3c9c5c5ee4e085091e401b6abf0d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:53:32 +1000 Subject: [PATCH 113/199] chore(release): 11.0.0-16 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4beb2f83c2..77399b9fcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-16](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-15...v11.0.0-16) (2026-06-25) + + +### Features + +* add sentry tunnel support ([3eecd2f](https://github.com/b0ink/doubtfire-deploy/commit/3eecd2f8e7733e79307830e173e6687fc4d2632f)) + ## [11.0.0-15](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-14...v11.0.0-15) (2026-06-24) From 3329076b6bdf52362b7b27cfac30041fa5c2aff1 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:10:45 +1000 Subject: [PATCH 114/199] refactor: move sentry tunnel to controller --- app/api/api_root.rb | 1 - app/api/sentry_tunnel_api.rb | 45 --------------------- app/controllers/sentry_tunnel_controller.rb | 44 ++++++++++++++++++++ config/routes.rb | 1 + 4 files changed, 45 insertions(+), 46 deletions(-) delete mode 100644 app/api/sentry_tunnel_api.rb create mode 100644 app/controllers/sentry_tunnel_controller.rb diff --git a/app/api/api_root.rb b/app/api/api_root.rb index 6f6b6f1669..3dbc682297 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -66,7 +66,6 @@ class ApiRoot < Grape::API mount GroupSetsApi mount LearningOutcomesApi mount ProjectsApi - mount SentryTunnelApi mount SettingsApi mount StudentsApi mount Submission::PortfolioApi diff --git a/app/api/sentry_tunnel_api.rb b/app/api/sentry_tunnel_api.rb deleted file mode 100644 index d2d6392cb2..0000000000 --- a/app/api/sentry_tunnel_api.rb +++ /dev/null @@ -1,45 +0,0 @@ -require 'grape' -require 'rest-client' -require 'uri' - -class SentryTunnelApi < Grape::API - helpers do - def sentry_envelope_url - dsn = ENV.fetch('SENTRY_DSN', nil) - return nil if dsn.blank? - - uri = URI.parse(dsn) - project_id = uri.path.delete_prefix('/') - public_key = uri.user - return nil if project_id.blank? || public_key.blank? - - "#{uri.scheme}://#{uri.host}/api/#{project_id}/envelope/?sentry_key=#{URI.encode_www_form_component(public_key)}" - rescue URI::InvalidURIError - nil - end - end - - desc 'Forward browser Sentry envelopes to Sentry' - post '/sentry/tunnel' do - envelope_url = sentry_envelope_url - status 204 - return if envelope_url.blank? - - body = request.body.read - return if body.blank? - - RestClient::Request.execute( - method: :post, - url: envelope_url, - payload: body, - headers: { content_type: request.content_type || 'application/x-sentry-envelope' }, - timeout: 5, - open_timeout: 2 - ) - - nil - rescue RestClient::Exception, SocketError, Timeout::Error => e - logger.warn "Unable to forward Sentry envelope: #{e.class}" - nil - end -end diff --git a/app/controllers/sentry_tunnel_controller.rb b/app/controllers/sentry_tunnel_controller.rb new file mode 100644 index 0000000000..122035e514 --- /dev/null +++ b/app/controllers/sentry_tunnel_controller.rb @@ -0,0 +1,44 @@ +require 'rest-client' +require 'uri' + +class SentryTunnelController < ApplicationController + skip_before_action :verify_authenticity_token + + def create + envelope_url = sentry_envelope_url + return head :no_content if envelope_url.blank? + + body = request.raw_post + return head :no_content if body.blank? + + RestClient::Request.execute( + method: :post, + url: envelope_url, + payload: body, + headers: { content_type: request.content_type || 'application/x-sentry-envelope' }, + timeout: 5, + open_timeout: 2 + ) + + head :no_content + rescue RestClient::Exception, SocketError, Timeout::Error => e + Rails.logger.warn "Unable to forward Sentry envelope: #{e.class}" + head :no_content + end + + private + + def sentry_envelope_url + dsn = ENV.fetch('SENTRY_DSN', nil) + return nil if dsn.blank? + + uri = URI.parse(dsn) + project_id = uri.path.delete_prefix('/') + public_key = uri.user + return nil if project_id.blank? || public_key.blank? + + "#{uri.scheme}://#{uri.host}/api/#{project_id}/envelope/?sentry_key=#{URI.encode_www_form_component(public_key)}" + rescue URI::InvalidURIError + nil + end +end diff --git a/config/routes.rb b/config/routes.rb index ea52a79000..6fd15a672f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -5,6 +5,7 @@ get 'api/submission/unit/:id/task_definitions/:task_def_id/download_submissions', to: 'task_downloads#index' get 'api/submission/unit/:id/task_definitions/:task_def_id/student_pdfs', to: 'task_submission_pdfs#index' get 'api/units/:id/all_resources', to: 'lecture_resource_downloads#index' + post 'api/sentry/tunnel', to: 'sentry_tunnel#create' mount ApiRoot => '/' mount GrapeSwaggerRails::Engine => '/api/docs' From 1834b41aeb1e84119dcb03966cca1d027dfb73bd Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:10:50 +1000 Subject: [PATCH 115/199] chore(release): 11.0.0-17 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77399b9fcf..43da15bf2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-17](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-16...v11.0.0-17) (2026-06-25) + ## [11.0.0-16](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-15...v11.0.0-16) (2026-06-25) From baf4f692be7558bca51fab8daa89b16f3daf2088 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:25:16 +1000 Subject: [PATCH 116/199] fix: rename tunnel route to avoid ad blockers --- config/routes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/routes.rb b/config/routes.rb index 6fd15a672f..14452920a1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -5,7 +5,7 @@ get 'api/submission/unit/:id/task_definitions/:task_def_id/download_submissions', to: 'task_downloads#index' get 'api/submission/unit/:id/task_definitions/:task_def_id/student_pdfs', to: 'task_submission_pdfs#index' get 'api/units/:id/all_resources', to: 'lecture_resource_downloads#index' - post 'api/sentry/tunnel', to: 'sentry_tunnel#create' + post 'api/client-reports', to: 'sentry_tunnel#create' mount ApiRoot => '/' mount GrapeSwaggerRails::Engine => '/api/docs' From 81d7b27caa9fd379bb9ba3e8584e52ba76ab2fdc Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:25:21 +1000 Subject: [PATCH 117/199] chore(release): 11.0.0-18 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43da15bf2c..69edcd6761 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-18](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-17...v11.0.0-18) (2026-06-25) + + +### Bug Fixes + +* rename tunnel route to avoid ad blockers ([baf4f69](https://github.com/b0ink/doubtfire-deploy/commit/baf4f692be7558bca51fab8daa89b16f3daf2088)) + ## [11.0.0-17](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-16...v11.0.0-17) (2026-06-25) ## [11.0.0-16](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-15...v11.0.0-16) (2026-06-25) From f461d2db71d563cb4971c0adcff76371dbb0fdfd Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:57:46 +1000 Subject: [PATCH 118/199] refactor: support session replay data --- app/controllers/sentry_tunnel_controller.rb | 44 -------------- app/middleware/sentry_tunnel_middleware.rb | 65 +++++++++++++++++++++ config/application.rb | 3 + config/routes.rb | 1 - 4 files changed, 68 insertions(+), 45 deletions(-) delete mode 100644 app/controllers/sentry_tunnel_controller.rb create mode 100644 app/middleware/sentry_tunnel_middleware.rb diff --git a/app/controllers/sentry_tunnel_controller.rb b/app/controllers/sentry_tunnel_controller.rb deleted file mode 100644 index 122035e514..0000000000 --- a/app/controllers/sentry_tunnel_controller.rb +++ /dev/null @@ -1,44 +0,0 @@ -require 'rest-client' -require 'uri' - -class SentryTunnelController < ApplicationController - skip_before_action :verify_authenticity_token - - def create - envelope_url = sentry_envelope_url - return head :no_content if envelope_url.blank? - - body = request.raw_post - return head :no_content if body.blank? - - RestClient::Request.execute( - method: :post, - url: envelope_url, - payload: body, - headers: { content_type: request.content_type || 'application/x-sentry-envelope' }, - timeout: 5, - open_timeout: 2 - ) - - head :no_content - rescue RestClient::Exception, SocketError, Timeout::Error => e - Rails.logger.warn "Unable to forward Sentry envelope: #{e.class}" - head :no_content - end - - private - - def sentry_envelope_url - dsn = ENV.fetch('SENTRY_DSN', nil) - return nil if dsn.blank? - - uri = URI.parse(dsn) - project_id = uri.path.delete_prefix('/') - public_key = uri.user - return nil if project_id.blank? || public_key.blank? - - "#{uri.scheme}://#{uri.host}/api/#{project_id}/envelope/?sentry_key=#{URI.encode_www_form_component(public_key)}" - rescue URI::InvalidURIError - nil - end -end diff --git a/app/middleware/sentry_tunnel_middleware.rb b/app/middleware/sentry_tunnel_middleware.rb new file mode 100644 index 0000000000..8556130117 --- /dev/null +++ b/app/middleware/sentry_tunnel_middleware.rb @@ -0,0 +1,65 @@ +require 'rest-client' +require 'uri' + +class SentryTunnelMiddleware + PATH = '/api/client-reports'.freeze + + def initialize(app) + @app = app + end + + def call(env) + return @app.call(env) unless env['REQUEST_METHOD'] == 'POST' && env['PATH_INFO'] == PATH + + forward_envelope(env) + [204, {}, []] + end + + private + + def forward_envelope(env) + envelope_url = sentry_envelope_url + return if envelope_url.blank? + + body = env['rack.input'].read + return if body.blank? + + RestClient::Request.execute( + method: :post, + url: envelope_url, + payload: body, + headers: sentry_headers(env), + timeout: 5, + open_timeout: 2 + ) + rescue RestClient::ExceptionWithResponse => e + Rails.logger.warn "Unable to forward Sentry envelope: #{e.class} #{e.response&.code}" + rescue RestClient::Exception, SocketError, Timeout::Error => e + Rails.logger.warn "Unable to forward Sentry envelope: #{e.class}" + ensure + env['rack.input'].rewind if env['rack.input'].respond_to?(:rewind) + end + + def sentry_headers(env) + headers = { + content_type: env['CONTENT_TYPE'].presence || 'application/x-sentry-envelope' + } + + headers[:content_encoding] = env['HTTP_CONTENT_ENCODING'] if env['HTTP_CONTENT_ENCODING'].present? + headers + end + + def sentry_envelope_url + dsn = ENV.fetch('SENTRY_DSN', nil) + return nil if dsn.blank? + + uri = URI.parse(dsn) + project_id = uri.path.delete_prefix('/') + public_key = uri.user + return nil if project_id.blank? || public_key.blank? + + "#{uri.scheme}://#{uri.host}/api/#{project_id}/envelope/?sentry_key=#{URI.encode_www_form_component(public_key)}" + rescue URI::InvalidURIError + nil + end +end diff --git a/config/application.rb b/config/application.rb index 2466620f08..a0d490ada2 100644 --- a/config/application.rb +++ b/config/application.rb @@ -5,6 +5,7 @@ require 'csv' require 'yaml' require 'bunny-pub-sub/services_manager' +require_relative '../app/middleware/sentry_tunnel_middleware' # Precompile assets before deploying to production if defined?(Bundler) @@ -287,6 +288,8 @@ def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) Rails.root.join('app/models/d2l') # CORS config + config.middleware.insert_before Rack::MethodOverride, SentryTunnelMiddleware + config.middleware.insert_before Warden::Manager, Rack::Cors do allow do origins '*' diff --git a/config/routes.rb b/config/routes.rb index 14452920a1..ea52a79000 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -5,7 +5,6 @@ get 'api/submission/unit/:id/task_definitions/:task_def_id/download_submissions', to: 'task_downloads#index' get 'api/submission/unit/:id/task_definitions/:task_def_id/student_pdfs', to: 'task_submission_pdfs#index' get 'api/units/:id/all_resources', to: 'lecture_resource_downloads#index' - post 'api/client-reports', to: 'sentry_tunnel#create' mount ApiRoot => '/' mount GrapeSwaggerRails::Engine => '/api/docs' From 113e9e48e6cdda14b46d84ccfa9b6185d27bd253 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:57:50 +1000 Subject: [PATCH 119/199] chore(release): 11.0.0-19 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69edcd6761..6ee7f21c39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-19](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-18...v11.0.0-19) (2026-06-25) + ## [11.0.0-18](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-17...v11.0.0-18) (2026-06-25) From 0e0ce34160dce4e23fc9deea2f035119c5d378dd Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 14:42:31 +1000 Subject: [PATCH 120/199] fix: ensure overseer image is pulled before running test --- app/sidekiq/accept_overseer_job.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/app/sidekiq/accept_overseer_job.rb b/app/sidekiq/accept_overseer_job.rb index de54c078a5..58adb3870c 100644 --- a/app/sidekiq/accept_overseer_job.rb +++ b/app/sidekiq/accept_overseer_job.rb @@ -56,6 +56,11 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment assessment_pass = true + overseer_image = task_definition.overseer_image || + task.unit.overseer_image || + OverseerImage.find_by(tag: docker_image_name_tag) + ensure_docker_image_present(docker_image_name_tag, overseer_image) + active_overseer_steps.each do |step| result = run_overseer_step( step: step, @@ -114,6 +119,18 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment raise e end + def ensure_docker_image_present(docker_image_name_tag, overseer_image) + _, _, inspect_status = Open3.capture3('docker', 'image', 'inspect', docker_image_name_tag) + return if inspect_status.success? + + raise "Docker image #{docker_image_name_tag} is not configured" if overseer_image.nil? + + overseer_image.pull_from_docker + return if overseer_image.success? + + raise "Unable to pull Docker image #{docker_image_name_tag}: #{overseer_image.pulled_image_text}" + end + def run_overseer_step(step:, work_dir:, work_dir_name:, task_id:, timestamp:, docker_image_name_tag:, overseer_assessment_id:) script_contents = step.run_command raise "Execution script is empty" if script_contents.blank? @@ -156,6 +173,7 @@ def run_overseer_step(step:, work_dir:, work_dir_name:, task_id:, timestamp:, do command = %( timeout #{timeout} docker run --rm -i \ + --pull never \ --cpus 1 \ --network none \ #{volume_mount} \ From 91402b02d058436ec74cea2d2a387e17311e66a0 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 14:46:00 +1000 Subject: [PATCH 121/199] refactor: revert change to preserve overdue statuses on failure --- app/sidekiq/accept_overseer_job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/sidekiq/accept_overseer_job.rb b/app/sidekiq/accept_overseer_job.rb index 58adb3870c..59ddd753fd 100644 --- a/app/sidekiq/accept_overseer_job.rb +++ b/app/sidekiq/accept_overseer_job.rb @@ -100,9 +100,9 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment end else oa.update!(status: :failed) - preserve_status_on_failure = [TaskStatus.time_exceeded.id, TaskStatus.assess_in_portfolio.id].include?(task.task_status_id) + # preserve_status_on_failure = [TaskStatus.time_exceeded.id, TaskStatus.assess_in_portfolio.id].include?(task.task_status_id) - unless failure_status.nil? || preserve_status_on_failure + unless failure_status.nil? # || preserve_status_on_failure # TODO: have an override status setting for the step? eg. if the task is overdue, let it remain overdue, otherwise use this task status task.update!(task_status: failure_status) task.add_status_comment(task.project.tutor_for(task.task_definition), failure_status) From 86b2040bf48f494b1e06919fcce50243fe5a0240 Mon Sep 17 00:00:00 2001 From: audrey <111032067+audreypho@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:02:47 +1000 Subject: [PATCH 122/199] feat: capture daily snapshots of aggregated task completion data (#607) * feat: create scheduled task-completion snapshots * feat: task completion stats aggregate by tutorial and job scheduled to 11:55pm * feat: add tests for task completion snapshots retrieval and capture * feat: add factory and tests for task_completion_snapshot model * feat: add tests for aggregating and capturing task-completion-stats snapshots * refactor: remove foreign key constraint from task_completion_snapshots, rerun migration * fix: `aggregate_task_complete_stats` uses existing status_for_task_definition * refactor: task completion stats uses async sidekiq job for snapshots * feat: add convenor permission for capturing task completion snapshots * feat: add rate limit to task completion snapshot (30mins) * refactor: change snapshots to be stored as JSON files * feat: task completion snapshot captures data in CSV and stores individual data * fix: task completion stats csv uses task status names instead of id * fix: task completion csv correctly uses task status names * feat: task completion snapshots include campus information * feat: add campus to task completion snapshot * feat: store task completion snapshots as zip * fix: simplify task_completion_csv_generator, keeping campus column for all generated csvs * fix: update task_completion related unit tests in `unit_model_test` * refactor: store task completion snapshots as zip containing csv files * fix: formatting * chore: bump migration * chore: bump migration --------- Co-authored-by: b0ink <40929320+b0ink@users.noreply.github.com> --- app/api/units_api.rb | 55 ++++++ app/helpers/file_helper.rb | 22 +++ app/models/task_completion_snapshot.rb | 176 ++++++++++++++++++ app/models/unit.rb | 68 +++++-- .../aggregate_task_completion_stats_job.rb | 32 ++++ config/schedule.yml | 4 + ...055323_create_task_completion_snapshots.rb | 12 ++ db/schema.rb | 11 +- test/api/units_api_test.rb | 165 ++++++++++++++++ .../task_completion_snapshot_factory.rb | 6 + test/models/task_completion_snapshot_test.rb | 109 +++++++++++ test/models/unit_model_test.rb | 120 +++++++++++- test/sidekiq/scheduled_job_test.rb | 3 +- 13 files changed, 764 insertions(+), 19 deletions(-) create mode 100644 app/models/task_completion_snapshot.rb create mode 100644 app/sidekiq/aggregate_task_completion_stats_job.rb create mode 100644 db/migrate/20260625055323_create_task_completion_snapshots.rb create mode 100644 test/factories/task_completion_snapshot_factory.rb create mode 100644 test/models/task_completion_snapshot_test.rb diff --git a/app/api/units_api.rb b/app/api/units_api.rb index 4d1e80b971..5ea553b8a7 100644 --- a/app/api/units_api.rb +++ b/app/api/units_api.rb @@ -604,6 +604,61 @@ class UnitsApi < Grape::API present unit.student_task_completion_stats, with: Grape::Presenters::Presenter end + desc 'Get historical task completion snapshots' + params do + optional :start_date, type: Date, desc: 'Include snapshots captured on or after this date' + optional :end_date, type: Date, desc: 'Include snapshots captured on or before this date' + optional :limit, type: Integer, desc: 'Maximum number of snapshots to return', default: 365 + end + get '/units/:id/stats/task_completion_snapshots' do + unit = Unit.find(params[:id]) + unless authorise? current_user, unit, :download_stats + error!({ error: "Not authorised to download stats of student tasks in #{unit.code}" }, 403) + end + + snapshots = unit.task_completion_snapshots.order(snapshot_timestamp: :desc) + if params[:start_date].present? + start_timestamp = params[:start_date].in_time_zone.beginning_of_day.to_i + snapshots = snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) >= ?', start_timestamp) + end + if params[:end_date].present? + end_timestamp = params[:end_date].in_time_zone.end_of_day.to_i + snapshots = snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) <= ?', end_timestamp) + end + snapshots = snapshots.limit([params[:limit].to_i, 365].min) + + present snapshots.map { |snapshot| + stats = snapshot.load_stats + + { + snapshot_date: snapshot.snapshot_date, + snapshot_timestamp: snapshot.snapshot_timestamp, + stats: stats + } + }, with: Grape::Presenters::Presenter + end + + desc 'Capture task completion snapshot immediately for this unit' + post '/units/:id/stats/task_completion_snapshots/capture' do + unit = Unit.find(params[:id]) + unless authorise? current_user, unit, :capture_task_completion_snapshot + error!({ error: "Not authorised to capture stats of student tasks in #{unit.code}" }, 403) + end + + # Check if a snapshot was captured within the past 30 minutes + recent_snapshot = unit.task_completion_snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) > ?', 30.minutes.ago.to_i).order(snapshot_timestamp: :desc).first + if recent_snapshot.present? + recent_snapshot_time = recent_snapshot.snapshot_time + remaining_seconds = [(recent_snapshot_time + 30.minutes - Time.zone.now).ceil, 0].max + remaining_minutes = [(remaining_seconds / 60.0).ceil, 1].max + error!({ error: "A snapshot was captured at #{recent_snapshot_time.strftime('%H:%M')}. Please wait #{remaining_minutes} more minute(s) before capturing another snapshot." }, 429) + end + + job_id = AggregateTaskCompletionStatsJob.perform_async(unit.id) + job = setup_job(job_id) + present job, with: Entities::SidekiqJobEntity + end + desc 'Download stats related to the number of tasks assessed by each tutor' get '/csv/units/:id/tutor_assessments' do unit = Unit.find(params[:id]) diff --git a/app/helpers/file_helper.rb b/app/helpers/file_helper.rb index 2f63d6b504..967d8ed302 100644 --- a/app/helpers/file_helper.rb +++ b/app/helpers/file_helper.rb @@ -304,6 +304,25 @@ def unit_portfolio_dir(unit, create: true, archived: true) dst end + def unit_analytics_dir(unit, create: true, archived: true) + dst = unit_work_root(unit, archived: archived) + dst << 'analytics/' + + FileUtils.mkdir_p(dst) if create + dst + end + + def unit_task_status_snapshot_path(unit, create: true, archived: true) + analytics_dir = unit_analytics_dir(unit, create: create, archived: archived) + FileUtils.mkdir_p(analytics_dir) if create + File.join(analytics_dir, 'task-status-snapshots.zip') + end + + def snapshot_csv_filename(snapshot_timestamp) + return nil if snapshot_timestamp.blank? + "#{sanitized_filename(snapshot_timestamp.to_s)}.csv" + end + # # Generates a path for storing student portfolios # @@ -999,6 +1018,9 @@ def line_wrap(path, width: 160) module_function :unit_dir module_function :root_portfolio_dir module_function :unit_portfolio_dir + module_function :unit_analytics_dir + module_function :unit_task_status_snapshot_path + module_function :snapshot_csv_filename module_function :unit_work_root module_function :project_work_root module_function :student_portfolio_dir diff --git a/app/models/task_completion_snapshot.rb b/app/models/task_completion_snapshot.rb new file mode 100644 index 0000000000..ee6ae30a00 --- /dev/null +++ b/app/models/task_completion_snapshot.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +require 'csv' +require 'zip' + +class TaskCompletionSnapshot < ApplicationRecord + include FileHelper + + belongs_to :unit + + validates :snapshot_timestamp, presence: true + validates :snapshot_timestamp, uniqueness: { scope: :unit_id } + + after_destroy :delete_snapshot_file + + def snapshot_file_path + return nil if unit.blank? + FileHelper.unit_task_status_snapshot_path(unit, create: true) + end + + def snapshot_contents + file_path = snapshot_file_path + return nil if file_path.blank? + if File.exist?(file_path) + return read_csv_from_zip(file_path, snapshot_timestamp) + end + nil + rescue Zip::Error + nil + end + + def snapshot_date + return nil if snapshot_timestamp.blank? + + snapshot_time.to_date + end + + def snapshot_time + return nil if snapshot_timestamp.blank? + + Time.zone.at(snapshot_timestamp.to_i) + end + + def load_stats + snapshot_contents = self.snapshot_contents + + return {} if snapshot_contents.blank? + + parse_csv_stats(snapshot_contents) + rescue CSV::MalformedCSVError + {} + end + + def store_stats!(payload) + file_path = snapshot_file_path + raise 'Cannot store stats without a unit' if file_path.blank? + + FileUtils.mkdir_p(File.dirname(file_path)) + + csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp) + raise 'Cannot store stats without a valid snapshot timestamp' if csv_filename.blank? + + tmp_path = "#{file_path}.tmp" + + # Read existing zip entries (if file exists) + existing_entries = {} + if File.exist?(file_path) + Zip::File.open(file_path) do |zip_file| + zip_file.each do |entry| + next if entry.directory? + existing_entries[entry.name] = entry.get_input_stream.read + end + end + end + + # Update or add the current snapshot entry + existing_entries[csv_filename] = payload.to_s + + # Write the zip file with all entries + Zip::OutputStream.open(tmp_path) do |zip| + existing_entries.each do |filename, content| + zip.put_next_entry(filename) + zip.write(content) + end + end + + FileUtils.mv(tmp_path, file_path) + ensure + FileUtils.rm_f(tmp_path) if defined?(tmp_path) && tmp_path + end + + private + + def parse_csv_stats(csv_text) + csv = CSV.parse(csv_text, headers: true) + return {} if csv.empty? + + stream_headers = unit.tutorial_streams.pluck(:abbreviation) + stream_headers = ['Tutorial'] if stream_headers.empty? + task_definitions = unit.task_definitions_by_grade + + stats = Hash.new { |hash, key| hash[key] = Hash.new { |tutorial_hash, tutorial_key| tutorial_hash[tutorial_key] = Hash.new { |task_hash, task_key| task_hash[task_key] = Hash.new(0) } } } + + csv.each do |row| + campus_abbreviation = row['Campus'].to_s.strip + next if campus_abbreviation.blank? + + campus_name = Campus.find_by(abbreviation: campus_abbreviation)&.name || campus_abbreviation + + stream_headers.each do |stream_header| + tutorial_name = row[stream_header].to_s.strip + next if tutorial_name.blank? + + task_definitions.each do |task_definition| + status_value = row[task_definition.abbreviation].to_s.strip + status_key = TaskStatus.id_to_key(status_value.to_i) || :not_started + stats[campus_name][tutorial_name][task_definition.abbreviation][status_key.to_s] += 1 + end + end + end + + stats + end + + def read_csv_from_zip(zip_path, snapshot_timestamp) + csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp) + Zip::File.open(zip_path) do |zip_file| + entry = zip_file.find_entry(csv_filename) + return nil if entry.nil? + + entry.get_input_stream.read + end + end + + def delete_snapshot_file + return if snapshot_timestamp.blank? + + file_path = snapshot_file_path + return if file_path.blank? || !File.exist?(file_path) + + csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp) + return if csv_filename.blank? + + tmp_path = "#{file_path}.tmp" + + begin + # Read existing zip entries excluding the one we want to delete + remaining_entries = {} + Zip::File.open(file_path) do |zip_file| + zip_file.each do |entry| + next if entry.directory? + next if entry.name == csv_filename + remaining_entries[entry.name] = entry.get_input_stream.read + end + end + + if remaining_entries.empty? + # If no entries left, just delete the zip file + FileUtils.rm_f(file_path) + else + # Write the zip file with remaining entries + Zip::OutputStream.open(tmp_path) do |zip| + remaining_entries.each do |filename, content| + zip.put_next_entry(filename) + zip.write(content) + end + end + FileUtils.mv(tmp_path, file_path) + end + rescue StandardError => e + # If anything goes wrong with zip operations, just clean up and log + logger.error("Error managing snapshot zip file: #{e.message}") + FileUtils.rm_f(tmp_path) + end + end +end diff --git a/app/models/unit.rb b/app/models/unit.rb index d1980a6abb..9d01e3dbaa 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -77,6 +77,7 @@ def self.permissions :get_marking_sessions, :upload_grades_csv, :get_staff_notes, + :capture_task_completion_snapshot, :mannage_communications, :delete_engagement ] @@ -169,6 +170,7 @@ def role_for(user) has_many :unit_roles, dependent: :destroy, inverse_of: :unit has_many :learning_outcomes, as: :context, dependent: :destroy # inverse_of: :unit has_many :marking_sessions, dependent: :destroy + has_many :task_completion_snapshots, dependent: :destroy, inverse_of: :unit has_many :communication_sets, class_name: 'CommunicationSet', dependent: :destroy has_many :communication_rules, through: :communication_sets, class_name: 'CommunicationRule' has_many :communication_set_schedules, through: :communication_sets, class_name: 'CommunicationSetSchedule' @@ -1933,23 +1935,31 @@ def days_awaiting_feedback_by_tutorial_csv end def task_completion_csv + task_completion_csv_generator() + end + + def task_completion_csv_generator(task_status_uses_id: false) task_def_by_grade = task_definitions_by_grade streams = tutorial_streams grp_sets = group_sets + base_headers = [ + 'Student ID', + 'Username', + 'Student Name', + ] + base_headers << 'Campus' + base_headers.push( + 'Target Grade', + 'Email', + 'Portfolio', + 'Grade', + 'Rationale', + 'Assessor', + ) CSV.generate() do |csv| # Add header row - csv << ([ - 'Student ID', - 'Username', - 'Student Name', - 'Target Grade', - 'Email', - 'Portfolio', - 'Grade', - 'Rationale', - 'Assessor', - ] + + csv << (base_headers + (streams.count > 0 ? streams.map { |t| t.abbreviation } : ['Tutorial']) + grp_sets.map(&:name) + task_def_by_grade.map do |task_definition| @@ -1964,7 +1974,11 @@ def task_completion_csv # Get the details to fetch for each task definition... td_select = task_def_by_grade.map do |td| result = [] - result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN (CASE WHEN task_statuses.name IS NULL THEN 'Not Started' ELSE task_statuses.name END) ELSE NULL END) AS status_#{td.id}" + if task_status_uses_id + result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN (CASE WHEN tasks.task_status_id IS NULL THEN #{TaskStatus.not_started.id} ELSE tasks.task_status_id END) ELSE NULL END) AS status_#{td.id}" + else + result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN (CASE WHEN task_statuses.name IS NULL THEN 'Not Started' ELSE task_statuses.name END) ELSE NULL END) AS status_#{td.id}" + end result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN tasks.grade ELSE NULL END) AS grade_#{td.id}" if td.is_graded? result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN tasks.quality_pts ELSE NULL END) AS stars_#{td.id}" if td.has_stars? result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN tasks.contribution_pts ELSE NULL END) AS people_#{td.id}" if td.is_group_task? @@ -1976,6 +1990,7 @@ def task_completion_csv .joins( :unit, 'INNER JOIN users ON projects.user_id = users.id', + 'LEFT OUTER JOIN campuses ON campuses.id = projects.campus_id', 'INNER JOIN task_definitions ON task_definitions.unit_id = units.id', 'LEFT OUTER JOIN tutorial_streams ON tutorial_streams.unit_id = units.id', 'LEFT OUTER JOIN tutorial_enrolments ON tutorial_enrolments.project_id = projects.id', @@ -1986,7 +2001,7 @@ def task_completion_csv 'LEFT OUTER JOIN groups ON groups.id = group_memberships.group_id' ).select( 'projects.id as project_id', 'users.student_id as student_id', 'users.username as username', 'users.first_name as first_name', 'projects.assessor_id as project_assessor', - 'users.last_name as last_name', 'projects.target_grade', 'users.email as email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale', + 'users.last_name as last_name', 'campuses.abbreviation as campus_abbreviation', 'projects.target_grade', 'users.email as email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale', *td_select, # Get tutorial for each stream in unit *streams.map { |s| "MAX(CASE WHEN tutorials.tutorial_stream_id = #{s.id} OR tutorials.tutorial_stream_id IS NULL THEN tutorials.abbreviation ELSE NULL END) AS tutorial_#{s.id}" }, @@ -1994,12 +2009,16 @@ def task_completion_csv "MAX(CASE WHEN tutorial_streams.id IS NULL THEN tutorials.abbreviation ELSE NULL END) AS tutorial", *grp_sets.map { |gs| "MAX(CASE WHEN groups.group_set_id = #{gs.id} THEN groups.name ELSE NULL END) AS grp_#{gs.id}" } ).group( - 'projects.id', 'student_id', 'username', 'first_name', 'last_name', 'target_grade', 'email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale' + 'projects.id', 'student_id', 'username', 'first_name', 'last_name', 'campus_abbreviation', 'target_grade', 'email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale' ).each do |row| - csv << ([ + student_details = [ row['student_id'], row['username'], "#{row['first_name']} #{row['last_name']}", + ] + student_details << row['campus_abbreviation'] + + csv << (student_details + [ grade_label(row['target_grade']), row['email'], row['portfolio_production_date'].present? && !row['compile_portfolio'] && File.exist?(FileHelper.student_portfolio_path(self, row['username'], create: true)), @@ -2015,7 +2034,11 @@ def task_completion_csv end.flatten + grp_sets.map do |gs| row["grp_#{gs.id}"] end + task_def_by_grade.map do |td| - result = [row["status_#{td.id}"].nil? ? TaskStatus.not_started.name : row["status_#{td.id}"]] + if task_status_uses_id + result = [row["status_#{td.id}"].nil? ? TaskStatus.not_started.id : row["status_#{td.id}"].to_i] + else + result = [row["status_#{td.id}"].nil? ? TaskStatus.not_started.name : row["status_#{td.id}"]] + end result << grade_abbreviation(row["grade_#{td.id}"]) if td.is_graded? result << row["stars_#{td.id}"] if td.has_stars? result << row["people_#{td.id}"] if td.is_group_task? @@ -3673,6 +3696,19 @@ def get_tutor_times_csv(start_date: nil, end_date: nil, timezone: nil, ignore_se end end + def capture_task_complete_stats_snapshot!(snapshot_time: Time.zone.now) + snapshot_payload = task_completion_csv_generator(task_status_uses_id: true) + + timestamp = snapshot_time.to_i.to_s + + task_completion_snapshots + .find_or_initialize_by(snapshot_timestamp: timestamp) + .tap do |snapshot| + snapshot.save! + snapshot.store_stats!(snapshot_payload) + end + end + private def delete_associated_files diff --git a/app/sidekiq/aggregate_task_completion_stats_job.rb b/app/sidekiq/aggregate_task_completion_stats_job.rb new file mode 100644 index 0000000000..4bb375ebec --- /dev/null +++ b/app/sidekiq/aggregate_task_completion_stats_job.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class AggregateTaskCompletionStatsJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + include ApplicationHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first] }, + on_conflict: :reject, + retry: false + + def perform(unit_id = nil) + logger.info 'Starting task completion stats aggregation...' + + at(0) + total(1) + + if unit_id.present? + Unit.find(unit_id).capture_task_complete_stats_snapshot! + else + Unit.active_units.find_each(&:capture_task_complete_stats_snapshot!) + end + + at(1) + logger.info 'Completed task completion stats aggregation!' + rescue StandardError => e + logger.error e + raise e + end +end diff --git a/config/schedule.yml b/config/schedule.yml index b26a1309d2..62fd893daf 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -16,6 +16,10 @@ refresh_moderation_feedback_timestamps: cron: "every 60 minutes" class: "RefreshModerationFeedbackTimestampsJob" +aggregate_task_completion_stats: + cron: "every day at 11:55pm" + class: "AggregateTaskCompletionStatsJob" + poll_communication_set_schedules: cron: "every 5 minutes" class: "PollCommunicationSetSchedulesJob" diff --git a/db/migrate/20260625055323_create_task_completion_snapshots.rb b/db/migrate/20260625055323_create_task_completion_snapshots.rb new file mode 100644 index 0000000000..7b66ae68d4 --- /dev/null +++ b/db/migrate/20260625055323_create_task_completion_snapshots.rb @@ -0,0 +1,12 @@ +class CreateTaskCompletionSnapshots < ActiveRecord::Migration[8.0] + def change + create_table :task_completion_snapshots do |t| + t.references :unit, null: false + t.string :snapshot_timestamp, null: false + + t.timestamps + end + + add_index :task_completion_snapshots, [:unit_id, :snapshot_timestamp], unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 8e5a22b10e..9b7572e806 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_06_18_033139) do +ActiveRecord::Schema[8.0].define(version: 2026_06_25_055323) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -552,6 +552,15 @@ t.index ["user_id"], name: "index_task_comments_on_user_id" end + create_table "task_completion_snapshots", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.string "snapshot_timestamp", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["unit_id", "snapshot_timestamp"], name: "idx_on_unit_id_snapshot_timestamp_e923c3ae10", unique: true + t.index ["unit_id"], name: "index_task_completion_snapshots_on_unit_id" + end + create_table "task_definition_grade_due_dates", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "task_definition_id", null: false t.integer "target_grade", null: false diff --git a/test/api/units_api_test.rb b/test/api/units_api_test.rb index 9ff3ed5d03..23add5b13e 100644 --- a/test/api/units_api_test.rb +++ b/test/api/units_api_test.rb @@ -622,4 +622,169 @@ def test_draft_learning_summary_upload_requirements unit.reload assert_equal task_def_doc.id, unit.draft_task_definition_id end + + def test_get_task_completion_snapshots + unit = FactoryBot.create :unit, with_students: false, task_count: 1, stream_count: 0, tutorials: 1, campus_count: 1 + tutorial = unit.tutorials.first + task_definition = unit.task_definitions_by_grade.first + + older_snapshot = TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-01 10:00:00').to_i.to_s + ) + + mid_snapshot = TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-02 10:00:00').to_i.to_s + ) + + latest_snapshot = TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-03 10:00:00').to_i.to_s + ) + + older_snapshot.store_stats!(build_task_completion_snapshot_csv(tutorial, task_definition, [TaskStatus.not_started.id])) + mid_snapshot.store_stats!(build_task_completion_snapshot_csv(tutorial, task_definition, [TaskStatus.complete.id, TaskStatus.complete.id])) + latest_snapshot.store_stats!(build_task_completion_snapshot_csv(tutorial, task_definition, [TaskStatus.complete.id, TaskStatus.complete.id, TaskStatus.complete.id])) + + add_auth_header_for(user: unit.main_convenor_user) + header 'Host', 'localhost' + get "/api/units/#{unit.id}/stats/task_completion_snapshots", { limit: 2 } + + assert_equal 200, last_response.status, last_response_body + assert_equal 2, last_response_body.length + + assert_equal latest_snapshot.snapshot_date.to_s, last_response_body[0]['snapshot_date'].to_date.to_s + assert_equal mid_snapshot.snapshot_date.to_s, last_response_body[1]['snapshot_date'].to_date.to_s + + latest_stats = last_response_body[0]['stats'] + assert_equal 3, latest_stats[tutorial.campus.name][tutorial.abbreviation][task_definition.abbreviation]['complete'] + + assert_not_equal older_snapshot.snapshot_date.to_s, last_response_body[1]['snapshot_date'].to_date.to_s + end + + def test_get_task_completion_snapshots_filters_by_date + unit = FactoryBot.create :unit, with_students: false, task_count: 1, stream_count: 0, tutorials: 1, campus_count: 1 + tutorial = unit.tutorials.first + task_definition = unit.task_definitions_by_grade.first + + TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-03-30 10:00:00').to_i.to_s + ) + + included_snapshot = TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-02 10:00:00').to_i.to_s + ) + + TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-05 10:00:00').to_i.to_s + ) + + unit.task_completion_snapshots.find_each do |snapshot| + snapshot.store_stats!(build_task_completion_snapshot_csv(tutorial, task_definition, [TaskStatus.complete.id])) + end + + add_auth_header_for(user: unit.main_convenor_user) + header 'Host', 'localhost' + get "/api/units/#{unit.id}/stats/task_completion_snapshots", { + start_date: Date.new(2026, 4, 1), + end_date: Date.new(2026, 4, 3) + } + + assert_equal 200, last_response.status, last_response_body + assert_equal 1, last_response_body.length + assert_equal included_snapshot.snapshot_date.to_s, last_response_body[0]['snapshot_date'].to_date.to_s + end + + def test_get_task_completion_snapshots_not_authorised + unit = FactoryBot.create :unit, with_students: false, task_count: 0 + TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.now.to_i.to_s + ) + + add_auth_header_for(user: User.where(role: Role.student).first) + header 'Host', 'localhost' + get "/api/units/#{unit.id}/stats/task_completion_snapshots" + + assert_equal 403, last_response.status + end + + def test_post_capture_task_completion_snapshot + Sidekiq::Testing.inline! do + unit = FactoryBot.create :unit + + count_before = TaskCompletionSnapshot.where(unit: unit).count + + add_auth_header_for(user: unit.main_convenor_user) + header 'Host', 'localhost' + post "/api/units/#{unit.id}/stats/task_completion_snapshots/capture" + + assert_equal 201, last_response.status, last_response_body + assert_not_nil last_response_body['id'] + + snapshot = TaskCompletionSnapshot.where(unit: unit).order(snapshot_timestamp: :desc).first + assert_not_nil snapshot + assert_equal count_before + 1, TaskCompletionSnapshot.where(unit: unit).count + + assert_equal Date.current.to_s, snapshot.snapshot_date.to_s + assert_not_empty snapshot.load_stats + assert File.exist?(snapshot.snapshot_file_path) + ensure + Sidekiq::Testing.fake! + end + end + + def test_post_capture_task_completion_snapshot_not_authorised + unit = FactoryBot.create :unit, with_students: false, task_count: 0 + + add_auth_header_for(user: User.where(role: Role.student).first) + header 'Host', 'localhost' + post "/api/units/#{unit.id}/stats/task_completion_snapshots/capture" + + assert_equal 403, last_response.status + end + + private + + def build_task_completion_snapshot_csv(tutorial, task_definition, statuses) + headers = [ + 'Student ID', + 'Username', + 'Student Name', + 'Campus', + 'Target Grade', + 'Email', + 'Portfolio', + 'Grade', + 'Rationale', + 'Assessor', + 'Tutorial', + task_definition.abbreviation, + ] + + CSV.generate do |csv| + csv << headers + + statuses.each_with_index do |status, index| + csv << [ + "#{index + 1}", + "student-#{index + 1}", + "Student #{index + 1}", + tutorial.campus.abbreviation, + '0', + "student-#{index + 1}@example.com", + 'false', + '', + '', + '', + tutorial.abbreviation, + status, + ] + end + end + end end diff --git a/test/factories/task_completion_snapshot_factory.rb b/test/factories/task_completion_snapshot_factory.rb new file mode 100644 index 0000000000..871bd0611a --- /dev/null +++ b/test/factories/task_completion_snapshot_factory.rb @@ -0,0 +1,6 @@ +FactoryBot.define do + factory :task_completion_snapshot do + unit + snapshot_timestamp { Time.current.to_i.to_s } + end +end diff --git a/test/models/task_completion_snapshot_test.rb b/test/models/task_completion_snapshot_test.rb new file mode 100644 index 0000000000..38300242bd --- /dev/null +++ b/test/models/task_completion_snapshot_test.rb @@ -0,0 +1,109 @@ +require 'test_helper' + +class TaskCompletionSnapshotTest < ActiveSupport::TestCase + setup do + @unit = FactoryBot.create(:unit, with_students: false, task_count: 1, stream_count: 0, tutorials: 1, campus_count: 1) + @snapshot = FactoryBot.create(:task_completion_snapshot, unit: @unit) + end + + test 'task_completion_snapshot belongs to unit' do + assert @snapshot.unit.is_a?(Unit) + assert_equal @unit, @snapshot.unit + end + + test 'task_completion_snapshot is valid with all attributes' do + assert @snapshot.valid? + end + + test 'task_completion_snapshot is invalid without snapshot_timestamp' do + snapshot = FactoryBot.build(:task_completion_snapshot, snapshot_timestamp: nil) + assert_not snapshot.valid? + assert snapshot.errors[:snapshot_timestamp].include?("can't be blank") + end + + test 'task_completion_snapshot enforces unique snapshot_timestamp per unit' do + duplicate = FactoryBot.build( + :task_completion_snapshot, + unit: @unit, + snapshot_timestamp: @snapshot.snapshot_timestamp + ) + + assert_not duplicate.valid? + assert duplicate.errors[:snapshot_timestamp].include?('has already been taken') + end + + test 'task_completion_snapshot allows same snapshot_timestamp for different units' do + other_unit = FactoryBot.create(:unit) + snapshot = FactoryBot.build( + :task_completion_snapshot, + unit: other_unit, + snapshot_timestamp: @snapshot.snapshot_timestamp + ) + + assert snapshot.valid? + end + + test 'store_stats! writes a csv file contained in a zip that can be loaded' do + tutorial = @unit.tutorials.first + task_definition = @unit.task_definitions_by_grade.first + + payload = CSV.generate do |csv| + csv << ['Student ID', 'Username', 'Student Name', 'Campus', 'Target Grade', 'Email', 'Portfolio', 'Grade', 'Rationale', 'Assessor', 'Tutorial', task_definition.abbreviation] + csv << ['1', 'student-1', 'Student 1', tutorial.campus.abbreviation, '0', 'student-1@example.com', 'false', '', '', '', tutorial.abbreviation, TaskStatus.complete.id] + csv << ['2', 'student-2', 'Student 2', tutorial.campus.abbreviation, '0', 'student-2@example.com', 'false', '', '', '', tutorial.abbreviation, TaskStatus.complete.id] + csv << ['3', 'student-3', 'Student 3', tutorial.campus.abbreviation, '0', 'student-3@example.com', 'false', '', '', '', tutorial.abbreviation, TaskStatus.complete.id] + csv << ['4', 'student-4', 'Student 4', tutorial.campus.abbreviation, '0', 'student-4@example.com', 'false', '', '', '', tutorial.abbreviation, TaskStatus.complete.id] + end + + expected = { + tutorial.campus.name => { + tutorial.abbreviation => { + task_definition.abbreviation => { + 'complete' => 4 + } + } + } + } + + @snapshot.store_stats!(payload) + + assert File.exist?(@snapshot.snapshot_file_path) + assert_equal expected, @snapshot.load_stats + end + + test 'load_stats returns empty hash if file missing' do + snapshot = FactoryBot.create( + :task_completion_snapshot, + unit: @unit, + snapshot_timestamp: (Time.zone.now.to_i + 100).to_s + ) + + FileUtils.rm_f(snapshot.snapshot_file_path) + assert_equal({}, snapshot.load_stats) + end + + test 'deleting snapshot deletes associated zip file' do + tutorial = @unit.tutorials.first + task_definition = @unit.task_definitions_by_grade.first + + payload = CSV.generate do |csv| + csv << ['Student ID', 'Username', 'Student Name', 'Target Grade', 'Email', 'Portfolio', 'Grade', 'Rationale', 'Assessor', 'Tutorial', task_definition.abbreviation] + csv << ['1', 'student-1', 'Student 1', '0', 'student-1@example.com', 'false', '', '', '', tutorial.abbreviation, TaskStatus.complete.id] + end + + @snapshot.store_stats!(payload) + + file_path = @snapshot.snapshot_file_path + assert File.exist?(file_path) + + @snapshot.destroy + assert_not File.exist?(file_path) + end + + test 'snapshot_date is derived from snapshot_timestamp' do + timestamp = Time.zone.local(2026, 4, 8, 23, 55, 0).to_i.to_s + snapshot = FactoryBot.build(:task_completion_snapshot, snapshot_timestamp: timestamp) + + assert_equal Date.new(2026, 4, 8), snapshot.snapshot_date + end +end diff --git a/test/models/unit_model_test.rb b/test/models/unit_model_test.rb index eca67c6755..051c893c38 100644 --- a/test/models/unit_model_test.rb +++ b/test/models/unit_model_test.rb @@ -736,7 +736,7 @@ def test_task_completion_csv end # 18 = 9 general + 2 streams + 3 task defs + 1 group details + 1 stars + 1 grade + 1 contrib - check_task_completion_csv unit, 18 + check_task_completion_csv unit, 19 end def test_task_completion_csv_no_task_data @@ -1271,4 +1271,122 @@ def test_cant_disable_aip_only_while_aip_tasks_exist assert_includes unit.errors[:mark_late_submissions_as_assess_in_portfolio], 'cannot be disabled while tasks are in the Assess in Portfolio state' end + + test 'capture-task-complete-stats-snapshot creates snapshot for date' do + data = build_unit_with_controlled_task_statuses + unit = data[:unit] + snapshot_time = Time.zone.local(2026, 4, 8, 23, 55, 0) + expected_stats = parse_task_completion_stats_csv(unit, unit.task_completion_csv_generator(task_status_uses_id: true)) + + count_before = unit.task_completion_snapshots.count + snapshot = unit.capture_task_complete_stats_snapshot!(snapshot_time: snapshot_time) + + assert_equal count_before + 1, unit.task_completion_snapshots.count + assert_equal snapshot_time.to_date, snapshot.snapshot_date + assert_equal snapshot_time.to_i.to_s, snapshot.snapshot_timestamp + assert_equal expected_stats, snapshot.load_stats + + persisted_snapshot = unit.task_completion_snapshots.find_by(snapshot_timestamp: snapshot_time.to_i.to_s) + assert_not_nil persisted_snapshot + assert_equal snapshot.id, persisted_snapshot.id + ensure + unit&.destroy + end + + test 'capture-task-complete-stats-snapshot creates a new snapshot for a new timestamp' do + data = build_unit_with_controlled_task_statuses + unit = data[:unit] + task_definitions = data[:task_definitions] + student2 = data[:student2] + + first_time = Time.zone.local(2026, 4, 8, 9, 0, 0) + second_time = Time.zone.local(2026, 4, 8, 20, 0, 0) + + first_snapshot = unit.capture_task_complete_stats_snapshot!(snapshot_time: first_time) + first_stats = first_snapshot.load_stats.deep_dup + count_before = unit.task_completion_snapshots.count + + # Change one task status so the new capture has different stats. + student2.task_for_task_definition(task_definitions[0]).update!(task_status: TaskStatus.fail) + expected_updated_stats = parse_task_completion_stats_csv(unit, unit.task_completion_csv_generator(task_status_uses_id: true)) + + updated_snapshot = unit.capture_task_complete_stats_snapshot!(snapshot_time: second_time) + + assert_equal count_before + 1, unit.task_completion_snapshots.count + assert_not_equal first_snapshot.id, updated_snapshot.id + assert_equal second_time.to_i.to_s, updated_snapshot.snapshot_timestamp + assert_not_equal first_stats, updated_snapshot.load_stats + assert_equal expected_updated_stats, updated_snapshot.load_stats + ensure + unit&.destroy + end + + private + + def parse_task_completion_stats_csv(unit, csv_text) + csv = CSV.parse(csv_text, headers: true) + streams = unit.tutorial_streams.pluck(:abbreviation) + streams = ['Tutorial'] if streams.empty? + task_definitions = unit.task_definitions_by_grade + campus_header = csv.headers.find { |header| header.to_s.casecmp('Campus').zero? } + + campus_names_by_abbreviation = if campus_header.present? + abbreviations = csv.map { |row| row[campus_header].to_s.strip }.reject(&:blank?).uniq + Campus.where(abbreviation: abbreviations).pluck(:abbreviation, :name).to_h + else + {} + end + + csv.each_with_object(Hash.new { |hash, key| hash[key] = {} }) do |row, stats| + streams.each do |stream_name| + tutorial_name = row[stream_name].to_s.strip + next if tutorial_name.blank? + + campus_abbreviation = campus_header.present? ? row[campus_header].to_s.strip : nil + + campus_name = if campus_abbreviation.present? + campus_names_by_abbreviation[campus_abbreviation] || campus_abbreviation + elsif stream_name == 'Tutorial' + unit.tutorials.find_by(abbreviation: tutorial_name)&.campus&.name || stream_name + else + stream_name + end + + stats[campus_name][tutorial_name] ||= {} + + task_definitions.each do |task_definition| + status_name = row[task_definition.abbreviation].to_s.strip + status_key = TaskStatus.id_to_key(status_name.to_i) || :not_started + stats[campus_name][tutorial_name][task_definition.abbreviation] ||= Hash.new(0) + stats[campus_name][tutorial_name][task_definition.abbreviation][status_key.to_s] += 1 + end + end + end + end + + def build_unit_with_controlled_task_statuses + unit = FactoryBot.create(:unit, with_students: false, task_count: 2, stream_count: 0, tutorials: 1, campus_count: 1) + tutorial = unit.tutorials.first + campus = tutorial.campus + task_definitions = unit.task_definitions.order(:id).to_a + + student1 = unit.enrol_student(FactoryBot.create(:user, :student), campus) + student2 = unit.enrol_student(FactoryBot.create(:user, :student), campus) + student1.enrol_in(tutorial) + student2.enrol_in(tutorial) + + student1.task_for_task_definition(task_definitions[0]).update!(task_status: TaskStatus.complete) + student2.task_for_task_definition(task_definitions[0]).update!(task_status: TaskStatus.complete) + student1.task_for_task_definition(task_definitions[1]).update!(task_status: TaskStatus.fail) + student2.task_for_task_definition(task_definitions[1]).update!(task_status: TaskStatus.not_started) + + { + unit: unit, + tutorial: tutorial, + task_definitions: task_definitions, + student1: student1, + student2: student2 + } + end + end diff --git a/test/sidekiq/scheduled_job_test.rb b/test/sidekiq/scheduled_job_test.rb index e56b1b7766..e21285fdf3 100644 --- a/test/sidekiq/scheduled_job_test.rb +++ b/test/sidekiq/scheduled_job_test.rb @@ -6,13 +6,14 @@ class TiiCheckProgressJobTest < ActiveSupport::TestCase def test_jobs_are_scheduled Sidekiq::Cron::Job.destroy_all! Sidekiq::Cron::Job.load_from_hash!(YAML.load_file(Rails.root.join('config/schedule.yml'))) - assert_equal 5, Sidekiq::Cron::Job.all.count, Sidekiq::Cron::Job.all.map(&:name) + assert_equal 6, Sidekiq::Cron::Job.all.count, Sidekiq::Cron::Job.all.map(&:name) Sidekiq::Cron::Job.all.each(&:enqueue!) assert_equal 1, TiiRegisterWebHookJob.jobs.count assert_equal 1, TiiCheckProgressJob.jobs.count assert_equal 1, ClearAccessTokensJob.jobs.count assert_equal 1, RefreshModerationFeedbackTimestampsJob.jobs.count + assert_equal 1, AggregateTaskCompletionStatsJob.jobs.count assert_equal 1, PollCommunicationSetSchedulesJob.jobs.count # assert_equal 1, ArchiveOldUnitsJob.jobs.count end From d9085b509b7995ae9d41e8e3973dd57b011a166d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:15:36 +1000 Subject: [PATCH 123/199] fix: relocate submission history on unit code change --- app/models/unit.rb | 10 ++++++++++ test/models/unit_model_test.rb | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/app/models/unit.rb b/app/models/unit.rb index 9d01e3dbaa..d9972869aa 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -3757,6 +3757,16 @@ def move_files_on_code_change FileUtils.mv(old_dir, new_dir) unless File.exist?(new_dir) end + old_submission_history_dir = File.join( + FileHelper.root_submission_history_dir(archived: archived), + FileHelper.sanitized_path("#{saved_change_to_code[0]}-#{id}") + ) + + if File.exist? old_submission_history_dir + new_submission_history_dir = FileHelper.unit_submission_history_dir(self) + FileUtils.mv(old_submission_history_dir, new_submission_history_dir) unless File.exist? new_submission_history_dir + end + # rubocop:disable Rails/SkipsModelValidations tasks.where('portfolio_evidence IS NOT NULL').update_all("portfolio_evidence = REPLACE(portfolio_evidence, '#{saved_change_to_code[0]}-#{id}', '#{code}-#{id}')") # rubocop:enable Rails/SkipsModelValidations diff --git a/test/models/unit_model_test.rb b/test/models/unit_model_test.rb index 051c893c38..f06a239b93 100644 --- a/test/models/unit_model_test.rb +++ b/test/models/unit_model_test.rb @@ -1012,6 +1012,12 @@ def test_change_unit_code_moves_files assert task_pdf.include?(unit.code) assert task_pdf.include?(unit.id.to_s) + old_submission_history_path = FileHelper.unit_submission_history_dir(unit, archived: false) + FileUtils.mkdir_p(old_submission_history_path) + submission_history_file = 'output.txt' + FileUtils.touch(File.join(old_submission_history_path, submission_history_file)) + assert File.exist?(File.join(old_submission_history_path, submission_history_file)) + unit.code = "New-#{unit.code}" unit.save! @@ -1029,6 +1035,12 @@ def test_change_unit_code_moves_files assert File.exist?(task.final_pdf_path), "Portfolio evidence file does not exist = #{task.final_pdf_path}" assert task.has_pdf + new_submission_history_path = FileHelper.unit_submission_history_dir(unit, archived: false) + assert_not File.exist?(old_submission_history_path), + "Old submission history still exists - #{old_submission_history_path}" + assert File.exist?(File.join(new_submission_history_path, submission_history_file)), + "New submission history file does not exist - #{new_submission_history_path}" + unit.destroy! end From 4ff4ff3baade4f56ddc7942f7c9e2a9f0d2c32cb Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:32:58 +1000 Subject: [PATCH 124/199] refactor: move jplag report dir under student work and remove config --- app/helpers/file_helper.rb | 30 ++++++++++++++++--- .../similarity/unit_similarity_module.rb | 14 ++++----- config/application.rb | 6 ---- test/models/file_helper_test.rb | 5 ++++ test/models/unit_model_test.rb | 7 +++++ 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/app/helpers/file_helper.rb b/app/helpers/file_helper.rb index 967d8ed302..8526c4b333 100644 --- a/app/helpers/file_helper.rb +++ b/app/helpers/file_helper.rb @@ -340,13 +340,33 @@ def student_portfolio_path(unit, username, create: true, archived: true) File.join(student_portfolio_dir(unit, username, create: create, archived: archived), FileHelper.sanitized_filename("#{username}-portfolio.pdf")) end - def task_jplag_report_dir(unit) - file_server = Doubtfire::Application.config.jplag_report_dir - "#{file_server}/#{unit.code}-#{unit.id}/" # trust the server config and passed in type for paths + def root_jplag_report_dir(archived: false) + file_server = if archived + archive_root + else + student_work_root + end + + "#{file_server}/jplag/results/" + end + + def unit_jplag_report_dir(unit, create: false, archived: true) + dst = if (unit.archived && archived) || (archived == :force) + File.join(root_jplag_report_dir(archived: true), sanitized_path("#{unit.code}-#{unit.id}")) + else + File.join(root_jplag_report_dir(archived: false), sanitized_path("#{unit.code}-#{unit.id}")) + end + + FileUtils.mkdir_p dst if create + "#{dst}/" + end + + def task_jplag_report_dir(unit, create: false, archived: true) + unit_jplag_report_dir(unit, create: create, archived: archived) end def task_jplag_report_path(unit, task) - File.join(task_jplag_report_dir(unit), FileHelper.sanitized_filename("#{task.abbreviation}-result.jplag")) + File.join(unit_jplag_report_dir(unit), FileHelper.sanitized_filename("#{task.abbreviation}-result.jplag")) end def comment_attachment_path(task_comment, attachment_extension) @@ -1071,6 +1091,8 @@ def line_wrap(path, width: 160) module_function :known_extension? module_function :pages_in_pdf module_function :line_wrap + module_function :root_jplag_report_dir + module_function :unit_jplag_report_dir module_function :task_jplag_report_dir module_function :task_jplag_report_path end diff --git a/app/models/similarity/unit_similarity_module.rb b/app/models/similarity/unit_similarity_module.rb index 1fd332d3ed..ecfd3ff63e 100644 --- a/app/models/similarity/unit_similarity_module.rb +++ b/app/models/similarity/unit_similarity_module.rb @@ -108,7 +108,6 @@ def check_jplag_similarity(force: false) # making temp directory for unit - jplag root_work_dir = Rails.root.join("tmp", "jplag", "#{code}-#{id}") - unit_code = "#{code}-#{id}" begin logger.info "Checking plagiarsm for unit #{code} - #{name} (id=#{id})" @@ -138,8 +137,8 @@ def check_jplag_similarity(force: false) FileUtils.mkdir_p(tasks_dir) # There are new tasks, check these with JPLAG - run_jplag_on_done_files(td, tasks_dir, tasks_with_files, unit_code) - report_path = "#{Doubtfire::Application.config.jplag_report_dir}/#{unit_code}/#{td.abbreviation}-result.jplag" + report_path = FileHelper.task_jplag_report_path(self, td) + run_jplag_on_done_files(td, tasks_dir, tasks_with_files, report_path) warn_pct = td.plagiarism_warn_pct || 50 logger.debug "Warn PCT: #{warn_pct}" @@ -230,17 +229,16 @@ def update_moss_plagiarism_stats # end # JPLAG Function - extracts "done" files for each task and packages them into a directory for JPLAG to run on - def run_jplag_on_done_files(task_definition, tasks_dir, tasks_with_files, unit_code) + def run_jplag_on_done_files(task_definition, tasks_dir, tasks_with_files, report_path) similarity_pct = task_definition.plagiarism_warn_pct return if similarity_pct.nil? # Check if the directory exists and create it if it doesn't - results_dir = "/jplag/results/#{unit_code}" + results_dir = File.dirname(report_path) system("docker exec -i jplag sh -c 'if [ ! -d \"#{results_dir}\" ]; then mkdir -p \"#{results_dir}\"; fi'") || raise('Failed to create JPlag results directory') # Remove existing result file if it exists - result_file = "#{results_dir}/#{task_definition.abbreviation}-result.jplag" - system("docker exec -i jplag sh -c 'if [ -f \"#{result_file}\" ]; then rm \"#{result_file}\"; fi'") || raise('Failed to remove previous JPlag report') + system("docker exec -i jplag sh -c 'if [ -f \"#{report_path}\" ]; then rm \"#{report_path}\"; fi'") || raise('Failed to remove previous JPlag report') # Extract task resources for base code use_base_code = false @@ -321,7 +319,7 @@ def run_jplag_on_done_files(task_definition, tasks_dir, tasks_with_files, unit_c min_token_string, skip_cluster_string, "-M RUN", - "-r #{results_dir}/#{task_definition.abbreviation}-result", + "-r #{report_path.delete_suffix('.jplag')}", "--overwrite" ].join(" ") diff --git a/config/application.rb b/config/application.rb index a0d490ada2..c4f3e6f3b7 100644 --- a/config/application.rb +++ b/config/application.rb @@ -89,12 +89,6 @@ def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) # Have rails report errors and log messages to the following email address where present config.email_errors_to = ENV.fetch('DF_EMAIL_ERRORS_TO', nil) - # ==> JPLAG report directory - # File server location for storing JPLAG reports. Defaults to `jplag/results` - # directory under root but is overridden using DF_JPLAG_REPORT_DIR environment - # variable. - config.jplag_report_dir = ENV['DF_JPLAG_REPORT_DIR'] || Rails.root.join('jplag/results').to_s - # Tunes the comparison sensitivity by adjusting the minimum token required to be # counted as a matching section. A smaller value increases the sensitivity # but might lead to more false-positives diff --git a/test/models/file_helper_test.rb b/test/models/file_helper_test.rb index 80eb68fd02..d77a026fa5 100644 --- a/test/models/file_helper_test.rb +++ b/test/models/file_helper_test.rb @@ -22,10 +22,15 @@ def test_archive_paths archive_portfolio_path = FileHelper.unit_portfolio_dir(unit, create: false, archived: :force) original_portfolio_path = FileHelper.unit_portfolio_dir(unit, create: false, archived: false) + archive_jplag_path = FileHelper.unit_jplag_report_dir(unit, archived: :force) + original_jplag_path = FileHelper.unit_jplag_report_dir(unit, archived: false) + assert_match %r{^#{FileHelper.archive_root}/}, archive_work_path assert_match %r{^#{FileHelper.archive_root}/portfolio/}, archive_portfolio_path + assert_match %r{^#{FileHelper.archive_root}/jplag/results/}, archive_jplag_path assert_match %r{^#{FileHelper.student_work_root}/}, original_work_path assert_match %r{^#{FileHelper.student_work_root}/portfolio/}, original_portfolio_path + assert_match %r{^#{FileHelper.student_work_root}/jplag/results/}, original_jplag_path end def test_accept_zip_upload diff --git a/test/models/unit_model_test.rb b/test/models/unit_model_test.rb index f06a239b93..33730635e4 100644 --- a/test/models/unit_model_test.rb +++ b/test/models/unit_model_test.rb @@ -1130,11 +1130,16 @@ def test_archive_unit FileUtils.mkdir_p(old_submission_history_path) FileUtils.touch(File.join(old_submission_history_path, 'output.txt')) + old_jplag_report_path = FileHelper.task_jplag_report_path(unit, td) + FileUtils.mkdir_p(File.dirname(old_jplag_report_path)) + FileUtils.touch(old_jplag_report_path) + assert File.exist?(old_path) assert File.exist?(task_pdf) assert File.exist?(old_portfolio_path) assert File.exist?(old_submission_history_path) assert File.exist?(File.join(old_submission_history_path, 'output.txt')) + assert File.exist?(old_jplag_report_path) unit.move_files_to_archive unit.archived = true @@ -1152,6 +1157,8 @@ def test_archive_unit assert_not File.exist?(old_submission_history_path), "Old submission history still exists - #{old_submission_history_path}" assert File.exist?(FileHelper.task_submission_identifier_path(:done, task)) assert File.exist?(File.join(FileHelper.task_submission_identifier_path_with_timestamp(:done, task, '123_45'), 'output.txt')) + assert_not File.exist?(old_jplag_report_path), "Old JPlag report still exists - #{old_jplag_report_path}" + assert File.exist?(FileHelper.task_jplag_report_path(unit, td)), "New JPlag report does not exist" assert File.exist?(task.final_pdf_path), "Portfolio evidence file does not exist - #{task.final_pdf_path}" From b495dc1b3f1122ddde397f4736817a72ecadea52 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:33:15 +1000 Subject: [PATCH 125/199] chore: relocate units jplag report dir on unit code change --- app/models/unit.rb | 21 +++++++++++++++++++++ test/models/unit_model_test.rb | 8 ++++++++ 2 files changed, 29 insertions(+) diff --git a/app/models/unit.rb b/app/models/unit.rb index d9972869aa..f75b0218bb 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -3592,6 +3592,15 @@ def move_files_to_archive FileUtils.mkdir_p(FileHelper.root_submission_history_dir(archived: true)) FileUtils.mv(original_submission_history_path, archive_submission_history_path) end + + # Move JPlag reports + archive_jplag_report_path = FileHelper.unit_jplag_report_dir(self, create: false, archived: :force) + original_jplag_report_path = FileHelper.unit_jplag_report_dir(self, create: false, archived: false) + + if File.exist?(original_jplag_report_path) && ! File.exist?(archive_jplag_report_path) + FileUtils.mkdir_p(FileHelper.root_jplag_report_dir(archived: true)) + FileUtils.mv(original_jplag_report_path, archive_jplag_report_path) + end end def get_tutor_times(start_date: nil, end_date: nil, timezone: nil, ignore_sessions_during_tutorials: false) @@ -3715,10 +3724,12 @@ def delete_associated_files unit_path = FileHelper.unit_dir(self, create: false) unit_portfolio_path = FileHelper.unit_portfolio_dir(self, create: false) submission_history_path = FileHelper.unit_submission_history_dir(self) + jplag_report_path = FileHelper.unit_jplag_report_dir(self, create: false) FileUtils.rm_rf unit_path FileUtils.rm_rf unit_portfolio_path FileUtils.rm_rf submission_history_path + FileUtils.rm_rf jplag_report_path FileUtils.cd FileHelper.student_work_dir end @@ -3767,6 +3778,16 @@ def move_files_on_code_change FileUtils.mv(old_submission_history_dir, new_submission_history_dir) unless File.exist? new_submission_history_dir end + old_jplag_report_dir = File.join( + FileHelper.root_jplag_report_dir(archived: archived), + FileHelper.sanitized_path("#{saved_change_to_code[0]}-#{id}") + ) + + if File.exist? old_jplag_report_dir + new_jplag_report_dir = FileHelper.unit_jplag_report_dir(self, create: false) + FileUtils.mv(old_jplag_report_dir, new_jplag_report_dir) unless File.exist? new_jplag_report_dir + end + # rubocop:disable Rails/SkipsModelValidations tasks.where('portfolio_evidence IS NOT NULL').update_all("portfolio_evidence = REPLACE(portfolio_evidence, '#{saved_change_to_code[0]}-#{id}', '#{code}-#{id}')") # rubocop:enable Rails/SkipsModelValidations diff --git a/test/models/unit_model_test.rb b/test/models/unit_model_test.rb index 33730635e4..b07e4807ed 100644 --- a/test/models/unit_model_test.rb +++ b/test/models/unit_model_test.rb @@ -1018,6 +1018,11 @@ def test_change_unit_code_moves_files FileUtils.touch(File.join(old_submission_history_path, submission_history_file)) assert File.exist?(File.join(old_submission_history_path, submission_history_file)) + old_jplag_report_path = FileHelper.task_jplag_report_path(unit, td) + FileUtils.mkdir_p(File.dirname(old_jplag_report_path)) + FileUtils.touch(old_jplag_report_path) + assert File.exist?(old_jplag_report_path) + unit.code = "New-#{unit.code}" unit.save! @@ -1041,6 +1046,9 @@ def test_change_unit_code_moves_files assert File.exist?(File.join(new_submission_history_path, submission_history_file)), "New submission history file does not exist - #{new_submission_history_path}" + assert_not File.exist?(old_jplag_report_path), "Old JPlag report still exists - #{old_jplag_report_path}" + assert File.exist?(FileHelper.task_jplag_report_path(unit, td)), "New JPlag report does not exist" + unit.destroy! end From 14c4240aaeaf7943bc3d24bd951bf390aed2b868 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:51:14 +1000 Subject: [PATCH 126/199] chore: minify test task names --- .../COS10001-ImportTasksWithOverseerSteps.csv | 2 +- ...COS10001-ImportTasksWithTutorialStream.csv | 68 +++++++++---------- ...10001-ImportTasksWithoutTutorialStream.csv | 68 +++++++++---------- test_files/COS10001-Tasks.csv | 68 +++++++++---------- .../COS10001-TasksUnorderedUploadReqs.csv | 6 +- .../COS10001-Tasks-Prerequisites.csv | 2 +- 6 files changed, 107 insertions(+), 107 deletions(-) diff --git a/test_files/COS10001-ImportTasksWithOverseerSteps.csv b/test_files/COS10001-ImportTasksWithOverseerSteps.csv index d990f899e9..c475a7c90e 100644 --- a/test_files/COS10001-ImportTasksWithOverseerSteps.csv +++ b/test_files/COS10001-ImportTasksWithOverseerSteps.csv @@ -1,2 +1,2 @@ name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts,overseer_steps -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,FALSE,FALSE,FALSE,FALSE,0,FALSE,[],[],"[{""name"":""Step 1"",""description"":""Step 1"",""display_name"":""Step 1 student"",""display_description"":""Step 1 student"",""run_command"":""b64:IyEvYmluL2Jhc2gKCmVjaG8gIkhlbGxvIHdvcmxkISI"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":null,""halt_on_success"":null,""halt_on_failure"":null,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" +P1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,FALSE,FALSE,FALSE,FALSE,0,FALSE,[],[],"[{""name"":""Step 1"",""description"":""Step 1"",""display_name"":""Step 1 student"",""display_description"":""Step 1 student"",""run_command"":""b64:IyEvYmluL2Jhc2gKCmVjaG8gIkhlbGxvIHdvcmxkISI"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":null,""halt_on_success"":null,""halt_on_failure"":null,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" diff --git a/test_files/COS10001-ImportTasksWithTutorialStream.csv b/test_files/COS10001-ImportTasksWithTutorialStream.csv index 3e6d413711..fecc3ff4c9 100644 --- a/test_files/COS10001-ImportTasksWithTutorialStream.csv +++ b/test_files/COS10001-ImportTasksWithTutorialStream.csv @@ -1,37 +1,37 @@ name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 2.1 - Hand Execute Assignment,2.1P,"Using the assignment statement, you can assign a value to a variable. In this task you will demonstrate how this action works within the computer.",2,0,FALSE,"[{""key"":""file0"",""name"":""Program Execution 1"",""type"":""image""},{""key"":""file1"",""name"":""Program Execution 2"",""type"":""image""},{""key"":""file2"",""name"":""Program Execution 3"",""type"":""image""},{""key"":""file3"",""name"":""Program Execution 4"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 2.2 - Hello User,2.2P,Now that we have variables we can create a program that reads in the users name from the Terminal and echoes back a welcome message.,4,0,FALSE,"[{""key"":""file0"",""name"":""HelloUser.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 2.3 - My Drawing Procedure,2.3P,Procedures are a great way of encapsulating the instructions needed to perform a task. In most cases the task will need some input data for it to work with. Use parameters to provide data to your procedures.,2,0,FALSE,"[{""key"":""file0"",""name"":""Shape Drawing Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 2.4 - My Functions,2.4P,Using functions you can now create artefacts to encapsulate the steps needed to calculate a value.,4,0,FALSE,"[{""key"":""file0"",""name"":""My Function Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 2.5 - Concept Maps,2.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",2,Tue,3,Tue,,,5,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 3.1 - Hand Execution of Control Flow,3.1P,In this task you will use the hand execution process to demonstrate how the control flow constructs operate within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Button Code"",""type"":""code""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 3.2 - Name Tester,3.2P,Control flow enables you to easily add conditions and loops to your programs. In this task you will create a small program that uses conditions and loops to output custom messages to users.,4,0,FALSE,"[{""key"":""file0"",""name"":""Name Tester code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 3.3 - Circle Moving,3.3P,In this task you will create a small program that allows the user to move a circle around on the screen.,4,0,FALSE,"[{""key"":""file0"",""name"":""Circle Mover code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 3.4 - User Input Functions,3.4C,So far we have provided you with a unit to read and check values entered by the user: the Terminal User Input unit. In this task you will extend this library so that it has a number of additional functions.,4,1,FALSE,"[{""key"":""file0"",""name"":""User Input unit code"",""type"":""code""},{""key"":""file1"",""name"":""Program code"",""type"":""code""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 3.5 - Concept Map,3.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Distinction Task 3.6 - Mandelbrot,3.6D,The Mandelbrot provides an interesting challenge in order to determine how to zoom in to and out of the section of the Mandelbrot being shown to the user.,4,2,FALSE,"[{""key"":""file0"",""name"":""Mandelbrot code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 4.1 - Using Records and Enumerations,4.1P,Effectively organising your data makes programs much easier to develop. By using records and enumerations you can start to model the entities associated with your programs.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,7,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 4.2 - Fruit Punch,4.2C,Create a program using the concepts covered so far.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 4.3 - Concept Map,4.3C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P2.1 - Hand Execute Assignment,2.1P,"Using the assignment statement, you can assign a value to a variable. In this task you will demonstrate how this action works within the computer.",2,0,FALSE,"[{""key"":""file0"",""name"":""Program Execution 1"",""type"":""image""},{""key"":""file1"",""name"":""Program Execution 2"",""type"":""image""},{""key"":""file2"",""name"":""Program Execution 3"",""type"":""image""},{""key"":""file3"",""name"":""Program Execution 4"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P2.2 - Hello User,2.2P,Now that we have variables we can create a program that reads in the users name from the Terminal and echoes back a welcome message.,4,0,FALSE,"[{""key"":""file0"",""name"":""HelloUser.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P2.3 - My Drawing Procedure,2.3P,Procedures are a great way of encapsulating the instructions needed to perform a task. In most cases the task will need some input data for it to work with. Use parameters to provide data to your procedures.,2,0,FALSE,"[{""key"":""file0"",""name"":""Shape Drawing Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P2.4 - My Functions,2.4P,Using functions you can now create artefacts to encapsulate the steps needed to calculate a value.,4,0,FALSE,"[{""key"":""file0"",""name"":""My Function Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C2.5 - Concept Maps,2.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",2,Tue,3,Tue,,,5,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P3.1 - Hand Execution of Control Flow,3.1P,In this task you will use the hand execution process to demonstrate how the control flow constructs operate within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Button Code"",""type"":""code""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P3.2 - Name Tester,3.2P,Control flow enables you to easily add conditions and loops to your programs. In this task you will create a small program that uses conditions and loops to output custom messages to users.,4,0,FALSE,"[{""key"":""file0"",""name"":""Name Tester code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P3.3 - Circle Moving,3.3P,In this task you will create a small program that allows the user to move a circle around on the screen.,4,0,FALSE,"[{""key"":""file0"",""name"":""Circle Mover code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C3.4 - User Input Functions,3.4C,So far we have provided you with a unit to read and check values entered by the user: the Terminal User Input unit. In this task you will extend this library so that it has a number of additional functions.,4,1,FALSE,"[{""key"":""file0"",""name"":""User Input unit code"",""type"":""code""},{""key"":""file1"",""name"":""Program code"",""type"":""code""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C3.5 - Concept Map,3.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +D3.6 - Mandelbrot,3.6D,The Mandelbrot provides an interesting challenge in order to determine how to zoom in to and out of the section of the Mandelbrot being shown to the user.,4,2,FALSE,"[{""key"":""file0"",""name"":""Mandelbrot code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P4.1 - Using Records and Enumerations,4.1P,Effectively organising your data makes programs much easier to develop. By using records and enumerations you can start to model the entities associated with your programs.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,7,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C4.2 - Fruit Punch,4.2C,Create a program using the concepts covered so far.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C4.3 - Concept Map,4.3C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] Test 1,T1,Test 1 covers weeks 1 to 3,1,0,TRUE,[],5,Fri,5,Fri,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 5.1 - Hand Execution of Arrays,5.1P,Demonstrate how arrays work within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 5.2 - Arrays of Records,5.2P,Add an array of records to your program that uses records.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 5.3 - Food Hunter,5.3C,Extend a small game to make use of arrays.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 5.4 - Concept Map,5.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Distinction Task 5.5 - Sort Visualiser,5.5D,Create a program to demonstrate sorting working within the computer.,4,2,FALSE,"[{""key"":""file0"",""name"":""Sort Visualiser"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,"[{""abbreviation"":""1.4C"",""task_status_id"":2},{""abbreviation"":""3.3P"",""task_status_id"":9}]",[] -Pass Task 6.1 - Structure Charts,6.1P,Illustrate the structure of your program using a structure chart.,2,0,FALSE,"[{""key"":""file0"",""name"":""Program structrue chart"",""type"":""image""}]",6,Tue,7,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 7.1 - Programming Principles,7.1P,"Describe the principles of structured, procedural, programming.",4,0,FALSE,"[{""key"":""file0"",""name"":""Program Principles Description"",""type"":""document""}]",7,Tue,8,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Distinction Task 7.2 - Game of Life,7.2D,Create the Game of Life,4,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",7,Tue,8,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 8.1 - Language Reference Sheet,8.1P,Create a reference sheet for C or C#,4,0,FALSE,"[{""key"":""file0"",""name"":""Reference Sheet"",""type"":""document""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 8.2 - Circle Moving 2,8.2P,Recreate your circle moving program using C,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P5.1 - Hand Execution of Arrays,5.1P,Demonstrate how arrays work within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P5.2 - Arrays of Records,5.2P,Add an array of records to your program that uses records.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C5.3 - Food Hunter,5.3C,Extend a small game to make use of arrays.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C5.4 - Concept Map,5.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +D5.5 - Sort Visualiser,5.5D,Create a program to demonstrate sorting working within the computer.,4,2,FALSE,"[{""key"":""file0"",""name"":""Sort Visualiser"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,"[{""abbreviation"":""1.4C"",""task_status_id"":2},{""abbreviation"":""3.3P"",""task_status_id"":9}]",[] +P6.1 - Structure Charts,6.1P,Illustrate the structure of your program using a structure chart.,2,0,FALSE,"[{""key"":""file0"",""name"":""Program structrue chart"",""type"":""image""}]",6,Tue,7,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P7.1 - Programming Principles,7.1P,"Describe the principles of structured, procedural, programming.",4,0,FALSE,"[{""key"":""file0"",""name"":""Program Principles Description"",""type"":""document""}]",7,Tue,8,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +D7.2 - Game of Life,7.2D,Create the Game of Life,4,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",7,Tue,8,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P8.1 - Language Reference Sheet,8.1P,Create a reference sheet for C or C#,4,0,FALSE,"[{""key"":""file0"",""name"":""Reference Sheet"",""type"":""document""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P8.2 - Circle Moving 2,8.2P,Recreate your circle moving program using C,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] Test 2,T2,Covers all core concepts.,1,0,TRUE,[],9,Fri,9,Fri,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 9.1 - Reading Another Language,9.1P,Demonstrate how programs written in C work within the computer,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Credit Task 9.2 - Another Language,9.2C,Create a program with C using the concepts covered.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -High Distinction Task 10.1 - Custom Program,10.1H,Extend your custom program to meet the High Distinction criteria.,4,3,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -High Distinction Task 10.2 - Research Report,10.2H,Start working on a research project,8,3,FALSE,"[{""key"":""file0"",""name"":""Research Report Document"",""type"":""document""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Pass Task 11.1 - Learning Summary Report,11.1P,Summarise your learning from the unit.,4,0,FALSE,"[{""key"":""file0"",""name"":""Learning Summary Report"",""type"":""document""}]",11,Tue,12,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] -Distinction Task 6.2 - Custom Program,6.2D,Start working on your custom program!,16,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",6,Tue,13,Tue,,,5,TRUE,90,,,import-tasks,,,,FALSE,[],[] +P9.1 - Reading Another Language,9.1P,Demonstrate how programs written in C work within the computer,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +C9.2 - Another Language,9.2C,Create a program with C using the concepts covered.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +HD10.1 - Custom Program,10.1H,Extend your custom program to meet the High Distinction criteria.,4,3,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +HD10.2 - Research Report,10.2H,Start working on a research project,8,3,FALSE,"[{""key"":""file0"",""name"":""Research Report Document"",""type"":""document""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +P11.1 - Learning Summary Report,11.1P,Summarise your learning from the unit.,4,0,FALSE,"[{""key"":""file0"",""name"":""Learning Summary Report"",""type"":""document""}]",11,Tue,12,Tue,,,0,FALSE,90,,,import-tasks,,,,FALSE,[],[] +D6.2 - Custom Program,6.2D,Start working on your custom program!,16,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",6,Tue,13,Tue,,,5,TRUE,90,,,import-tasks,,,,FALSE,[],[] diff --git a/test_files/COS10001-ImportTasksWithoutTutorialStream.csv b/test_files/COS10001-ImportTasksWithoutTutorialStream.csv index e23a686ac5..14880abf7f 100644 --- a/test_files/COS10001-ImportTasksWithoutTutorialStream.csv +++ b/test_files/COS10001-ImportTasksWithoutTutorialStream.csv @@ -1,37 +1,37 @@ name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 2.1 - Hand Execute Assignment,2.1P,"Using the assignment statement, you can assign a value to a variable. In this task you will demonstrate how this action works within the computer.",2,0,FALSE,"[{""key"":""file0"",""name"":""Program Execution 1"",""type"":""image""},{""key"":""file1"",""name"":""Program Execution 2"",""type"":""image""},{""key"":""file2"",""name"":""Program Execution 3"",""type"":""image""},{""key"":""file3"",""name"":""Program Execution 4"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 2.2 - Hello User,2.2P,Now that we have variables we can create a program that reads in the users name from the Terminal and echoes back a welcome message.,4,0,FALSE,"[{""key"":""file0"",""name"":""HelloUser.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 2.3 - My Drawing Procedure,2.3P,Procedures are a great way of encapsulating the instructions needed to perform a task. In most cases the task will need some input data for it to work with. Use parameters to provide data to your procedures.,2,0,FALSE,"[{""key"":""file0"",""name"":""Shape Drawing Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 2.4 - My Functions,2.4P,Using functions you can now create artefacts to encapsulate the steps needed to calculate a value.,4,0,FALSE,"[{""key"":""file0"",""name"":""My Function Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 2.5 - Concept Maps,2.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",2,Tue,3,Tue,,,5,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 3.1 - Hand Execution of Control Flow,3.1P,In this task you will use the hand execution process to demonstrate how the control flow constructs operate within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Button Code"",""type"":""code""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 3.2 - Name Tester,3.2P,Control flow enables you to easily add conditions and loops to your programs. In this task you will create a small program that uses conditions and loops to output custom messages to users.,4,0,FALSE,"[{""key"":""file0"",""name"":""Name Tester code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 3.3 - Circle Moving,3.3P,In this task you will create a small program that allows the user to move a circle around on the screen.,4,0,FALSE,"[{""key"":""file0"",""name"":""Circle Mover code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 3.4 - User Input Functions,3.4C,So far we have provided you with a unit to read and check values entered by the user: the Terminal User Input unit. In this task you will extend this library so that it has a number of additional functions.,4,1,FALSE,"[{""key"":""file0"",""name"":""User Input unit code"",""type"":""code""},{""key"":""file1"",""name"":""Program code"",""type"":""code""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 3.5 - Concept Map,3.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",3,Tue,4,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Distinction Task 3.6 - Mandelbrot,3.6D,The Mandelbrot provides an interesting challenge in order to determine how to zoom in to and out of the section of the Mandelbrot being shown to the user.,4,2,FALSE,"[{""key"":""file0"",""name"":""Mandelbrot code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 4.1 - Using Records and Enumerations,4.1P,Effectively organising your data makes programs much easier to develop. By using records and enumerations you can start to model the entities associated with your programs.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,7,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 4.2 - Fruit Punch,4.2C,Create a program using the concepts covered so far.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 4.3 - Concept Map,4.3C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",4,Tue,5,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +P1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +C1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +P2.1 - Hand Execute Assignment,2.1P,"Using the assignment statement, you can assign a value to a variable. In this task you will demonstrate how this action works within the computer.",2,0,FALSE,"[{""key"":""file0"",""name"":""Program Execution 1"",""type"":""image""},{""key"":""file1"",""name"":""Program Execution 2"",""type"":""image""},{""key"":""file2"",""name"":""Program Execution 3"",""type"":""image""},{""key"":""file3"",""name"":""Program Execution 4"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P2.2 - Hello User,2.2P,Now that we have variables we can create a program that reads in the users name from the Terminal and echoes back a welcome message.,4,0,FALSE,"[{""key"":""file0"",""name"":""HelloUser.pas"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P2.3 - My Drawing Procedure,2.3P,Procedures are a great way of encapsulating the instructions needed to perform a task. In most cases the task will need some input data for it to work with. Use parameters to provide data to your procedures.,2,0,FALSE,"[{""key"":""file0"",""name"":""Shape Drawing Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P2.4 - My Functions,2.4P,Using functions you can now create artefacts to encapsulate the steps needed to calculate a value.,4,0,FALSE,"[{""key"":""file0"",""name"":""My Function Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +C2.5 - Concept Maps,2.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",2,Tue,3,Tue,,,5,FALSE,90,,,,,,,FALSE,[],[] +P3.1 - Hand Execution of Control Flow,3.1P,In this task you will use the hand execution process to demonstrate how the control flow constructs operate within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Button Code"",""type"":""code""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P3.2 - Name Tester,3.2P,Control flow enables you to easily add conditions and loops to your programs. In this task you will create a small program that uses conditions and loops to output custom messages to users.,4,0,FALSE,"[{""key"":""file0"",""name"":""Name Tester code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P3.3 - Circle Moving,3.3P,In this task you will create a small program that allows the user to move a circle around on the screen.,4,0,FALSE,"[{""key"":""file0"",""name"":""Circle Mover code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +C3.4 - User Input Functions,3.4C,So far we have provided you with a unit to read and check values entered by the user: the Terminal User Input unit. In this task you will extend this library so that it has a number of additional functions.,4,1,FALSE,"[{""key"":""file0"",""name"":""User Input unit code"",""type"":""code""},{""key"":""file1"",""name"":""Program code"",""type"":""code""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +C3.5 - Concept Map,3.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",3,Tue,4,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +D3.6 - Mandelbrot,3.6D,The Mandelbrot provides an interesting challenge in order to determine how to zoom in to and out of the section of the Mandelbrot being shown to the user.,4,2,FALSE,"[{""key"":""file0"",""name"":""Mandelbrot code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +P4.1 - Using Records and Enumerations,4.1P,Effectively organising your data makes programs much easier to develop. By using records and enumerations you can start to model the entities associated with your programs.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,7,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +C4.2 - Fruit Punch,4.2C,Create a program using the concepts covered so far.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +C4.3 - Concept Map,4.3C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",4,Tue,5,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] Test 1,T1,Test 1 covers weeks 1 to 3,1,0,TRUE,[],5,Fri,5,Fri,,,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 5.1 - Hand Execution of Arrays,5.1P,Demonstrate how arrays work within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 5.2 - Arrays of Records,5.2P,Add an array of records to your program that uses records.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 5.3 - Food Hunter,5.3C,Extend a small game to make use of arrays.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 5.4 - Concept Map,5.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",5,Tue,6,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Distinction Task 5.5 - Sort Visualiser,5.5D,Create a program to demonstrate sorting working within the computer.,4,2,FALSE,"[{""key"":""file0"",""name"":""Sort Visualiser"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,,,,,FALSE,"[{""abbreviation"":""1.4C"",""task_status_id"":2},{""abbreviation"":""3.3P"",""task_status_id"":9}]",[] -Pass Task 6.1 - Structure Charts,6.1P,Illustrate the structure of your program using a structure chart.,2,0,FALSE,"[{""key"":""file0"",""name"":""Program structrue chart"",""type"":""image""}]",6,Tue,7,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 7.1 - Programming Principles,7.1P,"Describe the principles of structured, procedural, programming.",4,0,FALSE,"[{""key"":""file0"",""name"":""Program Principles Description"",""type"":""document""}]",7,Tue,8,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Distinction Task 7.2 - Game of Life,7.2D,Create the Game of Life,4,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",7,Tue,8,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 8.1 - Language Reference Sheet,8.1P,Create a reference sheet for C or C#,4,0,FALSE,"[{""key"":""file0"",""name"":""Reference Sheet"",""type"":""document""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 8.2 - Circle Moving 2,8.2P,Recreate your circle moving program using C,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P5.1 - Hand Execution of Arrays,5.1P,Demonstrate how arrays work within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P5.2 - Arrays of Records,5.2P,Add an array of records to your program that uses records.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +C5.3 - Food Hunter,5.3C,Extend a small game to make use of arrays.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +C5.4 - Concept Map,5.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",5,Tue,6,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +D5.5 - Sort Visualiser,5.5D,Create a program to demonstrate sorting working within the computer.,4,2,FALSE,"[{""key"":""file0"",""name"":""Sort Visualiser"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,,,,,FALSE,"[{""abbreviation"":""1.4C"",""task_status_id"":2},{""abbreviation"":""3.3P"",""task_status_id"":9}]",[] +P6.1 - Structure Charts,6.1P,Illustrate the structure of your program using a structure chart.,2,0,FALSE,"[{""key"":""file0"",""name"":""Program structrue chart"",""type"":""image""}]",6,Tue,7,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P7.1 - Programming Principles,7.1P,"Describe the principles of structured, procedural, programming.",4,0,FALSE,"[{""key"":""file0"",""name"":""Program Principles Description"",""type"":""document""}]",7,Tue,8,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +D7.2 - Game of Life,7.2D,Create the Game of Life,4,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",7,Tue,8,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +P8.1 - Language Reference Sheet,8.1P,Create a reference sheet for C or C#,4,0,FALSE,"[{""key"":""file0"",""name"":""Reference Sheet"",""type"":""document""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] +P8.2 - Circle Moving 2,8.2P,Recreate your circle moving program using C,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,,,,,FALSE,[],[] Test 2,T2,Covers all core concepts.,1,0,TRUE,[],9,Fri,9,Fri,,,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 9.1 - Reading Another Language,9.1P,Demonstrate how programs written in C work within the computer,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Credit Task 9.2 - Another Language,9.2C,Create a program with C using the concepts covered.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -High Distinction Task 10.1 - Custom Program,10.1H,Extend your custom program to meet the High Distinction criteria.,4,3,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",10,Tue,13,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -High Distinction Task 10.2 - Research Report,10.2H,Start working on a research project,8,3,FALSE,"[{""key"":""file0"",""name"":""Research Report Document"",""type"":""document""}]",10,Tue,13,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Pass Task 11.1 - Learning Summary Report,11.1P,Summarise your learning from the unit.,4,0,FALSE,"[{""key"":""file0"",""name"":""Learning Summary Report"",""type"":""document""}]",11,Tue,12,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] -Distinction Task 6.2 - Custom Program,6.2D,Start working on your custom program!,16,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",6,Tue,13,Tue,,,5,TRUE,90,,,,,,,FALSE,[],[] +P9.1 - Reading Another Language,9.1P,Demonstrate how programs written in C work within the computer,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +C9.2 - Another Language,9.2C,Create a program with C using the concepts covered.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +HD10.1 - Custom Program,10.1H,Extend your custom program to meet the High Distinction criteria.,4,3,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",10,Tue,13,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +HD10.2 - Research Report,10.2H,Start working on a research project,8,3,FALSE,"[{""key"":""file0"",""name"":""Research Report Document"",""type"":""document""}]",10,Tue,13,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +P11.1 - Learning Summary Report,11.1P,Summarise your learning from the unit.,4,0,FALSE,"[{""key"":""file0"",""name"":""Learning Summary Report"",""type"":""document""}]",11,Tue,12,Tue,,,0,FALSE,90,,,,,,,FALSE,[],[] +D6.2 - Custom Program,6.2D,Start working on your custom program!,16,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",6,Tue,13,Tue,,,5,TRUE,90,,,,,,,FALSE,[],[] diff --git a/test_files/COS10001-Tasks.csv b/test_files/COS10001-Tasks.csv index f1a1b36659..05965e0583 100644 --- a/test_files/COS10001-Tasks.csv +++ b/test_files/COS10001-Tasks.csv @@ -1,38 +1,38 @@ name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts,overseer_steps -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],,"[{""name"":""Hello World"",""description"":""Hello World"",""display_name"":""Hello World"",""display_description"":""Hello World"",""run_command"":""b64:IyEvYmluL3NoCgplY2hvICJIZWxsbyB3b3JsZCEi"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":null,""halt_on_success"":null,""halt_on_failure"":null,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" -Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.1P"",""task_status_id"":9}]",,"[{""name"":""Picture Drawing"",""description"":""Picture Drawing"",""display_name"":""Picture Drawing"",""display_description"":""Picture Drawing"",""run_command"":""b64:IyEvYmluL3NoCgplY2hvICJIZWxsbyB3b3JsZCEi"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":""fix_and_resubmit"",""halt_on_success"":null,""halt_on_failure"":true,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" -Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.2P"",""task_status_id"":9}]",, -Credit Task 1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.3P"",""task_status_id"":9}]",, -Pass Task 2.1 - Hand Execute Assignment,2.1P,"Using the assignment statement, you can assign a value to a variable. In this task you will demonstrate how this action works within the computer.",2,0,FALSE,"[{""key"":""file0"",""name"":""Program Execution 1"",""type"":""image""},{""key"":""file1"",""name"":""Program Execution 2"",""type"":""image""},{""key"":""file2"",""name"":""Program Execution 3"",""type"":""image""},{""key"":""file3"",""name"":""Program Execution 4"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.3P"",""task_status_id"":9}]",, -Pass Task 2.2 - Hello User,2.2P,Now that we have variables we can create a program that reads in the users name from the Terminal and echoes back a welcome message.,4,0,FALSE,"[{""key"":""file0"",""name"":""HelloUser.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.1P"",""task_status_id"":9}]",, -Pass Task 2.3 - My Drawing Procedure,2.3P,Procedures are a great way of encapsulating the instructions needed to perform a task. In most cases the task will need some input data for it to work with. Use parameters to provide data to your procedures.,2,0,FALSE,"[{""key"":""file0"",""name"":""Shape Drawing Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.2P"",""task_status_id"":9}]",, -Pass Task 2.4 - My Functions,2.4P,Using functions you can now create artefacts to encapsulate the steps needed to calculate a value.,4,0,FALSE,"[{""key"":""file0"",""name"":""My Function Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.3P"",""task_status_id"":9}]",, -Credit Task 2.5 - Concept Maps,2.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",2,Tue,3,Tue,,,5,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.4C"",""task_status_id"":2}]",, -Pass Task 3.1 - Hand Execution of Control Flow,3.1P,In this task you will use the hand execution process to demonstrate how the control flow constructs operate within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Button Code"",""type"":""code""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.4P"",""task_status_id"":9}]",, -Pass Task 3.2 - Name Tester,3.2P,Control flow enables you to easily add conditions and loops to your programs. In this task you will create a small program that uses conditions and loops to output custom messages to users.,4,0,FALSE,"[{""key"":""file0"",""name"":""Name Tester code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.1P"",""task_status_id"":9}]",, -Pass Task 3.3 - Circle Moving,3.3P,In this task you will create a small program that allows the user to move a circle around on the screen.,4,0,FALSE,"[{""key"":""file0"",""name"":""Circle Mover code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.2P"",""task_status_id"":9}]",, -Credit Task 3.4 - User Input Functions,3.4C,So far we have provided you with a unit to read and check values entered by the user: the Terminal User Input unit. In this task you will extend this library so that it has a number of additional functions.,4,1,FALSE,"[{""key"":""file0"",""name"":""User Input unit code"",""type"":""code""},{""key"":""file1"",""name"":""Program code"",""type"":""code""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.5C"",""task_status_id"":9}]",, -Credit Task 3.5 - Concept Map,3.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.4C"",""task_status_id"":8}]",, -Distinction Task 3.6 - Mandelbrot,3.6D,The Mandelbrot provides an interesting challenge in order to determine how to zoom in to and out of the section of the Mandelbrot being shown to the user.,4,2,FALSE,"[{""key"":""file0"",""name"":""Mandelbrot code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.5C"",""task_status_id"":2}]",, -Pass Task 4.1 - Using Records and Enumerations,4.1P,Effectively organising your data makes programs much easier to develop. By using records and enumerations you can start to model the entities associated with your programs.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.3P"",""task_status_id"":9}]",, -Credit Task 4.2 - Fruit Punch,4.2C,Create a program using the concepts covered so far.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Credit Task 4.3 - Concept Map,4.3C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +P1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file0"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],,"[{""name"":""Hello World"",""description"":""Hello World"",""display_name"":""Hello World"",""display_description"":""Hello World"",""run_command"":""b64:IyEvYmluL3NoCgplY2hvICJIZWxsbyB3b3JsZCEi"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":null,""halt_on_success"":null,""halt_on_failure"":null,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" +P1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.1P"",""task_status_id"":9}]",,"[{""name"":""Picture Drawing"",""description"":""Picture Drawing"",""display_name"":""Picture Drawing"",""display_description"":""Picture Drawing"",""run_command"":""b64:IyEvYmluL3NoCgplY2hvICJIZWxsbyB3b3JsZCEi"",""timeout"":30,""sort_order"":0,""step_type"":""status_check"",""partial_output_diff"":null,""stdin_input_file"":null,""expected_output_file"":null,""feedback_message"":null,""status_on_success"":null,""status_on_failure"":""fix_and_resubmit"",""halt_on_success"":null,""halt_on_failure"":true,""show_expected_output"":true,""show_stdin"":null,""show_stdout"":true,""enabled"":true}]" +P1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.2P"",""task_status_id"":9}]",, +C1.4 - Concept Map,1.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",1,Tue,2,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.3P"",""task_status_id"":9}]",, +P2.1 - Hand Execute Assignment,2.1P,"Using the assignment statement, you can assign a value to a variable. In this task you will demonstrate how this action works within the computer.",2,0,FALSE,"[{""key"":""file0"",""name"":""Program Execution 1"",""type"":""image""},{""key"":""file1"",""name"":""Program Execution 2"",""type"":""image""},{""key"":""file2"",""name"":""Program Execution 3"",""type"":""image""},{""key"":""file3"",""name"":""Program Execution 4"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.3P"",""task_status_id"":9}]",, +P2.2 - Hello User,2.2P,Now that we have variables we can create a program that reads in the users name from the Terminal and echoes back a welcome message.,4,0,FALSE,"[{""key"":""file0"",""name"":""HelloUser.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.1P"",""task_status_id"":9}]",, +P2.3 - My Drawing Procedure,2.3P,Procedures are a great way of encapsulating the instructions needed to perform a task. In most cases the task will need some input data for it to work with. Use parameters to provide data to your procedures.,2,0,FALSE,"[{""key"":""file0"",""name"":""Shape Drawing Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.2P"",""task_status_id"":9}]",, +P2.4 - My Functions,2.4P,Using functions you can now create artefacts to encapsulate the steps needed to calculate a value.,4,0,FALSE,"[{""key"":""file0"",""name"":""My Function Code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",2,Tue,3,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.3P"",""task_status_id"":9}]",, +C2.5 - Concept Maps,2.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",2,Tue,3,Tue,,,5,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""1.4C"",""task_status_id"":2}]",, +P3.1 - Hand Execution of Control Flow,3.1P,In this task you will use the hand execution process to demonstrate how the control flow constructs operate within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Button Code"",""type"":""code""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.4P"",""task_status_id"":9}]",, +P3.2 - Name Tester,3.2P,Control flow enables you to easily add conditions and loops to your programs. In this task you will create a small program that uses conditions and loops to output custom messages to users.,4,0,FALSE,"[{""key"":""file0"",""name"":""Name Tester code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.1P"",""task_status_id"":9}]",, +P3.3 - Circle Moving,3.3P,In this task you will create a small program that allows the user to move a circle around on the screen.,4,0,FALSE,"[{""key"":""file0"",""name"":""Circle Mover code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.2P"",""task_status_id"":9}]",, +C3.4 - User Input Functions,3.4C,So far we have provided you with a unit to read and check values entered by the user: the Terminal User Input unit. In this task you will extend this library so that it has a number of additional functions.,4,1,FALSE,"[{""key"":""file0"",""name"":""User Input unit code"",""type"":""code""},{""key"":""file1"",""name"":""Program code"",""type"":""code""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""2.5C"",""task_status_id"":9}]",, +C3.5 - Concept Map,3.5C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.4C"",""task_status_id"":8}]",, +D3.6 - Mandelbrot,3.6D,The Mandelbrot provides an interesting challenge in order to determine how to zoom in to and out of the section of the Mandelbrot being shown to the user.,4,2,FALSE,"[{""key"":""file0"",""name"":""Mandelbrot code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",3,Tue,4,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.5C"",""task_status_id"":2}]",, +P4.1 - Using Records and Enumerations,4.1P,Effectively organising your data makes programs much easier to develop. By using records and enumerations you can start to model the entities associated with your programs.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,7,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,"[{""abbreviation"":""3.3P"",""task_status_id"":9}]",, +C4.2 - Fruit Punch,4.2C,Create a program using the concepts covered so far.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +C4.3 - Concept Map,4.3C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",4,Tue,5,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, Test 1,T1,Test 1 covers weeks 1 to 3,1,0,TRUE,[],5,Fri,5,Fri,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Pass Task 5.1 - Hand Execution of Arrays,5.1P,Demonstrate how arrays work within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Pass Task 5.2 - Arrays of Records,5.2P,Add an array of records to your program that uses records.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Credit Task 5.3 - Food Hunter,5.3C,Extend a small game to make use of arrays.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Credit Task 5.4 - Concept Map,5.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Distinction Task 5.5 - Sort Visualiser,5.5D,Create a program to demonstrate sorting working within the computer.,4,2,FALSE,"[{""key"":""file0"",""name"":""Sort Visualiser"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,TRUE,"[]",[],,,,"[{""abbreviation"":""1.4C"",""task_status_id"":2},{""abbreviation"":""3.3P"",""task_status_id"":9}]",, -Pass Task 6.1 - Structure Charts,6.1P,Illustrate the structure of your program using a structure chart.,2,0,FALSE,"[{""key"":""file0"",""name"":""Program structrue chart"",""type"":""image""}]",6,Tue,7,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Pass Task 7.1 - Programming Principles,7.1P,"Describe the principles of structured, procedural, programming.",4,0,FALSE,"[{""key"":""file0"",""name"":""Program Principles Description"",""type"":""document""}]",7,Tue,8,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Distinction Task 7.2 - Game of Life,7.2D,Create the Game of Life,4,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",7,Tue,8,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Pass Task 8.1 - Language Reference Sheet,8.1P,Create a reference sheet for C or C#,4,0,FALSE,"[{""key"":""file0"",""name"":""Reference Sheet"",""type"":""document""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Pass Task 8.2 - Circle Moving 2,8.2P,Recreate your circle moving program using C,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +P5.1 - Hand Execution of Arrays,5.1P,Demonstrate how arrays work within the computer.,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +P5.2 - Arrays of Records,5.2P,Add an array of records to your program that uses records.,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +C5.3 - Food Hunter,5.3C,Extend a small game to make use of arrays.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +C5.4 - Concept Map,5.4C,A concept map visually shows the relationships between concepts. This task aims to help you think through the various relationships between the structured procedural programming concepts and the associated programming artefacts.,4,1,FALSE,"[{""key"":""file0"",""name"":""Concept map"",""type"":""document""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +D5.5 - Sort Visualiser,5.5D,Create a program to demonstrate sorting working within the computer.,4,2,FALSE,"[{""key"":""file0"",""name"":""Sort Visualiser"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",5,Tue,6,Tue,,,0,FALSE,90,,,import-tasks,TRUE,"[]",[],,,,"[{""abbreviation"":""1.4C"",""task_status_id"":2},{""abbreviation"":""3.3P"",""task_status_id"":9}]",, +P6.1 - Structure Charts,6.1P,Illustrate the structure of your program using a structure chart.,2,0,FALSE,"[{""key"":""file0"",""name"":""Program structrue chart"",""type"":""image""}]",6,Tue,7,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +P7.1 - Programming Principles,7.1P,"Describe the principles of structured, procedural, programming.",4,0,FALSE,"[{""key"":""file0"",""name"":""Program Principles Description"",""type"":""document""}]",7,Tue,8,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +D7.2 - Game of Life,7.2D,Create the Game of Life,4,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",7,Tue,8,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +P8.1 - Language Reference Sheet,8.1P,Create a reference sheet for C or C#,4,0,FALSE,"[{""key"":""file0"",""name"":""Reference Sheet"",""type"":""document""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +P8.2 - Circle Moving 2,8.2P,Recreate your circle moving program using C,4,0,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",8,Tue,9,Tue,10,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, Test 2,T2,Covers all core concepts.,1,0,TRUE,[],9,Fri,9,Fri,,,0,FALSE,90,,,import-tasks,FALSE,"[]",[],,,,"[{""abbreviation"":""T1"",""task_status_id"":2},{""abbreviation"":""8.1P"",""task_status_id"":9},{""abbreviation"":""8.2P"",""task_status_id"":9}]",, -Pass Task 9.1 - Reading Another Language,9.1P,Demonstrate how programs written in C work within the computer,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Credit Task 9.2 - Another Language,9.2C,Create a program with C using the concepts covered.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -High Distinction Task 10.1 - Custom Program,10.1H,Extend your custom program to meet the High Distinction criteria.,4,3,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -High Distinction Task 10.2 - Research Report,10.2H,Start working on a research project,8,3,FALSE,"[{""key"":""file0"",""name"":""Research Report Document"",""type"":""document""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Pass Task 11.1 - Learning Summary Report,11.1P,Summarise your learning from the unit.,4,0,FALSE,"[{""key"":""file0"",""name"":""Learning Summary Report"",""type"":""document""}]",11,Tue,12,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, -Distinction Task 6.2 - Custom Program,6.2D,Start working on your custom program!,16,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",6,Tue,13,Tue,,,5,TRUE,90,,,import-tasks,FALSE,[],[],,,,[],, +P9.1 - Reading Another Language,9.1P,Demonstrate how programs written in C work within the computer,2,0,FALSE,"[{""key"":""file0"",""name"":""Execution of Program 1"",""type"":""image""},{""key"":""file1"",""name"":""Execution of Program 2"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +C9.2 - Another Language,9.2C,Create a program with C using the concepts covered.,4,1,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",9,Tue,10,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +HD10.1 - Custom Program,10.1H,Extend your custom program to meet the High Distinction criteria.,4,3,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +HD10.2 - Research Report,10.2H,Start working on a research project,8,3,FALSE,"[{""key"":""file0"",""name"":""Research Report Document"",""type"":""document""}]",10,Tue,13,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +P11.1 - Learning Summary Report,11.1P,Summarise your learning from the unit.,4,0,FALSE,"[{""key"":""file0"",""name"":""Learning Summary Report"",""type"":""document""}]",11,Tue,12,Tue,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, +D6.2 - Custom Program,6.2D,Start working on your custom program!,16,2,FALSE,"[{""key"":""file0"",""name"":""Program code"",""type"":""code""},{""key"":""file1"",""name"":""Design overview"",""type"":""document""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",6,Tue,13,Tue,,,5,TRUE,90,,,import-tasks,FALSE,[],[],,,,[],, Test 10,T10,Test 10 tests the import task bug,1,0,TRUE,[],10,Fri,10,Fri,,,0,FALSE,90,,,import-tasks,FALSE,[],[],,,,[],, diff --git a/test_files/COS10001-TasksUnorderedUploadReqs.csv b/test_files/COS10001-TasksUnorderedUploadReqs.csv index e41817add2..753308fb52 100644 --- a/test_files/COS10001-TasksUnorderedUploadReqs.csv +++ b/test_files/COS10001-TasksUnorderedUploadReqs.csv @@ -1,4 +1,4 @@ name,abbreviation,description,weighting,target_grade,restrict_status_updates,upload_requirements,start_week,start_day,target_week,target_day,due_week,due_day,max_quality_pts,is_graded,plagiarism_warn_pct,plagiarism_checks,group_set,tutorial_stream,scorm_enabled,scorm_allow_review,scorm_bypass_test,scorm_time_delay_enabled,scorm_attempt_limit,assess_in_portfolio_only,task_prerequisites,discussion_prompts -Pass Task 1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file1"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file5"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] -Pass Task 1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file3"",""name"":""Screenshot"",""type"":""image""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] +P1.1 - Hello World,1.1P,"As a first step, create the classic 'Hello World' program. This will help ensure that you have all of the software installed correctly, and are ready to move on with creating other,,, programs.",1,0,FALSE,"[{""key"":""file1"",""name"":""HelloWorld.cpp"",""type"":""code""},{""key"":""file1"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] +P1.2 - Picture Drawing,1.2P,Create a program that calls procedures to draw a picture to a window (something other than a house which we use as the example).,2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file5"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] +P1.3 - Creating a Procedure,1.3P,"Now that you have created a program that uses procedures, you can learn how to create your own procedures. Creating procedures will allow you to group your program's actions into procedures that perform meaningful tasks.",2,0,FALSE,"[{""key"":""file0"",""name"":""PictureDrawing.pas"",""type"":""code""},{""key"":""file3"",""name"":""Screenshot"",""type"":""image""},{""key"":""file2"",""name"":""Screenshot"",""type"":""image""}]",1,Tue,2,Tue,5,Mon,0,FALSE,90,,,import-tasks,FALSE,[],[] diff --git a/test_files/csv_test_files/COS10001-Tasks-Prerequisites.csv b/test_files/csv_test_files/COS10001-Tasks-Prerequisites.csv index a399ef52e7..5bcf1752af 100644 --- a/test_files/csv_test_files/COS10001-Tasks-Prerequisites.csv +++ b/test_files/csv_test_files/COS10001-Tasks-Prerequisites.csv @@ -3,4 +3,4 @@ Assignment 12,A12,rerum ut fugit saepe ipsa in quidem,2,0,FALSE,0,FALSE,50,[],," Pass task 1,1.1P,rerum ut fugit saepe ipsa in quidem,2,0,FALSE,0,FALSE,50,[],,"[{""key"":""file0"",""name"":""Assumenda accusamus quas"",""type"":""image""}]",-1,Sat,1,Mon,12,Mon,,,,,,import-tasks,FALSE,[],[] Pass task 2,2.1P,rerum ut fugit saepe ipsa in quidem,2,0,FALSE,0,FALSE,50,[],,"[{""key"":""file0"",""name"":""Assumenda accusamus quas"",""type"":""image""}]",-1,Sat,1,Mon,12,Mon,,,,,,import-tasks,FALSE,[],[] Distinction Task,5.5D,rerum ut fugit saepe ipsa in quidem,2,0,FALSE,0,FALSE,50,[],,"[{""key"":""file0"",""name"":""Assumenda accusamus quas"",""type"":""image""}]",2,Sat,3,Mon,13,Mon,,,,,,import-tasks,FALSE,[],[] -Distinction Task 2,5.5D.new,rerum ut fugit saepe ipsa in quidem,2,0,FALSE,0,FALSE,50,[],,"[{""key"":""file0"",""name"":""Assumenda accusamus quas"",""type"":""image""}]",2,Sat,3,Mon,13,Mon,,,,,,import-tasks,FALSE,"[{""abbreviation"":""1.1P"",""task_status_id"":2},{""abbreviation"":""1.2P"",""task_status_id"":9}]",[] +D2,5.5D.new,rerum ut fugit saepe ipsa in quidem,2,0,FALSE,0,FALSE,50,[],,"[{""key"":""file0"",""name"":""Assumenda accusamus quas"",""type"":""image""}]",2,Sat,3,Mon,13,Mon,,,,,,import-tasks,FALSE,"[{""abbreviation"":""1.1P"",""task_status_id"":2},{""abbreviation"":""1.2P"",""task_status_id"":9}]",[] From 791f06dac18aa5fa0012689076c13f6a8b82963d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:53:22 +1000 Subject: [PATCH 127/199] chore(release): 11.0.0-20 --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ee7f21c39..7325d43982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-20](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-19...v11.0.0-20) (2026-06-25) + + +### Features + +* capture daily snapshots of aggregated task completion data ([#607](https://github.com/b0ink/doubtfire-deploy/issues/607)) ([86b2040](https://github.com/b0ink/doubtfire-deploy/commit/86b2040bf48f494b1e06919fcce50243fe5a0240)) + + +### Bug Fixes + +* ensure overseer image is pulled before running test ([0e0ce34](https://github.com/b0ink/doubtfire-deploy/commit/0e0ce34160dce4e23fc9deea2f035119c5d378dd)) +* relocate submission history on unit code change ([d9085b5](https://github.com/b0ink/doubtfire-deploy/commit/d9085b509b7995ae9d41e8e3973dd57b011a166d)) + ## [11.0.0-19](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-18...v11.0.0-19) (2026-06-25) ## [11.0.0-18](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-17...v11.0.0-18) (2026-06-25) From 40cefeeac294a3220e7f45e85b2c9a5de4bf4f3d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 26 Jun 2026 12:56:38 +1000 Subject: [PATCH 128/199] ci: remove jplag report dir env var --- .github/workflows/push.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1ec93a0761..dbd2b1061c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -29,7 +29,6 @@ env: DF_REDIS_SIDEKIQ_URL: "redis://redis:6379/0" LATEX_CONTAINER_NAME: doubtfire-texlive LATEX_BUILD_PATH: /texlive/shell/latex_build.sh - DF_JPLAG_REPORT_DIR: /jplag/results LTI_SHARED_API_SECRET: "abc123" LTI_ENABLED: true @@ -111,7 +110,6 @@ jobs: options: > --name jplag -v ${{ github.workspace }}/student-work:/student-work - -v ${{ github.workspace }}/jplag/results:${{ env.DF_JPLAG_REPORT_DIR }} -v ${{ github.workspace }}/tmp/jplag:/tmp/jplag -v ${{ github.workspace }}/test_files/submissions/jplag:/test_files --detach @@ -152,7 +150,6 @@ jobs: -e DF_REDIS_SIDEKIQ_URL -e LATEX_CONTAINER_NAME -e LATEX_BUILD_PATH - -e DF_JPLAG_REPORT_DIR -e LTI_SHARED_API_SECRET -e LTI_ENABLED run: bundle exec rake db:populate @@ -164,7 +161,6 @@ jobs: -v ${{ github.workspace }}:/doubtfire -v ${{ github.workspace }}/student-work:/student-work -v /var/run/docker.sock:/var/run/docker.sock - -v ${{ github.workspace }}/jplag/results:${{ env.DF_JPLAG_REPORT_DIR }} -v ${{ github.workspace }}/tmp/jplag:/tmp/jplag -e RAILS_ENV -e DF_STUDENT_WORK_DIR @@ -185,7 +181,6 @@ jobs: -e DF_REDIS_SIDEKIQ_URL -e LATEX_CONTAINER_NAME -e LATEX_BUILD_PATH - -e DF_JPLAG_REPORT_DIR -e LTI_SHARED_API_SECRET -e LTI_ENABLED run: TERM=xterm bundle exec rails test From 87fe8136ea5e9f7011cee3b304bbe5c8a68c7d6d Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:19:46 +1000 Subject: [PATCH 129/199] fix: support mysql 8 reserved keyword in task completion stats query --- app/models/unit.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index d9972869aa..387be61c14 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -1985,6 +1985,8 @@ def task_completion_csv_generator(task_status_uses_id: false) result end.flatten + group_table_alias = 'task_completion_groups' + # Query across all projects, joined to task's via definitions to ensure all definitions are covered active_projects .joins( @@ -1998,7 +2000,7 @@ def task_completion_csv_generator(task_status_uses_id: false) 'LEFT OUTER JOIN tasks ON tasks.task_definition_id = task_definitions.id AND projects.id = tasks.project_id', 'LEFT OUTER JOIN task_statuses ON tasks.task_status_id = task_statuses.id', 'LEFT OUTER JOIN group_memberships ON group_memberships.project_id = projects.id AND group_memberships.active = TRUE', - 'LEFT OUTER JOIN groups ON groups.id = group_memberships.group_id' + "LEFT OUTER JOIN #{Group.quoted_table_name} #{group_table_alias} ON #{group_table_alias}.id = group_memberships.group_id" ).select( 'projects.id as project_id', 'users.student_id as student_id', 'users.username as username', 'users.first_name as first_name', 'projects.assessor_id as project_assessor', 'users.last_name as last_name', 'campuses.abbreviation as campus_abbreviation', 'projects.target_grade', 'users.email as email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale', @@ -2007,7 +2009,7 @@ def task_completion_csv_generator(task_status_uses_id: false) *streams.map { |s| "MAX(CASE WHEN tutorials.tutorial_stream_id = #{s.id} OR tutorials.tutorial_stream_id IS NULL THEN tutorials.abbreviation ELSE NULL END) AS tutorial_#{s.id}" }, # Get tutorial for case when no stream "MAX(CASE WHEN tutorial_streams.id IS NULL THEN tutorials.abbreviation ELSE NULL END) AS tutorial", - *grp_sets.map { |gs| "MAX(CASE WHEN groups.group_set_id = #{gs.id} THEN groups.name ELSE NULL END) AS grp_#{gs.id}" } + *grp_sets.map { |gs| "MAX(CASE WHEN #{group_table_alias}.group_set_id = #{gs.id} THEN #{group_table_alias}.name ELSE NULL END) AS grp_#{gs.id}" } ).group( 'projects.id', 'student_id', 'username', 'first_name', 'last_name', 'campus_abbreviation', 'target_grade', 'email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale' ).each do |row| From 5981dfb57f1b7379737ccda8e347131310561084 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:19:55 +1000 Subject: [PATCH 130/199] chore(release): 11.0.0-21 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7325d43982..2e83e01e08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-21](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-20...v11.0.0-21) (2026-06-26) + + +### Bug Fixes + +* support mysql 8 reserved keyword in task completion stats query ([87fe813](https://github.com/b0ink/doubtfire-deploy/commit/87fe8136ea5e9f7011cee3b304bbe5c8a68c7d6d)) + ## [11.0.0-20](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-19...v11.0.0-20) (2026-06-25) From 7cc533f5cd62dfcdee092f89756a95e14888db74 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:30:51 +1000 Subject: [PATCH 131/199] chore(release): 11.0.0-22 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e83e01e08..49a6cfeea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-22](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-21...v11.0.0-22) (2026-06-26) + ## [11.0.0-21](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-20...v11.0.0-21) (2026-06-26) From 3215c0811aa8c9eb6425cb9d4e080f2fc042e6fb Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:55:46 +1000 Subject: [PATCH 132/199] fix: ensure jplag scan continues on failed report --- .../similarity/unit_similarity_module.rb | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/app/models/similarity/unit_similarity_module.rb b/app/models/similarity/unit_similarity_module.rb index ecfd3ff63e..819dc288c6 100644 --- a/app/models/similarity/unit_similarity_module.rb +++ b/app/models/similarity/unit_similarity_module.rb @@ -105,6 +105,7 @@ def check_jplag_similarity(force: false) # need pwd to restore after cding into submission folder (so the files do not have full path) pwd = FileUtils.pwd + completed_all_checks = true # making temp directory for unit - jplag root_work_dir = Rails.root.join("tmp", "jplag", "#{code}-#{id}") @@ -138,19 +139,26 @@ def check_jplag_similarity(force: false) # There are new tasks, check these with JPLAG report_path = FileHelper.task_jplag_report_path(self, td) - run_jplag_on_done_files(td, tasks_dir, tasks_with_files, report_path) - warn_pct = td.plagiarism_warn_pct || 50 - logger.debug "Warn PCT: #{warn_pct}" + begin + run_jplag_on_done_files(td, tasks_dir, tasks_with_files, report_path) + warn_pct = td.plagiarism_warn_pct || 50 + logger.debug "Warn PCT: #{warn_pct}" - # Remove any existing plagiarism links that are below the threshold, in case it has been updated since the last analysis - JplagTaskSimilarity.joins(:task) - .where("pct < ? AND tasks.task_definition_id = ?", warn_pct, td.id) - .delete_all + # Remove any existing plagiarism links that are below the threshold, in case it has been updated since the last analysis + JplagTaskSimilarity.joins(:task) + .where("pct < ? AND tasks.task_definition_id = ?", warn_pct, td.id) + .delete_all - process_jplag_plagiarism_report(report_path, warn_pct, td.group_set) + process_jplag_plagiarism_report(report_path, warn_pct, td.group_set) + rescue StandardError => e + completed_all_checks = false + logger.error "Failed to check JPlag similarity for task #{td.name} (id=#{td.id}). Error: #{e.message}" + end + end + if completed_all_checks + self.last_plagarism_scan = Time.zone.now + save! end - self.last_plagarism_scan = Time.zone.now - save! ensure FileUtils.chdir(pwd) if FileUtils.pwd != pwd end From 85e36f902a94aeea1f8766d4d44629aed27df12e Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:55:50 +1000 Subject: [PATCH 133/199] chore(release): 11.0.0-23 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49a6cfeea0..a01035a60f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-23](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-22...v11.0.0-23) (2026-06-26) + + +### Bug Fixes + +* ensure jplag scan continues on failed report ([3215c08](https://github.com/b0ink/doubtfire-deploy/commit/3215c0811aa8c9eb6425cb9d4e080f2fc042e6fb)) + ## [11.0.0-22](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-21...v11.0.0-22) (2026-06-26) ## [11.0.0-21](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-20...v11.0.0-21) (2026-06-26) From 964e957c4a00f1dadb2d992583d3f2bb717aa58b Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 26 Jun 2026 17:44:59 +1000 Subject: [PATCH 134/199] fix: ensure non text comments cant be deleted --- app/api/task_comments_api.rb | 15 ++++++++++++ test/api/comments/comment_test.rb | 40 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/app/api/task_comments_api.rb b/app/api/task_comments_api.rb index 1307051e96..cfe2500a87 100644 --- a/app/api/task_comments_api.rb +++ b/app/api/task_comments_api.rb @@ -178,6 +178,21 @@ class TaskCommentsApi < Grape::API error!({ error: 'Not authorised to delete this comment' }, 403) end + # Comments that don't reveal a delete button + protected_comment_types = [ + AssessmentComment, + ExtensionComment, + ScormComment, + TaskCheckedInComment, + TaskDiscussedComment, + TaskFeedbackReviewRequestComment, + TaskStatusComment + ] + + if protected_comment_types.any? { |comment_type| task_comment.is_a?(comment_type) } + error!({ error: 'This comment type cannot be deleted' }, 403) + end + task_comment.destroy SessionTracker.record_assessment_activity( diff --git a/test/api/comments/comment_test.rb b/test/api/comments/comment_test.rb index 8b2ce97204..961b2a5e0d 100644 --- a/test/api/comments/comment_test.rb +++ b/test/api/comments/comment_test.rb @@ -276,6 +276,46 @@ def test_student_cannot_edit_other_users_comment assert_equal 'Tutor comment', comment.reload.read_attribute(:comment) end + def test_special_task_comments_cannot_be_deleted + project = FactoryBot.create(:project) + task_definition = project.unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + tutor = project.tutor_for(task_definition) + student = project.student + + submission_history = FactoryBot.create(:submission_history, task: task) + overseer_assessment = FactoryBot.create( + :overseer_assessment, + task: task, + submission_history: submission_history, + submission_timestamp: submission_history.submission_timestamp, + status: :failed + ) + + protected_comments = [ + task.add_status_comment(student, TaskStatus.ready_for_feedback), + task.add_discussed_comment(tutor), + task.add_feedback_review_request_comment(student), + AssessmentComment.create!( + task: task, + user: tutor, + recipient: student, + comment: 'Automated tests failed', + commentable: overseer_assessment + ) + ] + + protected_comments.each do |comment| + add_auth_header_for(user: comment.user) + delete_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments/#{comment.id}" + assert_equal 403, last_response.status, "Expected #{comment.class.name} delete to be rejected" + end + + deleted_comment_types = protected_comments.reject { |comment| TaskComment.exists?(comment.id) }.map(&:content_type) + + assert_empty deleted_comment_types, "Expected protected comment types to remain after delete attempt: #{deleted_comment_types.join(', ')}" + end + def test_student_reply_to_other_student_in_same_group unit = FactoryBot.create :unit From 4404d9665235fb67a79ced3d0ec400eca36edc46 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:33:14 +1000 Subject: [PATCH 135/199] chore(release): 11.0.0-24 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a01035a60f..dfa796955a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-24](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-23...v11.0.0-24) (2026-06-29) + + +### Bug Fixes + +* ensure non text comments cant be deleted ([964e957](https://github.com/b0ink/doubtfire-deploy/commit/964e957c4a00f1dadb2d992583d3f2bb717aa58b)) + ## [11.0.0-23](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-22...v11.0.0-23) (2026-06-26) From d98d318adc1c5153af9a844d95fdbbe2fde3903b Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:13:29 +1000 Subject: [PATCH 136/199] fix: ensure pii is not submitted to sentry --- app/sidekiq/accept_submission_job.rb | 4 +- config/initializers/sentry.rb | 92 +++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/app/sidekiq/accept_submission_job.rb b/app/sidekiq/accept_submission_job.rb index 3010ea1ac1..eaaf9b380e 100644 --- a/app/sidekiq/accept_submission_job.rb +++ b/app/sidekiq/accept_submission_job.rb @@ -47,7 +47,6 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) extra: { task_id: task.id, task_definition_abbreviation: task.task_definition.abbreviation, - username: task.project.user.username, latex_log_message: e.respond_to?(:log_message) ? e.log_message.to_s.last(5000) : nil } ) @@ -90,8 +89,7 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) e, extra: { task_id: task&.id, - task_definition_abbreviation: task&.task_definition&.abbreviation, - username: task&.project&.user&.username + task_definition_abbreviation: task&.task_definition&.abbreviation } ) end diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb index 1cb50372f7..1f919db82e 100644 --- a/config/initializers/sentry.rb +++ b/config/initializers/sentry.rb @@ -1,12 +1,100 @@ +module OnTrackSentryRedaction + FILTERED = "[Filtered]".freeze + PII_KEY_PATTERN = /(^|_)(email|first_name|last_name|login_id|name|student_id|username|user_id)\z/i + RAILS_LATEX_TASK_WORK_DIR = %r{(tmp/rails-latex/task-\d{8}-\d{4}-)[^/\s]+?(-[^/\s-]+-\d+-\d+(?:-retry)?)(?=/|\s|$)} + RAILS_LATEX_PORTFOLIO_WORK_DIR = %r{(tmp/rails-latex/portfolio-\d{8}-\d{4}-)[^/\s]+?(-\d+-\d+(?:-retry)?)(?=/|\s|$)} + SCORM_PATH = %r{(/scorm/[^/?#]+/)[^/?#]+(/)[^/?#]+}i + + module_function + + def scrub_event(event) + scrub_request(event.request) if event.respond_to?(:request) + + event.message = scrub_string(event.message) if event.respond_to?(:message) && event.respond_to?(:message=) + event.user.replace(scrub_hash(event.user)) if event.respond_to?(:user) && event.user.respond_to?(:replace) + event.extra.replace(scrub_hash(event.extra)) if event.respond_to?(:extra) && event.extra.respond_to?(:replace) + + scrub_exception(event.exception) if event.respond_to?(:exception) + scrub_breadcrumbs(event.breadcrumbs) if event.respond_to?(:breadcrumbs) + + event + end + + def scrub_request(request) + return unless request + + request.url = scrub_string(request.url) if request.respond_to?(:url) && request.respond_to?(:url=) + request.headers&.each_key do |key| + request.headers[key] = FILTERED if key.casecmp("Username").zero? + end + end + + def scrub_exception(exception) + exception&.each_value do |value| + value.value = scrub_string(value.value) if value.respond_to?(:value) && value.respond_to?(:value=) + + next unless value.respond_to?(:stacktrace) + + value.stacktrace&.frames&.each do |frame| + %i[abs_path context_line filename function module pre_context post_context vars].each do |attribute| + next unless frame.respond_to?(attribute) && frame.respond_to?(:"#{attribute}=") + + frame.public_send(:"#{attribute}=", scrub_value(frame.public_send(attribute))) + end + end + end + end + + def scrub_breadcrumbs(breadcrumbs) + breadcrumbs&.each do |breadcrumb| + breadcrumb.message = scrub_string(breadcrumb.message) if breadcrumb.respond_to?(:message) && breadcrumb.respond_to?(:message=) + breadcrumb.data = scrub_hash(breadcrumb.data) if breadcrumb.respond_to?(:data) && breadcrumb.respond_to?(:data=) + end + end + + def scrub_value(value) + case value + when Hash + scrub_hash(value) + when Array + value.map { |item| scrub_value(item) } + when String + scrub_string(value) + else + value + end + end + + def scrub_hash(hash) + hash.each_with_object({}) do |(key, value), result| + key = key.to_s + result[key] = key.match?(PII_KEY_PATTERN) ? FILTERED : scrub_value(value) + end + end + + def scrub_string(value) + return value unless value.is_a?(String) + + value + .gsub(RAILS_LATEX_TASK_WORK_DIR, "\\1[username]\\2") + .gsub(RAILS_LATEX_PORTFOLIO_WORK_DIR, "\\1[username]\\2") + .gsub(SCORM_PATH, "\\1[username]\\2[Filtered]") + end +end + if ENV["SENTRY_DSN"].present? Sentry.init do |config| config.dsn = ENV.fetch("SENTRY_DSN", nil) - # get breadcrumbs from logs - config.breadcrumbs_logger = [:active_support_logger, :http_logger] + # Rails log breadcrumbs are excluded because app logs can include user details. + config.breadcrumbs_logger = [:http_logger] config.environment = ENV.fetch("SENTRY_ENVIRONMENT", Rails.env) config.release = ENV["SENTRY_RELEASE"] if ENV["SENTRY_RELEASE"].present? # Add data like request headers and IP for users, if applicable; # see https://docs.sentry.io/platforms/ruby/data-management/data-collected/ for more info config.send_default_pii = false + + config.before_send = lambda do |event, _hint| + OnTrackSentryRedaction.scrub_event(event) + end end end From 2ab0bb7c549f014f0ca2e032242f84912aca1931 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:14:12 +1000 Subject: [PATCH 137/199] chore(release): 11.0.0-25 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfa796955a..8d92a8ba5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-25](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-24...v11.0.0-25) (2026-06-29) + + +### Bug Fixes + +* ensure pii is not submitted to sentry ([d98d318](https://github.com/b0ink/doubtfire-deploy/commit/d98d318adc1c5153af9a844d95fdbbe2fde3903b)) + ## [11.0.0-24](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-23...v11.0.0-24) (2026-06-29) From f217c61cf666dae7c781a393fbd9b544dcb32f24 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:06:19 +1000 Subject: [PATCH 138/199] fix: improve error when adding prerequisite to non existing task def --- app/models/unit.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/unit.rb b/app/models/unit.rb index 5ad8edba64..06b6574e13 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -1774,6 +1774,14 @@ def import_tasks_from_csv(file) prerequisite_td = task_definitions.find_by(abbreviation: abbreviation) task_status_id = prerequisite['task_status_id'].to_i + if prerequisite_td.nil? + errors << { + row: "TaskDef '#{task_abbreviation}' prerequisites: #{prerequisites_list}", + message: "Unable to find prerequisite task definition with abbreviation #{abbreviation}." + } + next + end + TaskPrerequisite.create!({ task_definition_id: td.id, prerequisite: prerequisite_td, From 4c2e19472f73deeaee271ed5b7d5dc66b01f7836 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:06:28 +1000 Subject: [PATCH 139/199] fix: avoid skipping task names containing 'name' during import --- app/models/unit.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index 06b6574e13..61717d9c92 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -1731,7 +1731,7 @@ def import_tasks_from_csv(file) headers: true, header_converters: [->(i) { i.nil? ? '' : i }, :downcase, ->(hdr) { hdr.strip.tr(' ', '_').to_sym unless hdr.nil? }], converters: [->(body) { body&.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') }]).each do |row| - next if row[0] =~ /^(Task Name)|(name)/ # Skip header + next if ['Task Name', 'name'].include?(row[0].to_s.strip) # Skip header rows begin missing = missing_headers(row, TaskDefinition.required_csv_columns) From ae1b6203bb719ae5ea1c983065513569748f63b1 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:08:36 +1000 Subject: [PATCH 140/199] test: ensure tasks containing name dont get skipped --- test/models/task_definition_test.rb | 52 +++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test/models/task_definition_test.rb b/test/models/task_definition_test.rb index 8d3500e7cc..6312419108 100644 --- a/test/models/task_definition_test.rb +++ b/test/models/task_definition_test.rb @@ -316,6 +316,58 @@ def test_import_overseer_steps_from_csv_fixture assert imported_step.enabled end + def test_import_does_not_skip_task_name_containing_name + target_unit = Unit.create!( + code: 'CSVNAME1', + name: 'CSV Import With Name Substring', + description: 'Import target', + teaching_period: TeachingPeriod.find(3) + ) + + csv = CSV.generate do |rows| + rows << TaskDefinition.required_csv_columns + rows << [ + 'Coin Clash (Tournament Mini-Project)', + 'D4', + 'Build an adversarial game agent.', + 1, + 0, + false, + 0, + false, + 90, + false, + false, + false, + false, + 0, + nil, + [{ key: 'file0', name: 'coin_clash.rb', type: 'code' }].to_json, + 1, + 'Tue', + 1, + 'Tue', + 1, + 'Tue', + nil, + false, + [].to_json, + [].to_json + ] + end + + file = Tempfile.new(['task-definitions', '.csv']) + file.write(csv) + file.close + + result = target_unit.import_tasks_from_csv(file.path) + + assert_empty result[:errors], result + assert target_unit.task_definitions.exists?(abbreviation: 'D4') + ensure + file&.unlink + end + def test_export_without_tutorial_stream data = { code: 'COS10001', From 924bb36f76c8e96d6e5c5d567792dd1db468c4d9 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 29 Jun 2026 17:16:10 +1000 Subject: [PATCH 141/199] chore(release): 11.0.0-26 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d92a8ba5a..48f8762540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-26](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-25...v11.0.0-26) (2026-06-29) + + +### Bug Fixes + +* avoid skipping task names containing 'name' during import ([4c2e194](https://github.com/b0ink/doubtfire-deploy/commit/4c2e19472f73deeaee271ed5b7d5dc66b01f7836)) +* improve error when adding prerequisite to non existing task def ([f217c61](https://github.com/b0ink/doubtfire-deploy/commit/f217c61cf666dae7c781a393fbd9b544dcb32f24)) + ## [11.0.0-25](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-24...v11.0.0-25) (2026-06-29) From 19a459c544a14d3c155d665ba90ead11c3c50288 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:01:13 +1000 Subject: [PATCH 142/199] fix: prevent convenors with observer only perms to be made main convenor --- app/models/unit.rb | 1 + test/models/unit_model_test.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/app/models/unit.rb b/app/models/unit.rb index 61717d9c92..03e54aceec 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -385,6 +385,7 @@ def ensure_main_convenor_is_appropriate errors.add(:main_convenor, "must be a staff member from unit") unless id == main_convenor.unit_id errors.add(:main_convenor, "must be configured to administer unit") unless main_convenor.is_convenor? + errors.add(:main_convenor, "cannot be observer only") if main_convenor.observer_only? errors.add(:main_convenor, "must be capable of administering units - ensure user has appropriate permissions (contact admin staff to update)") unless main_convenor_user.has_convenor_capability? end diff --git a/test/models/unit_model_test.rb b/test/models/unit_model_test.rb index b07e4807ed..a67e779809 100644 --- a/test/models/unit_model_test.rb +++ b/test/models/unit_model_test.rb @@ -912,6 +912,20 @@ def test_change_main_convenor_success assert unit.valid?, 'It should be ok to change to the convenor user' end + def test_change_main_convenor_does_not_allow_observer_only_roles + unit = FactoryBot.create :unit, campus_count: 1, tutorials: 0, stream_count: 0, task_count: 0, with_students: false + + convenor_user = FactoryBot.create :user, :convenor + convenor_user_role = unit.employ_staff convenor_user, Role.convenor + convenor_user_role.update!(observer_only: true) + + unit.main_convenor_id = convenor_user_role.id + assert_not unit.valid?, 'It should not be ok to change to an observer-only convenor user' + + convenor_user_role.update!(observer_only: false) + assert unit.valid?, 'It should be ok once the convenor user is no longer observer only' + end + def test_change_main_convenor_does_not_allow_roles_from_other_units unit = FactoryBot.create :unit, campus_count: 1, tutorials:0, stream_count:0, task_count:0, with_students:false other_unit = FactoryBot.create :unit, campus_count: 1, tutorials:0, stream_count:0, task_count:0, with_students:false From a49698c7602a7d5a11f2406cd2729c5178f0ad1c Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:59:19 +1000 Subject: [PATCH 143/199] test: reload model --- test/models/unit_model_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/models/unit_model_test.rb b/test/models/unit_model_test.rb index a67e779809..b53659ea02 100644 --- a/test/models/unit_model_test.rb +++ b/test/models/unit_model_test.rb @@ -923,6 +923,7 @@ def test_change_main_convenor_does_not_allow_observer_only_roles assert_not unit.valid?, 'It should not be ok to change to an observer-only convenor user' convenor_user_role.update!(observer_only: false) + unit.main_convenor.reload assert unit.valid?, 'It should be ok once the convenor user is no longer observer only' end From 262a5c65dc9f1f7ef412d469bbf5f1b104f05437 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:46:33 +1000 Subject: [PATCH 144/199] chore(release): 11.0.0-27 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48f8762540..2ab7e67788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-27](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-26...v11.0.0-27) (2026-07-01) + + +### Bug Fixes + +* prevent convenors with observer only perms to be made main convenor ([19a459c](https://github.com/b0ink/doubtfire-deploy/commit/19a459c544a14d3c155d665ba90ead11c3c50288)) + ## [11.0.0-26](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-25...v11.0.0-26) (2026-06-29) From d5ca74843fa16040dec81fa9bd66d315736059a0 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:03:57 +1000 Subject: [PATCH 145/199] fix: only allow convenors to pull overflow stats --- app/api/units_api.rb | 2 +- app/models/unit.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/api/units_api.rb b/app/api/units_api.rb index 5ea553b8a7..411065f97e 100644 --- a/app/api/units_api.rb +++ b/app/api/units_api.rb @@ -553,7 +553,7 @@ class UnitsApi < Grape::API desc 'Download CSV of overflow task claims in this unit' get '/csv/units/:id/overflow_task_claims' do unit = Unit.find(params[:id]) - unless authorise? current_user, unit, :download_stats + unless authorise? current_user, unit, :download_overflow_stats error!({ error: "Not authorised to download overflow task claim stats for #{unit.code}" }, 403) end diff --git a/app/models/unit.rb b/app/models/unit.rb index 03e54aceec..671bb2325e 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -64,6 +64,7 @@ def self.permissions :provide_bulk_feedback, :change_project_enrolment, :download_stats, + :download_overflow_stats, :download_grades, :download_jplag_report, :rollover_unit, @@ -95,6 +96,7 @@ def self.permissions :add_tutorial, :add_task_def, :download_stats, + :download_overflow_stats, :download_unit_csv, :download_grades, :exceed_capacity, From 5b4bc718c477974d90650a8bddda4c766680341c Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:07:33 +1000 Subject: [PATCH 146/199] chore(release): 11.0.0-28 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ab7e67788..c42855f984 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-28](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-27...v11.0.0-28) (2026-07-01) + + +### Bug Fixes + +* only allow convenors to pull overflow stats ([d5ca748](https://github.com/b0ink/doubtfire-deploy/commit/d5ca74843fa16040dec81fa9bd66d315736059a0)) + ## [11.0.0-27](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-26...v11.0.0-27) (2026-07-01) From a4bb79ac27425fa64ed711e932d595cefa2a5598 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:04:07 +1000 Subject: [PATCH 147/199] fix: allow students to get submission history timestamps --- app/api/submission/portfolio_evidence_api.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/submission/portfolio_evidence_api.rb b/app/api/submission/portfolio_evidence_api.rb index 82f87088cf..0079816be4 100644 --- a/app/api/submission/portfolio_evidence_api.rb +++ b/app/api/submission/portfolio_evidence_api.rb @@ -187,7 +187,7 @@ def self.logger project = Project.find(params[:id]) task_definition = project.unit.task_definitions.find(params[:task_definition_id]) - unless authorise? current_user, project.unit, :provide_feedback + unless authorise? current_user, project, :get_submission error!({ error: "Not authorised to get submission history for task '#{task_definition.name}'" }, 401) end From 0ea20bce61b2d0710a95968b69b9aafe5d746bde Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:04:16 +1000 Subject: [PATCH 148/199] chore(release): 11.0.0-29 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c42855f984..affee5a01d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-29](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-28...v11.0.0-29) (2026-07-02) + + +### Bug Fixes + +* allow students to get submission history timestamps ([a4bb79a](https://github.com/b0ink/doubtfire-deploy/commit/a4bb79ac27425fa64ed711e932d595cefa2a5598)) + ## [11.0.0-28](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-27...v11.0.0-28) (2026-07-01) From 99c854738ab12ed36a8fa7923600604486f6c136 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 3 Jul 2026 12:46:11 +1000 Subject: [PATCH 149/199] test: ensure pdf with excessive text wrapping has correct page count --- test/models/task_test.rb | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/test/models/task_test.rb b/test/models/task_test.rb index ab80938026..e2a0c737c4 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -243,6 +243,61 @@ def test_pdf_creation_with_gif assert_not File.exist? path end + def test_pdf_creation_with_code_csv_and_gif_has_stable_last_page_footer + unit = Unit.first + td = TaskDefinition.new({ + unit_id: unit.id, + tutorial_stream: unit.tutorial_streams.first, + name: 'Task with code and image', + description: 'Code and image task', + weighting: 4, + target_grade: 0, + start_date: unit.start_date + 1.week, + target_date: unit.start_date + 2.weeks, + abbreviation: 'TaskPdfWithCodeCsvAndGif', + restrict_status_updates: false, + upload_requirements: [ + { "key" => 'file0', "name" => 'Code file', "type" => 'code' }, + { "key" => 'file1', "name" => 'An Image', "type" => 'image' } + ], + plagiarism_warn_pct: 0.8, + is_graded: false, + max_quality_pts: 0 + }) + td.save! + + data_to_post = with_files( + [ + { path: 'test_files/COS10001-ImportTasksWithTutorialStream.csv', type: 'text/csv' }, + { path: 'test_files/submissions/unbelievable.gif', type: 'image/gif' } + ], + { trigger: 'ready_for_feedback' } + ) + + project = unit.active_projects.first + + add_auth_header_for user: unit.main_convenor_user + + post "/api/projects/#{project.id}/task_def_id/#{td.id}/submission", data_to_post + + assert_equal 201, last_response.status, last_response_body + + task = project.task_for_task_definition(td) + assert task.convert_submission_to_pdf(log_to_stdout: true) + path = task.zip_file_path_for_done_task + assert path + assert File.exist? path + assert File.exist? task.final_pdf_path + + reader = PDF::Reader.new(task.final_pdf_path) + + assert_equal 6, reader.pages.count # 1 cover page + 5 pages + assert_includes reader.pages.last.text.gsub(/\s+/, ' '), 'Page 5 of 5' + + td.destroy + assert_not File.exist? path + end + def test_image_upload unit = Unit.first td = TaskDefinition.new({ From fc40bdea6414870c6457f5266838cb44d6296fcd Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 3 Jul 2026 12:47:29 +1000 Subject: [PATCH 150/199] fix: run lualatex a third time to ensure correct page references --- lib/shell/latex_build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/shell/latex_build.sh b/lib/shell/latex_build.sh index c853bdb20f..a729f11414 100644 --- a/lib/shell/latex_build.sh +++ b/lib/shell/latex_build.sh @@ -15,7 +15,12 @@ cd work lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex RESULT=$? if [ $RESULT -eq 0 ]; then - echo "Running lualatex a second time to remove temporary last page..." + echo "Running lualatex a second time to remove temporary last page and update references..." + lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex + RESULT=$? +fi +if [ $RESULT -eq 0 ]; then + echo "Running lualatex a third time to stabilise page references..." lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex RESULT=$? fi From 25bcafcdc00d76523080f6a198170b452f646923 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:42:32 +1000 Subject: [PATCH 151/199] fix: ensure admins can manage communications --- app/models/unit.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/unit.rb b/app/models/unit.rb index 671bb2325e..5b7a8b1423 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -107,7 +107,8 @@ def self.permissions :download_jplag_report, :get_marking_sessions, :get_staff_notes, - :get_tutor_times + :get_tutor_times, + :mannage_communications, ] # What can auditors do with units? From 8d2cd59e12f74bfca213a70e9231a851c91d9da4 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:42:37 +1000 Subject: [PATCH 152/199] chore(release): 11.0.0-30 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index affee5a01d..86d72a7fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-30](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-29...v11.0.0-30) (2026-07-04) + + +### Bug Fixes + +* ensure admins can manage communications ([25bcafc](https://github.com/b0ink/doubtfire-deploy/commit/25bcafcdc00d76523080f6a198170b452f646923)) + ## [11.0.0-29](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-28...v11.0.0-29) (2026-07-02) From 0eb873e11bd51f7f891a065743307b634d713bea Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:35:54 +1000 Subject: [PATCH 153/199] chore(release): 11.0.0-31 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86d72a7fae..1459963b2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-31](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-30...v11.0.0-31) (2026-07-08) + + +### Bug Fixes + +* run lualatex a third time to ensure correct page references ([fc40bde](https://github.com/b0ink/doubtfire-deploy/commit/fc40bdea6414870c6457f5266838cb44d6296fcd)) + ## [11.0.0-30](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-29...v11.0.0-30) (2026-07-04) From 05a44fa8e9954ed82ea149072288ebc546d5f9e9 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:04:30 +1000 Subject: [PATCH 154/199] feat: add rediscuss status --- app/api/discussion_prompts_api.rb | 2 +- app/api/submission/portfolio_evidence_api.rb | 3 +++ app/mailers/notifications_mailer.rb | 2 +- .../communication/communication_condition.rb | 1 + app/models/project.rb | 2 +- app/models/task.rb | 14 +++++++---- app/models/task_status.rb | 10 +++++++- app/models/unit.rb | 2 +- app/models/unit_role.rb | 2 +- .../20260709014859_add_rediscuss_status.rb | 23 +++++++++++++++++++ db/schema.rb | 2 +- lib/tasks/init.rake | 3 ++- test/models/task_status_test.rb | 9 +++++--- test/models/task_test.rb | 18 +++++++++++++++ 14 files changed, 77 insertions(+), 16 deletions(-) create mode 100644 db/migrate/20260709014859_add_rediscuss_status.rb diff --git a/app/api/discussion_prompts_api.rb b/app/api/discussion_prompts_api.rb index 77e9aa4987..34b11641be 100644 --- a/app/api/discussion_prompts_api.rb +++ b/app/api/discussion_prompts_api.rb @@ -108,7 +108,7 @@ class DiscussionPromptsApi < Grape::API error!({ error: 'You do not have permission to access this project' }, 403) end - tasks_to_discuss = project.tasks.where(task_status: [TaskStatus.discuss, TaskStatus.attention_required, TaskStatus.demonstrate]) + tasks_to_discuss = project.tasks.where(task_status: [TaskStatus.discuss, TaskStatus.rediscuss, TaskStatus.attention_required, TaskStatus.demonstrate]) task_definition_ids = tasks_to_discuss.pluck(:task_definition_id) result = DiscussionPrompt diff --git a/app/api/submission/portfolio_evidence_api.rb b/app/api/submission/portfolio_evidence_api.rb index 0079816be4..8a6d36fe84 100644 --- a/app/api/submission/portfolio_evidence_api.rb +++ b/app/api/submission/portfolio_evidence_api.rb @@ -22,6 +22,7 @@ def self.logger ready_for_feedback: 1, assess_in_portfolio: 1, discuss: 2, + rediscuss: 2, attention_required: 0, demonstrate: 2, complete: 3 @@ -67,6 +68,8 @@ def self.logger 'completed' when TaskStatus.discuss 'discussed' + when TaskStatus.rediscuss + 'rediscussed' when TaskStatus.demonstrate 'demonstrated' when TaskStatus.ready_for_feedback diff --git a/app/mailers/notifications_mailer.rb b/app/mailers/notifications_mailer.rb index 748d5bd029..f4aefa1499 100644 --- a/app/mailers/notifications_mailer.rb +++ b/app/mailers/notifications_mailer.rb @@ -61,7 +61,7 @@ def weekly_student_summary(project, summary_stats, did_revert_to_pass) @student_engagements = @engagements.select { |e| [TaskStatus.not_started.name, TaskStatus.need_help.name, TaskStatus.working_on_it.name, TaskStatus.ready_for_feedback.name].include? e.engagement }.count - @staff_engagements = @engagements.select { |e| [TaskStatus.complete.name, TaskStatus.feedback_exceeded.name, TaskStatus.redo.name, TaskStatus.discuss.name, TaskStatus.attention_required.name, TaskStatus.demonstrate.name, TaskStatus.fail.name].include? e.engagement }.count + @staff_engagements = @engagements.select { |e| [TaskStatus.complete.name, TaskStatus.feedback_exceeded.name, TaskStatus.redo.name, TaskStatus.discuss.name, TaskStatus.rediscuss.name, TaskStatus.attention_required.name, TaskStatus.demonstrate.name, TaskStatus.fail.name].include? e.engagement }.count @task_states = project.tasks.joins(:task_status).select("count(tasks.id) as number, task_statuses.name as status").group("task_statuses.name") diff --git a/app/models/communication/communication_condition.rb b/app/models/communication/communication_condition.rb index 108667ce41..6623b0f4c5 100644 --- a/app/models/communication/communication_condition.rb +++ b/app/models/communication/communication_condition.rb @@ -37,6 +37,7 @@ class CommunicationCondition < ApplicationRecord time_exceeded assess_in_portfolio attention_required + rediscuss ].freeze belongs_to :communication, diff --git a/app/models/project.rb b/app/models/project.rb index 9ea9e9d2db..64dc33ed4e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -506,7 +506,7 @@ def self.create_task_stats_from(total_task_counts, project_task_counts, target_g red_pct = ((project_task_counts.fail_count + project_task_counts.feedback_exceeded_count + project_task_counts.time_exceeded_count) / total_task_counts[target_grade]).signif(2) orange_pct = ((project_task_counts.redo_count + project_task_counts.need_help_count + project_task_counts.fix_and_resubmit_count) / total_task_counts[target_grade]).signif(2) - green_pct = ((project_task_counts.discuss_count + project_task_counts.demonstrate_count + project_task_counts.complete_count) / total_task_counts[target_grade]).signif(2) + green_pct = ((project_task_counts.discuss_count + project_task_counts.rediscuss_count + project_task_counts.demonstrate_count + project_task_counts.complete_count) / total_task_counts[target_grade]).signif(2) blue_pct = (project_task_counts.ready_for_feedback_count / total_task_counts[target_grade]).signif(2) grey_pct = (1 - red_pct - orange_pct - green_pct - blue_pct).signif(2) diff --git a/app/models/task.rb b/app/models/task.rb index 9533bda8f7..a48148c8a3 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -458,11 +458,11 @@ def complete? end def discuss_or_demonstrate? - [:discuss, :demonstrate].include?(status) + [:discuss, :rediscuss, :demonstrate].include?(status) end def discuss? - status == :discuss + [:discuss, :rediscuss].include?(status) end def demonstrate? @@ -482,7 +482,7 @@ def ready_for_feedback? end def ready_or_complete? - [:complete, :discuss, :demonstrate, :ready_for_feedback, :assess_in_portfolio].include? status + [:complete, :discuss, :rediscuss, :demonstrate, :ready_for_feedback, :assess_in_portfolio].include? status end def submitted_status? @@ -622,6 +622,10 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: return nil end + if status == TaskStatus.rediscuss && task_status != TaskStatus.discuss + return nil + end + if check_feedback if status == TaskStatus.complete && !has_manual_feedback_since_first_ready_for_feedback? errors.add(:task_status, "cannot be moved to '#{status.name}' until feedback has been given") @@ -644,7 +648,7 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: # Can only be graded if task_def is not assess_in_portfolio_only if task_definition.max_quality_pts > 0 case status - when TaskStatus.complete, TaskStatus.discuss, TaskStatus.demonstrate, TaskStatus.attention_required + when TaskStatus.complete, TaskStatus.discuss, TaskStatus.rediscuss, TaskStatus.demonstrate, TaskStatus.attention_required update(quality_pts: quality) end end @@ -779,7 +783,7 @@ def assess(task_status, assessor, assess_date = Time.zone.now, recursive_fix = f # Grant an extension on fix if due date is within 1 week case task_status - when TaskStatus.fix_and_resubmit, TaskStatus.discuss, TaskStatus.demonstrate + when TaskStatus.fix_and_resubmit, TaskStatus.discuss, TaskStatus.rediscuss, TaskStatus.demonstrate if to_same_day_anywhere_on_earth(due_date) < Time.zone.now + 7.days && can_apply_for_extension? && unit.extension_weeks_on_resubmit_request > 0 grant_extension(assessor, unit.extension_weeks_on_resubmit_request) end diff --git a/app/models/task_status.rb b/app/models/task_status.rb index b4dbac8cd4..bc6eec9b99 100644 --- a/app/models/task_status.rb +++ b/app/models/task_status.rb @@ -72,6 +72,10 @@ def self.attention_required TaskStatus.find(14) end + def self.rediscuss + TaskStatus.find(15) + end + class << self # Provide access to the count from the database via a new db_count method alias_method :db_count, :count @@ -84,7 +88,7 @@ class << self # Keep this hard coded! Saves cache load time. # Important: count must equal the largest id in the database def self.count - 14 + 15 end def self.status_for_name(name) @@ -103,6 +107,8 @@ def self.status_for_name(name) TaskStatus.working_on_it when 'discuss', 'd' TaskStatus.discuss + when 'rediscuss', 're-discuss', 're discuss' + TaskStatus.rediscuss when 'demonstrate', 'demo' TaskStatus.demonstrate when 'ready for feedback', 'ready_for_feedback', 'ready to mark', 'ready_to_mark', 'rtm', 'rff' @@ -142,6 +148,7 @@ def self.id_to_key(id) when 12 then :time_exceeded when 13 then :assess_in_portfolio when 14 then :attention_required + when 15 then :rediscuss else :not_started end end @@ -161,6 +168,7 @@ def status_key return :time_exceeded if self == TaskStatus.time_exceeded return :assess_in_portfolio if self == TaskStatus.assess_in_portfolio return :attention_required if self == TaskStatus.attention_required + return :rediscuss if self == TaskStatus.rediscuss return :not_started end diff --git a/app/models/unit.rb b/app/models/unit.rb index 5b7a8b1423..19e0098298 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -2377,7 +2377,7 @@ def get_all_tasks_for(user, my_tutorials_only = false) # def tasks_awaiting_feedback(user) get_all_tasks_for(user) - .where('task_statuses.id IN (:ids)', ids: [TaskStatus.discuss, TaskStatus.attention_required, TaskStatus.redo, TaskStatus.demonstrate, TaskStatus.fix_and_resubmit]) + .where('task_statuses.id IN (:ids)', ids: [TaskStatus.discuss, TaskStatus.rediscuss, TaskStatus.attention_required, TaskStatus.redo, TaskStatus.demonstrate, TaskStatus.fix_and_resubmit]) .order('task_definition_id') end diff --git a/app/models/unit_role.rb b/app/models/unit_role.rb index dd0e0c7b48..68fc0f5e09 100644 --- a/app/models/unit_role.rb +++ b/app/models/unit_role.rb @@ -136,7 +136,7 @@ def populate_summary_stats(summary_stats, tutorial_stream, tutorial, row) data[:engagements] = all_engagements data[:total_staff_engagements] = all_engagements.count - data[:staff_engagements] = weekly_engagements.where(engagement: [TaskStatus.complete.name, TaskStatus.feedback_exceeded.name, TaskStatus.redo.name, TaskStatus.discuss.name, TaskStatus.attention_required.name, TaskStatus.demonstrate.name, TaskStatus.fail.name]) + data[:staff_engagements] = weekly_engagements.where(engagement: [TaskStatus.complete.name, TaskStatus.feedback_exceeded.name, TaskStatus.redo.name, TaskStatus.discuss.name, TaskStatus.rediscuss.name, TaskStatus.attention_required.name, TaskStatus.demonstrate.name, TaskStatus.fail.name]) # Weekly task engagements for this tutorial data[:weekly_engagements_count] = weekly_engagements.count diff --git a/db/migrate/20260709014859_add_rediscuss_status.rb b/db/migrate/20260709014859_add_rediscuss_status.rb new file mode 100644 index 0000000000..509144a88c --- /dev/null +++ b/db/migrate/20260709014859_add_rediscuss_status.rb @@ -0,0 +1,23 @@ +class AddRediscussStatus < ActiveRecord::Migration[8.0] + DESCRIPTION = "You attempted to discuss this task, but it was not adequate. " \ + "Brush up your knowledge and return for another discussion " \ + "to get the task signed off.".freeze + + def up + status = TaskStatus.find_by(id: 15) || TaskStatus.find_by(name: "Rediscuss") + + if status + status.update!(name: "Rediscuss", description: DESCRIPTION) + else + TaskStatus.create!(id: 15, name: "Rediscuss", description: DESCRIPTION) + end + + Rails.cache.delete("task_statuses/15") + end + + def down + status = TaskStatus.find_by(id: 15, name: "Rediscuss") + status&.destroy! + Rails.cache.delete("task_statuses/15") + end +end diff --git a/db/schema.rb b/db/schema.rb index 9b7572e806..b8ec5659b3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_06_25_055323) do +ActiveRecord::Schema[8.0].define(version: 2026_07_09_014859) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false diff --git a/lib/tasks/init.rake b/lib/tasks/init.rake index 2023e509a3..ab9eaa5613 100644 --- a/lib/tasks/init.rake +++ b/lib/tasks/init.rake @@ -42,7 +42,8 @@ namespace :db do Fail: "You did not successfully demonstrate the required learning in this task.", "Time Exceeded": "You did not submit or complete the task before the appropriate deadline.", "Assess in Portfolio": "This task will not be signed off as complete by your tutor, and will be marked directly in your portfolio.", - "Attention Required": "This task needs to be discussed with your tutor so that you can get back on track." + "Attention Required": "This task needs to be discussed with your tutor so that you can get back on track.", + Rediscuss: "You attempted to discuss this task, but it was not adequate. Brush up your knowledge and return for another discussion to get the task signed off." } statuses.each do |name, desc| print "." diff --git a/test/models/task_status_test.rb b/test/models/task_status_test.rb index 0c61e627ba..f5f23c1aef 100644 --- a/test/models/task_status_test.rb +++ b/test/models/task_status_test.rb @@ -470,6 +470,9 @@ def test_status_for_name assert_equal TaskStatus.status_for_name('working on it').name, TaskStatus.working_on_it.name assert_equal TaskStatus.status_for_name('discuss').name, TaskStatus.discuss.name assert_equal TaskStatus.status_for_name('d').name, TaskStatus.discuss.name + assert_equal TaskStatus.status_for_name('rediscuss').name, TaskStatus.rediscuss.name + assert_equal TaskStatus.status_for_name('re-discuss').name, TaskStatus.rediscuss.name + assert_equal TaskStatus.status_for_name('re discuss').name, TaskStatus.rediscuss.name assert_equal TaskStatus.status_for_name('demonstrate').name, TaskStatus.demonstrate.name assert_equal TaskStatus.status_for_name('demo').name, TaskStatus.demonstrate.name @@ -492,11 +495,11 @@ def test_status_for_name end def test_staff_assigned_statuses - assert_equal TaskStatus.staff_assigned_statuses.count, 10 # number of staff tasks + assert_equal TaskStatus.staff_assigned_statuses.count, 11 # number of staff tasks end - def test_id_to_key_not_started - assert_equal TaskStatus.id_to_key(15), :not_started + def test_id_to_key_rediscuss + assert_equal TaskStatus.id_to_key(15), :rediscuss end end diff --git a/test/models/task_test.rb b/test/models/task_test.rb index e2a0c737c4..3597d8a86d 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -78,6 +78,24 @@ def test_trigger_transition_allows_assessment_outcomes_without_feedback_check_by assert_equal TaskStatus.complete, task.task_status end + def test_trigger_transition_only_allows_rediscuss_from_discuss + project = FactoryBot.create(:project) + unit = project.unit + task_definition = unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + tutor = unit.main_convenor_user + + task.update!(task_status: TaskStatus.ready_for_feedback) + + assert_nil task.trigger_transition(trigger: 'rediscuss', by_user: tutor) + assert_equal TaskStatus.ready_for_feedback, task.task_status + + task.update!(task_status: TaskStatus.discuss) + + assert task.trigger_transition(trigger: 'rediscuss', by_user: tutor) + assert_equal TaskStatus.rediscuss, task.task_status + end + def test_trigger_transition_requires_manual_feedback_before_assessment_outcomes_when_checking_feedback project = FactoryBot.create(:project) unit = project.unit From 1732604fde702ae9e523d8818b817e5281ea9900 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:54:12 +1000 Subject: [PATCH 155/199] chore: add coverpage css --- vendor/assets/stylesheets/doubtfire-coverpage.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vendor/assets/stylesheets/doubtfire-coverpage.css b/vendor/assets/stylesheets/doubtfire-coverpage.css index ba3974885e..6310e057e5 100644 --- a/vendor/assets/stylesheets/doubtfire-coverpage.css +++ b/vendor/assets/stylesheets/doubtfire-coverpage.css @@ -76,6 +76,10 @@ button.task-status.discuss { background-color: #31b0d5; color: white; } +button.task-status.rediscuss { + background-color: #126352; + color: white; +} button.task-status.attention-required { background-color: #f1814d; color: white; From 6196b11075f73fe6d6e503b5d3ea1e9697dfd99a Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:15:27 +1000 Subject: [PATCH 156/199] docs: update development and deployment documentation --- README.md | 169 +++++++++--------------------------------------------- 1 file changed, 28 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index e1207a2a90..ad80ec5b40 100644 --- a/README.md +++ b/README.md @@ -1,157 +1,44 @@ -![Doubtfire Logo](https://github.com/doubtfire-lms/doubtfire-web/raw/6.2.x/src/assets/icons/android-chrome-192x192.png) +

    + OnTrack logo +

    -# Doubtfire API [![test-doubtfire-api](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/push.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/push.yml) [![CodeQL](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/codeql.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/codeql.yml) [![RuboCop](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/rubocop.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/rubocop.yml) +# OnTrack API -Doubtfire is a feedback-driven learning support system. +[![test-doubtfire-api](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/push.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/push.yml) [![CodeQL](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/codeql.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/codeql.yml) [![RuboCop](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/rubocop.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/rubocop.yml) -## Table of Contents +OnTrack (formerly Doubtfire) is a feedback-driven learning support system. This repository contains +the Rails and Grape API used by OnTrack. -- [Doubtfire API](#doubtfire-api--) - - [Table of Contents](#table-of-contents) - - [Getting started](#getting-started) - - [Clone Repository](#clone-repository) - - [Install script](#install-script) - - [Manual install](#manual-install) - - [Environment variables](#environment-variables) - - [Get it up and running!](#get-it-up-and-running) -- [Running Rake Tasks](#running-rake-tasks) -- [Testing](#testing) -- [Contributing](#contributing) -- [License](#license) +## Development and deployment -## Getting started +Use the +[doubtfire-deploy](https://github.com/doubtfire-lms/doubtfire-deploy) +repository for the supported development environment and the deployment guide. +It checks out this API as a submodule and provides the database, Redis, PDF +services, web app, and required configuration in one place. -See [Doubtfire Deploy](https://github.com/doubtfire-lms/doubtfire-deploy) for instructions on deploying, and contributing, to the Doubtfire project. +When the development environment is running, the API documentation is available +at . -## Environment variables +## Testing -Doubtfire requires multiple environment variables that help define settings about the Doubtfire instance running. Whilst these will default to other values, you may want to override them in production. +Run API commands inside the `doubtfire-deploy` Dev Container: -| Key | Description | Default | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | -| `DF_AUTH_METHOD` | The authentication method you would like Doubtfire to use. Possible values are `database` for standard authentication with the database, `ldap` | `database` | -| | for [LDAP](https://www.freebsd.org/doc/en/articles/ldap-auth/), `aaf` for [AAF Rapid Connect](https://rapid.aaf.edu.au/), or `SAML2` for [SAML2.0 auth](https://en.wikipedia.org/wiki/SAML_2.0). | | -| `DF_STUDENT_WORK_DIR` | The directory to store uploaded student work for processing. | `student_work` | -| `DF_ARCHIVE_DIR` | The directory to move archived unit files to, and access from. | `DF_STUDENT_WORK_DIR/archive` | -| `DF_INSTITUTION_NAME` | The name of your institution running Doubtfire. | _Doubtfire University_ | -| `DF_INSTITUTION_EMAIL_DOMAIN` | The email domain from which emails are sent to and from in your institution. | `doubtfire.com` | -| `DF_INSTITUTION_HOST` | The host running the Doubtfire instance. | `localhost:3000` | -| `DF_COOKIE_DOMAIN` | The domain to be associated with secure cookies. | Attempts to read from host | -| `DF_INSTITUTION_PRODUCT_NAME` | The name of the product (i.e. Doubtfire) at your institution. | _Doubtfire_ | -| `DF_INSTITUTION_HAS_LOGO` | Set to true (or 1) if there is an associated institution logo to be included in the header. | false | -| `DF_INSTITUTION_LOGO_URL` | The url of the logo to include in the header if there is a logo. | /assets/images/institution-logo.png | -| `DF_INSTITUTION_LOGO_LINK_URL` | The url used for the hyperlink associated with clicking the logo. | / | -| `DF_SECRET_KEY_BASE` | The Rails secret key. | Default key provided. | -| `DF_SECRET_KEY_ATTR` | The secret key to encrypt certain database fields. | Default key provided. | -| `DF_SECRET_KEY_DEVISE` | The secret key provided to Devise. | Default key provided. | -| `DF_SECRET_KEY_MOSS` | The secret key provided to [Moss](http://theory.stanford.edu/~aiken/moss/) for plagiarism detection. This value will need to be set to run `rake submission:check_plagiarism` (otherwise you **won't** need it). You will need to register for a Moss account to use this. | No default. | -| `DF_INSTITUTION_PRIVACY` | A statement related to the need for students to submit their own work, and that this work may be uploaded to 3rd parties for the purpose of plagiarism detection. | Default statement provided | -| `DF_INSTITUTION_PLAGIARISM` | A statement clarifying the terms plagiarism and collusion. | Default statement provided | -| `DF_INSTITUTION_SETTINGS_RB` | The path of the institution specific settings rb code - used to map student imports from institutional exports to a format understood by Doubtfire. | No default | -| `DF_FFMPEG_PATH` | The path of to the ffmpeg binary for audio processing. | ffmpeg | -| `DF_REDIS_CACHE_URL` | The redis URL for rails used for development and production, ignored in the test env. | `redis://localhost:6379/0` | -| `DF_REDIS_SIDEKIQ_URL` | The redis URL for sidekiq. A working redis server is **mandatory** for sidekiq in all environments. | `redis://localhost:6379/1` | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | -| **Turn It In Integration** | | | -| `TII_ENABLED` | Whether or not Turn It In integration is enabled. | 0 / false | -| `TII_INDEX_SUBMISSIONS` | Whether or not to index submissions in Turn It In. Should be set to 1 or true in production environments | 0 / false | -| `TII_REGISTER_WEBHOOK` | Whether or not to register a webhook with Turn It In. Should be set to 1 or true in production environments | 0 / false | -| `TCA_API_KEY` | The API key for Turn It In integration, acquire from the Turn It In administration interface. | No default | -| `TCA_HOST` | The host for the Turn It In integration, eg: https://institution.turnitin.com | No default | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | -| **D2L Integration** | | | -| `D2L_ENABLED` | Whether or not D2L integration is enabled. | 0 / false | -| `D2L_CLIENT_ID` | The client ID for D2L integration - from the oauth registration in D2L | No default | -| `D2L_CLIENT_SECRET` | The client secret for D2L integration - from the oauth registration in D2L | No default | -| `D2L_REDIRECT_URI` | The redirect URI for D2L integration. Must redirect to https://host/api/d2l/callback which must match the oauth registration in D2L | No default | -| `D2L_API_HOST` | The specific institutional URL for the D2L server, eg: https://d2l.institution.edu | No default | -| `D2L_OAUTH_SITE` | The location of the D2L authentication server. | `https://auth.brightspace.com` | -| `D2L_OAUTH_SITE_AUTHORIZE_URL` | The URL to authorize the D2L integration. | `/oauth2/auth` | -| `D2L_OAUTH_SITE_TOKEN_URL` | The URL to get the token for the D2L integration. | `/core/connect/token` | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | -| **Latex Configuration** | | | -| `LATEX_CONTAINER_NAME` | The name of the container housing the latex image. Used when pdfs are generated. | No default | -| `LATEX_BUILD_PATH` | The path to the latex build script within the container. | /texlive/shell/latex_build.sh | +```sh +cd /workspace/doubtfire-api +rails test -If you have chosen to use AAF Rapid Connect authentication, then you will also need to provide the following: - -| Key | Description | Default | -| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | -| `DF_AAF_ISSUER_URL` | The URL of the AFF issuer, either `https://rapid.test.aaf.edu.au` for testing or `https://rapid.aaf.edu.au` for production. | `https://rapid.test.aaf.edu.au` | -| `DF_AAF_AUDIENCE_URL` | The URL of the AAF registered application. | No default - required | -| `DF_AAF_CALLBACK_URL` | The secure endpoint within your application that AAF Rapid Connect should POST responses to. It **must end with `/api/auth/jwt`** to access the Doubtfire JWT authentication endpoint. | No default - required | -| `DF_AAF_UNIQUE_URL` | The unique URL provided by AAF Rapid Connect used for redirection out of Doubtfire. | No default - required | -| `DF_AAF_IDENTITY_PROVIDER_URL` | The URL of the AAF-registered identity provider. | No default - required | -| `DF_AAF_AUTH_SIGNOUT_URL` | The URL to redirect to on sign out in order to log out of AAF Rapid Connect. | No default - required | -| `DF_SECRET_KEY_AAF` | The secret used to register your application with AAF. | `secretsecret12345` | - -If you are authenticating using SAML2, then you will also need to provide the following: - -| Key | Description | Default | -| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | -| `DF_SAML_METADATA_URL` | The URL to getch the SAML metadata. Either the url or a file path to the meta data must be provided. Where the url is provided it will be used. | No default | -| `DF_SAML_METADATA_FILE_PATH` | The path to the metadata xml file. When the url is not set, this path will be used to get the metadata for the saml settings. | No default | -| `DF_SAML_CONSUMER_SERVICE_URL` | The URL of the SAML application. | No default - required | -| `DF_SAML_IDP_TARGET_URL` | The IDP SAML login URL, (e.g., "https://login.microsoftonline.com/xxxx/saml2") - OnTrack will redirect to this for login. | No default - required | -| `DF_SAML_IDP_SIGNOUT_URL` | The IDP SAML logout URL, (e.g., "https://login.microsoftonline.com/xxxx/saml2") - OnTrack will redirect to this for logout. | the SAML login url | -| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | -| `DF_AAF_CALLBACK_URL` | The secure endpoint within your application that AAF Rapid Connect should POST responses to. It **must end with `/api/auth/jwt`** to access the Doubtfire JWT authentication endpoint. | No default - required | -| `DF_AAF_UNIQUE_URL` | The unique URL provided by AAF Rapid Connect used for redirection out of Doubtfire. | No default - required | -| `DF_AAF_IDENTITY_PROVIDER_URL` | The URL of the AAF-registered identity provider. | No default - required | -| `DF_AAF_AUTH_SIGNOUT_URL` | The URL to redirect to on sign out in order to log out of AAF Rapid Connect. | No default - required | -| `DF_SECRET_KEY_AAF` | The secret used to register your application with AAF. | `secretsecret12345` | - -You may choose to keep your environment variables inside a `.env` file using key-value pairs: - -``` -DF_INSTITUTION_HOST=doubtfire.unifoo.edu.au -DF_INSTITUTION_NAME="University of Foo" -``` - -You can also keep multiple `.env` files for different environments, e.g.: `.env.production` is different to `.env.develoment`. Doubtfire uses the [dotenv](https://github.com/bkeepers/dotenv) gem to make this happen. - -### Get it up and running! - -Once you've installed using either in install script or the manual install steps. - -``` -$ bundle exec rails s +# Running an individual test +rails test test/api/settings_test:14 # test_get_config_details ``` -You should see all the Doubtfire endpoints at **[http://localhost:3000/api/docs/](http://localhost:3000/api/docs/)**, which means the API is running. - -# Running Rake Tasks - -You can perform developer-specific tasks using `rake`. For a list of all tasks, execute in the root directory: - -``` -rake --tasks -``` - -# Testing - -Our aim with testing Doubtfire is to migrate to a [Test-Driven Development](https://en.wikipedia.org/wiki/Test-driven_development) -strategy, testing all new models and API endpoints (although we plan on writing -more tests for _existing_ models and API endpoints). If you are writing a new -API endpoint or model, we strongly suggest you include unit tests in the -appropriate folders (see below). - -To run unit tests, execute: - -```bash -$ rake test -``` - -Unit tests are located in the `test` directory, where **model** tests are under -the `model` subdirectory and **API** tests are under the `api` subdirectory. - -Any **helpers** should be included in the `helpers` subdirectory and helper -modules should be written under the `TestHelpers` module. +Tests are grouped under `test/models`, `test/api`, and `test/helpers`. List the +available maintenance and development tasks with `rails --tasks`. -# Contributing +## Contributing -Refer to CONTRIBUTING.md +See [CONTRIBUTING.md](CONTRIBUTING.md). -# License +## License -Licensed under GNU Affero General Public License (AGPL) v3 +Licensed under the GNU Affero General Public License (AGPL) v3. From 444b0f7811a75208411741507800ee236160ab82 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:23:29 +1000 Subject: [PATCH 157/199] docs: restore env var config --- README.md | 189 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ad80ec5b40..c81d68e9cf 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,191 @@ services, web app, and required configuration in one place. When the development environment is running, the API documentation is available at . +## Environment variables + +The API supports the environment variables below. The production deployment +template is maintained in +[`doubtfire-deploy`](https://github.com/doubtfire-lms/doubtfire-deploy/blob/main/production/api/.env.production). +An unset variable uses its application default where one exists; an empty value +may override that default with an empty string. + +### Application and database + +| Variable | Purpose | Default | +| --------------------------- | ----------------------------------------- | -------------------------- | +| `RAILS_ENV` | Rails runtime environment. | `development` | +| `DF_LOG_TO_STDOUT` | Send Rails logs to standard output. | `false` | +| `DF_PRODUCTION_DB_ADAPTER` | Production database adapter. | Deployment-specific | +| `DF_PRODUCTION_DB_HOST` | Production database host. | Deployment-specific | +| `DF_PRODUCTION_DB_DATABASE` | Production database name. | Deployment-specific | +| `DF_PRODUCTION_DB_USERNAME` | Production database username. | Deployment-specific | +| `DF_PRODUCTION_DB_PASSWORD` | Production database password. | Deployment-specific | +| `DF_REDIS_CACHE_URL` | Redis connection used by the Rails cache. | `redis://localhost:6379/0` | +| `DF_REDIS_SIDEKIQ_URL` | Redis connection used by Sidekiq. | `redis://localhost:6379/1` | + +### Storage and processing + +| Variable | Purpose | Default | +| ------------------------------------- | ---------------------------------------------------------------- | ------------------------------- | +| `DF_STUDENT_WORK_DIR` | Directory containing uploaded student work. | `student_work` | +| `DF_ARCHIVE_DIR` | Directory containing archived student work. | `DF_STUDENT_WORK_DIR/archive` | +| `DF_ARCHIVE_UNITS` | Enable automatic unit archiving. | `false` | +| `DF_UNIT_ARCHIVE_PERIOD` | Years to retain units before archiving. | `2` | +| `DF_MAX_PDF_GEN_PROCESSES` | Maximum concurrent PDF-generation processes. | `2` | +| `DF_MAX_FILE_SIZE` | Maximum uploaded file size in bytes. | `10000000` | +| `DF_ZIP_ENTRY_LIMIT` | Maximum number of entries in an uploaded ZIP. | `1000` | +| `DF_ZIP_COMPRESSION_RATIO_LIMIT` | Maximum permitted ZIP compression ratio. | `100` | +| `DF_ZIP_UNCOMPRESSED_SIZE_MULTIPLIER` | Maximum expanded ZIP size as a multiple of `DF_MAX_FILE_SIZE`. | `10` | +| `DF_AUDITOR_UNIT_ACCESS_YEARS` | Number of years of units visible to auditors. | `2` | +| `DF_IMPORT_STUDENTS_WEEKS_BEFPRE` | Weeks before a teaching period that student imports are allowed. | `1` | +| `DF_FFMPEG_PATH` | Path to FFmpeg for audio processing. | `ffmpeg` | +| `LATEX_CONTAINER_NAME` | Container used for LaTeX PDF generation. | Unset | +| `LATEX_BUILD_PATH` | LaTeX build script inside that container. | `/texlive/shell/latex_build.sh` | + +### Institution + +| Variable | Purpose | Default | +| ------------------------------ | --------------------------------------------- | ------------------------------------- | +| `DF_INSTITUTION_NAME` | Institution display name. | Institution configuration | +| `DF_INSTITUTION_EMAIL_DOMAIN` | Institution email domain. | Institution configuration | +| `DF_INSTITUTION_HOST` | Public application URL. | Institution configuration | +| `DF_COOKIE_DOMAIN` | Domain assigned to secure cookies. | Derived from `DF_INSTITUTION_HOST` | +| `DF_INSTITUTION_PRODUCT_NAME` | Product name shown to users. | Institution configuration | +| `DF_INSTITUTION_HAS_LOGO` | Enable an institution logo. | `false` | +| `DF_INSTITUTION_LOGO_URL` | Institution logo URL. | `/assets/images/institution-logo.png` | +| `DF_INSTITUTION_LOGO_LINK_URL` | Destination opened from the institution logo. | `/` | +| `DF_INSTITUTION_PRIVACY` | Submission privacy and authorship statement. | Institution configuration | +| `DF_INSTITUTION_PLAGIARISM` | Plagiarism and collusion statement. | Institution configuration | +| `DF_INSTITUTION_SETTINGS_RB` | Institution-specific Ruby configuration file. | Unset | + +### Authentication and encryption + +Rails credentials take precedence over the corresponding environment fallback +for `DF_SECRET_KEY_BASE`, `DF_SECRET_KEY_ATTR`, `DF_SECRET_KEY_DEVISE`, +`DF_SECRET_KEY_AAF`, `DF_SECRET_KEY_MOSS`, and `LTI_SHARED_API_SECRET`. + +| Variable | Purpose | Default | +| ----------------------------------- | ------------------------------------------------------------ | -------------------------------- | +| `DF_AUTH_METHOD` | Authentication method: `database`, `ldap`, `aaf`, or `saml`. | `database` | +| `DF_ACCESS_TOKEN_EXPIRY_SECONDS` | Access-token lifetime in seconds. | `7200` | +| `DF_REFRESH_TOKEN_EXPIRY_SECONDS` | Refresh-token lifetime in seconds. | `604800` | +| `DF_SECRET_KEY_BASE` | Rails secret key base fallback. | Required in production | +| `DF_SECRET_KEY_ATTR` | Legacy encrypted-attribute key fallback. | Required in production | +| `DF_SECRET_KEY_DEVISE` | Devise secret fallback. | Required in production | +| `DF_SECRET_KEY_MOSS` | MOSS integration secret fallback. | Unset | +| `DF_ENCRYPTION_PRIMARY_KEY` | Active Record Encryption primary key. | Required when encryption is used | +| `DF_ENCRYPTION_DETERMINISTIC_KEY` | Active Record Encryption deterministic key. | Required when encryption is used | +| `DF_ENCRYPTION_KEY_DERIVATION_SALT` | Active Record Encryption derivation salt. | Required when encryption is used | + +#### AAF Rapid Connect + +| Variable | Purpose | Default | +| ------------------------------ | ----------------------------------------- | ------------------------------- | +| `DF_AAF_ISSUER_URL` | AAF issuer URL. | `https://rapid.test.aaf.edu.au` | +| `DF_AAF_AUDIENCE_URL` | Registered application URL. | Required for AAF | +| `DF_AAF_CALLBACK_URL` | API JWT callback URL. | Required for AAF | +| `DF_AAF_IDENTITY_PROVIDER_URL` | Registered identity-provider URL. | Required for AAF | +| `DF_AAF_UNIQUE_URL` | Rapid Connect authentication-request URL. | Required for AAF | +| `DF_AAF_AUTH_SIGNOUT_URL` | URL used after sign-out. | Unset | +| `DF_SECRET_KEY_AAF` | AAF shared-secret fallback. | Required for AAF in production | + +#### SAML + +| Variable | Purpose | Default | +| ----------------------------------------- | ----------------------------------------------------------- | ---------------------- | +| `DF_SAML_METADATA_URL` | Identity-provider metadata URL. | Unset | +| `DF_SAML_METADATA_FILE_PATH` | Local identity-provider metadata file. | Unset | +| `DF_SAML_CONSUMER_SERVICE_URL` | SAML assertion consumer URL. | Required for SAML | +| `DF_SAML_SP_ENTITY_ID` | Service-provider entity ID. | Required for SAML | +| `DF_SAML_IDP_TARGET_URL` | Identity-provider login URL. | Required for SAML | +| `DF_SAML_IDP_SIGNOUT_URL` | Identity-provider logout URL. | Unset | +| `DF_SAML_IDP_CERT` | Identity-provider certificate when metadata is unavailable. | Conditionally required | +| `DF_SAML_IDP_SAML_NAME_IDENTIFIER_FORMAT` | SAML NameID format. | Email address | + +#### LDAP + +| Variable | Purpose | Default | +| --------------------------- | ---------------------------------------------------- | -------------------- | +| `DF_LDAP_HOST` | LDAP server host. | Required for LDAP | +| `DF_LDAP_PORT` | LDAP server port. | LDAP library default | +| `DF_LDAP_ATTRIBUTE` | LDAP attribute used as the login identifier. | Required for LDAP | +| `DF_LDAP_BASE` | LDAP search base. | Required for LDAP | +| `DF_LDAP_SSL` | Enable an encrypted LDAP connection. | `false` | +| `DF_LDAP_USE_ADMIN_TO_BIND` | Bind with an administrator account before searching. | `false` | +| `DF_LDAP_ADMIN_USER` | LDAP administrator bind user. | Unset | +| `DF_LDAP_ADMIN_PWD` | LDAP administrator bind password. | Unset | + +#### D2L + +| Variable | Purpose | Default | +| ------------------------------ | ----------------------------------------------------- | ------------------------------ | +| `D2L_ENABLED` | Enable D2L integration. | `false` | +| `D2L_CLIENT_ID` | D2L OAuth client ID. | Unset | +| `D2L_CLIENT_SECRET` | D2L OAuth client secret. | Unset | +| `D2L_REDIRECT_URI` | D2L OAuth callback URL ending in `/api/d2l/callback`. | Unset | +| `D2L_API_HOST` | Institution D2L API host. | Unset | +| `D2L_OAUTH_SITE` | D2L authorization server. | `https://auth.brightspace.com` | +| `D2L_OAUTH_SITE_AUTHORIZE_URL` | D2L authorization path. | `/oauth2/auth` | +| `D2L_OAUTH_SITE_TOKEN_URL` | D2L token path. | `/core/connect/token` | + +### Email + +| Variable | Purpose | Default | +| ---------------------------- | ---------------------------------------- | ------------------- | +| `DF_MAIL_PERFORM_DELIVERIES` | Enable outgoing email delivery. | Disabled | +| `DF_MAIL_DELIVERY_METHOD` | Action Mailer delivery method. | Deployment-specific | +| `DF_SMTP_ADDRESS` | SMTP server address. | Unset | +| `DF_SMTP_PORT` | SMTP server port. | Unset | +| `DF_SMTP_DOMAIN` | SMTP HELO domain. | Unset | +| `DF_SMTP_USERNAME` | SMTP username. | Unset | +| `DF_SMTP_PASSWORD` | SMTP password. | Unset | +| `DF_SMTP_AUTHENTICATION` | SMTP authentication method. | Unset | +| `DF_EMAIL_ERRORS_TO` | Address receiving PDF-generation errors. | Unset | + +### Integrations and background services + +| Variable | Purpose | Default | +| -------------------------------- | --------------------------------------------------------------- | ------- | +| `TII_ENABLED` | Enable Turnitin integration. | `false` | +| `TII_INDEX_SUBMISSIONS` | Index submissions in Turnitin. | `false` | +| `TII_REGISTER_WEBHOOK` | Register the Turnitin webhook. | `false` | +| `TCA_API_KEY` | Turnitin Core API key. | Unset | +| `TCA_HOST` | Turnitin institution host. | Unset | +| `DF_JPLAG_MIN_TOKENS` | Minimum matching-token threshold used by JPlag. | `-1` | +| `DF_JPLAG_SKIP_CLUSTER_CHECK` | Skip JPlag cluster calculation. | `false` | +| `DF_JPLAG_MAX_SHOWN_COMPARISONS` | Maximum comparisons retained in a JPlag report; `-1` means all. | `2500` | +| `LTI_ENABLED` | Enable LTI routes and authentication. | `false` | +| `LTI_SHARED_API_SECRET` | Shared secret between the Rails and LTI APIs. | Unset | +| `MODERATION_SCORE_FACTOR` | Multiplier applied to moderation score changes. | `1.0` | + +### Overseer and container access + +| Variable | Purpose | Default | +| ---------------------------------------------------- | ------------------------------------------------------------- | --------------------------------- | +| `OVERSEER_ENABLED` | Enable Overseer assessment services. | `false` | +| `OVERSEER_WORKDIR_VOLUME_MOUNT` | Host directory used for isolated Overseer work. | Required when Overseer is enabled | +| `OVERSEER_FALLBACK_VOLUME_CONTAINER` | Shared-container fallback when a host mount cannot be used. | Unset | +| `OVERSEER_STUDENT_NOTIFICATION_GRACE_PERIOD_MINUTES` | Delay before notifying students of unread failed assessments. | `30` | +| `DISK_SPACE_ENDPOINT_ENABLED` | Expose host storage availability to Overseer administrators. | `false` | +| `DOCKER_REGISTRY_URL` | Registry used for Overseer images. | Unset | +| `DOCKER_PROXY_URL` | Docker proxy or registry login endpoint. | Unset | +| `DOCKER_USER` | Docker registry username. | Unset | +| `DOCKER_TOKEN` | Docker registry access token. | Unset | + +### Error reporting + +| Variable | Purpose | Default | +| -------------------- | -------------------------------------------- | ----------------- | +| `SENTRY_DSN` | Sentry project data-source name. | Unset | +| `SENTRY_ENVIRONMENT` | Environment label attached to Sentry events. | Rails environment | + ## Testing +OnTrack aims to follow a +[Test-Driven Development](https://en.wikipedia.org/wiki/Test-driven_development) +approach. New or changed models and API endpoints should be accompanied by +tests that describe and verify their expected behaviour. + Run API commands inside the `doubtfire-deploy` Dev Container: ```sh @@ -32,8 +215,10 @@ rails test rails test test/api/settings_test:14 # test_get_config_details ``` -Tests are grouped under `test/models`, `test/api`, and `test/helpers`. List the -available maintenance and development tasks with `rails --tasks`. +Tests are grouped under `test/models`, `test/api`, and `test/helpers`. Shared +test helpers should be placed in `test/helpers`, with helper modules defined +under the `TestHelpers` namespace. List the available maintenance and +development tasks with `rails --tasks`. ## Contributing From b3ef36abb48d60225a95c3dd2e0d72bb5121546c Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:42:21 +1000 Subject: [PATCH 158/199] chore: extend observer only permissions --- app/helpers/authorisation_helpers.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/helpers/authorisation_helpers.rb b/app/helpers/authorisation_helpers.rb index b27fd59024..c88c252f56 100644 --- a/app/helpers/authorisation_helpers.rb +++ b/app/helpers/authorisation_helpers.rb @@ -26,7 +26,9 @@ def get_permission_hash(role, perm_hash, _other) :get_staff_note, :get_members, :get_groups, - :get_discussion_prompt + :get_discussion_prompt, + :get_engagements, + :get_tutor_times ].freeze # From 7640a36f682f6dd7240cb764dea6e46b7d01df35 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:03:14 +1000 Subject: [PATCH 159/199] feat: prompt for admin password when creating initial account --- lib/tasks/init.rake | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/tasks/init.rake b/lib/tasks/init.rake index ab9eaa5613..733092aa80 100644 --- a/lib/tasks/init.rake +++ b/lib/tasks/init.rake @@ -1,4 +1,27 @@ +require 'io/console' + namespace :db do + def prompt_for_admin_password + console = IO.console + raise 'An interactive terminal is required to set the initial admin password' if console.nil? + + loop do + password = console.getpass('Enter password for the initial admin account: ') + confirmation = console.getpass('Confirm password: ') + + raise 'Password entry was cancelled' if password.nil? || confirmation.nil? + + unless Devise.password_length.cover?(password.length) + puts "Password must be #{Devise.password_length} characters long." + next + end + + return password if password == confirmation + + puts 'Passwords do not match. Try again.' + end + end + # # Generate roles # @@ -74,8 +97,9 @@ namespace :db do profile[:login_id] ||= username if AuthenticationHelpers.db_auth? - profile[:password] = 'password' - profile[:password_confirmation] = 'password' + password = prompt_for_admin_password + profile[:password] = password + profile[:password_confirmation] = password end user = User.create!(profile) From 610daa37511f37e13a431cb81decc8c34ebb283a Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:10:27 +1000 Subject: [PATCH 160/199] chore: set default password in ci --- .github/workflows/push.yml | 2 ++ lib/tasks/init.rake | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index dbd2b1061c..7a2f506391 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -31,6 +31,7 @@ env: LATEX_BUILD_PATH: /texlive/shell/latex_build.sh LTI_SHARED_API_SECRET: "abc123" LTI_ENABLED: true + DF_INITIAL_ADMIN_PASSWORD: "password" jobs: unit-tests: @@ -152,6 +153,7 @@ jobs: -e LATEX_BUILD_PATH -e LTI_SHARED_API_SECRET -e LTI_ENABLED + -e DF_INITIAL_ADMIN_PASSWORD run: bundle exec rake db:populate - name: Run unit tests uses: maus007/docker-run-action-fork@207a4e2a8ebf7e4b985656ba990b1e53715dce2a diff --git a/lib/tasks/init.rake b/lib/tasks/init.rake index 733092aa80..74d215820f 100644 --- a/lib/tasks/init.rake +++ b/lib/tasks/init.rake @@ -2,8 +2,20 @@ require 'io/console' namespace :db do def prompt_for_admin_password + password = ENV['DF_INITIAL_ADMIN_PASSWORD'] + if password.present? + unless Devise.password_length.cover?(password.length) + raise "DF_INITIAL_ADMIN_PASSWORD must be #{Devise.password_length} characters long" + end + + return password + end + console = IO.console - raise 'An interactive terminal is required to set the initial admin password' if console.nil? + if console.nil? + raise 'An interactive terminal is required to set the initial admin password. ' \ + 'Set DF_INITIAL_ADMIN_PASSWORD for non-interactive setup.' + end loop do password = console.getpass('Enter password for the initial admin account: ') From 923701dfd3877286922acda9985caac13369ed1b Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:19:11 +1000 Subject: [PATCH 161/199] chore: fix rubocop --- lib/tasks/init.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/init.rake b/lib/tasks/init.rake index 74d215820f..9823ef5382 100644 --- a/lib/tasks/init.rake +++ b/lib/tasks/init.rake @@ -2,7 +2,7 @@ require 'io/console' namespace :db do def prompt_for_admin_password - password = ENV['DF_INITIAL_ADMIN_PASSWORD'] + password = ENV.fetch('DF_INITIAL_ADMIN_PASSWORD', nil) if password.present? unless Devise.password_length.cover?(password.length) raise "DF_INITIAL_ADMIN_PASSWORD must be #{Devise.password_length} characters long" From 18123d4d29d38f02d0918d038d5b13894da5c7e9 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:34:33 +1000 Subject: [PATCH 162/199] chore: only prompt for admin password in production --- lib/tasks/init.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/init.rake b/lib/tasks/init.rake index 9823ef5382..4137a32533 100644 --- a/lib/tasks/init.rake +++ b/lib/tasks/init.rake @@ -109,7 +109,7 @@ namespace :db do profile[:login_id] ||= username if AuthenticationHelpers.db_auth? - password = prompt_for_admin_password + password = Rails.env.production? ? prompt_for_admin_password : 'password' profile[:password] = password profile[:password_confirmation] = password end From a21daf846a46557331c3235f86f28a5c8a020786 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:43:08 +1000 Subject: [PATCH 163/199] feat: support word document submission (#647) * feat: support word document submission * chore: fix assert match * fix: use fallback container * feat: ensure word doc is not encrypted * refactor: use dynamic containers instead of a single active container * ci: use correct gotenberg image * chore: deny word documents if gotenberg is not configured * chore: default gotenberg image to null * chore: simple * refactor: clean up gotenberg script --- .github/workflows/push.yml | 26 ++++ app/helpers/file_helper.rb | 161 ++++++++++++++++++++- app/models/submission_history.rb | 40 +++++- app/models/task.rb | 38 ++++- config/application.rb | 14 ++ gotenberg.Dockerfile | 12 ++ lib/shell/word_document_build.sh | 86 +++++++++++ test/models/file_helper_test.rb | 199 ++++++++++++++++++++++++++ test/models/task_test.rb | 120 ++++++++++++++++ test_files/submissions/encrypted.docx | Bin 0 -> 19968 bytes 10 files changed, 687 insertions(+), 9 deletions(-) create mode 100644 gotenberg.Dockerfile create mode 100755 lib/shell/word_document_build.sh create mode 100644 test_files/submissions/encrypted.docx diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7a2f506391..3d7f226270 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -29,6 +29,8 @@ env: DF_REDIS_SIDEKIQ_URL: "redis://redis:6379/0" LATEX_CONTAINER_NAME: doubtfire-texlive LATEX_BUILD_PATH: /texlive/shell/latex_build.sh + GOTENBERG_IMAGE: doubtfire-gotenberg-development:local + GOTENBERG_FALLBACK_VOLUME_CONTAINER: doubtfire-gotenberg-volumes LTI_SHARED_API_SECRET: "abc123" LTI_ENABLED: true DF_INITIAL_ADMIN_PASSWORD: "password" @@ -64,6 +66,16 @@ jobs: tags: doubtfire-texlive-development:local cache-from: type=gha,scope=texlive cache-to: type=gha,mode=max,scope=texlive + - name: Build Gotenberg image + uses: docker/build-push-action@v5 + with: + context: . + file: gotenberg.Dockerfile + push: false + load: true + tags: doubtfire-gotenberg-development:local + cache-from: type=gha,scope=gotenberg + cache-to: type=gha,mode=max,scope=gotenberg - name: Build JPlag image uses: docker/build-push-action@v5 with: @@ -104,6 +116,13 @@ jobs: -v ${{ github.workspace }}:/doubtfire -v /var/run/docker.sock:/var/run/docker.sock run: docker exec -t ${{ env.LATEX_CONTAINER_NAME }} lualatex -v + - name: Start Gotenberg volume container + run: > + docker run --detach + --name ${{ env.GOTENBERG_FALLBACK_VOLUME_CONTAINER }} + --network none + --volume ${{ github.workspace }}/tmp/gotenberg:/workdir/gotenberg + alpine sleep infinity - name: Start JPlag service uses: maus007/docker-run-action-fork@207a4e2a8ebf7e4b985656ba990b1e53715dce2a with: @@ -151,6 +170,8 @@ jobs: -e DF_REDIS_SIDEKIQ_URL -e LATEX_CONTAINER_NAME -e LATEX_BUILD_PATH + -e GOTENBERG_IMAGE + -e GOTENBERG_FALLBACK_VOLUME_CONTAINER -e LTI_SHARED_API_SECRET -e LTI_ENABLED -e DF_INITIAL_ADMIN_PASSWORD @@ -183,6 +204,8 @@ jobs: -e DF_REDIS_SIDEKIQ_URL -e LATEX_CONTAINER_NAME -e LATEX_BUILD_PATH + -e GOTENBERG_IMAGE + -e GOTENBERG_FALLBACK_VOLUME_CONTAINER -e LTI_SHARED_API_SECRET -e LTI_ENABLED run: TERM=xterm bundle exec rails test @@ -190,3 +213,6 @@ jobs: run: docker rm -f ${{ env.LATEX_CONTAINER_NAME }} - name: Stop JPlag service run: docker rm -f jplag + - name: Stop Gotenberg volume container + if: always() + run: docker rm -f ${{ env.GOTENBERG_FALLBACK_VOLUME_CONTAINER }} diff --git a/app/helpers/file_helper.rb b/app/helpers/file_helper.rb index 8526c4b333..0953907850 100644 --- a/app/helpers/file_helper.rb +++ b/app/helpers/file_helper.rb @@ -17,9 +17,17 @@ module FileHelper ZIP_NESTED_ARCHIVE_EXTENSIONS = %w[ .7z .bz2 .ear .gz .jar .rar .tar .tar.bz2 .tar.gz .tar.xz .tbz .tbz2 .tgz .txz .war .xz .zip ].freeze + WORD_DOCUMENT_EXTENSION = '.docx' + WORD_DOCUMENT_MIME_TYPE = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + COMPOUND_FILE_SIGNATURE = "\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b.freeze + WORD_DOCUMENT_ENCRYPTION_STREAM_NAMES = %w[EncryptedPackage EncryptionInfo].map do |name| + name.encode(Encoding::UTF_16LE).b.freeze + end.freeze + + class DocumentConversionError < StandardError; end def known_extension?(extn) - allow_extensions = %w(pdf ps csv xls xlsx pas cpp c cs csv h hpp java py js html coffee scss yaml yml xml json ts r rb rmd rnw rhtml rpres tex vb sql txt md jack hack asm hdl tst out cmp vm sh bat dat ipynb css png bmp tiff tif jpeg jpg gif zip gz tgz tar wav ogg mp3 mp4 webm aac pcm aiff flac wma alac pml vue) + allow_extensions = %w(pdf ps docx csv xls xlsx pas cpp c cs csv h hpp java py js html coffee scss yaml yml xml json ts r rb rmd rnw rhtml rpres tex vb sql txt md jack hack asm hdl tst out cmp vm sh bat dat ipynb css png bmp tiff tif jpeg jpg gif zip gz tgz tar wav ogg mp3 mp4 webm aac pcm aiff flac wma alac pml vue) # Allow empty or nil extensions for blobs otherwise check that it matches the allowed list extn.blank? || allow_extensions.include?(extn) @@ -30,6 +38,8 @@ def known_extension?(extn) # - file is passed the file uploaded to Doubtfire (a hash with all relevant data about the file) # def accept_file(file, name, kind) + word_document = kind == 'document' && word_document?(file[:filename] || file['tempfile'].path) + case kind when 'image' mime_allow_list = ['image/png', 'image/gif', 'image/bmp', 'image/tiff', 'image/jpeg', 'image/x-ms-bmp'] @@ -39,7 +49,11 @@ def accept_file(file, name, kind) 'text/x-yaml', 'application/xml', 'text/x-typescript', 'text/x-vhdl', 'text/x-asm', 'text/x-jack', 'application/x-httpd-php', 'application/tst', 'text/x-cmp', 'text/x-vm', 'application/x-sh', 'application/x-bat', 'application/dat', 'application/x-wine-extension-ini'] when 'document' - mime_allow_list = [ 'application/pdf' ] + mime_allow_list = if word_document + [WORD_DOCUMENT_MIME_TYPE] + else + ['application/pdf'] + end when 'zip', 'archive' mime_allow_list = [ 'application/zip', @@ -70,6 +84,24 @@ def accept_file(file, name, kind) } end + if word_document && !word_document_conversion_configured? + msg = 'Word documents are currently not supported. Please export your document to PDF.' + logger.error 'Word document upload rejected because conversion is not configured' + return { + accepted: false, + msg: msg + } + end + + if word_document && encrypted_word_document?(file['tempfile'].path) + msg = 'Word document is encrypted or password protected. Remove the password protection and upload it again.' + logger.debug 'Word document is encrypted or password protected' + return { + accepted: false, + msg: msg + } + end + mime_check = mime_in_list?(file['tempfile'].path, mime_allow_list) unless mime_check msg = 'invalid file MIME type, file is likely corrupted.' @@ -80,8 +112,9 @@ def accept_file(file, name, kind) } end - # Extra checks for PDF documents - if kind == 'document' + # Extra checks for PDF documents. DOCX files are converted and validated as + # PDFs when the submission is processed asynchronously. + if kind == 'document' && !word_document pdf_validation_result = validate_pdf(file['tempfile'].path) if pdf_validation_result[:encrypted] @@ -124,6 +157,119 @@ def accept_file(file, name, kind) } end + def word_document?(path) + File.extname(path.to_s).casecmp(WORD_DOCUMENT_EXTENSION).zero? + end + + def word_document_conversion_configured? + config = Doubtfire::Application.config + config.gotenberg_image.present? && + (config.gotenberg_workdir_volume_mount.present? || config.gotenberg_fallback_volume_container.present?) + end + + # Password-protected OOXML files are stored in an OLE compound file rather + # than the ZIP container used by normal DOCX files. Confirm the compound-file + # signature and both encryption stream names to avoid treating any malformed + # or incorrectly named DOCX as encrypted. + def encrypted_word_document?(path) + longest_stream_name = WORD_DOCUMENT_ENCRYPTION_STREAM_NAMES.map(&:bytesize).max + matched_stream_names = Array.new(WORD_DOCUMENT_ENCRYPTION_STREAM_NAMES.length, false) + + File.open(path, 'rb') do |file| + return false unless file.read(COMPOUND_FILE_SIGNATURE.bytesize) == COMPOUND_FILE_SIGNATURE + + buffer = ''.b + while (chunk = file.read(16 * 1024)) + buffer << chunk + WORD_DOCUMENT_ENCRYPTION_STREAM_NAMES.each_with_index do |stream_name, index| + matched_stream_names[index] ||= buffer.include?(stream_name) + end + return true if matched_stream_names.all? + + buffer = buffer.byteslice(-(longest_stream_name - 1), longest_stream_name - 1) || ''.b + end + end + + false + rescue Errno::ENOENT, Errno::EACCES, IOError + false + end + + def convert_word_document_to_pdf(source_path, destination_path, work_id: SecureRandom.uuid) + work_id = work_id.to_s + unless work_id.match?(/\A[A-Za-z0-9_-]+\z/) + raise DocumentConversionError, 'Invalid Gotenberg work id.' + end + + work_root = Rails.root.join("tmp/gotenberg") + work_dir = work_root.join(work_id) + input_path = work_dir.join('input.docx') + output_path = work_dir.join('output.pdf') + + FileUtils.mkdir_p(work_dir) + FileUtils.chmod(0o777, work_dir) + FileUtils.cp(source_path, input_path) + + stdout, stderr, status = run_word_document_conversion(work_id) + unless status.success? + details = [stdout, stderr].compact.join("\n").strip.first(1_000) + message = "Word document conversion failed with exit status #{status.exitstatus}." + message = "#{message} #{details}" if details.present? + raise DocumentConversionError, message + end + + unless File.exist?(output_path) && validate_pdf(output_path)[:valid] + raise DocumentConversionError, 'Gotenberg did not produce a valid PDF for the Word document.' + end + + FileUtils.mv(output_path, destination_path) + destination_path + rescue SystemCallError => e + raise DocumentConversionError, "Word document conversion failed: #{e.message}" + ensure + FileUtils.rm_rf(work_dir) if defined?(work_dir) && work_dir + end + + def run_word_document_conversion(work_id) + Open3.capture3(*word_document_conversion_command(work_id)) + end + + def word_document_conversion_command(work_id) + config = Doubtfire::Application.config + raise DocumentConversionError, 'GOTENBERG_IMAGE is not configured.' if config.gotenberg_image.blank? + + timeout_seconds = config.word_document_conversion_timeout_seconds.to_i + timeout_seconds = 120 unless timeout_seconds.positive? + + [ + 'docker', 'run', '--rm', + '--cpus', '1', + '--network', 'none', + *gotenberg_volume_arguments(work_id), + '--name', "gotenberg-word-#{work_id}", + '--env', "WORD_DOCUMENT_CONVERSION_TIMEOUT_SECONDS=#{timeout_seconds}", + '--entrypoint', config.word_document_build_path, + config.gotenberg_image, + work_id + ] + end + + def gotenberg_volume_arguments(work_id) + config = Doubtfire::Application.config + mount = config.gotenberg_workdir_volume_mount + + if mount.present? + host_work_dir = File.join(mount, work_id) + container_work_dir = "/workdir/gotenberg/#{work_id}" + ['--volume', "#{host_work_dir}:#{container_work_dir}"] + elsif config.gotenberg_fallback_volume_container.present? + ['--volumes-from', config.gotenberg_fallback_volume_container] + else + raise DocumentConversionError, + 'Set GOTENBERG_WORKDIR_VOLUME_MOUNT or GOTENBERG_FALLBACK_VOLUME_CONTAINER.' + end + end + # # Sanitize the passed in paths, and ensure each part is valid # Will kill things like ../ etc or spaces in paths @@ -1025,6 +1171,13 @@ def line_wrap(path, width: 160) end # Export functions as module functions module_function :accept_file + module_function :word_document? + module_function :word_document_conversion_configured? + module_function :encrypted_word_document? + module_function :convert_word_document_to_pdf + module_function :run_word_document_conversion + module_function :word_document_conversion_command + module_function :gotenberg_volume_arguments module_function :sanitized_path module_function :sanitized_filename module_function :task_file_dir_for_unit diff --git a/app/models/submission_history.rb b/app/models/submission_history.rb index 745925ac99..d1f1faecc9 100644 --- a/app/models/submission_history.rb +++ b/app/models/submission_history.rb @@ -42,10 +42,19 @@ def self.create_archive!(task, submission_timestamp) file_name = entry.name.split('/').last next unless file_name&.match?(/^\d{3}-(?:document|code|image|zip|archive)/) - next unless enabled_indexes.include?(file_name.to_i) - - destination.get_output_stream(File.join(history.entry_prefix, entry.name)) do |output| - entry.get_input_stream { |input| IO.copy_stream(input, output) } + upload_index = file_name.to_i + next unless enabled_indexes.include?(upload_index) + + preview_path = document_preview_path(task, upload_index) + if file_name.match?(/^\d{3}-document.*\.docx\z/i) && File.exist?(preview_path) + preview_entry_name = File.join(history.entry_prefix, entry.name.sub(/\.docx\z/i, '.pdf')) + destination.get_output_stream(preview_entry_name) do |output| + File.open(preview_path, 'rb') { |input| IO.copy_stream(input, output) } + end + else + destination.get_output_stream(File.join(history.entry_prefix, entry.name)) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end end copied_files += 1 end @@ -156,6 +165,25 @@ def self.pending_marker_path(task) File.join(FileHelper.task_submission_identifier_path(:pending, task), 'submission-history') end + def self.document_preview_dir(task) + File.join(FileHelper.task_submission_identifier_path(:pending, task), 'document-previews') + end + + def self.document_preview_path(task, upload_index) + File.join(document_preview_dir(task), "#{upload_index.to_s.rjust(3, '0')}-document.pdf") + end + + def self.stage_document_preview!(task, upload_index, source_path) + raise "Converted document preview was not found: #{source_path}" unless File.exist?(source_path) + + FileUtils.mkdir_p(document_preview_dir(task)) + FileUtils.cp(source_path, document_preview_path(task, upload_index)) + end + + def self.clear_document_previews(task) + FileUtils.rm_rf(document_preview_dir(task)) + end + def self.mark_pending(task) marker_path = pending_marker_path(task) FileUtils.mkdir_p(File.dirname(marker_path)) @@ -164,6 +192,10 @@ def self.mark_pending(task) def self.clear_pending(task) FileUtils.rm_f(pending_marker_path(task)) + clear_document_previews(task) + + pending_dir = FileHelper.task_submission_identifier_path(:pending, task) + FileUtils.rm_rf(pending_dir) if Dir.exist?(pending_dir) && Dir.empty?(pending_dir) end def self.pending?(task) diff --git a/app/models/task.rb b/app/models/task.rb index a48148c8a3..4a09286d4c 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -1486,6 +1486,8 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end begin + converted_word_documents = convert_word_documents_to_pdf + tac = TaskAppController.new tac.init(self, false) @@ -1523,7 +1525,7 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end end - raise LatexError.new(log_message), 'Failed to convert your submission to PDF. Check code files submitted for invalid characters, that documents are valid pdfs, images are valid, and zip files are valid.' + raise LatexError.new(log_message), 'Failed to convert your submission to PDF. Check code files submitted for invalid characters, that documents are valid PDFs or DOCX files, images are valid, and zip files are valid.' end end @@ -1544,9 +1546,11 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end end + stage_word_document_previews(converted_word_documents) save return true rescue => e + SubmissionHistory.clear_document_previews(self) trigger_transition trigger: 'fix', by_user: project.tutor_for(task_definition) add_text_comment project.tutor_for(task_definition), "**Automated Comment**: Something went wrong with your submission. Check the files and resubmit this task. #{e.message}" raise e @@ -1558,6 +1562,38 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end end + def convert_word_documents_to_pdf + in_process_dir = student_work_dir(:in_process, false) + return [] unless Dir.exist?(in_process_dir) + + converted_documents = [] + + Dir.glob(File.join(in_process_dir, '*-document.*')).each do |source_path| + next unless FileHelper.word_document?(source_path) + + upload_index = File.basename(source_path).to_i + destination_path = source_path.sub(/\.docx\z/i, '.pdf') + FileHelper.convert_word_document_to_pdf( + source_path, + destination_path, + work_id: "task-#{id}-#{SecureRandom.uuid}" + ) + FileUtils.rm_f(source_path) + converted_documents << { upload_index: upload_index, path: destination_path } + end + + converted_documents + end + + def stage_word_document_previews(converted_documents) + converted_documents.each do |document| + requirement = upload_requirements[document[:upload_index]] + next unless requirement&.dig('submission_history') == true + + SubmissionHistory.stage_document_preview!(self, document[:upload_index], document[:path]) + end + end + # # The student has uploaded new work... # diff --git a/config/application.rb b/config/application.rb index c4f3e6f3b7..ee62d44efe 100644 --- a/config/application.rb +++ b/config/application.rb @@ -63,6 +63,20 @@ class Application < Rails::Application # Limit number of pdf generators to run at once config.pdfgen_max_processes = ENV['DF_MAX_PDF_GEN_PROCESSES'] || 2 + # Each Word document conversion runs a short-lived, network-isolated + # Gotenberg container. The image includes the conversion entrypoint. + config.gotenberg_image = ENV.fetch('GOTENBERG_IMAGE', nil) + + # Absolute host path to tmp/gotenberg. Production uses this to mount only + # the current conversion's work directory into its one-shot container. + config.gotenberg_workdir_volume_mount = ENV.fetch('GOTENBERG_WORKDIR_VOLUME_MOUNT', nil) + + # Development fallback matching Overseer. This exposes the fallback + # container's entire tmp/gotenberg mount to each conversion container. + config.gotenberg_fallback_volume_container = ENV.fetch('GOTENBERG_FALLBACK_VOLUME_CONTAINER', nil) + config.word_document_build_path = ENV.fetch('WORD_DOCUMENT_BUILD_PATH', '/gotenberg/word_document_build.sh') + config.word_document_conversion_timeout_seconds = ENV.fetch('WORD_DOCUMENT_CONVERSION_TIMEOUT_SECONDS', 120) + # Date range for auditors to view config.auditor_unit_access_years = ENV.fetch('DF_AUDITOR_UNIT_ACCESS_YEARS', 2).to_f * 1.year diff --git a/gotenberg.Dockerfile b/gotenberg.Dockerfile new file mode 100644 index 0000000000..6a23edcbbc --- /dev/null +++ b/gotenberg.Dockerfile @@ -0,0 +1,12 @@ +FROM gotenberg/gotenberg:8-libreoffice + +USER root + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY lib/shell/word_document_build.sh /gotenberg/word_document_build.sh +RUN chmod 755 /gotenberg/word_document_build.sh + +USER gotenberg diff --git a/lib/shell/word_document_build.sh b/lib/shell/word_document_build.sh new file mode 100755 index 0000000000..42bcb917e1 --- /dev/null +++ b/lib/shell/word_document_build.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +set -eu + +WORK_ID=${1:-} + +if [ -z "$WORK_ID" ]; then + echo "Usage: word_document_build.sh " >&2 + exit 2 +fi + +case "$WORK_ID" in + *[!A-Za-z0-9_-]*) + echo "Invalid Gotenberg work id" >&2 + exit 2 + ;; +esac + +WORK_DIR="/workdir/gotenberg/$WORK_ID" +INPUT_FILE="$WORK_DIR/input.docx" +OUTPUT_FILE="$WORK_DIR/output.pdf" +TEMP_OUTPUT="$WORK_DIR/output.pdf.tmp" +GOTENBERG_URL="http://localhost:3000" +GOTENBERG_PID= + +if [ ! -f "$INPUT_FILE" ]; then + echo "Word document input was not found" >&2 + exit 1 +fi + +cleanup() { + exit_status=$? + trap - EXIT INT TERM + + if [ -n "$GOTENBERG_PID" ] && kill -0 "$GOTENBERG_PID" 2>/dev/null; then + kill "$GOTENBERG_PID" 2>/dev/null || true + wait "$GOTENBERG_PID" 2>/dev/null || true + fi + + rm -f "$TEMP_OUTPUT" + exit "$exit_status" +} + +start_gotenberg() { + # Docker replaces the image's normal command with this script, so start the + # bundled API before making the local conversion request. + gotenberg --gotenberg-graceful-shutdown-duration=0s & + GOTENBERG_PID=$! + + curl \ + --fail \ + --silent \ + --show-error \ + --retry 30 \ + --retry-connrefused \ + --retry-delay 1 \ + --connect-timeout 1 \ + --max-time 30 \ + "$GOTENBERG_URL/health" \ + >/dev/null +} + +convert_document() { + curl \ + --fail-with-body \ + --silent \ + --show-error \ + --connect-timeout 5 \ + --max-time "${WORD_DOCUMENT_CONVERSION_TIMEOUT_SECONDS:-120}" \ + --request POST \ + --form "files=@$INPUT_FILE" \ + --output "$TEMP_OUTPUT" \ + "$GOTENBERG_URL/forms/libreoffice/convert" + + if [ ! -s "$TEMP_OUTPUT" ]; then + echo "Gotenberg did not produce a PDF" >&2 + exit 1 + fi + + mv "$TEMP_OUTPUT" "$OUTPUT_FILE" +} + +trap cleanup EXIT INT TERM + +start_gotenberg +convert_document diff --git a/test/models/file_helper_test.rb b/test/models/file_helper_test.rb index d77a026fa5..bfc5cb105b 100644 --- a/test/models/file_helper_test.rb +++ b/test/models/file_helper_test.rb @@ -3,6 +3,21 @@ require "zip" class FileHelperTest < ActiveSupport::TestCase + def with_word_document_conversion_configured + config = Doubtfire::Application.config + original_image = config.gotenberg_image + original_mount = config.gotenberg_workdir_volume_mount + original_fallback = config.gotenberg_fallback_volume_container + config.gotenberg_image = 'doubtfire-gotenberg:test' + config.gotenberg_workdir_volume_mount = nil + config.gotenberg_fallback_volume_container = 'fallback-container' + yield + ensure + config.gotenberg_image = original_image + config.gotenberg_workdir_volume_mount = original_mount + config.gotenberg_fallback_volume_container = original_fallback + end + def test_convert_use_with_gif in_file = "#{Rails.root}/test_files/submissions/unbelievable.gif" @@ -13,6 +28,190 @@ def test_convert_use_with_gif end end + def test_accepts_docx_as_a_document + with_word_document_conversion_configured do + Tempfile.create(['submission', '.docx']) do |docx_file| + FileUtils.cp(Rails.root.join('test_files/TestWordDoc.docx'), docx_file.path) + + result = FileHelper.accept_file( + { + filename: 'submission.docx', + 'tempfile' => docx_file + }, + 'Report', + 'document' + ) + + assert result[:accepted], result[:msg] + end + end + end + + def test_rejects_docx_when_word_document_conversion_is_not_configured + config = Doubtfire::Application.config + original_image = config.gotenberg_image + original_mount = config.gotenberg_workdir_volume_mount + original_fallback = config.gotenberg_fallback_volume_container + config.gotenberg_image = nil + config.gotenberg_workdir_volume_mount = nil + config.gotenberg_fallback_volume_container = nil + + File.open(Rails.root.join('test_files/TestWordDoc.docx')) do |docx_file| + result = FileHelper.accept_file( + { + filename: 'submission.docx', + 'tempfile' => docx_file + }, + 'Report', + 'document' + ) + + assert_not result[:accepted] + assert_equal( + 'Word documents are currently not supported. Please export your document to PDF.', + result[:msg] + ) + end + ensure + config.gotenberg_image = original_image + config.gotenberg_workdir_volume_mount = original_mount + config.gotenberg_fallback_volume_container = original_fallback + end + + def test_rejects_encrypted_docx_with_an_explicit_error + with_word_document_conversion_configured do + File.open(Rails.root.join('test_files/submissions/encrypted.docx')) do |docx_file| + result = FileHelper.accept_file( + { + filename: 'submission.docx', + 'tempfile' => docx_file + }, + 'Submission', + 'document' + ) + + assert_not result[:accepted] + assert_equal( + 'Word document is encrypted or password protected. Remove the password protection and upload it again.', + result[:msg] + ) + end + end + end + + def test_word_document_conversion_requires_an_image_and_work_directory_source + config = Doubtfire::Application.config + original_image = config.gotenberg_image + original_mount = config.gotenberg_workdir_volume_mount + original_fallback = config.gotenberg_fallback_volume_container + + config.gotenberg_image = nil + config.gotenberg_workdir_volume_mount = '/host/gotenberg' + assert_not FileHelper.word_document_conversion_configured? + + config.gotenberg_image = 'doubtfire-gotenberg:test' + config.gotenberg_workdir_volume_mount = nil + config.gotenberg_fallback_volume_container = nil + assert_not FileHelper.word_document_conversion_configured? + + config.gotenberg_workdir_volume_mount = '/host/gotenberg' + assert FileHelper.word_document_conversion_configured? + + config.gotenberg_workdir_volume_mount = nil + config.gotenberg_fallback_volume_container = 'fallback-container' + assert FileHelper.word_document_conversion_configured? + ensure + config.gotenberg_image = original_image + config.gotenberg_workdir_volume_mount = original_mount + config.gotenberg_fallback_volume_container = original_fallback + end + + def test_converts_docx_to_pdf_with_gotenberg + successful_status = Struct.new(:exitstatus) do + def success? + true + end + end.new(0) + runner = lambda do |work_id| + FileUtils.cp( + Rails.root.join('test_files/submissions/valid.pdf'), + Rails.root.join('tmp/gotenberg', work_id, 'output.pdf') + ) + ['', '', successful_status] + end + + Dir.mktmpdir do |dir| + source_path = File.join(dir, 'submission.docx') + destination_path = File.join(dir, 'submission.pdf') + FileUtils.cp(Rails.root.join('test_files/TestWordDoc.docx'), source_path) + + original_runner = FileHelper.method(:run_word_document_conversion) + FileHelper.define_singleton_method(:run_word_document_conversion, runner) + begin + result = FileHelper.convert_word_document_to_pdf( + source_path, + destination_path, + work_id: 'test-work-id' + ) + ensure + FileHelper.define_singleton_method(:run_word_document_conversion, original_runner) + end + + assert_equal destination_path, result + assert FileHelper.validate_pdf(destination_path)[:valid] + end + end + + def test_gotenberg_uses_an_isolated_host_work_directory_mount_when_configured + config = Doubtfire::Application.config + original_mount = config.gotenberg_workdir_volume_mount + original_fallback = config.gotenberg_fallback_volume_container + config.gotenberg_workdir_volume_mount = '/host/gotenberg' + config.gotenberg_fallback_volume_container = 'fallback-container' + + assert_equal( + ['--volume', '/host/gotenberg/test-work-id:/workdir/gotenberg/test-work-id'], + FileHelper.gotenberg_volume_arguments('test-work-id') + ) + ensure + config.gotenberg_workdir_volume_mount = original_mount + config.gotenberg_fallback_volume_container = original_fallback + end + + def test_gotenberg_uses_the_development_volume_container_fallback + config = Doubtfire::Application.config + original_mount = config.gotenberg_workdir_volume_mount + original_fallback = config.gotenberg_fallback_volume_container + config.gotenberg_workdir_volume_mount = nil + config.gotenberg_fallback_volume_container = 'fallback-container' + + assert_equal( + ['--volumes-from', 'fallback-container'], + FileHelper.gotenberg_volume_arguments('test-work-id') + ) + ensure + config.gotenberg_workdir_volume_mount = original_mount + config.gotenberg_fallback_volume_container = original_fallback + end + + def test_gotenberg_worker_runs_in_its_own_network_namespace + config = Doubtfire::Application.config + original_image = config.gotenberg_image + original_mount = config.gotenberg_workdir_volume_mount + config.gotenberg_image = 'doubtfire-gotenberg:test' + config.gotenberg_workdir_volume_mount = '/host/gotenberg' + + command = FileHelper.word_document_conversion_command('test-work-id') + network_index = command.index('--network') + + assert_equal 'none', command[network_index + 1] + assert_not(command.any? { |argument| argument.start_with?('container:') }) + assert_equal 'doubtfire-gotenberg:test', command[-2] + ensure + config.gotenberg_image = original_image + config.gotenberg_workdir_volume_mount = original_mount + end + def test_archive_paths unit = FactoryBot.create(:unit, with_students: false) diff --git a/test/models/task_test.rb b/test/models/task_test.rb index 3597d8a86d..cc6b60a1f9 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -938,6 +938,126 @@ def test_pdf_validation_on_submit unit.destroy! end + def test_docx_submission_preserves_original_and_stores_pdf_history_preview + unit = FactoryBot.create(:unit, student_count: 1, task_count: 0) + task_definition = TaskDefinition.create!( + unit_id: unit.id, + tutorial_stream: unit.tutorial_streams.first, + name: 'Word Document Test Task', + description: 'Test task', + weighting: 4, + target_grade: 0, + start_date: unit.start_date + 1.week, + target_date: unit.start_date + 2.weeks, + abbreviation: 'WordDocTestTask', + restrict_status_updates: false, + upload_requirements: [ + { + 'key' => 'file0', + 'name' => 'A Word document', + 'type' => 'document', + 'submission_history' => true + } + ], + plagiarism_warn_pct: 0.8, + is_graded: false, + max_quality_pts: 0 + ) + + data_to_post = with_file( + 'test_files/TestWordDoc.docx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + { trigger: 'ready_for_feedback' } + ) + project = unit.active_projects.first + add_auth_header_for user: unit.main_convenor_user + + post "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/submission", data_to_post + + assert_equal 201, last_response.status, last_response_body + + task = project.task_for_task_definition(task_definition) + conversion_work_id = nil + converter = lambda do |_source_path, destination_path, work_id:| + conversion_work_id = work_id + FileUtils.cp(Rails.root.join('test_files/submissions/valid.pdf'), destination_path) + destination_path + end + original_converter = FileHelper.method(:convert_word_document_to_pdf) + FileHelper.define_singleton_method(:convert_word_document_to_pdf, converter) + begin + converted = task.convert_submission_to_pdf(log_to_stdout: true) + ensure + FileHelper.define_singleton_method(:convert_word_document_to_pdf, original_converter) + end + + assert converted + assert_match(/\Atask-#{task.id}-/, conversion_work_id) + assert FileHelper.validate_pdf(task.final_pdf_path)[:valid] + + Zip::File.open(task.zip_file_path_for_done_task) do |archive| + assert archive.find_entry("#{task.id}/000-document.docx") + assert_nil archive.find_entry("#{task.id}/000-document.pdf") + end + + history = SubmissionHistory.create_archive!(task, '12345') + Zip::File.open(history.archive_file_name) do |archive| + assert archive.find_entry("12345/#{task.id}/000-document.pdf") + assert_nil archive.find_entry("12345/#{task.id}/000-document.docx") + end + ensure + task_definition&.destroy! + unit&.destroy! + end + + def test_docx_submission_text_is_included_in_final_pdf + unit = FactoryBot.create(:unit, student_count: 1, task_count: 0) + task_definition = TaskDefinition.create!( + unit_id: unit.id, + tutorial_stream: unit.tutorial_streams.first, + name: 'Word Document PDF Text Test Task', + description: 'Test task', + weighting: 4, + target_grade: 0, + start_date: unit.start_date + 1.week, + target_date: unit.start_date + 2.weeks, + abbreviation: 'WordDocPdfText', + restrict_status_updates: false, + upload_requirements: [ + { + 'key' => 'file0', + 'name' => 'A Word document', + 'type' => 'document' + } + ], + plagiarism_warn_pct: 0.8, + is_graded: false, + max_quality_pts: 0 + ) + + data_to_post = with_file( + 'test_files/TestWordDoc.docx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + { trigger: 'ready_for_feedback' } + ) + project = unit.active_projects.first + add_auth_header_for user: unit.main_convenor_user + + post "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/submission", data_to_post + + assert_equal 201, last_response.status, last_response_body + + task = project.task_for_task_definition(task_definition) + assert task.convert_submission_to_pdf(log_to_stdout: true) + assert File.exist?(task.final_pdf_path) + + pdf_text = PDF::Reader.new(task.final_pdf_path).pages.map(&:text).join(' ').gsub(/\s+/, ' ') + assert_includes pdf_text, 'This is a test word document, with at least six words.' + ensure + task_definition&.destroy! + unit&.destroy! + end + def test_pdf_creation_fails_on_invalid_pdf unit = FactoryBot.create(:unit, student_count: 1, task_count: 0) td = TaskDefinition.new({ diff --git a/test_files/submissions/encrypted.docx b/test_files/submissions/encrypted.docx new file mode 100644 index 0000000000000000000000000000000000000000..2ee05b536f2a734e8cc17cbb48af5cb499e272a8 GIT binary patch literal 19968 zcmeIZ1yo&KmM)3}cXxujySux)1a}VZ1b26LCj^4Ky9W#IuE8xK+{0ggx~sbD-m30* z#~b~|t9iz^*V@bGTGRI4Fy_~3l&ykV+-;EmKms7(AirK=Kp_9B9UNE({e6!N0s{8u z{jb;8*Wb-hzzQ@Fz@~rE{|6-S0^H*}puXO8_@--!KkH<`@~!U;u)sPT5b!`C0D%Yu z5)eQW06_r)6$msS(1E}J0uu-5g*b>kuyg|P1Zv=)CrqHl{NCo9o&2BZgFylG1o}GYTO9F! z3luGZ0e{B@iQEiw*E<0lfc5L;{RLzxltF|7iSMKMWAzY=0RE2_gpM zGE!h8c z$3Iy|1_1;8&HtZ0|J%}Xf3E))&)@v+O($>V?>O)~j=$ai-}e8vHUCBbGbF$V)Vn;; z`rdpH0JK8@h&j+37(hh-u*WyQumalUn~gI3T~B$lpWgsh|I76DuJAVB^3U>Z0U{v& zQCwgFj#2$1UcLE^#eW$87jOE{o(-JzALuv#e`Y;3pcIUN+?;_j0s!lFz_ql1c-#M* zKY4>l0Ds&d!XT``k_mXF2Vn={0Pal!SYrd$nEz-K0ZLK~*z?va3~XlxN}Czj!UVzz z!t%#BUl0Z0djG6m{#I6hIs4z+>u){1mA~mf%Sax0Mu*?s|FFab#-F#?@%G;EX3_s} z0&u|Fd*g2o|CzruP{MEW{pScD1oV&i_ZF}I`d#&}XMa8XOZ@#^{&nB~QyXIYZ^YlX z=Y-IooA}Rq`Y)gV-=05jw*2?#_qqSx^XDI(|3B~VZ#l62DgQU!zdcv~9{r#ALjs@o zPQVyr2ei8X>bu7qFVz2r{NHSKU7SXn+vN|Mz#$KWqPQa)No=#Q!CF9*iQLSboUH)%{3Of_Oe92R_9g&RD|-um5_L5RdM*+o zXBQ)TQzKh|y%|4=r; z>F%6Vn!ms2jn4U zT!05>VFz|v8aZ43rH_M+gvi9o!P3l0*wzBzWaVOM$4??GuKK%G)Y8bx{tx;7uE_&T z%|t~+|FXzmO;VK-W@lyrDgd|*4I^7u;66k=lsG+ggn!%3zsi3};9nB>|DOc>|NIVq zLS?>N&r}Lu>_$+zqTthU0nzProgY=%4zywq9dMmqIf{WtIaJ}P2;sT?jBXlZBV{Zk z!{hYA>%)fExB|7#h`_Yrf$2QmewtD0z1)8*UA{KWKda=btA@uY?M}>4&7VW(0XiO& z9R)UFt!i4pgbaQJS6)sIogs$h+MP<&LV*@oZfV~*{OL36PEg9FEur&u3Hd0Gn7rO5 z23}x;C?%h!T-~kR{(=x=->-6=jPM}=mb)DsWle4{LW7w#hdk~V2AOfx`3tomm9u); zaC@W96`q}TDg0n8^HSw{K%!lQ-O*shcXHWq^^}wGs;A4;+cXjXOE5wvggGbIWE8oQ zWQ1Df1LJD~Q@Lu)b-J^##rm0S;U&UZv7E_MajRC!Z2u9#{5VARudf9lXIZpKan~53 zc_7p;rsbZv#^*+w5bw`pwF=c$(|n+o))mP%2mw?dRnsT5C>xD<6V|HwBnwZV!+kHh zRHQvwxHtN{Zxpj^@8E?&pg43^QvDdXv>BB6#@uFa#M0RN*>Z>U?YoUQChb58`10HB z#a!NYSJK)037nno{fkoFFBL?k2Ecq11PI{5R#8aCV7y2htFU5KelbwG>H(rI~@ zR%~e!4wl>cFa$l83B@7b?(1vNPDrGaAJGx)kj~eFk;OpWwM`=_xrhwHscd5#%^gDZ znEPV+apN)crKJ$fvhS%l&kP@s{Bq+FzhisKqoF;Ej%ls6}#=OJkLA{G=gJD&>vL%wm22prnU05T&#`pyNTs zH>KikLQ0Z?|MDKczkD9%oradiI(MAyg>sfwB2k}K@8`6vUxd0NAzV6GGD8v4A#}49 zKPM5nx8w~z5t*fz0(ss3JaD; zY2mToa`B-F>f5qTW&!s3qvngz=x!)-l0&?HI8j$U#tNyf0^S#|Lhj_Ji& zM9|3m8KQby<-C(|D5B8@1Q~ziIO&?DuWNGHa!U;mio2p4!5zQ3tLItCQkx$NL&La1{hgu?TL14zsNS)Q= z+Fj};%%GWp9;N+lE}l?)&sLCID)bjbq6+p@xh0VW)N@~1s8p&PJ$y%WX!Z>a4NdvFCRRIY6p$JuW`RBwb@}x;sPsa zhyLJ)SPxnpFhW!$k4+e_RrjTM7#g@QDD$^;F>4!v`!V6n47XK9^QYVtITnW7&1WHF~5Cck^pV>ko~wy2oE4U^DyT1fE37y9v8CA2UxO zH5cSSwoY*H;=~R)zq%=lc7jRLSyoq2NM|H`itE3h7P)Kkjn#dO2`KRY1bbjzkrY-^Up`VT!2W+bf+5UF-JqC;ZxyC%ErQ> zxa%Ndj~fWON{CW?NHrhI9z4jE;UuilF z&Z#8cFP!aQ>HIo!i!!4hIjbN#bW}^8SXGWDL#_>Xf=7x8ggF>}v~3nvH9v%U&AM{r zw#vNu!~!6JE~5&wIww=;4y~lQ)S6x?5^2zA6yt%!0CATQ5n&J^MyWv!LKMl>klG`N zw}kJAY2uXd)yKJ)eH~!f0%zS$#c!;Cy*$(?Fw9*GWN_<)L^>oKYhxsOEVFCgat}ojVV|n|0Woy~6^^()8WH}{U1j%<~QQK6Dx-wO_ zE=&fo3nKL*x012kvy@vaQVjd9qM{>_cCmW!^t?8Irp&gCXsiFACdo?Rf%r ze0;@E@LuoA!Ci_LuRvkq z+@8jnWr(-j(%m*GMX8ax&*5RelWwD%>)IpOM@1wEZs_*TXWu>7-db#J3uC8wEFnZp`o|^Rh<1vdq_Go7bRb%0#4LghkM1j?@%An42BX}!Tbl|{MD^=8#>sf)@)&o0=0f8y)ApvTlM^i7_Us-G z{AQE7jGH97#$CvXuH{&3(N}*dawL#*&~{52a~VCAeQd%bfvi}!w6SCAPaRyARZ#R(gH%$m%J;g zx3)hfTKl6v5_b=zILu>ciqj}hW~p(3gnD0EV~+rr57vA9;*>36!?SYg(QL4kuEv9p z8JaaT+6ekz!TPeKPkgKyF$|Fql{9@-t?vW@evv-yHBzr$_sGE$v{>I6UF4rFwy!cU zK$vWet$&J~vq$xM2n)hGe?l7|*VdWcU{kVkBi?dVYs6)?iwqKy5kuqL9$^qqAYh21 zvjAI-J2Wts!Qjz5XBPCxJ0%(#XCKfVz+)5aNTL#otQV-;qq9~5B_!!!@zMpm8k4-q zFvWCC=XVh*dNrt$YGQb>UxoSB0IQYmUd~B5)OYSzA!7Q4shM>lmRgynG}Jk9J@TOi zp%jEGjdbY4!=v95r<4h|R!S+Q}pTfMd@EAB6;n)w+ z3o~fVRE3ayRqeUnAAq7c{WU^7!n4d8yA{$Yh_zbIwI@gz&9$BQ`8A&aBve9Y&!l`L6 z-kiJrIA|0};by~MWO^$4fCJ+kUQ*S4S>?|`Dz=iI&)$Hl=V%|RLt0q5_AE56Z+{OM zZGVVl>W^ON*be>Bt*qQ`Yv_xTMw*|__X!_FNa2aH+r3{qH6$4lwGnl16hBD(oehD%k)M{TBjK;y~Pv-;;inB2PUr=}0FUD0q$xQQJIMzr|c zU0fR@h>E&u4#s#g2SJ#w$xV07ZCCe%!nY_*vrfb-fJ1!!2WnS4PgIOdR13AO=MQ>z zL=VtvVnckRdzLXs3v=aT0u0qS@@DArI<5M19GB!o(3cTvmJZupOz*d(G6{HnD>oi* z5QQjqVuO-XNtVg4vZ!hMl%vDv{e%V7o{QeculmSN+E0@eIY%jX%j5m@C6=YT}ZG5DxP@5md7A!i61-0URO?$=3a+Hh2Od-)}q z&=FDl`m~K=_nKbJRgm&eVw^i*RB0u4D)k(l<0^5ub*=o>}`zVKWs=#RY;vz*m zD6luA*fqXFl7t9|j_{l0lb_IsXl5EAj19qesf${)<6@8u;C5!!q-$RXkTb9`LVrgx zI5ywCbw|u2LgjmkU83F7<0`AJgR@|&()o~80lJu(pXZg3!JdK7y|fM~N*C|q@$Hcc zAh_6$A_+~9L-lgoLMh_PGN*bMK0T!Hl6u%Gq^0?j=%|A=+Tse#4kLKDNjZBH@M*>) z$N6fXsDhad0{VOYXO_F%jLTYLqRB0xpGP_B2X_T8^||J!Rt<;k#|xHE-BOK~y{=fpRVt!l`A8 zni^#fVAvQl$%#dxLee`>(q#wF1HUZ#K?19n0ivb@kO_=7$6N1vHeye5mt1**_2hiF zkE!Eh1ZvfCl7a+c>0(uUS}=JYN5(jg?0bgNS($m%- zG%D3h*<|(Pxit=^9rL+^ofJmHX`5&0=`y~cMGanUj)kT;erDQQ$HMitehzbYgf|a8 z-_4;Q=bxzpQV(JKRljDkW88;EswjM84l2{ttIQ;R|LuaN0G*9q1WVjU^^4sX=wqaYd}zJ1DSFB;$=6Cnr-xlH=Pa1Okrx8EcUmV?#{Kn$imjp7rVO zVv5NFb|@VNGjVzER(nI<8k3jmmC}c||6P%Gxyn(~gRBJ#uSG}#)JyV|+QW5Xw=?y! zZ2^6!A2^ys`BR00gtl9sT!M9MbgUo?-=Mc2{+t681ydYxgI(qWs95s&Bnj&fYE}JS z-Y~_lN!Sn7vjEu~K&=(}b+(iO?~GkJxREp6R79Z^T+d z0CN3#0cRZ!OV)iIS*qr;`-Ic-fZb#Lp{i=zPbo>f+3TvT1XtxoA-)~o3~+7eYsKH2 zVpr%&nJG;&#CXFDgOhYEyCD-Zm=HhCCCecaTq8CaiC|xr&X;rpVwzVRu4Ln3w5^?v zyHByJsfHdIQGitaBpYY&or1WK8|k~ql?#fyzHN~1`*-1ECLV5n#Bf-|a?jCXaTo}v z>vCvFaeAu)KMr?qGYto&P>eV^=J^+UntK_^eBs#xSy_g@#Fq41@*pSuSPs`7=b?0> zwZpB^rvaU=LshUL_W3pOagnXoM<(I9`gYM%sPfTtr;qS|>D<3VyR1xY4O=P+H5Ha67-P;k~M#b4dKfv#_n z`yG|#5k3WKvoUj2>Q9rn`5M>+u9={IGWxY>S^rZp90_WAOcYUog6t?WP@>>Acn-%& zcdnV+1fi*Y$JEZeS*}I5#a=9+@7Y}krng{f< zP)THI@Hx~41#A|ChF&Y#b`B;iwEz(}-ReC>sb7nL4~xa%9UvhHpY$Cy^=^fGys?`LFCIli+qZd6o(ziu zS#<6ih~C813Y=^yZa*byRWGlb#-Ep_@U_k~Ii2+U6H%{m?qqZkg*1px)F!682v+{- z=_9&3CiKr-4cj9zAPryhSg)^12i6cSY1<@cMekHXumGLtHR~di+!@y42BDowg3)z2 zsoQ$UGtu!z#IX;zEV$6g<$OVw$<42=2e5LW)HF%r>qi^DN8}B@3Hl#?Evl22fX7hT zP!NrVkLU0B=np%v_|88hl-GSl8s#K>&t|cwP!KTOA{i0BM&58WxblkbJ8v$sd5wPg zN=7Pn{N7~5o(GElh}x+(465kYko9aIY31^{+^4Qd)!0fswS~K9qhq&_8=Sf3l_5Bc zAI=w;{)BtgAB^%f&h6}=LB%3oPKz?qTJS%JJ1gqruF8bO%p#MjA>ycdmWk2PR533p5}}Fx~?~%1%R{ALZ6;BEEg8uNq4I zv7n)Ny^0h6jjp`2U`GWRin+s%lvZc^j|NCpy&x?MQDLP)Cg#z)>8 z%^XV}R1KC6Fz@$8Y@TtZ68&m(tT4bnFHCe%|zSJ@xR0QKc1`;>cW~%uX#l?QK*c<~7bM zi)}2(LOGIVyt6da0v+?W_aPU94XSz^)4Im&>L`Jn$DP(;^g6Be2#hU9u{+OM&8+!y zo=Zy`0votKTK-FH>@=ph5aN43wgKpTS*RkO6IdqCM-?x0nkfz=3CK3J#DWH)af(K&dI5WT@;T*$`&4M)6{L9;l*633eBJKqrWTo{F- z&9y>DoyoI}h7@IL5$&7C{=pQ&RDu10(x8O&Vtyl=+>4oS9dd&&i$b_5GT{6N&FtXa z6vUJ#9#KjgtbFm^bjte_K6#gDQH5Rb;zmLY#|S@;DO5lDUQj{pF~^BVR^T{SZY3kR zgwXmh#=|QTU{qpUXcL(SR%cMPKQ(r>o=Owy;Zx~!^LFK- z%s2YnhxqZer~}ZX6S8Y(i`J3wU&LE{)L<&5>AjiKVq~jF4yNLRqIfF2J19yF5DzRq zOZ1Jdn7K?6oQt>4N#jeqxOgY0?)E!j9TM~D$BlYumclAYsN(N&uKPvd%Y-|UDclzl zVU$h0VLTx%5^{QfRm<;Y9(Ym|9Bwyw?3kv1=0}5kM42PVV~E#I53a4fQG~|9_KFGr zbUm5jilG6@VXwLIWo=5U@Di=6GVBS2jh#B!W(Btz&YShjguxvNFv(}iy{!{M{lmXn zdm6okJcfhN-o@Zy_(f*+{=mjW?9RD_fhIx=4G+rKAMcQCSgJJfoG;wSR?eJpsY|a_ zLyvf>4}5+3U{4_T-2ZWoLz0K|-5H_}Rem%xB#Ri7&_mhqHi6vYiOgYY4hloPbC5w# z6AJokKT#RBqPxg|?654&If+M8Y^E`q;F4Bt7zyqG*!v<)QL){?l`-SA0NlH|JcGn3VHLX;+ld`Z|$L{Hwodv;M30!hB+e+p>*_y zgr>ra1;NZk=hez60_+xpE7JB#t?*1;MZ$nz4%A(YptoJM>j>&}&Y=hyq9S^5(+7Hf z1QKC&w5p6IMoF&iayJAcMK0l#*N@8rU_G?&T3U8KokX@$(fv$FND0P_o*6g=p-SPj z$CHcj&?EQ$%=+0--@@h+!3vqeFZiZ>`J9+vt zPn9R%kxVQGJsP~`iSEEdcJb6IiN+MP0vb;{g{)fV=i|Uuv5!Sszps+Qy-d7_wQljb zS+Jeab}dSOQ%ieHaZ;$s;&rbC#lHCvb9107-wM~+IfmQPSt0{_x&b8xCsaFXe|7%> zx?wqCEVAFvJacy*kH+jeDFAhEfgsFcOp0_s zNgqO1EtT=mMYhI8DQjMu&o~c45MRgn<5wD z`^>Mf(ClkXdpW*EI0Fg0ur{!s{If`!D&KQH?%zXdE6$<$C+g8QXV`f|hp+R(?sDz? z5@@qSK6c9^nToCplibmWR|5sDKTOrIqOtOXPUuXsUYc!5&qjmvCGw`;(7O>mX?Aj%lV42&ukJs`I*jokDnoa*0$2? z?puQWqrwWabIbVALU-nhu%?)g{XwH0j&!`N%pIMaFIvZ-`njjW7AQ2vZdh`EOn z#G#)`%&2A_M)u_w2qm)oV`8&$PqjMbbf5FF7$JC!vV;dnA6rXOx2F3rmWZFgC^auM zvJDYle;l|LL^Z?sq-dyHr?k~@6ms=GaVKKQ zSn<@)*m?^?ZDXK^8QyXtwZ-Y`ss0mSFCARTShaI$ZSiN1j1TuXKPDh3(EC+UEipjo z$72CT?>%m=*$TBklc(YN26|pz=Q&l8zFSwfv2OWeTUhJdp2aJJ))Z;YQG;X{*u2i3 zl_Z8xV?Ylboi@3_pw(r0AFJKfNJa6I%K^NysU^n}P46Yl2?i>9SILsfH*iXrYYDEdNRq;(pH(63d~Kj_;Bd#6)(!Z@j;TwP`p%0W z+D%Q%YP2CsNCVx4OC-DZ<2%mt%Z~VlgnR9H_YG;?&!&7Y^Ur}g*j)t`t}&G+IHB1> zy+wI5k1|0FkHz*#{@9Oq=|8;cnhZObaFO4+-*TiLBlPp(I)L|VX^@?$=xyQaSeMs| z3m*l|`8*CD7NFuc#~J0yh&7Ycg;n<1nr7%R((p#ouP$&VsM}k`enj=$Om7e3m(m-f z`es^rLmzcBPCB=o9jvj=ljxN)-hAlZ+`3&iHgn(F_+=&f7yLfVjThz(o*C@%OHoix z*(fOo)T$xqYpW}`cuYbFOapBe?DZmDAq@{=cDUT&Yu!1LZ}zcvBO|SA1hzRw&$IyV z(>9x~v~bH^C4{O>&X@0{uo-H@KD$u*2gI8eU1;A@3lHbg;T|0BG8)W6TV0&E4?>YQ zJ{YQ`ct2)Uuc>p0_yMBNFZ1g^see&6n1<@Fx_BmF;=`?Nsqk$^%IOlSfQ_#=Ry;N3nD&lZ9MZsb;YEck?@x>-CF^VGt5 zxz#1C*-n8#1ZOFVXFsCJ4@PGtG%q(2j_-8yVI&#j?kR)j~zQj#Io?ihTOb5mpi=5k{SI z#KODO{hE6m7sS;2e1ArLx?6y;NLQ#?qO^KBncZRF*Eq>Ep)HI)KraUChqVVW7>$Yu zMa02)EfMULr}J4nQt>JwQ_-i7pQ=}K;E+d$8DYqHXOFYSlLOT%kM66iaJZ%l(x~*q zJcYgi|N4ODf#7;GAffg_0IN=5rn^BXouP&#g#q`fu2hO*>(0^Y)@l#UB=%Fktc$xT zPxb3mF(K0&iY!>C@(|~_VKzn!!Z#sSV`5p(G0bPKO4LBi^0@w5HJxpwIkqM~PQF`w z!;%ggbV*#!2Y&3Z0em^ue*R+JT@l25J&XD;1O=hiK5Qu@ngOpT(T1!bpHahW{gh+32K(zMW*Npa|3W$@=zPHi6RSQ~d+Sz14emIW0 zZ?PbSQk@MUkHz12$?LR@E6Y^9M@Yx;-f{l8Fn^wK)ucz24lYH5Hc;o3cxvwORqmwl z`waS_V#HIPK<7;Zmz>QkKbxSM5VV6tY$#LndR@^ENmeynFaKZ{w89B*t@LNgIRttu zEGRU$wZdw>U@cny80l>fB!}!<`83%4`KkU28xDY7sR8dNwQ%B^O8Guq6LfNyR+#NKH7r6r1D)1aBeSD&itIQ`?BpC zXpQPS!V?yKjrTxmX>#`#SA;Q5r`*7>?Sk1}271_;b6T&VpCnP@*Nb~=Rc*)UNKnjh z)uFdxjazI@pnp7nyAi3X2m#mZaAv+-K(Mh9_L*YzUiCOc&1iX#O1`jr39pLJiD7)- zj}&wcW)rb=*>XH;I|EUOwzGbn2=xPZ?m>zDLt?fc@dtg*YH!D{%y2M@Nr*Fgj$L^y z58Qc^ps$-$J1BU*$mz~t#p9x+pc-o8FzWo6Ceo@s{PRxr*l?UkcEo-jD!3)Fk6w32 z3uJJ2_Klww(T`YPst!h$C~HP&ua+zz0GW&OV5x{;{Mumi`v$yVJ|*zUI0*FY)RZk> z<>hk0b(Aqs8YfohuUw<9I}wslFhMz=`BYM0&_5YKS0MRzUP7jYfC!a0oq>?ol+-`t zXat57_0XHiq4EIw>wMwSW8{?&p-#5dzeK|#iOY0#@ntRE!W<-m36>+Haow{!GY8A; z@h)5;R2!|MI7;}r;zr1eb{gQMN_2$lUV0Q=CTLuVeRs+sywO{$4fu@b&^L;0c+eSA zc#0q*o!Fxln&Cg#xeV(A4gDlH_C!Lf}OLyb|Hx|)mURO&|~A)??fB+y_mRR1ZA#5#q_ z!q2xgCsBdGuddqw(JYsO@(pXtHpZY0+BF@qLc-hWIs9kEyJHQM@;urzclL9uUO{=Y z4i(;IklNnc=<))@uWOmzyt=ki9P>AL(%N2P5kRIHE8=H(`@d1yH7FXB+WGBQaFnbQYJaCrV9h`P^*?s8+^hPc<`VTJ9PC# zG_tYE8vwy|g4dQ}#B|fV13nSdPm9LX5+0fMpA~|rOt^|)HFQ2@S zZGZvVY;6Qw!^Xs&H^9aeczu?K3UMiU3pa?zA4=ykkr;YxzD@gynb7F2vRL>!L#GS` zQ)>us;eKT{$TZgoIU=@MhsqS$rE_$@usMZyqhMh0tjx86M43oKQKSKd4Z~c>QiSpC zEmty}QrD_+sA_k@T4|^)E$=B)6|fig%;;qI);2jx*;{_SOl@fI+dPvUj7-Lfa-%8^rBI}cX);T*Ri@-HrA=q(M+0xzwktbDqFTNhSp!( zUrnOCorRxJe^KB5A`0m>!-j}}$w%+>f7O)EZ-YS9r9@5H;I*E(fCGH836eperxe{! z&cc7O8x59szl32!qi?Lh@K@N`mDQsctIitt$%PRGFUXZs{aoE_Tqyu+Ki~SZR~-q1 zi?9`(c*~?{ybEm}8fwtJtVAp2Y9*KprDIjIkPE|vv^rJOEVksBpspZJHO2=BHCj<9yN8;o)$f! z=saQvyG@ePZd8A>Wuc!A^m9trt2f_L7B+N##g{g!FPVhrmOS)xdp{b_=?536jce%q zFBA7MEg*YSP!0MnNwQoc%kjhum5JQm$5q_NjJA=Lnsw?w`#MqFXbxz?#vqyz$=9Mr zR6~Wc=4i_RK;4Z$dCVR=-t&bK%F@4lfkeQ4Z*_X_fLS!E#cz5B&w~3srkr?cRiQrE z1HtN>7E0TmhTK{0?0$DSd;L)Sa4|!<^kKZ!U`UVCHY&C3{2e))95)7gTNJM z^UX)*EI#3|C#4Ht$W0)>189q4ipzZe{`t_25|e0=n+8sQfbB=w4@cnVaZw%OYl#Q! zMmhM-EXmmqh>K&1{Ie151nI3`?!}6D4mMc_5w2$fxw(zWK~owo17~r)T;q-A2c`h} zOL=->NYU431>2eSd`6qLSmZMh#CTPj#HynBPpnqEi(2o~Syx7npL<|n9T#m7<23{ziIDk?)Zz2%VL+%`W&Td!{1cYUh+gLugf_o#}(*+u|r zyw%mDKvmXJi8m7AU`c>lqmtolC4C&LqH2u8L7p%T-x!sIDT0JBCYZ5^3_#t3{XM_o z(*fCCSQiLq1(xxW!XBD+Mf66;z1P7Pmk%-kgqPra-w0g$v9Ks zUMT8LDUXp^FV(c!n2vL8La-i7kG(`kkHT3)_ zBGTcfhDG3uV+tOrZApJ2oNP4WZ|j>N(%Y04{qpvsg%{i-i0c9Qv9fxtRinnS;LDrs zn`N-T z|3XNqd|fG!IFOlh9?$!>x&(TG*<9yyGaOhjWx~;T{>Aeh*p|>9`|(p$*}*S0#Y?Xl zyBD9XxTu8_lKE=#C3Pn$m%+Am5ATZbk|Mr**HY>Dku6vO{!vCOhR=R0Ru9Akqah~_ z+;TKek{^%QEJ{YiRBcpPLO#Zaif&bc`=mgS%XkEuR*uFyg5itMAWMOOnu~^{sk@CM z-f(_L*#&=Dgcyrj*RDF5K(jiwP&2QGu-B8^{b@dixliof z6axMzYVg%cQ&x0=b0PX&Jr6HOksy%=CxYWQ^5tvE);LiY&K9-1V0wGg*zn1Ig0jg0 z?*nrJ$M09KgYDLJRQL>O7Sk0r?99J5_99_chv%ewwpFar1M}c)r-WtNDD&)v1kd z6FEd`|0vhC^Zg;qDWYi&p@8yp20opxln#3jt7PLxsqKBl-V-#fe2Wn3HJn52eU)M# zqHa}%yPo~sa-ZT&JXS+%7k+G5EH{FUGYc@^b??tzlQ=Ef_!LM!RJ0ji*(u5_g1BC5 zPg!TkVpo{@-FL4hy`n_*&1}==@bI%w)SF840%qh35e+O z5xz4JC};Ls*}mE|B3Ct8@Gi|X%U^JKo@|G=+N!oiH87>S_*`UpGa*@1@1~kN(A-g_ z6PLM}VuuM<2Mg6@B9c8hX2tN;{=gC2vkD6RCg&Z)C?FERY1`?c`l43-WG#IlDwj*v zur=hCxA0Y#rff!$!F@rIQJ63=?CBcSm9*_Wsln_psZ_1t3w^l@y32VaxY@co{8%Nz z_g{pf?e7E1Oxm$L3GGkHX4N#(@A=+Mtw$QJS1M$xQg6CHTDx3{BK=6ux4l*}cHkFk z$QQB|u_WRtBBq7{O;NGVm2_{R6m*Fs`I*o|<@ zyXN-xXw<(EmtTACF;9!mRK+4A_UCT%sjS%W#)%s)Lme;${S!_YRw618h)icDcG)4nyp*<`}|)%Mcb z$(4Mog=WeT(r{c26zaxudtDBn37sHaUomK~Yt3enOl<0V7Wh(zmjg%0T!FpJ#}kd% zA5N58&CE5QU61D2yx03tzEX3svE(xnrh*5`7z{$}`GpGgzw5lu$!fmaf6qT#g$@;F z(X-{jL~LkqQ#%X?N$Z53mOry-0seViR8K44o9MK^bLwVbys<#6v7y*^v-1)1ph{sT z`}4ThMCUjENZHV#qt>|P^469mkbHXDn{|o@+;RWoz1a!kJn^A!kod340klH}j|S$B zWDPWX7X=~iTmbT#qrN!pTJ%BvWr0E=TWOQVs5o(tPy#_l6Sw~J>J$DbSpo*_Ev!b&EbEh&+ zw0LuwQ9SNVa*Rqe&#)_}q?afTiX$Tm5DDF%?>fHbFzTAw&M@d;rpI8R6^I^!TDN)y zDK%R^;ai|utgu`n`&GZJe4V3cAFW$~?MRlpq6^JN**T~d_~Pxyq{hN2D$i|crO3h| zA_XuN=I1A26yP%jrmRZayO>!3b51?q(svXccvgS!HCLrEJtCgeA?L#H`Jo9Jn=P6r2D~N=n>p zoE|cg_AKIDQmh>EW|q#@)=GBj%0S+K%j?ffRZf7Jow2gEGcB_@qlO5hqLqfUlbD5@ zriBf+9Gg718LgX}oi@PT#a&EGOI=%sOUjtT%GzGmgH1xs%hS=-#l%BOR9Quq!%JRD zSxm{*O4-%cQOV*jC;jUjVCO&5b^k3Tn8?-1>c5)={BPOB0(=fU|0X}6?1-Eltn5Vr zuJ$g#Okk!r`b$#ozb6C#Psk0{umFll*4|0d!J3;zfz!%O*iH)YxBAgAa{^{Ln>k4V zvz4Xo9bAFuaj=upP@vUdSG6)$1{hnLNZRW-TL157{i$6y5iL_=IYlQg2}NyN6;CHN zNq1vWS{Hd!d1(hOVKrwjHycZFc5!!4DS0OqV`Xbo6=w-e4ih;QWo1rf2L(w}J7Hr9 z4G%UaRc;-Cu&f=Ms;Q*?n;rkVEdk~JCo@xLCv#DNqdCCVoR!5*&cx1DRl?3%R^HS^ vTftmKQp?pshRZ?Gor_h1?GHy_{A>FV;QO0fpt<~20=mFIlPv!~aLoS#G7w?M literal 0 HcmV?d00001 From b48edb71fc58eb7ee8e878ce8036b13f5b2f8cf4 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:43:25 +1000 Subject: [PATCH 164/199] chore(release): 11.0.0-32 --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1459963b2c..57d3c0bd7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-32](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-31...v11.0.0-32) (2026-07-16) + + +### Features + +* add rediscuss status ([05a44fa](https://github.com/b0ink/doubtfire-deploy/commit/05a44fa8e9954ed82ea149072288ebc546d5f9e9)) +* prompt for admin password when creating initial account ([7640a36](https://github.com/b0ink/doubtfire-deploy/commit/7640a36f682f6dd7240cb764dea6e46b7d01df35)) +* support word document submission ([#647](https://github.com/b0ink/doubtfire-deploy/issues/647)) ([a21daf8](https://github.com/b0ink/doubtfire-deploy/commit/a21daf846a46557331c3235f86f28a5c8a020786)) + ## [11.0.0-31](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-30...v11.0.0-31) (2026-07-08) From a87b250303b0aff8a431334e8b19a40f790cbd35 Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:59:13 +1000 Subject: [PATCH 165/199] feat: content management system (#642) * chore: init cms * chore: fix rubocop * feat: expose if task definition has linked content * chore: fix rubocop * feat: ability to replace existing site * fix: ensure site name is uniqe * feat: expose units content main site * chore: reset schema * chore: bump migration * feat: support task resource via content site path --- app/api/api_root.rb | 1 + app/api/entities/task_definition_entity.rb | 6 + app/api/entities/unit_content_link_entity.rb | 13 ++ app/api/entities/unit_content_site_entity.rb | 14 ++ app/api/entities/unit_entity.rb | 7 + app/api/task_definitions_api.rb | 10 +- app/api/unit_contents_api.rb | 213 ++++++++++++++++++ app/api/units_api.rb | 1 + app/helpers/file_stream_helper.rb | 10 +- .../similarity/unit_similarity_module.rb | 22 +- app/models/task_definition.rb | 63 +++++- .../turn_it_in/task_definition_tii_module.rb | 13 +- app/models/unit.rb | 23 +- app/models/unit_content_link.rb | 27 +++ app/models/unit_content_site.rb | 180 +++++++++++++++ ...60713232535_add_unit_content_management.rb | 31 +++ db/schema.rb | 28 ++- .../models/unit_content_task_resource_test.rb | 123 ++++++++++ 18 files changed, 765 insertions(+), 20 deletions(-) create mode 100644 app/api/entities/unit_content_link_entity.rb create mode 100644 app/api/entities/unit_content_site_entity.rb create mode 100644 app/api/unit_contents_api.rb create mode 100644 app/models/unit_content_link.rb create mode 100644 app/models/unit_content_site.rb create mode 100644 db/migrate/20260713232535_add_unit_content_management.rb create mode 100644 test/models/unit_content_task_resource_test.rb diff --git a/app/api/api_root.rb b/app/api/api_root.rb index 3dbc682297..cb583d7a2c 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -96,6 +96,7 @@ class ApiRoot < Grape::API mount TutorialStreamsApi mount TutorialEnrolmentsApi mount UnitRolesApi + mount UnitContentsApi mount UnitsApi mount TutorNotesApi diff --git a/app/api/entities/task_definition_entity.rb b/app/api/entities/task_definition_entity.rb index 6027226c3d..9ea759bafc 100644 --- a/app/api/entities/task_definition_entity.rb +++ b/app/api/entities/task_definition_entity.rb @@ -1,3 +1,5 @@ +require 'entities/unit_content_link_entity' + module Entities class TaskDefinitionEntity < Grape::Entity format_with(:date_only) do |date| @@ -43,6 +45,10 @@ def staff?(my_role) expose :has_task_assessment_resources?, as: :has_task_assessment_resources, if: ->(unit, options) { staff?(options[:my_role]) } expose :has_task_assessment_script?, as: :has_task_assessment_script, if: ->(unit, options) { staff?(options[:my_role]) } expose :has_scorm_data?, as: :has_scorm_data + expose :has_content_link?, as: :has_content_link + expose :content_link, using: UnitContentLinkEntity, expose_nil: false + expose :has_task_resource_link?, as: :has_task_resource_link + expose :task_resource_link, using: UnitContentLinkEntity, expose_nil: false expose :scorm_enabled expose :scorm_allow_review expose :scorm_bypass_test diff --git a/app/api/entities/unit_content_link_entity.rb b/app/api/entities/unit_content_link_entity.rb new file mode 100644 index 0000000000..6e548428ad --- /dev/null +++ b/app/api/entities/unit_content_link_entity.rb @@ -0,0 +1,13 @@ +require 'entities/unit_content_site_entity' + +module Entities + class UnitContentLinkEntity < Grape::Entity + expose :id + expose :unit_id + expose :unit_content_site_id + expose :context_type + expose :context_key + expose :route + expose :unit_content_site, as: :site, using: Entities::UnitContentSiteEntity + end +end diff --git a/app/api/entities/unit_content_site_entity.rb b/app/api/entities/unit_content_site_entity.rb new file mode 100644 index 0000000000..dc03e4f45c --- /dev/null +++ b/app/api/entities/unit_content_site_entity.rb @@ -0,0 +1,14 @@ +module Entities + class UnitContentSiteEntity < Grape::Entity + expose :id + expose :unit_id + expose :name + expose :original_filename + expose :root_dir + expose :root_dir_options + expose :file_paths, if: ->(_site, options) { options[:include_file_paths] } + expose :is_main + expose :created_at + expose :updated_at + end +end diff --git a/app/api/entities/unit_entity.rb b/app/api/entities/unit_entity.rb index 46f26976be..4f7a887cfb 100644 --- a/app/api/entities/unit_entity.rb +++ b/app/api/entities/unit_entity.rb @@ -1,3 +1,5 @@ +require 'entities/unit_content_link_entity' + module Entities class UnitEntity < Grape::Entity format_with(:date_only) do |date| @@ -41,6 +43,11 @@ def can_read_unit_config?(my_role) expose :active expose :grade_values expose :grade_definitions + expose :has_main_content_site?, as: :has_main_content_site, unless: :summary_only + expose :unit_content_links, + as: :content_links, + using: UnitContentLinkEntity, + unless: :summary_only expose :overseer_image_id, unless: :summary_only, if: lambda { |unit, options| can_read_unit_config?(options[:my_role]) } expose :assessment_enabled, unless: :summary_only diff --git a/app/api/task_definitions_api.rb b/app/api/task_definitions_api.rb index 5e0be83ee7..239efa51b4 100644 --- a/app/api/task_definitions_api.rb +++ b/app/api/task_definitions_api.rb @@ -1,4 +1,5 @@ require 'grape' +require 'mime/types' class TaskDefinitionsApi < Grape::API helpers AuthenticationHelpers @@ -658,7 +659,14 @@ class TaskDefinitionsApi < Grape::API error!({ error: 'Not authorised to download task details of unit' }, 403) end - if task_def.has_task_resources? + resource = task_def.linked_task_resource + + if resource + path = resource[:path] + filename = File.basename(resource[:filename]).gsub(/[\r\n"]/, '_') + content_type MIME::Types.type_for(filename).first&.content_type || 'application/octet-stream' + header['Content-Disposition'] = "attachment; filename=\"#{filename}\"" + elsif task_def.has_uploaded_task_resources? path = task_def.task_resources content_type 'application/octet-stream' header['Content-Disposition'] = "attachment; filename=#{task_def.abbreviation}-resources.zip" diff --git a/app/api/unit_contents_api.rb b/app/api/unit_contents_api.rb new file mode 100644 index 0000000000..1b4d9f924e --- /dev/null +++ b/app/api/unit_contents_api.rb @@ -0,0 +1,213 @@ +require 'grape' +require 'entities/unit_content_link_entity' +require 'entities/unit_content_site_entity' + +class UnitContentsApi < Grape::API + helpers AuthenticationHelpers + helpers AuthorisationHelpers + helpers MimeCheckHelpers + helpers FileStreamHelper + + helpers do + def unit_content_link_for_route(unit, content_route) + normalized_route = "/#{content_route.to_s.gsub(%r{\A/+|/+\z}, '')}" + normalized_route = '/' if normalized_route.blank? + + unit.unit_content_links + .where.not(context_type: 'task_definition_resource') + .find_by(route: normalized_route) + end + + def authorise_unit_content_management!(unit) + return if authorise?(current_user, unit, :manage_unit_content) || + authorise?(current_user, User, :admin_units) + + error!({ error: 'Not authorised to manage unit content' }, 403) + end + end + + before do + authenticated? + end + + desc 'Get unit content archive' + params do + optional :content_route, type: String, desc: 'The content route being loaded' + optional :content_site_id, type: Integer, desc: 'Specific content site to load' + end + get '/units/:id/content' do + unit = Unit.find(params[:id]) + + unless authorise?(current_user, unit, :get_unit) || authorise?(current_user, User, :admin_units) + error!({ error: "Couldn't find Unit with id=#{params[:id]}" }, 403) + end + + content_src_dir = 'src' + link = nil + site = if params[:content_site_id].present? + unit.unit_content_sites.find(params[:content_site_id]) + else + link = unit_content_link_for_route(unit, params[:content_route]) + link&.unit_content_site || + unit.unit_content_sites.find_by(is_main: true) + end + + error!({ error: 'Unit content archive is not configured' }, 404) unless site + + content_archive_path = site.archive_path + + error!({ error: 'Unit content archive is not available' }, 404) unless File.exist?(content_archive_path) + + content_type 'application/zip' + header['Content-Disposition'] = "inline; filename=#{File.basename(site.original_filename)}" + header['X-Content-Source-Directory'] = content_src_dir + header['X-Content-Site-Id'] = site.id.to_s + header['X-Content-Route'] = link&.route || params[:content_route] || '/' + header['X-Content-Root-Dir'] = site.root_dir + header['Access-Control-Expose-Headers'] = + 'Content-Disposition,X-Content-Source-Directory,X-Content-Site-Id,X-Content-Route,X-Content-Root-Dir' + env['api.format'] = :binary + + stream_file content_archive_path + end + + desc 'List unit content sites' + get '/units/:id/content/sites' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + present unit.unit_content_sites.order(created_at: :desc), + with: Entities::UnitContentSiteEntity, + include_file_paths: true + end + + desc 'Upload a unit content site archive' + params do + requires :file, type: File, desc: 'The static content site zip' + optional :name, type: String, desc: 'Display name for the uploaded site' + end + post '/units/:id/content/sites' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + file = params[:file] + check_mime_against_list! file[:tempfile].path, + 'zip', + ['application/zip', + 'multipart/x-gzip', + 'multipart/x-zip', + 'application/x-gzip', + 'application/octet-stream'] + + site = UnitContentSite.store_upload!(unit, file, name: params[:name]) + present site, with: Entities::UnitContentSiteEntity, include_file_paths: true + end + + desc 'Delete a unit content site' + delete '/units/:id/content/sites/:site_id' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + unit.unit_content_sites.find(params[:site_id]).destroy! + true + end + + desc 'Update a unit content site' + params do + optional :file, type: File, desc: 'Replacement static content site zip' + optional :name, type: String, desc: 'Display name for the uploaded site' + optional :root_dir, type: String, desc: 'Folder within the zip to serve as the site root' + optional :is_main, type: Boolean, desc: 'Use this as the default site for unit content' + end + put '/units/:id/content/sites/:site_id' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + site = unit.unit_content_sites.find(params[:site_id]) + update_params = declared(params, include_missing: false).slice(:name, :root_dir, :is_main) + file = params[:file] + root_dir = update_params[:root_dir] + + if file.present? + check_mime_against_list! file[:tempfile].path, + 'zip', + ['application/zip', + 'multipart/x-gzip', + 'multipart/x-zip', + 'application/x-gzip', + 'application/octet-stream'] + end + + root_dir_options = + file.present? ? UnitContentSite.root_dir_options_for(file[:tempfile].path) : site.root_dir_options + + if root_dir.present? && !root_dir_options.include?(root_dir) + error!({ error: 'Root directory is not available in this content site archive' }, 422) + end + + if update_params[:is_main] + unit.unit_content_sites.where.not(id: site.id).find_each do |content_site| + content_site.update!(is_main: false) + end + end + + if file.present? + site.replace_upload!(file, root_dir: root_dir) + update_params.except!(:root_dir) + end + + site.update!(update_params) + present site, with: Entities::UnitContentSiteEntity, include_file_paths: true + end + + desc 'List unit content links' + get '/units/:id/content/links' do + unit = Unit.find(params[:id]) + + unless authorise?(current_user, unit, :get_unit) || authorise?(current_user, User, :admin_units) + error!({ error: "Couldn't find Unit with id=#{params[:id]}" }, 403) + end + + present unit.unit_content_links.includes(:unit_content_site).order(:context_type, :context_key), + with: Entities::UnitContentLinkEntity + end + + desc 'Replace unit content links' + params do + requires :links, type: Array do + requires :context_type, type: String + requires :context_key, type: String + requires :unit_content_site_id, type: Integer + optional :route, type: String + end + end + put '/units/:id/content/links' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + submitted_contexts = params[:links].map do |link_params| + [link_params[:context_type], link_params[:context_key]] + end + + unit.unit_content_links + .where(context_type: %w[grade grade_overview task_definition task_definition_resource]) + .find_each do |link| + link.destroy! unless submitted_contexts.include?([link.context_type, link.context_key]) + end + + links = params[:links].map do |link_params| + site = unit.unit_content_sites.find(link_params[:unit_content_site_id]) + link = unit.unit_content_links.find_or_initialize_by( + context_type: link_params[:context_type], + context_key: link_params[:context_key] + ) + + link.unit_content_site = site + link.route = link_params[:route].presence || '/' + link.save! + link + end + + present links, with: Entities::UnitContentLinkEntity + end +end diff --git a/app/api/units_api.rb b/app/api/units_api.rb index 411065f97e..8c6fe71754 100644 --- a/app/api/units_api.rb +++ b/app/api/units_api.rb @@ -46,6 +46,7 @@ class UnitsApi < Grape::API { unit_roles: [:role, :user] }, { task_definitions: :tutorial_stream }, :learning_outcomes, + { unit_content_links: :unit_content_site }, { tutorial_streams: :activity_type }, { tutorials: [:tutor, :tutorial_stream] }, :tutorial_enrolments, diff --git a/app/helpers/file_stream_helper.rb b/app/helpers/file_stream_helper.rb index 1644b3fe4b..8317b9d393 100644 --- a/app/helpers/file_stream_helper.rb +++ b/app/helpers/file_stream_helper.rb @@ -1,4 +1,9 @@ module FileStreamHelper + def expose_headers(*headers_to_expose) + exposed_headers = header['Access-Control-Expose-Headers'].to_s.split(',').map(&:strip) + header['Access-Control-Expose-Headers'] = (exposed_headers + headers_to_expose).compact_blank.uniq.join(',') + end + # Extract part of the contents so that is can be streamed to the client # file_path is the path to the file to be streamed # this will set the headers and return the content @@ -33,7 +38,7 @@ def stream_file(file_path) begin_point = 0 end_point = 10_485_760 else - header['Access-Control-Expose-Headers'] = 'Content-Disposition' if header.key?('Content-Disposition') + expose_headers('Content-Disposition') if header.key?('Content-Disposition') sendfile file_path return @@ -41,7 +46,7 @@ def stream_file(file_path) # Return the requested content content_length = [end_point - begin_point + 1, 0].max # Ensure we don't attempt to read a negative length - header['Access-Control-Expose-Headers'] = header.key?('Content-Disposition') ? 'Content-Disposition,Content-Range,Accept-Ranges' : 'Content-Range,Accept-Ranges' + expose_headers(('Content-Disposition' if header.key?('Content-Disposition')), 'Content-Range', 'Accept-Ranges') header['Content-Range'] = "bytes #{begin_point}-#{end_point}/#{file_size}" header['Content-Length'] = content_length.to_s header['Accept-Ranges'] = 'bytes' @@ -52,5 +57,6 @@ def stream_file(file_path) body = File.binread(file_path, content_length, begin_point) end + module_function :expose_headers module_function :stream_file end diff --git a/app/models/similarity/unit_similarity_module.rb b/app/models/similarity/unit_similarity_module.rb index 819dc288c6..041040d711 100644 --- a/app/models/similarity/unit_similarity_module.rb +++ b/app/models/similarity/unit_similarity_module.rb @@ -252,13 +252,21 @@ def run_jplag_on_done_files(task_definition, tasks_dir, tasks_with_files, report use_base_code = false if task_definition.has_task_resources? && task_definition.use_resources_for_jplag_base_code use_base_code = true - path = task_definition.task_resources - - Zip::File.open(path) do |zip_file| - zip_file.each do |entry| - dest = File.join(tasks_dir, 'base', entry.name) - FileUtils.mkdir_p(File.dirname(dest)) - entry.extract(dest) { true } + linked_resource = task_definition.linked_task_resource + + if linked_resource && !task_definition.task_resource_zip?(linked_resource) + base_dir = File.join(tasks_dir, 'base') + FileUtils.mkdir_p(base_dir) + FileUtils.cp(linked_resource[:path], File.join(base_dir, linked_resource[:filename])) + else + path = linked_resource ? linked_resource[:path] : task_definition.task_resources + + Zip::File.open(path) do |zip_file| + zip_file.each do |entry| + dest = File.join(tasks_dir, 'base', entry.name) + FileUtils.mkdir_p(File.dirname(dest)) + entry.extract(dest) { true } + end end end end diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index 7ef377811f..de21e06901 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -125,6 +125,33 @@ def grade_due_date_overrides end end + def content_link + return @content_link if defined?(@content_link) + + @content_link = unit.unit_content_links.find do |link| + link.context_type == 'task_definition' && link.context_key == abbreviation + end || unit.unit_content_links.find_by(context_type: 'task_definition', context_key: abbreviation) + end + + def has_content_link? + content_link.present? + end + + def task_resource_link + return @task_resource_link if defined?(@task_resource_link) + + @task_resource_link = unit.unit_content_links.find do |link| + link.context_type == 'task_definition_resource' && link.context_key == abbreviation + end || unit.unit_content_links.find_by( + context_type: 'task_definition_resource', + context_key: abbreviation + ) + end + + def has_task_resource_link? + task_resource_link.present? && task_resource_link.unit_content_site.file?(task_resource_link.route) + end + def grade_target_date(target_grade) grade_due_dates.find { |g| g.target_grade == target_grade.to_i }&.target_due_date end @@ -204,7 +231,7 @@ def copy_to(other_unit) FileUtils.cp(task_sheet, new_td.task_sheet()) end - if has_task_resources? + if has_uploaded_task_resources? # Copy the task resources, and trigger tii integration if needed new_td.add_task_resources(task_resources, copy: true) end @@ -271,6 +298,10 @@ def reset_overdue_tasks def move_files_on_abbreviation_change old_abbr = saved_change_to_abbreviation[0] # 0 is original abbreviation + unit.unit_content_links + .where(context_type: %w[task_definition task_definition_resource], context_key: old_abbr) + .find_each { |link| link.update!(context_key: abbreviation) } + if File.exist? task_sheet_with_abbreviation(old_abbr, false) FileUtils.mv(task_sheet_with_abbreviation(old_abbr), task_sheet()) end @@ -735,6 +766,10 @@ def is_group_task? end def has_task_resources? + has_task_resource_link? || has_uploaded_task_resources? + end + + def has_uploaded_task_resources? File.exist? task_resources(false) end @@ -811,10 +846,10 @@ def add_task_resources(file, copy: false) end def remove_task_resources() - if has_task_resources? + if has_uploaded_task_resources? FileUtils.rm task_resources - tii_group_attachments.destroy_all if tii_checks? + tii_group_attachments.destroy_all if tii_checks? && !has_task_resource_link? end end @@ -912,12 +947,18 @@ def related_tasks_with_files(consolidate_groups = true) # Read a file from the task definition resources. # - # @param filename [String] The name of the file to read from the zipfile. + # @param filename [String] The linked filename or path within the resource zip. # @return [String] The contents of the file, or nil if the file does not exist. def read_file_from_resources(filename) - return nil unless has_task_resources? + linked_resource = linked_task_resource + return nil unless linked_resource || has_uploaded_task_resources? + + if linked_resource && !task_resource_zip?(linked_resource) + return filename == linked_resource[:filename] ? File.binread(linked_resource[:path]) : nil + end - Zip::File.open(task_resources) do |zip_file| + resource_path = linked_resource ? linked_resource[:path] : task_resources + Zip::File.open(resource_path) do |zip_file| entry = zip_file.glob(filename).first return entry.get_input_stream.read if entry end @@ -925,6 +966,16 @@ def read_file_from_resources(filename) nil end + def linked_task_resource + return unless has_task_resource_link? + + task_resource_link.unit_content_site.extract_file(task_resource_link.route) + end + + def task_resource_zip?(resource) + resource.present? && File.extname(resource[:filename]).casecmp('.zip').zero? + end + private def target_grade_enabled_for_unit diff --git a/app/models/turn_it_in/task_definition_tii_module.rb b/app/models/turn_it_in/task_definition_tii_module.rb index 8e9d9e3097..0f7340b933 100644 --- a/app/models/turn_it_in/task_definition_tii_module.rb +++ b/app/models/turn_it_in/task_definition_tii_module.rb @@ -37,10 +37,21 @@ def send_group_attachments_to_tii return if tii_group_id.blank? return unless has_task_resources? + linked_resource = linked_task_resource + if linked_resource && !task_resource_zip?(linked_resource) + filename = linked_resource[:filename] + return unless filename.downcase.end_with?('.doc', '.docx') + return if filename.include?('__MACOSX') || File.size(linked_resource[:path]) < 50 + + TiiGroupAttachment.find_or_create_from_task_definition(self, filename) + return + end + count = 0 # loop through files in the task resources zip file - Zip::File.open(task_resources) do |zip_file| + resource_path = linked_resource ? linked_resource[:path] : task_resources + Zip::File.open(resource_path) do |zip_file| zip_file.each do |entry| next unless entry.file? next unless entry.name.end_with?('.doc', '.docx') diff --git a/app/models/unit.rb b/app/models/unit.rb index 19e0098298..eff8740ec8 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -79,6 +79,7 @@ def self.permissions :upload_grades_csv, :get_staff_notes, :capture_task_completion_snapshot, + :manage_unit_content, :mannage_communications, :delete_engagement ] @@ -108,6 +109,7 @@ def self.permissions :get_marking_sessions, :get_staff_notes, :get_tutor_times, + :manage_unit_content, :mannage_communications, ] @@ -177,6 +179,8 @@ def role_for(user) has_many :communication_sets, class_name: 'CommunicationSet', dependent: :destroy has_many :communication_rules, through: :communication_sets, class_name: 'CommunicationRule' has_many :communication_set_schedules, through: :communication_sets, class_name: 'CommunicationSetSchedule' + has_many :unit_content_sites, dependent: :destroy + has_many :unit_content_links, dependent: :destroy has_many :comments, through: :projects has_many :tasks, through: :projects @@ -326,6 +330,10 @@ def has_teaching_period? self.teaching_period.present? end + def has_main_content_site? + unit_content_sites.exists?(is_main: true) + end + def grade_values grade_definitions.filter_map { |definition| definition['value'] unless definition['value'] == -1 } end @@ -2148,8 +2156,19 @@ def get_task_resources_zip end if td.has_task_resources? - dst_path = FileHelper.sanitized_filename(td.abbreviation.to_s) + '.zip' - zip.add(dst_path, td.task_resources) + linked_resource = td.linked_task_resource + + if linked_resource && !td.task_resource_zip?(linked_resource) + dst_path = File.join( + FileHelper.sanitized_filename(td.abbreviation.to_s), + FileHelper.sanitized_filename(linked_resource[:filename]) + ) + zip.add(dst_path, linked_resource[:path]) + else + dst_path = FileHelper.sanitized_filename(td.abbreviation.to_s) + '.zip' + resource_path = linked_resource ? linked_resource[:path] : td.task_resources + zip.add(dst_path, resource_path) + end end end end # zip diff --git a/app/models/unit_content_link.rb b/app/models/unit_content_link.rb new file mode 100644 index 0000000000..6115ca7676 --- /dev/null +++ b/app/models/unit_content_link.rb @@ -0,0 +1,27 @@ +class UnitContentLink < ApplicationRecord + CONTEXT_TYPES = %w[grade grade_overview task_definition task_definition_resource].freeze + + belongs_to :unit + belongs_to :unit_content_site + + validates :context_type, inclusion: { in: CONTEXT_TYPES } + validates :context_key, :route, presence: true + validates :context_key, uniqueness: { scope: [:unit_id, :context_type] } + validate :resource_path_must_exist + + before_validation :normalise_route + + private + + def normalise_route + self.route = "/#{route.to_s.gsub(%r{\A/+|/+\z}, '')}" + self.route = '/' if route.blank? + end + + def resource_path_must_exist + return unless context_type == 'task_definition_resource' + return if unit_content_site&.file?(route) + + errors.add(:route, 'must be a file in the selected content site') + end +end diff --git a/app/models/unit_content_site.rb b/app/models/unit_content_site.rb new file mode 100644 index 0000000000..3b1a02ecf8 --- /dev/null +++ b/app/models/unit_content_site.rb @@ -0,0 +1,180 @@ +require 'fileutils' +require 'digest/sha1' +require 'securerandom' +require 'set' +require 'zip' + +class UnitContentSite < ApplicationRecord + include FileHelper + + belongs_to :unit + has_many :unit_content_links, dependent: :destroy + + validates :name, :original_filename, :archive_path, presence: true + validates :name, uniqueness: { scope: :unit_id, case_sensitive: false } + validates :root_dir, presence: true + + after_destroy :delete_archive + + def self.archive_dir_for(unit) + File.join(FileHelper.unit_dir(unit), 'content_sites') + end + + def self.store_upload!(unit, file, name: nil) + original_filename = file[:filename] || file[:name] || 'content.zip' + site_name = name.presence || File.basename(original_filename, '.*') + archive_dir = archive_dir_for(unit) + FileUtils.mkdir_p archive_dir + + site = unit.unit_content_sites.create!( + name: site_name, + original_filename: original_filename, + root_dir: '/', + is_main: unit.unit_content_sites.none?, + archive_path: File.join( + archive_dir, + "#{SecureRandom.hex(8)}-#{FileHelper.sanitized_filename(original_filename)}" + ) + ) + + FileUtils.cp file[:tempfile].path, site.archive_path + site + end + + def replace_upload!(file, root_dir: nil) + original_archive_path = archive_path + replacement_original_filename = file[:filename] || file[:name] || original_filename + replacement_archive_path = File.join( + self.class.archive_dir_for(unit), + "#{SecureRandom.hex(8)}-#{FileHelper.sanitized_filename(replacement_original_filename)}" + ) + + FileUtils.cp file[:tempfile].path, replacement_archive_path + replacement_root_options = self.class.root_dir_options_for(replacement_archive_path) + replacement_root_dir = + root_dir.presence || + (replacement_root_options.include?(self.root_dir) ? self.root_dir : '/') + + update!( + original_filename: replacement_original_filename, + archive_path: replacement_archive_path, + root_dir: replacement_root_dir + ) + FileUtils.rm_f original_archive_path if original_archive_path.present? + self + rescue StandardError + FileUtils.rm_f replacement_archive_path if replacement_archive_path.present? + raise + end + + def self.root_dir_options_for(archive_path) + root_dir_options_from_entries(archive_entries_for(archive_path)) + rescue Zip::Error + ['/'] + end + + def self.archive_entries_for(archive_path) + entries = [] + + Zip::File.open(archive_path) do |zip| + zip.each do |entry| + entries << entry.name unless entry.directory? + end + end + + entries + end + + def self.root_dir_options_from_entries(entries) + dirs = entries.each_with_object(Set.new(['/'])) do |entry, paths| + parts = entry.split('/').reject(&:blank?) + next if parts.any? { |part| ignored_archive_path?(part) } + + parts[0...-1].each_index do |index| + paths << "/#{parts[0..index].join('/')}" + end + end + + dirs.to_a.sort + end + + def self.ignored_archive_path?(path) + path.start_with?('__MACOSX') || path == '.DS_Store' || path.start_with?('._') + end + + def root_dir_options + self.class.root_dir_options_for(archive_path) + end + + def file_paths + root_prefix = normalized_root_dir + + self.class.archive_entries_for(archive_path).filter_map do |entry| + parts = entry.split('/').reject(&:blank?) + next if parts.any? { |part| self.class.ignored_archive_path?(part) } + next unless root_prefix.blank? || entry.start_with?("#{root_prefix}/") + + relative_path = root_prefix.blank? ? entry : entry.delete_prefix("#{root_prefix}/") + "/#{relative_path}" if relative_path.present? + end.sort + rescue Zip::Error, Errno::ENOENT + [] + end + + def file?(path) + entry_name = archive_entry_name(path) + return false if entry_name.blank? || !File.exist?(archive_path) + + Zip::File.open(archive_path) do |zip| + entry = zip.find_entry(entry_name) + entry.present? && entry.file? + end + rescue Zip::Error + false + end + + def extract_file(path) + entry_name = archive_entry_name(path) + return nil if entry_name.blank? || !File.exist?(archive_path) + + Zip::File.open(archive_path) do |zip| + entry = zip.find_entry(entry_name) + return nil unless entry&.file? + + filename = File.basename(entry.name) + cache_key = Digest::SHA1.hexdigest("#{archive_path}:#{entry.name}:#{updated_at.to_f}") + extracted_path = FileHelper.tmp_file( + "unit-content-#{id}-#{cache_key}-#{FileHelper.sanitized_filename(filename)}" + ) + + unless File.exist?(extracted_path) && File.size(extracted_path) == entry.size + temporary_path = "#{extracted_path}.#{SecureRandom.hex(6)}.tmp" + entry.extract(temporary_path) { true } + FileUtils.mv(temporary_path, extracted_path) + end + + { path: extracted_path, filename: filename } + ensure + FileUtils.rm_f(temporary_path) if defined?(temporary_path) && temporary_path.present? + end + rescue Zip::Error + nil + end + + private + + def archive_entry_name(path) + relative_path = path.to_s.gsub(%r{\A/+|/+\z}, '') + return nil if relative_path.blank? + + [normalized_root_dir, relative_path].compact_blank.join('/') + end + + def normalized_root_dir + root_dir.to_s.gsub(%r{\A/+|/+\z}, '') + end + + def delete_archive + FileUtils.rm_f archive_path if archive_path.present? + end +end diff --git a/db/migrate/20260713232535_add_unit_content_management.rb b/db/migrate/20260713232535_add_unit_content_management.rb new file mode 100644 index 0000000000..076a54b8d6 --- /dev/null +++ b/db/migrate/20260713232535_add_unit_content_management.rb @@ -0,0 +1,31 @@ +class AddUnitContentManagement < ActiveRecord::Migration[8.0] + def change + create_table :unit_content_sites do |t| + t.references :unit, null: false + t.string :name, null: false + t.string :original_filename, null: false + t.string :archive_path, null: false + t.string :root_dir, null: false, default: '/' + t.boolean :is_main, null: false, default: false + + t.timestamps + end + + add_index :unit_content_sites, [:unit_id, :name], unique: true + + create_table :unit_content_links do |t| + t.references :unit, null: false + t.references :unit_content_site, null: false + t.string :context_type, null: false + t.string :context_key, null: false + t.string :route, null: false, default: '/' + + t.timestamps + end + + add_index :unit_content_links, + [:unit_id, :context_type, :context_key], + unique: true, + name: 'index_unit_content_links_on_context' + end +end diff --git a/db/schema.rb b/db/schema.rb index b8ec5659b3..f1ccbfcd2e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_07_09_014859) do +ActiveRecord::Schema[8.0].define(version: 2026_07_13_232535) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -852,6 +852,32 @@ t.index ["unit_role_id"], name: "index_tutorials_on_unit_role_id" end + create_table "unit_content_links", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.bigint "unit_content_site_id", null: false + t.string "context_type", null: false + t.string "context_key", null: false + t.string "route", default: "/", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["unit_content_site_id"], name: "index_unit_content_links_on_unit_content_site_id" + t.index ["unit_id", "context_type", "context_key"], name: "index_unit_content_links_on_context", unique: true + t.index ["unit_id"], name: "index_unit_content_links_on_unit_id" + end + + create_table "unit_content_sites", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.string "name", null: false + t.string "original_filename", null: false + t.string "archive_path", null: false + t.string "root_dir", default: "/", null: false + t.boolean "is_main", default: false, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["unit_id", "name"], name: "index_unit_content_sites_on_unit_id_and_name", unique: true + t.index ["unit_id"], name: "index_unit_content_sites_on_unit_id" + end + create_table "unit_roles", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "user_id" t.bigint "tutorial_id" diff --git a/test/models/unit_content_task_resource_test.rb b/test/models/unit_content_task_resource_test.rb new file mode 100644 index 0000000000..5faca585eb --- /dev/null +++ b/test/models/unit_content_task_resource_test.rb @@ -0,0 +1,123 @@ +require 'test_helper' + +class UnitContentTaskResourceTest < ActiveSupport::TestCase + def test_linked_file_takes_priority_over_uploaded_resource_zip + unit = FactoryBot.create(:unit, with_students: false, stream_count: 0) + task_definition = FactoryBot.create( + :task_definition, + unit: unit, + abbreviation: 'P1', + outcome_count: 0 + ) + + Tempfile.create(['native-resources', '.zip']) do |native_zip| + write_zip(native_zip.path, 'native.txt' => 'native resource') + task_definition.add_task_resources(native_zip.path, copy: true) + end + + with_content_site(unit, 'dist/P1 Worksheet.docx' => 'linked worksheet') do |site| + unit.unit_content_links.create!( + unit_content_site: site, + context_type: 'task_definition_resource', + context_key: 'P1', + route: '/P1 Worksheet.docx' + ) + + task_definition.reload + resource = task_definition.linked_task_resource + + assert task_definition.has_task_resources? + assert task_definition.has_uploaded_task_resources? + assert task_definition.has_task_resource_link? + assert_not task_definition.task_resource_zip?(resource) + assert_equal 'P1 Worksheet.docx', resource[:filename] + assert_equal 'linked worksheet', task_definition.read_file_from_resources(resource[:filename]) + end + end + + def test_linked_zip_is_used_as_a_task_resource_archive + unit = FactoryBot.create(:unit, with_students: false, stream_count: 0) + task_definition = FactoryBot.create( + :task_definition, + unit: unit, + abbreviation: 'P1', + outcome_count: 0 + ) + + Tempfile.create(['linked-resources', '.zip']) do |resource_zip| + write_zip(resource_zip.path, 'starter/main.py' => 'print("Hello")') + + with_content_site_from_files(unit, 'dist/P1-resources.zip' => resource_zip.path) do |site| + unit.unit_content_links.create!( + unit_content_site: site, + context_type: 'task_definition_resource', + context_key: 'P1', + route: '/P1-resources.zip' + ) + + task_definition.reload + resource = task_definition.linked_task_resource + + assert task_definition.task_resource_zip?(resource) + assert_equal 'print("Hello")', task_definition.read_file_from_resources('starter/main.py') + end + end + end + + def test_resource_link_requires_an_existing_site_file + unit = FactoryBot.create(:unit, with_students: false, stream_count: 0) + + with_content_site(unit, 'dist/existing.txt' => 'content') do |site| + link = unit.unit_content_links.build( + unit_content_site: site, + context_type: 'task_definition_resource', + context_key: 'P1', + route: '/missing.txt' + ) + + assert_not link.valid? + assert_includes link.errors[:route], 'must be a file in the selected content site' + end + end + + private + + def with_content_site(unit, entries) + Tempfile.create(['unit-content', '.zip']) do |archive| + write_zip(archive.path, entries) + site = unit.unit_content_sites.create!( + name: 'Content', + original_filename: 'content.zip', + archive_path: archive.path, + root_dir: '/dist' + ) + + yield site + end + end + + def with_content_site_from_files(unit, entries) + Tempfile.create(['unit-content', '.zip']) do |archive| + Zip::File.open(archive.path, Zip::File::CREATE) do |zip| + entries.each { |entry_name, source_path| zip.add(entry_name, source_path) } + end + + site = unit.unit_content_sites.create!( + name: 'Content', + original_filename: 'content.zip', + archive_path: archive.path, + root_dir: '/dist' + ) + + yield site + end + end + + def write_zip(path, entries) + Zip::File.open(path, Zip::File::CREATE) do |zip| + entries.each do |entry_name, contents| + zip.get_output_stream(entry_name) { |stream| stream.write(contents) } + end + end + end +end From 184e01ab2467a002e13cd020aca79b89f8ac7cf9 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:17:27 +1000 Subject: [PATCH 166/199] chore(release): 11.0.0-33 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57d3c0bd7c..d4d434f6e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-33](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-32...v11.0.0-33) (2026-07-19) + + +### Features + +* content management system ([#642](https://github.com/b0ink/doubtfire-deploy/issues/642)) ([a87b250](https://github.com/b0ink/doubtfire-deploy/commit/a87b250303b0aff8a431334e8b19a40f790cbd35)) + ## [11.0.0-32](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-31...v11.0.0-32) (2026-07-16) From 4f60eff0eea770d20cee2226a13b1ad10dfafc0e Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:06:58 +1000 Subject: [PATCH 167/199] fix: pass saml settings correctly to logout response --- app/api/authentication_api.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/api/authentication_api.rb b/app/api/authentication_api.rb index e81de94d1f..67c946dd80 100644 --- a/app/api/authentication_api.rb +++ b/app/api/authentication_api.rb @@ -163,8 +163,10 @@ class AuthenticationApi < Grape::API requires :SAMLResponse, type: String, desc: 'SAML logout response data.' end post '/auth/saml_logout' do - response = OneLogin::RubySaml::Logoutresponse.new(params[:SAMLResponse], allowed_clock_drift: 1.second, - settings: AuthenticationHelpers.saml_settings) + response = OneLogin::RubySaml::Logoutresponse.new( + params[:SAMLResponse], + AuthenticationHelpers.saml_settings + ) # Check if the SAML response is valid - if not log an error unless response.is_valid? From 2141fd6876f50c5705f16acdd1f27d9b65152078 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:03:32 +1000 Subject: [PATCH 168/199] feat: track last access to site and unit --- app/api/authentication_api.rb | 4 +++ app/api/projects_api.rb | 1 + app/models/user.rb | 12 ++++++++ .../20260722015317_add_activity_timestamps.rb | 6 ++++ db/schema.rb | 4 ++- test/api/auth_test.rb | 30 +++++++++++++++++++ test/api/projects_api_test.rb | 24 +++++++++++++++ 7 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20260722015317_add_activity_timestamps.rb diff --git a/app/api/authentication_api.rb b/app/api/authentication_api.rb index e81de94d1f..92f4a966a5 100644 --- a/app/api/authentication_api.rb +++ b/app/api/authentication_api.rb @@ -77,6 +77,7 @@ class AuthenticationApi < Grape::API token&.destroy! token = user.generate_authentication_token! + user.record_sign_in! # Return user details present :user, user, with: Entities::UserEntity @@ -371,6 +372,7 @@ class AuthenticationApi < Grape::API # Invalidate the token and regenrate a new one token.destroy! token = user.generate_authentication_token! + user.record_sign_in! logger.info "Login #{params[:username]} from #{request.ip}" @@ -487,6 +489,8 @@ class AuthenticationApi < Grape::API end post '/auth/access-token' do if authenticated_via_refresh_token? + current_user.record_access! + # Check if we have a auth token as well if params[:delete_auth_token] user_param, auth_param = get_user_and_token_from(:header) diff --git a/app/api/projects_api.rb b/app/api/projects_api.rb index a895007ff3..e220ca5211 100644 --- a/app/api/projects_api.rb +++ b/app/api/projects_api.rb @@ -28,6 +28,7 @@ class ProjectsApi < Grape::API project = Project.eager_load(:unit, :user).find(params[:id]) if authorise? current_user, project, :get + project.update!(last_viewed_at: Time.current) if project.user_id == current_user.id present project, with: Entities::ProjectEntity, user: current_user, for_student: true, in_project: true else error!({ error: "Couldn't find Project with id=#{params[:id]}" }, 403) diff --git a/app/models/user.rb b/app/models/user.rb index 159b9aab7f..e49d796c4b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -90,6 +90,18 @@ def authenticate?(data) end end + # Record a completed interactive sign in. Signing in is also an access event. + def record_sign_in! + now = Time.current + update!(last_sign_in_at: now, last_access_at: now) + end + + # Refresh-token exchanges provide a low-write indication that the user is + # still accessing OnTrack without updating the user on every API request. + def record_access! + update!(last_access_at: Time.current) + end + # # Force-generates a new authentication token, regardless of whether or not # it is actually expired diff --git a/db/migrate/20260722015317_add_activity_timestamps.rb b/db/migrate/20260722015317_add_activity_timestamps.rb new file mode 100644 index 0000000000..fe781cc5f8 --- /dev/null +++ b/db/migrate/20260722015317_add_activity_timestamps.rb @@ -0,0 +1,6 @@ +class AddActivityTimestamps < ActiveRecord::Migration[8.0] + def change + add_column :users, :last_access_at, :datetime + add_column :projects, :last_viewed_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index f1ccbfcd2e..0c036357a0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_07_13_232535) do +ActiveRecord::Schema[8.0].define(version: 2026_07_22_015317) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -467,6 +467,7 @@ t.integer "spec_con_days", default: 0, null: false t.bigint "assessor_id" t.datetime "portfolio_submission_date" + t.datetime "last_viewed_at" t.index ["assessor_id"], name: "index_projects_on_assessor_id" t.index ["campus_id"], name: "index_projects_on_campus_id" t.index ["enrolled"], name: "index_projects_on_enrolled" @@ -982,6 +983,7 @@ t.string "tii_eula_version" t.datetime "tii_eula_date" t.boolean "tii_eula_version_confirmed", default: false, null: false + t.datetime "last_access_at" t.index ["email"], name: "index_users_on_email", unique: true t.index ["login_id"], name: "index_users_on_login_id", unique: true t.index ["role_id"], name: "index_users_on_role_id" diff --git a/test/api/auth_test.rb b/test/api/auth_test.rb index cc3f737601..00aaa66c6f 100644 --- a/test/api/auth_test.rb +++ b/test/api/auth_test.rb @@ -2,6 +2,7 @@ class AuthTest < ActiveSupport::TestCase include Rack::Test::Methods + include ActiveSupport::Testing::TimeHelpers include TestHelpers::AuthHelper include TestHelpers::JsonHelper @@ -62,6 +63,19 @@ def test_auth_post assert_match(/username=#{User.first.username};/, last_response.cookies['username'].to_s, 'Expect username to be set') end + def test_auth_records_sign_in_and_access_time + user = User.find_by!(username: 'aadmin') + sign_in_time = Time.zone.parse('2026-07-21 10:00:00 UTC') + + travel_to sign_in_time do + post_json '/api/auth.json', username: user.username, password: 'password' + end + + assert_equal 201, last_response.status + assert_equal sign_in_time, user.reload.last_sign_in_at + assert_equal sign_in_time, user.last_access_at + end + def test_auth_no_remember data_to_post = { username: 'aadmin', @@ -231,6 +245,22 @@ def test_refresh_token assert_equal last_response_body['auth_token'], new_new_token.authentication_token end + def test_refresh_token_updates_access_but_not_sign_in_time + user = FactoryBot.create(:user, last_sign_in_at: 1.day.ago, last_access_at: 1.hour.ago) + original_sign_in_time = user.last_sign_in_at + token = user.generate_authentication_token!(token_type: :refresh_token) + access_time = Time.zone.parse('2026-07-21 11:00:00 UTC') + + set_cookie "username=#{user.username}" + set_cookie "refresh_token=#{token.authentication_token}" + + travel_to(access_time) { post '/api/auth/access-token' } + + assert_equal 201, last_response.status + assert_equal original_sign_in_time, user.reload.last_sign_in_at + assert_equal access_time, user.last_access_at + end + def test_token_signout_works_with_multiple user = FactoryBot.create(:user) # Create 2 auth tokens diff --git a/test/api/projects_api_test.rb b/test/api/projects_api_test.rb index 30407838e6..7b471f8663 100644 --- a/test/api/projects_api_test.rb +++ b/test/api/projects_api_test.rb @@ -4,6 +4,7 @@ class ProjectsApiTest < ActiveSupport::TestCase include Rack::Test::Methods + include ActiveSupport::Testing::TimeHelpers include TestHelpers::AuthHelper include TestHelpers::JsonHelper include TestHelpers::TestFileHelper @@ -86,6 +87,29 @@ def test_get_project_response_is_correct assert_json_matches_model project, last_response_body, key_test end + def test_get_project_records_when_student_viewed_it + user = FactoryBot.create(:user, :student, enrol_in: 1) + project = user.projects.first + viewed_at = Time.zone.parse('2026-07-21 12:00:00 UTC') + add_auth_header_for(user: user) + + travel_to(viewed_at) { get "/api/projects/#{project.id}" } + + assert_equal 200, last_response.status + assert_equal viewed_at, project.reload.last_viewed_at + end + + def test_get_project_does_not_record_staff_view_as_student_view + project = FactoryBot.create(:project) + admin = FactoryBot.create(:user, :admin) + add_auth_header_for(user: admin) + + get "/api/projects/#{project.id}" + + assert_equal 200, last_response.status + assert_nil project.reload.last_viewed_at + end + def test_projects_works_with_inactive_units user = FactoryBot.create(:user, :student, enrol_in: 2) Unit.last.update(active: false) From 50fc8215598766fa8927ca480b122a361803d689 Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:06:26 +1000 Subject: [PATCH 169/199] feat: add last sign in and last unit access communication conditions --- app/api/communication_rules_api.rb | 6 ++ .../communication_condition_entity.rb | 1 + .../communication/communication_condition.rb | 3 +- .../communication/communication_rule.rb | 18 ++-- .../communication/login_status_condition.rb | 6 +- .../unit_viewed_status_condition.rb | 6 ++ app/sidekiq/communication_rule_job.rb | 3 +- app/sidekiq/execute_communication_set_job.rb | 14 ++- ...tivity_days_to_communication_conditions.rb | 5 ++ db/schema.rb | 3 +- test/api/communication_rules_api_test.rb | 57 ++++++++++++ test/models/communication_condition_test.rb | 33 +++++++ test/models/communication_set_test.rb | 89 +++++++++++++++++++ 13 files changed, 231 insertions(+), 13 deletions(-) create mode 100644 app/models/communication/unit_viewed_status_condition.rb create mode 100644 db/migrate/20260722030000_add_activity_days_to_communication_conditions.rb create mode 100644 test/api/communication_rules_api_test.rb diff --git a/app/api/communication_rules_api.rb b/app/api/communication_rules_api.rb index af64417750..e0a16e23a7 100644 --- a/app/api/communication_rules_api.rb +++ b/app/api/communication_rules_api.rb @@ -122,6 +122,7 @@ def sync_set_schedules!(communication_set, raw_schedules) target_grade: project.target_grade, spec_con_days: project.spec_con_days, last_sign_in_at: project.user&.last_sign_in_at, + last_viewed_at: project.last_viewed_at, campus: project.campus&.name } end @@ -513,6 +514,7 @@ def sync_set_schedules!(communication_set, raw_schedules) full_name: [project.user&.first_name, project.user&.last_name].compact.join(' '), target_grade: project.target_grade, last_sign_in_at: project.user&.last_sign_in_at, + last_viewed_at: project.last_viewed_at, campus: project.campus&.name } end @@ -568,6 +570,7 @@ def sync_set_schedules!(communication_set, raw_schedules) optional :task_status_count, type: Integer optional :task_target_grade, type: Integer optional :last_sign_in_at, type: DateTime + optional :activity_days, type: Integer optional :spec_con_days, type: Integer optional :tutorial_id, type: Integer optional :tutorial_stream_id, type: Integer @@ -591,6 +594,7 @@ def sync_set_schedules!(communication_set, raw_schedules) task_status_count: raw_condition_params[:task_status_count], task_target_grade: raw_condition_params[:task_target_grade], last_sign_in_at: raw_condition_params[:last_sign_in_at], + activity_days: raw_condition_params[:activity_days], spec_con_days: raw_condition_params[:spec_con_days], tutorial_id: raw_condition_params[:tutorial_id], tutorial_stream_id: raw_condition_params[:tutorial_stream_id], @@ -618,6 +622,7 @@ def sync_set_schedules!(communication_set, raw_schedules) optional :task_status_count, type: Integer optional :task_target_grade, type: Integer optional :last_sign_in_at, type: DateTime + optional :activity_days, type: Integer optional :spec_con_days, type: Integer optional :tutorial_id, type: Integer optional :tutorial_stream_id, type: Integer @@ -642,6 +647,7 @@ def sync_set_schedules!(communication_set, raw_schedules) task_status_count: raw_condition_params[:task_status_count], task_target_grade: raw_condition_params[:task_target_grade], last_sign_in_at: raw_condition_params[:last_sign_in_at], + activity_days: raw_condition_params[:activity_days], spec_con_days: raw_condition_params[:spec_con_days], tutorial_id: raw_condition_params[:tutorial_id], tutorial_stream_id: raw_condition_params[:tutorial_stream_id], diff --git a/app/api/entities/communication_condition_entity.rb b/app/api/entities/communication_condition_entity.rb index 045b58d96b..467aa787ee 100644 --- a/app/api/entities/communication_condition_entity.rb +++ b/app/api/entities/communication_condition_entity.rb @@ -10,6 +10,7 @@ class CommunicationConditionEntity < Grape::Entity expose :task_status_count expose :task_target_grade expose :last_sign_in_at + expose :activity_days expose :spec_con_days expose :tutorial_id expose :tutorial_stream_id diff --git a/app/models/communication/communication_condition.rb b/app/models/communication/communication_condition.rb index 6623b0f4c5..9c11a96a91 100644 --- a/app/models/communication/communication_condition.rb +++ b/app/models/communication/communication_condition.rb @@ -4,6 +4,7 @@ class CommunicationCondition < ApplicationRecord TaskDefinitionStatusCondition TaskStatusCountCondition LoginStatusCondition + UnitViewedStatusCondition SpecConCondition TutorialEnrolmentCondition TutorialStreamEnrolmentCondition @@ -20,7 +21,7 @@ class CommunicationCondition < ApplicationRecord ].freeze EQUALITY_OPERATORS = %w[equal_to not_equal_to].freeze - DATE_OPERATORS = %w[before after].freeze + ACTIVITY_OPERATORS = %w[more_than within_last].freeze ENROLMENT_OPERATORS = %w[enrolled_in not_enrolled_in].freeze TASK_STATUS_KEYS = %w[ not_started diff --git a/app/models/communication/communication_rule.rb b/app/models/communication/communication_rule.rb index 205192560e..d49148945e 100644 --- a/app/models/communication/communication_rule.rb +++ b/app/models/communication/communication_rule.rb @@ -19,8 +19,9 @@ def matching_projects(projects = nil) projects ||= communication_set.eligible_projects return projects if communication_conditions.empty? + evaluated_at = Time.current projects.select do |project| - matches = communication_conditions.map { |condition| condition_match?(project, condition) } + matches = communication_conditions.map { |condition| condition_match?(project, condition, evaluated_at) } operator == 'or' ? matches.any? : matches.all? end @@ -28,7 +29,7 @@ def matching_projects(projects = nil) private - def condition_match?(project, condition) + def condition_match?(project, condition, evaluated_at) case condition.type when 'TargetGradeCondition' target_grade_condition_match?(project, condition) @@ -37,7 +38,9 @@ def condition_match?(project, condition) when 'TaskStatusCountCondition' task_status_count_condition_match?(project, condition) when 'LoginStatusCondition' - login_status_condition_match?(project, condition) + activity_condition_match?(project.user&.last_sign_in_at, condition, evaluated_at) + when 'UnitViewedStatusCondition' + activity_condition_match?(project.last_viewed_at, condition, evaluated_at) when 'SpecConCondition' spec_con_condition_match?(project, condition) when 'TutorialEnrolmentCondition' @@ -86,12 +89,13 @@ def task_status_count_condition_match?(project, condition) compare_value(count, condition.task_status_count, condition.operator) end - def login_status_condition_match?(project, condition) - last_sign_in_at = project.user&.last_sign_in_at + def activity_condition_match?(last_activity_at, condition, evaluated_at) + return false if condition.activity_days.blank? + threshold = evaluated_at - condition.activity_days.days case condition.operator - when 'before' then last_sign_in_at.present? && last_sign_in_at < condition.last_sign_in_at - when 'after' then last_sign_in_at.present? && last_sign_in_at > condition.last_sign_in_at + when 'more_than' then last_activity_at.nil? || last_activity_at < threshold + when 'within_last' then last_activity_at.present? && last_activity_at >= threshold else false end end diff --git a/app/models/communication/login_status_condition.rb b/app/models/communication/login_status_condition.rb index c972c76fdc..cf6cbebea5 100644 --- a/app/models/communication/login_status_condition.rb +++ b/app/models/communication/login_status_condition.rb @@ -1,4 +1,6 @@ class LoginStatusCondition < CommunicationCondition - validates :last_sign_in_at, presence: true - validates :operator, inclusion: { in: DATE_OPERATORS } + validates :activity_days, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 1 } + validates :operator, inclusion: { in: ACTIVITY_OPERATORS } end diff --git a/app/models/communication/unit_viewed_status_condition.rb b/app/models/communication/unit_viewed_status_condition.rb new file mode 100644 index 0000000000..9ce4152885 --- /dev/null +++ b/app/models/communication/unit_viewed_status_condition.rb @@ -0,0 +1,6 @@ +class UnitViewedStatusCondition < CommunicationCondition + validates :activity_days, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 1 } + validates :operator, inclusion: { in: ACTIVITY_OPERATORS } +end diff --git a/app/sidekiq/communication_rule_job.rb b/app/sidekiq/communication_rule_job.rb index 3d1b02c6ce..af8e5831fd 100644 --- a/app/sidekiq/communication_rule_job.rb +++ b/app/sidekiq/communication_rule_job.rb @@ -25,7 +25,8 @@ def perform(rule_id) username: project.user&.username, student_id: project.user&.student_id, target_grade: project.target_grade, - last_sign_in_at: project.user&.last_sign_in_at + last_sign_in_at: project.user&.last_sign_in_at, + last_viewed_at: project.last_viewed_at } end ) diff --git a/app/sidekiq/execute_communication_set_job.rb b/app/sidekiq/execute_communication_set_job.rb index fa279e1124..5f99bd3fe6 100644 --- a/app/sidekiq/execute_communication_set_job.rb +++ b/app/sidekiq/execute_communication_set_job.rb @@ -510,7 +510,9 @@ def human_condition_summary(condition) statuses = Array(condition.task_statuses).map { |status| status.to_s.titleize }.join(', ') "Students that have #{operator_label(condition.operator)} #{condition.task_status_count} #{grade_label} tasks in [#{statuses}]" when 'LoginStatusCondition' - "Students whose last sign in is #{condition.operator.to_s.humanize.downcase} #{condition.last_sign_in_at}" + relative_activity_summary(condition, 'signed in') + when 'UnitViewedStatusCondition' + relative_activity_summary(condition, 'viewed this unit') when 'SpecConCondition' "Students with Special Consideration Days #{operator_label(condition.operator)} #{condition.spec_con_days}" when 'TutorialEnrolmentCondition' @@ -540,6 +542,16 @@ def human_condition_summary(condition) end end + def relative_activity_summary(condition, activity) + duration = "#{condition.activity_days} #{'day'.pluralize(condition.activity_days)}" + + if condition.operator == 'more_than' + "Students who have not #{activity} for more than #{duration}" + else + "Students who #{activity} within the last #{duration}" + end + end + def human_action_summary(action) case action.type when 'EmailStudentAction' diff --git a/db/migrate/20260722030000_add_activity_days_to_communication_conditions.rb b/db/migrate/20260722030000_add_activity_days_to_communication_conditions.rb new file mode 100644 index 0000000000..14adb011b5 --- /dev/null +++ b/db/migrate/20260722030000_add_activity_days_to_communication_conditions.rb @@ -0,0 +1,5 @@ +class AddActivityDaysToCommunicationConditions < ActiveRecord::Migration[8.0] + def change + add_column :communication_conditions, :activity_days, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index 0c036357a0..7a38c49aca 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_07_22_015317) do +ActiveRecord::Schema[8.0].define(version: 2026_07_22_030000) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -105,6 +105,7 @@ t.string "operator", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.integer "activity_days" t.index ["campus_id"], name: "index_communication_conditions_on_campus_id" t.index ["communication_id"], name: "index_communication_conditions_on_communication_id" t.index ["task_definition_id"], name: "index_communication_conditions_on_task_definition_id" diff --git a/test/api/communication_rules_api_test.rb b/test/api/communication_rules_api_test.rb new file mode 100644 index 0000000000..a9ff372c49 --- /dev/null +++ b/test/api/communication_rules_api_test.rb @@ -0,0 +1,57 @@ +require 'test_helper' + +class CommunicationRulesApiTest < ActiveSupport::TestCase + include Rack::Test::Methods + include TestHelpers::AuthHelper + include TestHelpers::JsonHelper + + def app + Rails.application + end + + def test_create_and_update_relative_activity_condition + admin = FactoryBot.create(:user, :admin) + unit = FactoryBot.create(:unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0) + communication_set = unit.communication_sets.create!(name: 'Activity', active: true) + rule = communication_set.communication_rules.create!(name: 'Inactive students', operator: 'and', position: 0) + add_auth_header_for(user: admin) + + post_json "/api/units/#{unit.id}/communication_rules/#{rule.id}/conditions", + communication_condition: { + type: 'LoginStatusCondition', + operator: 'more_than', + activity_days: 7 + } + + assert_equal 201, last_response.status + condition_response = last_response_body + assert_equal 'LoginStatusCondition', condition_response['type'] + assert_equal 'more_than', condition_response['operator'] + assert_equal 7, condition_response['activity_days'] + + put_json "/api/units/#{unit.id}/communication_rules/#{rule.id}/conditions/#{condition_response['id']}", + communication_condition: { + type: 'UnitViewedStatusCondition', + operator: 'within_last', + activity_days: 3 + } + + assert_equal 200, last_response.status + condition_response = last_response_body + assert_equal 'UnitViewedStatusCondition', condition_response['type'] + assert_equal 'within_last', condition_response['operator'] + assert_equal 3, condition_response['activity_days'] + + viewed_at = 1.day.ago.change(usec: 0) + project = FactoryBot.create(:project, unit: unit, last_viewed_at: viewed_at) + project.user.update!(last_sign_in_at: 2.days.ago.change(usec: 0)) + + get "/api/units/#{unit.id}/communication_sets/#{communication_set.id}" + + assert_equal 200, last_response.status + student = last_response_body['previews'].first['allocations'].first['students'].first + assert_equal project.user.username, student['username'] + assert_equal viewed_at.iso8601(3), Time.zone.parse(student['last_viewed_at']).iso8601(3) + assert_equal project.user.last_sign_in_at.iso8601(3), Time.zone.parse(student['last_sign_in_at']).iso8601(3) + end +end diff --git a/test/models/communication_condition_test.rb b/test/models/communication_condition_test.rb index 428aaa0410..a6755605da 100644 --- a/test/models/communication_condition_test.rb +++ b/test/models/communication_condition_test.rb @@ -37,4 +37,37 @@ def test_spec_con_condition_accepts_integer_spec_con_days condition.reload assert_equal 4, condition.spec_con_days end + + def test_activity_conditions_require_supported_operator_and_positive_integer_days + rule = communication_rule + + %w[LoginStatusCondition UnitViewedStatusCondition].each do |type| + condition = CommunicationCondition.new( + type: type, + communication: rule, + operator: 'more_than', + activity_days: 7 + ) + + assert condition.valid?, condition.errors.full_messages + + condition.activity_days = 0 + assert_not condition.valid? + + condition.activity_days = 1.5 + assert_not condition.valid? + + condition.activity_days = 7 + condition.operator = 'before' + assert_not condition.valid? + end + end + + private + + def communication_rule + unit = FactoryBot.create(:unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0) + communication_set = unit.communication_sets.create!(name: 'Test Set', active: true) + communication_set.communication_rules.create!(name: 'Test Rule', operator: 'and', position: 0) + end end diff --git a/test/models/communication_set_test.rb b/test/models/communication_set_test.rb index 931c201762..d4a3bd7e09 100644 --- a/test/models/communication_set_test.rb +++ b/test/models/communication_set_test.rb @@ -1,6 +1,8 @@ require 'test_helper' class CommunicationSetTest < ActiveSupport::TestCase + include ActiveSupport::Testing::TimeHelpers + def test_preview_projects_for_rule_excludes_students_claimed_by_earlier_rules unit = FactoryBot.create( :unit, @@ -66,4 +68,91 @@ def test_preview_projects_for_rule_matches_spec_con_days assert_equal [matching_project.id], matched_projects.map(&:id) end + + def test_login_status_condition_uses_relative_sign_in_time_and_handles_never + travel_to Time.zone.parse('2026-07-22 12:00:00 UTC') do + unit, rule = unit_and_rule('Login activity') + old_project = activity_project(unit, last_sign_in_at: 8.days.ago, last_viewed_at: 1.day.ago) + recent_project = activity_project(unit, last_sign_in_at: 6.days.ago, last_viewed_at: 10.days.ago) + boundary_project = activity_project(unit, last_sign_in_at: 7.days.ago) + never_project = activity_project(unit) + condition = rule.communication_conditions.create!( + type: 'LoginStatusCondition', + operator: 'more_than', + activity_days: 7 + ) + + assert_equal [old_project.id, never_project.id].sort, rule.matching_projects.map(&:id).sort + + condition.update!(operator: 'within_last') + assert_equal [recent_project.id, boundary_project.id].sort, rule.matching_projects.map(&:id).sort + end + end + + def test_unit_viewed_status_condition_uses_project_view_time_and_handles_never + travel_to Time.zone.parse('2026-07-22 12:00:00 UTC') do + unit, rule = unit_and_rule('Unit activity') + old_project = activity_project(unit, last_sign_in_at: 1.day.ago, last_viewed_at: 8.days.ago) + recent_project = activity_project(unit, last_sign_in_at: 10.days.ago, last_viewed_at: 6.days.ago) + boundary_project = activity_project(unit, last_viewed_at: 7.days.ago) + never_project = activity_project(unit, last_sign_in_at: 1.day.ago) + condition = rule.communication_conditions.create!( + type: 'UnitViewedStatusCondition', + operator: 'more_than', + activity_days: 7 + ) + + assert_equal [old_project.id, never_project.id].sort, rule.matching_projects.map(&:id).sort + + condition.update!(operator: 'within_last') + assert_equal [recent_project.id, boundary_project.id].sort, rule.matching_projects.map(&:id).sort + end + end + + def test_copy_to_preserves_relative_activity_conditions + source_unit, rule = unit_and_rule('Reusable activity') + destination_unit = FactoryBot.create(:unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0) + rule.communication_conditions.create!( + type: 'LoginStatusCondition', + operator: 'within_last', + activity_days: 3 + ) + rule.communication_conditions.create!( + type: 'UnitViewedStatusCondition', + operator: 'more_than', + activity_days: 14 + ) + + copied_set = source_unit.communication_sets.first.copy_to(destination_unit) + copied_conditions = copied_set.communication_rules.first.communication_conditions.index_by(&:type) + + assert_equal 3, copied_conditions.fetch('LoginStatusCondition').activity_days + assert_equal 'within_last', copied_conditions.fetch('LoginStatusCondition').operator + assert_equal 14, copied_conditions.fetch('UnitViewedStatusCondition').activity_days + assert_equal 'more_than', copied_conditions.fetch('UnitViewedStatusCondition').operator + end + + private + + def unit_and_rule(name) + unit = FactoryBot.create( + :unit, + with_students: false, + task_count: 0, + stream_count: 0, + tutorials: 0, + outcome_count: 0, + staff_count: 0, + campus_count: 1 + ) + communication_set = unit.communication_sets.create!(name: name, active: true) + rule = communication_set.communication_rules.create!(name: name, operator: 'and', position: 0) + + [unit, rule] + end + + def activity_project(unit, last_sign_in_at: nil, last_viewed_at: nil) + user = FactoryBot.create(:user, :student, last_sign_in_at: last_sign_in_at) + FactoryBot.create(:project, unit: unit, user: user, last_viewed_at: last_viewed_at) + end end From f26c9c573894543a50220dc7708fc4540e764aaf Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:33:28 +1000 Subject: [PATCH 170/199] fix: prevent deleting task definitions used as prerequisites --- app/api/task_definitions_api.rb | 3 ++- app/models/task_definition.rb | 12 ++++++++++++ test/api/units/task_definitions_api_test.rb | 21 +++++++++++++++++++++ test/models/task_definition_test.rb | 15 +++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/app/api/task_definitions_api.rb b/app/api/task_definitions_api.rb index 239efa51b4..c50fa07b4b 100644 --- a/app/api/task_definitions_api.rb +++ b/app/api/task_definitions_api.rb @@ -322,7 +322,8 @@ class TaskDefinitionsApi < Grape::API end task_def.destroy - task_def.destroyed? + error!({ error: task_def.errors.full_messages.last }, 403) unless task_def.destroyed? + true end desc 'Upload the task sheet for a given task' diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index de21e06901..fb14fa2303 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -55,6 +55,7 @@ def self.permissions delegate :role_for, to: :unit + before_destroy :ensure_not_used_as_prerequisite, prepend: true before_destroy :delete_associated_files after_update :move_files_on_abbreviation_change, if: :saved_change_to_abbreviation? @@ -184,6 +185,8 @@ def cant_disable_aip_only_if_aip_tasks_exist def check_existing_prerequisites prereqs = TaskPrerequisite.where(task_definition_id: id) prereqs.each do |dp| + next if dp.prerequisite.nil? + if target_grade < dp.prerequisite.target_grade errors.add(:target_grade, "cannot be lower than prerequisite #{dp.prerequisite.abbreviation}'s target grade") end @@ -191,12 +194,21 @@ def check_existing_prerequisites dependents = TaskPrerequisite.where(prerequisite_id: id) dependents.each do |pr| + next if pr.task_definition.nil? + if target_grade > pr.task_definition.target_grade errors.add(:target_grade, "cannot exceed the target grade #{pr.task_definition.abbreviation} because this is a prerequisite") end end end + def ensure_not_used_as_prerequisite + return unless TaskPrerequisite.exists?(prerequisite_id: id) + + errors.add(:base, "Cannot delete task definition while it is used as a prerequisite. Remove the prerequisite links first.") + throw :abort + end + # In the rollover process, copy this definition into another unit # Copy this task into the other unit def copy_to(other_unit) diff --git a/test/api/units/task_definitions_api_test.rb b/test/api/units/task_definitions_api_test.rb index 00967f32b0..8b630667a9 100644 --- a/test/api/units/task_definitions_api_test.rb +++ b/test/api/units/task_definitions_api_test.rb @@ -937,6 +937,27 @@ def test_task_prerequisites end end + def test_cannot_delete_task_definition_used_as_a_prerequisite + unit = FactoryBot.create(:unit, task_count: 0) + prerequisite = FactoryBot.create(:task_definition, unit: unit, target_grade: 0) + dependent = FactoryBot.create(:task_definition, unit: unit, target_grade: 0) + task_prerequisite = TaskPrerequisite.create!( + task_definition: dependent, + prerequisite: prerequisite, + task_status_id: TaskStatus.complete.id + ) + + add_auth_header_for(user: unit.main_convenor_user) + delete "/api/units/#{unit.id}/task_definitions/#{prerequisite.id}" + + assert_equal 403, last_response.status, last_response_body + assert_equal "Cannot delete task definition while it is used as a prerequisite. Remove the prerequisite links first.", + last_response_body['error'] + assert TaskDefinition.exists?(prerequisite.id) + assert TaskDefinition.exists?(dependent.id) + assert TaskPrerequisite.exists?(task_prerequisite.id) + end + def test_download_student_submission_jobs unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) diff --git a/test/models/task_definition_test.rb b/test/models/task_definition_test.rb index 6312419108..c82e639cb7 100644 --- a/test/models/task_definition_test.rb +++ b/test/models/task_definition_test.rb @@ -8,6 +8,21 @@ def app Rails.application end + def test_validation_ignores_orphaned_prerequisite_records + unit = FactoryBot.create(:unit, task_count: 0) + prerequisite = FactoryBot.create(:task_definition, unit: unit, target_grade: 0) + dependent = FactoryBot.create(:task_definition, unit: unit, target_grade: 0) + TaskPrerequisite.create!( + task_definition: dependent, + prerequisite: prerequisite, + task_status_id: TaskStatus.complete.id + ) + + prerequisite.delete + + assert_nothing_raised { dependent.update!(name: 'Updated task definition') } + end + def test_overseer_requires_a_submission_history_upload task_definition = FactoryBot.build( :task_definition, From 2b3752ac6b888cffc38cefa86a58bbebfb844aff Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:59:12 +1000 Subject: [PATCH 171/199] fix: allow unit deletion with task prerequisites --- app/models/task_definition.rb | 1 + test/api/units/task_definitions_api_test.rb | 1 + test/models/task_definition_test.rb | 17 +++++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index fb14fa2303..e7f3c7289b 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -203,6 +203,7 @@ def check_existing_prerequisites end def ensure_not_used_as_prerequisite + return if destroyed_by_association&.name == :task_definitions return unless TaskPrerequisite.exists?(prerequisite_id: id) errors.add(:base, "Cannot delete task definition while it is used as a prerequisite. Remove the prerequisite links first.") diff --git a/test/api/units/task_definitions_api_test.rb b/test/api/units/task_definitions_api_test.rb index 8b630667a9..d0068d350a 100644 --- a/test/api/units/task_definitions_api_test.rb +++ b/test/api/units/task_definitions_api_test.rb @@ -215,6 +215,7 @@ def test_post_task_resources assert_requested upload_stub, times: 1 assert_requested delete_stub, times: 0 + TaskPrerequisite.where(prerequisite_id: td.id).destroy_all td.destroy! assert_requested delete_stub, times: 1 end diff --git a/test/models/task_definition_test.rb b/test/models/task_definition_test.rb index c82e639cb7..ff1fdd4c9c 100644 --- a/test/models/task_definition_test.rb +++ b/test/models/task_definition_test.rb @@ -23,6 +23,23 @@ def test_validation_ignores_orphaned_prerequisite_records assert_nothing_raised { dependent.update!(name: 'Updated task definition') } end + def test_unit_can_be_destroyed_when_its_tasks_have_prerequisites + unit = FactoryBot.create(:unit, task_count: 0) + prerequisite = FactoryBot.create(:task_definition, unit: unit, target_grade: 0) + dependent = FactoryBot.create(:task_definition, unit: unit, target_grade: 0) + task_prerequisite = TaskPrerequisite.create!( + task_definition: dependent, + prerequisite: prerequisite, + task_status_id: TaskStatus.complete.id + ) + + unit.destroy! + + assert_not TaskDefinition.exists?(prerequisite.id) + assert_not TaskDefinition.exists?(dependent.id) + assert_not TaskPrerequisite.exists?(task_prerequisite.id) + end + def test_overseer_requires_a_submission_history_upload task_definition = FactoryBot.build( :task_definition, From 48180497875529f6ac843a00519a6ce69f4ac4bc Mon Sep 17 00:00:00 2001 From: b0ink <40929320+b0ink@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:05:45 +1000 Subject: [PATCH 172/199] fix: iterate sentry exception interface values correctly --- config/initializers/sentry.rb | 3 ++- test/config/sentry_redaction_test.rb | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 test/config/sentry_redaction_test.rb diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb index 1f919db82e..f09d28752b 100644 --- a/config/initializers/sentry.rb +++ b/config/initializers/sentry.rb @@ -30,7 +30,8 @@ def scrub_request(request) end def scrub_exception(exception) - exception&.each_value do |value| + exception_values = exception&.values + exception_values&.each do |value| value.value = scrub_string(value.value) if value.respond_to?(:value) && value.respond_to?(:value=) next unless value.respond_to?(:stacktrace) diff --git a/test/config/sentry_redaction_test.rb b/test/config/sentry_redaction_test.rb new file mode 100644 index 0000000000..46c63a7f26 --- /dev/null +++ b/test/config/sentry_redaction_test.rb @@ -0,0 +1,18 @@ +require "test_helper" + +class SentryRedactionTest < ActiveSupport::TestCase + test "scrubs exception values from Sentry's exception interface" do + exception_value = Sentry::SingleExceptionInterface.new( + exception: RuntimeError.new("tmp/rails-latex/task-20260724-1100-student-name-task-1-2/file.tex"), + mechanism: nil + ) + exception = Sentry::ExceptionInterface.new(exceptions: [exception_value]) + + OnTrackSentryRedaction.scrub_exception(exception) + + assert_equal( + "tmp/rails-latex/task-20260724-1100-[username]-task-1-2/file.tex (RuntimeError)", + exception_value.value + ) + end +end From f6dfd806570f6b30384571210402a3e92c405b2f Mon Sep 17 00:00:00 2001 From: Boink <40929320+b0ink@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:50:26 +1000 Subject: [PATCH 173/199] feat: discuss timeout (#646) * feat: discuss timeout * refactor: track when student was notified of warning and expiry * chore: test for notify discuss timeout job * chore: bump migration * chore: reset schema * chore: bump migration * chore: improve wording * feat: expose tasks discussion timeout expiry date * feat: send email with discussion timeout notification * feat: expose discuss timeout thresholds instead * fix: allow move to fix and resubmit * fix: ensure task transition to fix is authorised * refator: fix wording * chore: restore schema.rb * chore: bump migration * chore: fix test * feat: factor in teaching period breaks to pause discussion deadlines * refactor: move mail styling to shared partial * chore: fix test --- .../entities/minimal/minimal_unit_entity.rb | 2 + app/api/entities/task_entity.rb | 3 + app/api/entities/unit_entity.rb | 4 + app/api/units_api.rb | 12 ++ app/mailers/notifications_mailer.rb | 36 ++++++ .../comments/discuss_timeout_comment.rb | 12 ++ app/models/project.rb | 4 + app/models/task.rb | 90 +++++++++++--- app/models/unit.rb | 117 ++++++++++++++++++ app/sidekiq/notify_discuss_timeout_job.rb | 12 ++ app/sidekiq/send_discuss_timeout_email_job.rb | 24 ++++ .../discussion_deadline_mailer.html.erb | 59 +++++++++ .../discussion_deadline_mailer.text.erb | 1 + .../discussion_deadline_approaching.html.erb | 16 +++ .../discussion_deadline_approaching.text.erb | 14 +++ .../discussion_deadline_missed.html.erb | 13 ++ .../discussion_deadline_missed.text.erb | 14 +++ config/schedule.yml | 4 + .../20260722065239_add_discuss_timeout.rb | 14 +++ db/schema.rb | 11 +- lib/tasks/notifications.rake | 7 ++ test/mailers/unit_mail_test.rb | 57 +++++++++ test/models/discuss_timeout_test.rb | 67 ++++++++++ test/models/task_test.rb | 15 +++ test/sidekiq/scheduled_job_test.rb | 3 +- 25 files changed, 593 insertions(+), 18 deletions(-) create mode 100644 app/models/comments/discuss_timeout_comment.rb create mode 100644 app/sidekiq/notify_discuss_timeout_job.rb create mode 100644 app/sidekiq/send_discuss_timeout_email_job.rb create mode 100644 app/views/layouts/discussion_deadline_mailer.html.erb create mode 100644 app/views/layouts/discussion_deadline_mailer.text.erb create mode 100644 app/views/notifications_mailer/discussion_deadline_approaching.html.erb create mode 100644 app/views/notifications_mailer/discussion_deadline_approaching.text.erb create mode 100644 app/views/notifications_mailer/discussion_deadline_missed.html.erb create mode 100644 app/views/notifications_mailer/discussion_deadline_missed.text.erb create mode 100644 db/migrate/20260722065239_add_discuss_timeout.rb create mode 100644 lib/tasks/notifications.rake create mode 100644 test/models/discuss_timeout_test.rb diff --git a/app/api/entities/minimal/minimal_unit_entity.rb b/app/api/entities/minimal/minimal_unit_entity.rb index 06f9659f2a..7e71b425b3 100644 --- a/app/api/entities/minimal/minimal_unit_entity.rb +++ b/app/api/entities/minimal/minimal_unit_entity.rb @@ -22,6 +22,8 @@ class MinimalUnitEntity < Grape::Entity expose :active expose :grade_values expose :grade_definitions + expose :discuss_timeout_enabled, unless: :summary_only + expose :discuss_timeout_expire_days, unless: :summary_only end end end diff --git a/app/api/entities/task_entity.rb b/app/api/entities/task_entity.rb index 1de8fb6b39..93d6190b6f 100644 --- a/app/api/entities/task_entity.rb +++ b/app/api/entities/task_entity.rb @@ -18,6 +18,9 @@ class TaskEntity < Grape::Entity expose :target_start_date, expose_nil: false end + expose :moved_to_discuss_at, expose_nil: false + expose :discuss_timeout_expiry_at, expose_nil: false + expose :extensions expose :scorm_extensions diff --git a/app/api/entities/unit_entity.rb b/app/api/entities/unit_entity.rb index 4f7a887cfb..c820f2e362 100644 --- a/app/api/entities/unit_entity.rb +++ b/app/api/entities/unit_entity.rb @@ -81,6 +81,10 @@ def can_read_unit_config?(my_role) expose :feedback_warning_threshold_days, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) } expose :feedback_overflow_threshold_days, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) } + expose :discuss_timeout_enabled, unless: :summary_only + expose :discuss_timeout_warning_days, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) } + expose :discuss_timeout_expire_days, unless: :summary_only + expose :enforce_feedback_before_discussed_in_class, if: lambda { |unit, options| is_staff?(options[:my_role]) } end end diff --git a/app/api/units_api.rb b/app/api/units_api.rb index 8c6fe71754..3cc96fbe5c 100644 --- a/app/api/units_api.rb +++ b/app/api/units_api.rb @@ -93,6 +93,9 @@ class UnitsApi < Grape::API optional :assessment_enabled, type: Boolean optional :feedback_warning_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its highlighted in the tutors inbox' optional :feedback_overflow_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its added to overflow marking' + optional :discuss_timeout_enabled, type: Boolean, desc: 'Move stale Discuss tasks back to Fix and Resubmit after a warning period' + optional :discuss_timeout_warning_days, type: Integer, desc: 'Number of days in Discuss before warning the student' + optional :discuss_timeout_expire_days, type: Integer, desc: 'Number of days in Discuss before moving the task to Fix and Resubmit' optional :enforce_feedback_before_discussed_in_class, type: Boolean, desc: 'Require feedback to be completed before tasks can be marked discussed in class' optional :grade_definitions, type: Array do requires :id, type: String @@ -135,6 +138,9 @@ class UnitsApi < Grape::API :assessment_enabled, :feedback_warning_threshold_days, :feedback_overflow_threshold_days, + :discuss_timeout_enabled, + :discuss_timeout_warning_days, + :discuss_timeout_expire_days, :enforce_feedback_before_discussed_in_class, grade_definitions: [:id, :value, :label, :abbreviation] ) @@ -184,6 +190,9 @@ class UnitsApi < Grape::API optional :allow_student_change_tutorial, type: Boolean, desc: 'Can turn on/off student ability to change tutorials', default: true optional :feedback_warning_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its highlighted in the tutors inbox' optional :feedback_overflow_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its added to overflow marking' + optional :discuss_timeout_enabled, type: Boolean, desc: 'Move stale Discuss tasks back to Fix and Resubmit after a warning period', default: false + optional :discuss_timeout_warning_days, type: Integer, desc: 'Number of days in Discuss before warning the student', default: 7 + optional :discuss_timeout_expire_days, type: Integer, desc: 'Number of days in Discuss before moving the task to Fix and Resubmit', default: 14 optional :enforce_feedback_before_discussed_in_class, type: Boolean, desc: 'Require feedback to be completed before tasks can be marked discussed in class', default: false optional :grade_definitions, type: Array do requires :id, type: String @@ -223,6 +232,9 @@ class UnitsApi < Grape::API :allow_student_change_tutorial, :feedback_warning_threshold_days, :feedback_overflow_threshold_days, + :discuss_timeout_enabled, + :discuss_timeout_warning_days, + :discuss_timeout_expire_days, :enforce_feedback_before_discussed_in_class, grade_definitions: [:id, :value, :label, :abbreviation] ) diff --git a/app/mailers/notifications_mailer.rb b/app/mailers/notifications_mailer.rb index f4aefa1499..f4b0d255ad 100644 --- a/app/mailers/notifications_mailer.rb +++ b/app/mailers/notifications_mailer.rb @@ -1,4 +1,7 @@ class NotificationsMailer < ApplicationMailer + layout 'discussion_deadline_mailer', + only: %i[discussion_deadline_approaching discussion_deadline_missed] + def add_general @doubtfire_host = Doubtfire::Application.config.institution[:host] @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] @@ -80,6 +83,27 @@ def weekly_student_summary(project, summary_stats, did_revert_to_pass) mail(to: email_with_name, from: tutor_email, subject: subject) end + def discussion_deadline_approaching(task, sender, expiry_date) + add_discussion_deadline_details(task, sender) + @deadline = task.unit.formatted_discuss_timeout_date(expiry_date) + + mail( + to: %("#{@student.name}" <#{@student.email}>), + from: %("#{@sender.name}" <#{@sender.email}>), + subject: "#{@unit.code}: Discussion deadline approaching for #{@task.task_definition.abbreviation}" + ) + end + + def discussion_deadline_missed(task, sender) + add_discussion_deadline_details(task, sender) + + mail( + to: %("#{@student.name}" <#{@student.email}>), + from: %("#{@sender.name}" <#{@sender.email}>), + subject: "#{@unit.code}: Discussion deadline missed for #{@task.task_definition.abbreviation}" + ) + end + def top_task_desc(tt) "#{tt[:task_definition].abbreviation} - #{tt[:task_definition].name} #{"- which you need to discuss with your tutor" if tt[:status] == :discuss}" end @@ -112,4 +136,16 @@ def this_these(num) helper_method :were_was helper_method :are_is helper_method :this_these + + private + + def add_discussion_deadline_details(task, sender) + add_general + @task = task + @project = task.project + @unit = task.unit + @student = @project.student + @sender = sender + @task_url = "#{@doubtfire_host}/projects/#{@project.id}/dashboard/#{@task.task_definition.abbreviation}" + end end diff --git a/app/models/comments/discuss_timeout_comment.rb b/app/models/comments/discuss_timeout_comment.rb new file mode 100644 index 0000000000..a45b8aadd6 --- /dev/null +++ b/app/models/comments/discuss_timeout_comment.rb @@ -0,0 +1,12 @@ +class DiscussTimeoutComment < TaskComment + WARNING_CONTENT_TYPE = 'discuss_timeout_warning'.freeze + EXPIRED_CONTENT_TYPE = 'discuss_timeout_expired'.freeze + + def self.warning + WARNING_CONTENT_TYPE + end + + def self.expired + EXPIRED_CONTENT_TYPE + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 64dc33ed4e..b74cc84add 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -278,6 +278,8 @@ def reference_date end def task_details_for_shallow_serializer(user) + teaching_breaks = unit.teaching_period&.breaks.to_a + tasks .joins(:task_status) .joins("LEFT JOIN task_comments ON task_comments.task_id = tasks.id AND (task_comments.type IS NULL OR task_comments.type <> 'TaskStatusComment')") @@ -309,6 +311,8 @@ def task_details_for_shallow_serializer(user) extensions: t.extensions, scorm_extensions: t.scorm_extensions, due_date: t.due_date, + moved_to_discuss_at: t.moved_to_discuss_at, + discuss_timeout_expiry_at: t.discuss_timeout_expiry_at(teaching_breaks: teaching_breaks), submission_date: t.submission_date, completion_date: t.completion_date, target_start_date: t.target_start_date, diff --git a/app/models/task.rb b/app/models/task.rb index 4a09286d4c..9d605a7254 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -147,6 +147,7 @@ def specific_permission_hash(role, perm_hash, _other) delegate :target_date, to: :task_definition delegate :update_task_stats, to: :project + before_save :set_discuss_timeout_tracking, if: :will_save_change_to_task_status_id? after_update :update_task_stats, if: :saved_change_to_task_status_id? # TODO: consider moving to async task validates :task_definition_id, uniqueness: { scope: :project, @@ -260,6 +261,12 @@ def comments_for_user(user) ) end + def set_discuss_timeout_tracking + self.moved_to_discuss_at = task_status_id == TaskStatus.discuss.id ? Time.zone.now : nil + self.notified_discuss_warning_at = nil + self.notified_discuss_expiry_at = nil + end + def current_task_similarities task_similarities.where(dismissed: false) end @@ -446,6 +453,36 @@ def days_awaiting_feedback(now_time = Time.zone.now) ([0, current_time - submission_time - paused_seconds].max / 1.day).floor end + def discuss_timeout_elapsed_days(now_time = Time.zone.now, teaching_breaks: nil) + return 0 if moved_to_discuss_at.blank? + + discussion_time = moved_to_discuss_at.to_f + current_time = now_time.to_f + return 0 if current_time <= discussion_time + + teaching_breaks ||= unit&.teaching_period&.breaks || [] + paused_seconds = break_overlap_seconds(discussion_time, current_time, teaching_breaks) + + ([0, current_time - discussion_time - paused_seconds].max / 1.day).floor + end + + def discuss_timeout_expiry_at(timeout_days = unit.discuss_timeout_expire_days, teaching_breaks: nil) + return nil if moved_to_discuss_at.blank? + + deadline = moved_to_discuss_at + timeout_days.days + teaching_breaks ||= unit&.teaching_period&.breaks || [] + + teaching_breaks.sort_by(&:start_date).each do |teaching_break| + break_start = teaching_break.start_date + break_end = break_start + teaching_break.number_of_weeks.to_i.weeks + next if break_end <= moved_to_discuss_at || break_start >= deadline + + deadline += break_end - [break_start, moved_to_discuss_at].max + end + + deadline + end + # Excludes any breaks that would otherwise "pause" feedback def calendar_days_awaiting_feedback(now_time = Time.zone.now) return 0 if submission_date.blank? @@ -552,6 +589,25 @@ def active_overflow_task_claim claim end + def transition_assignment_allowed?(by_user, system_transition) + return true if system_transition + + if task_definition.lock_assessments_to_tutorial_stream + unit_role = unit.unit_role_for(by_user) + return false unless task_definition.tutorial_stream.tutorials.any? { |tutorial| tutorial.unit_role == unit_role } + end + + claim = active_overflow_task_claim + return true if claim.blank? + + unit_role = unit.unit_role_for(by_user) + unit_role.nil? || unit_role.id == claim.claimed_by_unit_role_id + end + + def transition_feedback_check_required?(check_feedback, system_transition) + check_feedback && !system_transition + end + def group return nil unless group_task? @@ -568,11 +624,12 @@ def ensured_group_submission end def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: false, quality: 1, recursive_fix: false, - check_feedback: false) + check_feedback: false, system_transition: false) # # Ensure that assessor is allowed to update the task in the indicated way # role = role_for(by_user) + role = :tutor if system_transition && by_user.present? return nil if role.nil? @@ -587,20 +644,7 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: # Protect closed states from student changes return nil if [:student, :group_member].include?(role) && task_submission_closed? - if task_definition.lock_assessments_to_tutorial_stream - unit_role = unit.unit_role_for(by_user) - tutorial_stream = task_definition.tutorial_stream - tutorials = tutorial_stream.tutorials - return nil unless tutorials.any? { |t| t.unit_role == unit_role } - end - - # Check to see if another tutor has claimed this task from overflow - if active_overflow_task_claim - unit_role = unit.unit_role_for(by_user) - if unit_role && unit_role.id != active_overflow_task_claim.claimed_by_unit_role_id - return nil - end - end + return nil unless transition_assignment_allowed?(by_user, system_transition) # # State transitions based upon the trigger # @@ -626,7 +670,7 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: return nil end - if check_feedback + if transition_feedback_check_required?(check_feedback, system_transition) if status == TaskStatus.complete && !has_manual_feedback_since_first_ready_for_feedback? errors.add(:task_status, "cannot be moved to '#{status.name}' until feedback has been given") return nil @@ -987,6 +1031,20 @@ def add_discussed_comment(current_user) discussed end + def add_discuss_timeout_comment(current_user, content_type, text) + return nil unless individual_task_or_submitter_of_group_task? + + comment = DiscussTimeoutComment.create + comment.task = self + comment.user = current_user + comment.comment = text + comment.content_type = content_type + comment.recipient = project.student + comment.save! + + comment + end + def add_checked_in_comment(current_user) discussed = TaskCheckedInComment.create discussed.task = self diff --git a/app/models/unit.rb b/app/models/unit.rb index eff8740ec8..79bb29a4a3 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -220,7 +220,13 @@ def role_for(user) validates :feedback_overflow_threshold_days, numericality: { greater_than_or_equal_to: 0 } + validates :discuss_timeout_warning_days, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :discuss_timeout_expire_days, + numericality: { only_integer: true, greater_than_or_equal_to: 1 } + validate :warning_not_greater_than_overflow + validate :discuss_timeout_warning_before_expiry validate :validate_end_date_after_start_date validate :ensure_teaching_period_dates_match, if: :has_teaching_period? @@ -256,6 +262,117 @@ def warning_not_greater_than_overflow ) end + def discuss_timeout_warning_before_expiry + return unless discuss_timeout_enabled + return if discuss_timeout_warning_days < discuss_timeout_expire_days + + errors.add(:discuss_timeout_warning_days, 'must be less than the expiry days') + end + + def self.notify_discuss_timeouts! + set_active.find_each(&:notify_discuss_timeouts!) + end + + def notify_discuss_timeouts! + return 0 unless discuss_timeout_enabled + + teaching_breaks = teaching_period&.breaks.to_a + discuss_timeout_tasks.find_each.sum do |task| + notify_discuss_timeout_for(task, teaching_breaks: teaching_breaks) + end + end + + def discuss_timeout_tasks + tasks + .includes(:project, :task_definition) + .where(task_status_id: TaskStatus.discuss.id) + .where.not(moved_to_discuss_at: nil) + .where('moved_to_discuss_at <= ?', discuss_timeout_warning_days.days.ago) + end + + def notify_discuss_timeout_for(task, teaching_breaks: nil, now_time: Time.zone.now) + return 0 if task.moved_to_discuss_at.blank? + + actor = task.project.tutor_for(task.task_definition) || main_convenor&.user + return 0 if actor.blank? + + elapsed_days = task.discuss_timeout_elapsed_days(now_time, teaching_breaks: teaching_breaks) + if elapsed_days >= discuss_timeout_expire_days + expire_discuss_timeout_task(task, actor) + elsif elapsed_days >= discuss_timeout_warning_days + warn_discuss_timeout_task(task, actor, teaching_breaks: teaching_breaks) + else + 0 + end + end + + def warn_discuss_timeout_task(task, actor, teaching_breaks: nil) + return 0 if task.notified_discuss_warning_at.present? + + expiry_date = discuss_timeout_expiry_date(task, teaching_breaks: teaching_breaks) + created_comment = false + Task.transaction do + comment = task.add_discuss_timeout_comment( + actor, + DiscussTimeoutComment.warning, + "You must discuss this task with your tutor before #{formatted_discuss_timeout_date(expiry_date)}. If it has not been discussed by then, it will move to Fix and Resubmit, and you will need to resubmit your work." + ) + raise ActiveRecord::Rollback if comment.blank? + + task.update!(notified_discuss_warning_at: Time.zone.now) + queue_discuss_timeout_email(task, actor, :approaching, expiry_date) + created_comment = true + end + + created_comment ? 1 : 0 + end + + def expire_discuss_timeout_task(task, actor) + return 0 if task.notified_discuss_expiry_at.present? + + created_comment = false + Task.transaction do + task.update!(notified_discuss_expiry_at: Time.zone.now) + unless task.trigger_transition(trigger: 'fix', by_user: actor, system_transition: true) + raise ActiveRecord::Rollback + end + + comment = task.add_discuss_timeout_comment( + actor, + DiscussTimeoutComment.expired, + "This task moved to Fix and Resubmit because it was not discussed by the deadline. Review any feedback and resubmit it when you are ready." + ) + unless comment + raise ActiveRecord::Rollback + end + + queue_discuss_timeout_email(task, actor, :missed) + created_comment = true + end + + created_comment ? 1 : 0 + end + + def discuss_timeout_expiry_date(task, teaching_breaks: nil) + task + .discuss_timeout_expiry_at(discuss_timeout_expire_days, teaching_breaks: teaching_breaks) + &.to_date + end + + def formatted_discuss_timeout_date(date) + result = "the #{date.day.ordinalize} of #{Date::MONTHNAMES[date.month]}" + return result if date.year == Time.zone.today.year + + "#{result} #{date.year}" + end + + def queue_discuss_timeout_email(task, actor, type, expiry_date = nil) + return unless send_notifications + return unless task.project.student.receive_feedback_notifications + + SendDiscussTimeoutEmailJob.perform_async(task.id, actor.id, type.to_s, expiry_date&.iso8601) + end + def detailed_name "#{name} #{teaching_period.present? ? teaching_period.detailed_name : start_date.strftime('%Y-%m-%d')}" end diff --git a/app/sidekiq/notify_discuss_timeout_job.rb b/app/sidekiq/notify_discuss_timeout_job.rb new file mode 100644 index 0000000000..3a1de38a04 --- /dev/null +++ b/app/sidekiq/notify_discuss_timeout_job.rb @@ -0,0 +1,12 @@ +class NotifyDiscussTimeoutJob + include Sidekiq::Job + + sidekiq_options lock: :until_executed, + lock_args_method: ->(_args) { ['notify-discuss-timeout'] }, + on_conflict: :reject, + retry: 1 + + def perform + Unit.notify_discuss_timeouts! + end +end diff --git a/app/sidekiq/send_discuss_timeout_email_job.rb b/app/sidekiq/send_discuss_timeout_email_job.rb new file mode 100644 index 0000000000..76349e7835 --- /dev/null +++ b/app/sidekiq/send_discuss_timeout_email_job.rb @@ -0,0 +1,24 @@ +class SendDiscussTimeoutEmailJob + include Sidekiq::Job + + sidekiq_options retry: 5 + + def perform(task_id, sender_id, notification_type, expiry_date = nil) + task = Task.find_by(id: task_id) + sender = User.find_by(id: sender_id) + return if task.blank? || sender.blank? + return unless task.unit.send_notifications + return unless task.project.student.receive_feedback_notifications + + mail = case notification_type + when 'approaching' + NotificationsMailer.discussion_deadline_approaching(task, sender, Date.iso8601(expiry_date)) + when 'missed' + NotificationsMailer.discussion_deadline_missed(task, sender) + else + raise ArgumentError, "Unknown discussion deadline notification type: #{notification_type}" + end + + mail.deliver_now + end +end diff --git a/app/views/layouts/discussion_deadline_mailer.html.erb b/app/views/layouts/discussion_deadline_mailer.html.erb new file mode 100644 index 0000000000..36b8b3f822 --- /dev/null +++ b/app/views/layouts/discussion_deadline_mailer.html.erb @@ -0,0 +1,59 @@ + + + + + + +
    + <%= yield %> + +

    + Cheers,
    + The <%= @doubtfire_product_name %> Team on behalf of <%= @sender.name %> +

    +
    +
    + Unsubscribe | Generated with <%= @doubtfire_product_name %> +
    + + diff --git a/app/views/layouts/discussion_deadline_mailer.text.erb b/app/views/layouts/discussion_deadline_mailer.text.erb new file mode 100644 index 0000000000..37f0bddbd7 --- /dev/null +++ b/app/views/layouts/discussion_deadline_mailer.text.erb @@ -0,0 +1 @@ +<%= yield %> diff --git a/app/views/notifications_mailer/discussion_deadline_approaching.html.erb b/app/views/notifications_mailer/discussion_deadline_approaching.html.erb new file mode 100644 index 0000000000..80f1278ab6 --- /dev/null +++ b/app/views/notifications_mailer/discussion_deadline_approaching.html.erb @@ -0,0 +1,16 @@ +

    <%= @unit.name %> - Discussion Deadline Approaching

    +

    <%= @task.task_definition.abbreviation %> - <%= @task.task_definition.name %>

    + +

    Hi <%= @student.first_name %>,

    + +

    + You must discuss task <%= @task.task_definition.abbreviation %> with your tutor before + <%= @deadline %>. +

    + +

    + If it is not discussed by then, it will move to Fix and Resubmit, and you will + need to resubmit your work. +

    + +

    View this task in <%= @doubtfire_product_name %>.

    diff --git a/app/views/notifications_mailer/discussion_deadline_approaching.text.erb b/app/views/notifications_mailer/discussion_deadline_approaching.text.erb new file mode 100644 index 0000000000..37d86d5171 --- /dev/null +++ b/app/views/notifications_mailer/discussion_deadline_approaching.text.erb @@ -0,0 +1,14 @@ +Hi <%= @student.first_name %>, + +You must discuss task <%= @task.task_definition.abbreviation %> with your tutor before <%= @deadline %>. + +If it is not discussed by then, it will move to Fix and Resubmit, and you will need to resubmit your work. + +View this task in <%= @doubtfire_product_name %>: <%= @task_url %> + +Cheers, +The <%= @doubtfire_product_name %> Team on behalf of <%= @sender.name %> + +--- + +Visit <%= @unsubscribe_url %> to unsubscribe from these notifications. diff --git a/app/views/notifications_mailer/discussion_deadline_missed.html.erb b/app/views/notifications_mailer/discussion_deadline_missed.html.erb new file mode 100644 index 0000000000..eade95411a --- /dev/null +++ b/app/views/notifications_mailer/discussion_deadline_missed.html.erb @@ -0,0 +1,13 @@ +

    <%= @unit.name %> - Discussion Deadline Missed

    +

    <%= @task.task_definition.abbreviation %> - <%= @task.task_definition.name %>

    + +

    Hi <%= @student.first_name %>,

    + +

    + Task <%= @task.task_definition.abbreviation %> moved to + Fix and Resubmit because it was not discussed by the deadline. +

    + +

    Review any feedback and resubmit it when you are ready.

    + +

    View this task in <%= @doubtfire_product_name %>.

    diff --git a/app/views/notifications_mailer/discussion_deadline_missed.text.erb b/app/views/notifications_mailer/discussion_deadline_missed.text.erb new file mode 100644 index 0000000000..bbc19c18a5 --- /dev/null +++ b/app/views/notifications_mailer/discussion_deadline_missed.text.erb @@ -0,0 +1,14 @@ +Hi <%= @student.first_name %>, + +<%= @task.task_definition.abbreviation %> - <%= @task.task_definition.name %> moved to Fix and Resubmit because it was not discussed by the deadline. + +Review any feedback and resubmit it when you are ready. + +View this task in <%= @doubtfire_product_name %>: <%= @task_url %> + +Cheers, +The <%= @doubtfire_product_name %> Team on behalf of <%= @sender.name %> + +--- + +Visit <%= @unsubscribe_url %> to unsubscribe from these notifications. diff --git a/config/schedule.yml b/config/schedule.yml index 62fd893daf..adb2a1f282 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -24,6 +24,10 @@ poll_communication_set_schedules: cron: "every 5 minutes" class: "PollCommunicationSetSchedulesJob" +notify_discuss_timeout: + cron: "every day at 8am" + class: "NotifyDiscussTimeoutJob" + # archive_old_units: # cron: "every 6 months" # class: "ArchiveOldUnitsJob" diff --git a/db/migrate/20260722065239_add_discuss_timeout.rb b/db/migrate/20260722065239_add_discuss_timeout.rb new file mode 100644 index 0000000000..413de251e5 --- /dev/null +++ b/db/migrate/20260722065239_add_discuss_timeout.rb @@ -0,0 +1,14 @@ +class AddDiscussTimeout < ActiveRecord::Migration[8.0] + def change + add_column :units, :discuss_timeout_enabled, :boolean, null: false, default: false + add_column :units, :discuss_timeout_warning_days, :integer, null: false, default: 7 + add_column :units, :discuss_timeout_expire_days, :integer, null: false, default: 14 + add_column :tasks, :moved_to_discuss_at, :datetime + add_column :tasks, :notified_discuss_warning_at, :datetime + add_column :tasks, :notified_discuss_expiry_at, :datetime + + add_index :tasks, [:task_status_id, :moved_to_discuss_at] + add_index :tasks, [:task_status_id, :notified_discuss_warning_at] + add_index :tasks, [:task_status_id, :notified_discuss_expiry_at] + end +end diff --git a/db/schema.rb b/db/schema.rb index 7a38c49aca..d68849bea9 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_07_22_030000) do +ActiveRecord::Schema[8.0].define(version: 2026_07_22_065239) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -704,10 +704,16 @@ t.datetime "target_start_date" t.datetime "target_due_date" t.datetime "last_tutor_feedback_at" + t.datetime "moved_to_discuss_at" + t.datetime "notified_discuss_warning_at" + t.datetime "notified_discuss_expiry_at" t.index ["group_submission_id"], name: "index_tasks_on_group_submission_id" t.index ["project_id", "task_definition_id"], name: "tasks_uniq_proj_task_def", unique: true t.index ["project_id"], name: "index_tasks_on_project_id" t.index ["task_definition_id"], name: "index_tasks_on_task_definition_id" + t.index ["task_status_id", "moved_to_discuss_at"], name: "index_tasks_on_task_status_id_and_moved_to_discuss_at" + t.index ["task_status_id", "notified_discuss_expiry_at"], name: "index_tasks_on_task_status_id_and_notified_discuss_expiry_at" + t.index ["task_status_id", "notified_discuss_warning_at"], name: "index_tasks_on_task_status_id_and_notified_discuss_warning_at" t.index ["task_status_id"], name: "index_tasks_on_task_status_id" end @@ -929,6 +935,9 @@ t.integer "feedback_overflow_threshold_days", default: 7 t.boolean "enforce_feedback_before_discussed_in_class", default: false, null: false t.text "grade_values", size: :long, collation: "utf8mb4_bin" + t.boolean "discuss_timeout_enabled", default: false, null: false + t.integer "discuss_timeout_warning_days", default: 7, null: false + t.integer "discuss_timeout_expire_days", default: 14, null: false t.index ["draft_task_definition_id"], name: "index_units_on_draft_task_definition_id" t.index ["main_convenor_id"], name: "index_units_on_main_convenor_id" t.index ["overseer_image_id"], name: "index_units_on_overseer_image_id" diff --git a/lib/tasks/notifications.rake b/lib/tasks/notifications.rake new file mode 100644 index 0000000000..577b0f3a48 --- /dev/null +++ b/lib/tasks/notifications.rake @@ -0,0 +1,7 @@ +namespace :notifications do + desc 'Warn students about stale Discuss tasks and expire overdue Discuss tasks' + task notify_discuss_timeout: :environment do + count = Unit.notify_discuss_timeouts! + Rails.logger.info "Discuss timeout notification pass created #{count} comment(s)." + end +end diff --git a/test/mailers/unit_mail_test.rb b/test/mailers/unit_mail_test.rb index c84b18a16d..86285d28f0 100644 --- a/test/mailers/unit_mail_test.rb +++ b/test/mailers/unit_mail_test.rb @@ -73,4 +73,61 @@ def test_send_overseer_assessment_failed_email assert mail.html_part.body.include? "projects/#{project.id}/dashboard/#{task.task_definition.abbreviation}" end + def test_send_discussion_deadline_emails + unit = FactoryBot.create(:unit) + project = unit.active_projects.first + task = project.task_for_task_definition(unit.task_definitions.first) + sender = unit.main_convenor_user + deadline = 7.days.from_now.to_date + + approaching = NotificationsMailer.discussion_deadline_approaching(task, sender, deadline) + missed = NotificationsMailer.discussion_deadline_missed(task, sender) + + assert_equal project.student.email, approaching.to.first + assert_includes approaching.subject, 'Discussion deadline approaching' + assert_includes approaching.text_part.body.to_s, unit.formatted_discuss_timeout_date(deadline) + assert_includes approaching.text_part.body.to_s, "projects/#{project.id}/dashboard/#{task.task_definition.abbreviation}" + assert_equal 1, approaching.html_part.body.to_s.scan('