diff --git a/.ci-setup/crontab b/.ci-setup/crontab index 87c9c955d8..b1298d5e39 100644 --- a/.ci-setup/crontab +++ b/.ci-setup/crontab @@ -3,6 +3,7 @@ BASH_ENV=/container.env PATH=/tmp/texlive/bin/x86_64-linux:/tmp/texlive/bin/aarch64-linux:/usr/local/bundle/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bundle/bin 10,15,20,25,30,35,40,45,50,55 * * * * /doubtfire/lib/shell/generate_pdfs.sh +0,10,20,30,40,50 * * * * /doubtfire/lib/shell/send_overseer_notifications.sh 0 5 * * * /doubtfire/lib/shell/check_plagiarism.sh 0 8 * * * /doubtfire/lib/shell/portfolio_autogen_check.sh 0 7 * * 1 /doubtfire/lib/shell/send_weekly_emails.sh diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..53de1f9cbd --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Jan 11 2022: Repository wide format (support zeitwerk mode loader) +f4050e826af83166e45b0b4f2af14179b40305dd diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2640e2f00c..3d7f226270 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -22,16 +22,18 @@ env: DF_TEST_DB_DATABASE: "doubtfire-test" DF_TEST_DB_USERNAME: "dfire" DF_TEST_DB_PASSWORD: "pwd" - OVERSEER_ENABLED: "0" + OVERSEER_ENABLED: "true" DF_ENCRYPTION_PRIMARY_KEY: "AMLOMYA5GV8B4fTK3VKMhVGn8WdvUW8g" DF_ENCRYPTION_DETERMINISTIC_KEY: "anlmuJ6cB3bN3biXRbYvmPsC5ALPFqGG" DF_ENCRYPTION_KEY_DERIVATION_SALT: "hzPR8D4qpOnAg7VeAhkhWw6JmmzKJB10" DF_REDIS_SIDEKIQ_URL: "redis://redis:6379/0" LATEX_CONTAINER_NAME: doubtfire-texlive LATEX_BUILD_PATH: /texlive/shell/latex_build.sh - DF_JPLAG_REPORT_DIR: /jplag/results + GOTENBERG_IMAGE: doubtfire-gotenberg-development:local + GOTENBERG_FALLBACK_VOLUME_CONTAINER: doubtfire-gotenberg-volumes LTI_SHARED_API_SECRET: "abc123" LTI_ENABLED: true + DF_INITIAL_ADMIN_PASSWORD: "password" jobs: unit-tests: @@ -64,6 +66,16 @@ jobs: tags: doubtfire-texlive-development:local cache-from: type=gha,scope=texlive cache-to: type=gha,mode=max,scope=texlive + - name: Build Gotenberg image + uses: docker/build-push-action@v5 + with: + context: . + file: gotenberg.Dockerfile + push: false + load: true + tags: doubtfire-gotenberg-development:local + cache-from: type=gha,scope=gotenberg + cache-to: type=gha,mode=max,scope=gotenberg - name: Build JPlag image uses: docker/build-push-action@v5 with: @@ -104,6 +116,13 @@ jobs: -v ${{ github.workspace }}:/doubtfire -v /var/run/docker.sock:/var/run/docker.sock run: docker exec -t ${{ env.LATEX_CONTAINER_NAME }} lualatex -v + - name: Start Gotenberg volume container + run: > + docker run --detach + --name ${{ env.GOTENBERG_FALLBACK_VOLUME_CONTAINER }} + --network none + --volume ${{ github.workspace }}/tmp/gotenberg:/workdir/gotenberg + alpine sleep infinity - name: Start JPlag service uses: maus007/docker-run-action-fork@207a4e2a8ebf7e4b985656ba990b1e53715dce2a with: @@ -111,7 +130,6 @@ jobs: options: > --name jplag -v ${{ github.workspace }}/student-work:/student-work - -v ${{ github.workspace }}/jplag/results:${{ env.DF_JPLAG_REPORT_DIR }} -v ${{ github.workspace }}/tmp/jplag:/tmp/jplag -v ${{ github.workspace }}/test_files/submissions/jplag:/test_files --detach @@ -152,9 +170,11 @@ jobs: -e DF_REDIS_SIDEKIQ_URL -e LATEX_CONTAINER_NAME -e LATEX_BUILD_PATH - -e DF_JPLAG_REPORT_DIR + -e GOTENBERG_IMAGE + -e GOTENBERG_FALLBACK_VOLUME_CONTAINER -e LTI_SHARED_API_SECRET -e LTI_ENABLED + -e DF_INITIAL_ADMIN_PASSWORD run: bundle exec rake db:populate - name: Run unit tests uses: maus007/docker-run-action-fork@207a4e2a8ebf7e4b985656ba990b1e53715dce2a @@ -164,7 +184,6 @@ jobs: -v ${{ github.workspace }}:/doubtfire -v ${{ github.workspace }}/student-work:/student-work -v /var/run/docker.sock:/var/run/docker.sock - -v ${{ github.workspace }}/jplag/results:${{ env.DF_JPLAG_REPORT_DIR }} -v ${{ github.workspace }}/tmp/jplag:/tmp/jplag -e RAILS_ENV -e DF_STUDENT_WORK_DIR @@ -185,7 +204,8 @@ jobs: -e DF_REDIS_SIDEKIQ_URL -e LATEX_CONTAINER_NAME -e LATEX_BUILD_PATH - -e DF_JPLAG_REPORT_DIR + -e GOTENBERG_IMAGE + -e GOTENBERG_FALLBACK_VOLUME_CONTAINER -e LTI_SHARED_API_SECRET -e LTI_ENABLED run: TERM=xterm bundle exec rails test @@ -193,3 +213,6 @@ jobs: run: docker rm -f ${{ env.LATEX_CONTAINER_NAME }} - name: Stop JPlag service run: docker rm -f jplag + - name: Stop Gotenberg volume container + if: always() + run: docker rm -f ${{ env.GOTENBERG_FALLBACK_VOLUME_CONTAINER }} diff --git a/.gitignore b/.gitignore index 977065f4e4..35a46c3ae7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ student-work/ .DS_Store .env .env* +/config/credentials/*.yml.enc +/config/credentials/*.key +/config/master.key # IDE configs .idea/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 34e30380ce..04d14af22f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,446 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [11.0.0-44](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-43...v11.0.0-44) (2026-08-08) + + +### Features + +* bulk upload staff notes ([6c3b3e0](https://github.com/b0ink/doubtfire-deploy/commit/6c3b3e0b1b67ad3226718e9e7925b849802e0ae3)) +* bulk upload staff notes ([#665](https://github.com/b0ink/doubtfire-deploy/issues/665)) ([3e0f32d](https://github.com/b0ink/doubtfire-deploy/commit/3e0f32d724f3664deddbff8fef55ca10c8fed3b1)) + +## [11.0.0-43](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-42...v11.0.0-43) (2026-08-03) + + +### Features + +* group engagement ([#658](https://github.com/b0ink/doubtfire-deploy/issues/658)) ([6f02a33](https://github.com/b0ink/doubtfire-deploy/commit/6f02a33942ccbd5107eb4a88f9274b848196e3c5)) + +## [11.0.0-42](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-41...v11.0.0-42) (2026-07-31) + + +### Bug Fixes + +* ensure screenshots embedded in markdown compiles correctly ([9b624d6](https://github.com/b0ink/doubtfire-deploy/commit/9b624d68576710e40abe22ffdcac79b6277b9f02)) + +## [11.0.0-41](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-40...v11.0.0-41) (2026-07-30) + +## [11.0.0-40](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-39...v11.0.0-40) (2026-07-30) + + +### Features + +* enable unit content download ([1757731](https://github.com/b0ink/doubtfire-deploy/commit/17577319e897641cbc8ba20af029ee448fc61379)) + +## [11.0.0-39](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-38...v11.0.0-39) (2026-07-28) + + +### Bug Fixes + +* only record discussion after successful transition ([c8bd418](https://github.com/b0ink/doubtfire-deploy/commit/c8bd4188c758bb1829c46fb3e076d803329a41a9)) + +## [11.0.0-38](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-37...v11.0.0-38) (2026-07-28) + + +### Bug Fixes + +* ensure feedback templates rollover to new unit ([df3bc30](https://github.com/b0ink/doubtfire-deploy/commit/df3bc306da42d46f7187ff40f0fbc0e9d3f2122d)) + +## [11.0.0-37](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-36...v11.0.0-37) (2026-07-24) + + +### Features + +* automated engagement stamps ([#650](https://github.com/b0ink/doubtfire-deploy/issues/650)) ([59641bd](https://github.com/b0ink/doubtfire-deploy/commit/59641bd390da394b8aff20c21433dc885419c319)) + + +### Bug Fixes + +* pass saml settings correctly to logout response ([4f60eff](https://github.com/b0ink/doubtfire-deploy/commit/4f60eff0eea770d20cee2226a13b1ad10dfafc0e)) +* pass saml settings correctly to logout response ([#648](https://github.com/b0ink/doubtfire-deploy/issues/648)) ([8df3ea9](https://github.com/b0ink/doubtfire-deploy/commit/8df3ea9a4ec5dd45b7697e6ae11ab88e54fe2b64)) + +## [11.0.0-36](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-35...v11.0.0-36) (2026-07-24) + + +### Features + +* communication condition to check if student has portfolio ([#651](https://github.com/b0ink/doubtfire-deploy/issues/651)) ([90d362e](https://github.com/b0ink/doubtfire-deploy/commit/90d362e4ff2d32d0e3791d0bec3f5f2d6f649ee1)) + + +### Bug Fixes + +* normalize upload requirement file keys on save ([#653](https://github.com/b0ink/doubtfire-deploy/issues/653)) ([bf457a1](https://github.com/b0ink/doubtfire-deploy/commit/bf457a1a2137e7586b36eb9bc9b14045caad28f4)) + +## [11.0.0-35](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-34...v11.0.0-35) (2026-07-24) + + +### Bug Fixes + +* enable youtube embeds ([a8da36c](https://github.com/b0ink/doubtfire-deploy/commit/a8da36c3fbf64af3fad6bfaf02cd66e9921b903e)) + +## [11.0.0-34](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-33...v11.0.0-34) (2026-07-24) + + +### Features + +* add last sign in and last unit access communication conditions ([50fc821](https://github.com/b0ink/doubtfire-deploy/commit/50fc8215598766fa8927ca480b122a361803d689)) +* discuss timeout ([#646](https://github.com/b0ink/doubtfire-deploy/issues/646)) ([f6dfd80](https://github.com/b0ink/doubtfire-deploy/commit/f6dfd806570f6b30384571210402a3e92c405b2f)) +* track last access to site and unit ([2141fd6](https://github.com/b0ink/doubtfire-deploy/commit/2141fd6876f50c5705f16acdd1f27d9b65152078)) + + +### Bug Fixes + +* allow unit deletion with task prerequisites ([2b3752a](https://github.com/b0ink/doubtfire-deploy/commit/2b3752ac6b888cffc38cefa86a58bbebfb844aff)) +* iterate sentry exception interface values correctly ([4818049](https://github.com/b0ink/doubtfire-deploy/commit/48180497875529f6ac843a00519a6ce69f4ac4bc)) +* prevent deleting task definitions used as prerequisites ([f26c9c5](https://github.com/b0ink/doubtfire-deploy/commit/f26c9c573894543a50220dc7708fc4540e764aaf)) + +## [11.0.0-33](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-32...v11.0.0-33) (2026-07-19) + + +### Features + +* content management system ([#642](https://github.com/b0ink/doubtfire-deploy/issues/642)) ([a87b250](https://github.com/b0ink/doubtfire-deploy/commit/a87b250303b0aff8a431334e8b19a40f790cbd35)) + +## [11.0.0-32](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-31...v11.0.0-32) (2026-07-16) + + +### Features + +* add rediscuss status ([05a44fa](https://github.com/b0ink/doubtfire-deploy/commit/05a44fa8e9954ed82ea149072288ebc546d5f9e9)) +* prompt for admin password when creating initial account ([7640a36](https://github.com/b0ink/doubtfire-deploy/commit/7640a36f682f6dd7240cb764dea6e46b7d01df35)) +* support word document submission ([#647](https://github.com/b0ink/doubtfire-deploy/issues/647)) ([a21daf8](https://github.com/b0ink/doubtfire-deploy/commit/a21daf846a46557331c3235f86f28a5c8a020786)) + +## [11.0.0-31](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-30...v11.0.0-31) (2026-07-08) + + +### Bug Fixes + +* run lualatex a third time to ensure correct page references ([fc40bde](https://github.com/b0ink/doubtfire-deploy/commit/fc40bdea6414870c6457f5266838cb44d6296fcd)) + +## [11.0.0-30](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-29...v11.0.0-30) (2026-07-04) + + +### Bug Fixes + +* ensure admins can manage communications ([25bcafc](https://github.com/b0ink/doubtfire-deploy/commit/25bcafcdc00d76523080f6a198170b452f646923)) + +## [11.0.0-29](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-28...v11.0.0-29) (2026-07-02) + + +### Bug Fixes + +* allow students to get submission history timestamps ([a4bb79a](https://github.com/b0ink/doubtfire-deploy/commit/a4bb79ac27425fa64ed711e932d595cefa2a5598)) + +## [11.0.0-28](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-27...v11.0.0-28) (2026-07-01) + + +### Bug Fixes + +* only allow convenors to pull overflow stats ([d5ca748](https://github.com/b0ink/doubtfire-deploy/commit/d5ca74843fa16040dec81fa9bd66d315736059a0)) + +## [11.0.0-27](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-26...v11.0.0-27) (2026-07-01) + + +### Bug Fixes + +* prevent convenors with observer only perms to be made main convenor ([19a459c](https://github.com/b0ink/doubtfire-deploy/commit/19a459c544a14d3c155d665ba90ead11c3c50288)) + +## [11.0.0-26](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-25...v11.0.0-26) (2026-06-29) + + +### Bug Fixes + +* avoid skipping task names containing 'name' during import ([4c2e194](https://github.com/b0ink/doubtfire-deploy/commit/4c2e19472f73deeaee271ed5b7d5dc66b01f7836)) +* improve error when adding prerequisite to non existing task def ([f217c61](https://github.com/b0ink/doubtfire-deploy/commit/f217c61cf666dae7c781a393fbd9b544dcb32f24)) + +## [11.0.0-25](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-24...v11.0.0-25) (2026-06-29) + + +### Bug Fixes + +* ensure pii is not submitted to sentry ([d98d318](https://github.com/b0ink/doubtfire-deploy/commit/d98d318adc1c5153af9a844d95fdbbe2fde3903b)) + +## [11.0.0-24](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-23...v11.0.0-24) (2026-06-29) + + +### Bug Fixes + +* ensure non text comments cant be deleted ([964e957](https://github.com/b0ink/doubtfire-deploy/commit/964e957c4a00f1dadb2d992583d3f2bb717aa58b)) + +## [11.0.0-23](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-22...v11.0.0-23) (2026-06-26) + + +### Bug Fixes + +* ensure jplag scan continues on failed report ([3215c08](https://github.com/b0ink/doubtfire-deploy/commit/3215c0811aa8c9eb6425cb9d4e080f2fc042e6fb)) + +## [11.0.0-22](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-21...v11.0.0-22) (2026-06-26) + +## [11.0.0-21](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-20...v11.0.0-21) (2026-06-26) + + +### Bug Fixes + +* support mysql 8 reserved keyword in task completion stats query ([87fe813](https://github.com/b0ink/doubtfire-deploy/commit/87fe8136ea5e9f7011cee3b304bbe5c8a68c7d6d)) + +## [11.0.0-20](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-19...v11.0.0-20) (2026-06-25) + + +### Features + +* capture daily snapshots of aggregated task completion data ([#607](https://github.com/b0ink/doubtfire-deploy/issues/607)) ([86b2040](https://github.com/b0ink/doubtfire-deploy/commit/86b2040bf48f494b1e06919fcce50243fe5a0240)) + + +### Bug Fixes + +* ensure overseer image is pulled before running test ([0e0ce34](https://github.com/b0ink/doubtfire-deploy/commit/0e0ce34160dce4e23fc9deea2f035119c5d378dd)) +* relocate submission history on unit code change ([d9085b5](https://github.com/b0ink/doubtfire-deploy/commit/d9085b509b7995ae9d41e8e3973dd57b011a166d)) + +## [11.0.0-19](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-18...v11.0.0-19) (2026-06-25) + +## [11.0.0-18](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-17...v11.0.0-18) (2026-06-25) + + +### Bug Fixes + +* rename tunnel route to avoid ad blockers ([baf4f69](https://github.com/b0ink/doubtfire-deploy/commit/baf4f692be7558bca51fab8daa89b16f3daf2088)) + +## [11.0.0-17](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-16...v11.0.0-17) (2026-06-25) + +## [11.0.0-16](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-15...v11.0.0-16) (2026-06-25) + + +### Features + +* add sentry tunnel support ([3eecd2f](https://github.com/b0ink/doubtfire-deploy/commit/3eecd2f8e7733e79307830e173e6687fc4d2632f)) + +## [11.0.0-15](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-14...v11.0.0-15) (2026-06-24) + + +### Bug Fixes + +* check grade value is valid json ([3dd6106](https://github.com/b0ink/doubtfire-deploy/commit/3dd6106bb65eb739c81f8eab5332684a915d0c9f)) + +## [11.0.0-14](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-13...v11.0.0-14) (2026-06-24) + + +### Bug Fixes + +* avoid serialisation ([1a47564](https://github.com/b0ink/doubtfire-deploy/commit/1a475640e211d3dec7e3b5034b3f0efddaa6e29d)) +* check grade value is valid json ([ea7c5db](https://github.com/b0ink/doubtfire-deploy/commit/ea7c5db44e3f1751cf6d93028a07f0f82cca3bec)) + +## [11.0.0-13](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-12...v11.0.0-13) (2026-06-24) + + +### Bug Fixes + +* allow nullable grade values ([8b586f9](https://github.com/b0ink/doubtfire-deploy/commit/8b586f9b76c9cc82f71a33bcace5532e4ad9c4ec)) +* avoid json default ([3a0c879](https://github.com/b0ink/doubtfire-deploy/commit/3a0c879398a62469524f6e49410cccf5da0a2a33)) + +## [11.0.0-12](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-11...v11.0.0-12) (2026-06-24) + + +### Features + +* customisable grades ([#638](https://github.com/b0ink/doubtfire-deploy/issues/638)) ([57cb9e2](https://github.com/b0ink/doubtfire-deploy/commit/57cb9e250b7fbfe9eb477021c9077d86d4b696ab)) + +## [11.0.0-11](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-10...v11.0.0-11) (2026-06-18) + + +### Features + +* engagement passport ([#631](https://github.com/b0ink/doubtfire-deploy/issues/631)) ([d32145e](https://github.com/b0ink/doubtfire-deploy/commit/d32145ef62c7ad014524e5e3610844b7bda20b76)) + + +### Bug Fixes + +* serve public web cal as raw text ([6836a20](https://github.com/b0ink/doubtfire-deploy/commit/6836a20e582473eff4edb41e2fd1f6c4a5a9751d)) + +## [11.0.0-10](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-9...v11.0.0-10) (2026-06-17) + + +### Features + +* add column for days awaiting feedback including breaks ([f7b877c](https://github.com/b0ink/doubtfire-deploy/commit/f7b877c9759089ce1bb4c4e99490a039c0761181)) +* submission history ([#636](https://github.com/b0ink/doubtfire-deploy/issues/636)) ([4f95d61](https://github.com/b0ink/doubtfire-deploy/commit/4f95d61c8d897183ce83ca6d96b06570e51e4def)) + +## [11.0.0-9](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-8...v11.0.0-9) (2026-06-11) + + +### Features + +* expose route to fetch tasks waiting for pdf gen ([5d3407e](https://github.com/b0ink/doubtfire-deploy/commit/5d3407e2c897fa05f6e5c6ee41d8e0379fb6a367)) +* overflow task claim logs ([#614](https://github.com/b0ink/doubtfire-deploy/issues/614)) ([065180f](https://github.com/b0ink/doubtfire-deploy/commit/065180f248db7e80dd86f0491822e2051a9fa220)) + + +### Bug Fixes + +* ensure more reliable test ([93599d2](https://github.com/b0ink/doubtfire-deploy/commit/93599d27d736ebfaf7b997dfd82d764a45195881)) + +## [11.0.0-8](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-7...v11.0.0-8) (2026-06-09) + + +### Features + +* communications system ([#617](https://github.com/b0ink/doubtfire-deploy/issues/617)) ([9724909](https://github.com/b0ink/doubtfire-deploy/commit/97249090eac3f7e214991f7c665f92768948b6db)) + +## [11.0.0-7](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-6...v11.0.0-7) (2026-06-08) + + +### Bug Fixes + +* ensure start and target dates set correctly if unit start date is a monday ([0717d90](https://github.com/b0ink/doubtfire-deploy/commit/0717d9051181811c7d58dd453abf076b87cb0444)) +* handle incomplete notebook metadata and markdown lists in pdfs ([a01e4d7](https://github.com/b0ink/doubtfire-deploy/commit/a01e4d7e7ca7ca255b2080f1f3e44da48db4c270)) +* normalize notebook source lines ([b950e55](https://github.com/b0ink/doubtfire-deploy/commit/b950e55ad801fbdd6f3dafb3732aefc0b6cc6987)) + +## [11.0.0-6](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-5...v11.0.0-6) (2026-06-04) + +## [11.0.0-5](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-4...v11.0.0-5) (2026-06-04) + + +### Features + +* notify students when overseer tests fail ([#627](https://github.com/b0ink/doubtfire-deploy/issues/627)) ([4280df3](https://github.com/b0ink/doubtfire-deploy/commit/4280df3a6446920b4ae3c9108d28421ab9b3e6c5)) + + +### Bug Fixes + +* correctly get task ([1949b42](https://github.com/b0ink/doubtfire-deploy/commit/1949b42258355b843d7d0e5563be040d4d859164)) + +## [11.0.0-4](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-3...v11.0.0-4) (2026-06-04) + + +### Features + +* suspend any stale overseer assessments ([678f386](https://github.com/b0ink/doubtfire-deploy/commit/678f3866632d174713ab46bc7876c43dd2df56cd)) + + +### Bug Fixes + +* ensure we check for queued jobs ([3790c9f](https://github.com/b0ink/doubtfire-deploy/commit/3790c9f3dc4899d9785f6996a77a5b550f1c5d87)) + +## [11.0.0-3](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-2...v11.0.0-3) (2026-06-04) + + +### Bug Fixes + +* allow error log emails with no backtrace ([f5b383f](https://github.com/b0ink/doubtfire-deploy/commit/f5b383fdeda016dd7d391325f04700356cc58da5)) + +## [11.0.0-2](https://github.com/b0ink/doubtfire-deploy/compare/v11.0.0-1...v11.0.0-2) (2026-06-04) + + +### Features + +* clear submission in process with no active sidekiq job ([#626](https://github.com/b0ink/doubtfire-deploy/issues/626)) ([bb453ce](https://github.com/b0ink/doubtfire-deploy/commit/bb453ce00f540bcf1d64d70e18c6d97b6f8bdc5a)) + +## [11.0.0-1](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-110...v11.0.0-1) (2026-06-03) + + +### Features + +* support zip file submissions ([#624](https://github.com/b0ink/doubtfire-deploy/issues/624)) ([d5d8742](https://github.com/b0ink/doubtfire-deploy/commit/d5d874214fe31f494b5a30f67f7286e711ba45b2)) + +## [10.0.0](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-110...v10.0.0) (2026-06-03) + + +### Features + +* support zip file submissions ([#624](https://github.com/b0ink/doubtfire-deploy/issues/624)) ([d5d8742](https://github.com/b0ink/doubtfire-deploy/commit/d5d874214fe31f494b5a30f67f7286e711ba45b2)) + +## [10.0.0-110](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-109...v10.0.0-110) (2026-06-03) + +## [10.0.0-109](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-108...v10.0.0-109) (2026-05-13) + + +### Bug Fixes + +* ensure recent feedback has been left for fix ([a824b1b](https://github.com/b0ink/doubtfire-api/commit/a824b1b5fc4f47a12be33d410962af31fe810c89)) + +## [10.0.0-108](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-107...v10.0.0-108) (2026-05-13) + + +### Features + +* enable sentry error logging ([#620](https://github.com/b0ink/doubtfire-api/issues/620)) ([11cc264](https://github.com/b0ink/doubtfire-api/commit/11cc264b83d2c5e161b1926e12c496f2fd9add13)) + +## [10.0.0-107](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-106...v10.0.0-107) (2026-05-05) + +## [10.0.0-106](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-105...v10.0.0-106) (2026-04-28) + + +### Features + +* enable comment editing ([#613](https://github.com/b0ink/doubtfire-api/issues/613)) ([4096f87](https://github.com/b0ink/doubtfire-api/commit/4096f8716b63f4ca5239f1f0ff7a4591096ee10d)) + +## [10.0.0-105](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-104...v10.0.0-105) (2026-04-27) + +## [10.0.0-104](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-103...v10.0.0-104) (2026-04-23) + + +### Features + +* import/export overseer steps via task definition csv ([#604](https://github.com/b0ink/doubtfire-api/issues/604)) ([5dbd575](https://github.com/b0ink/doubtfire-api/commit/5dbd57588c06cb69eae35c5eab12d757bbe0bec5)) + + +### Bug Fixes + +* prevent duplicate submission requests ([#611](https://github.com/b0ink/doubtfire-api/issues/611)) ([1d056ca](https://github.com/b0ink/doubtfire-api/commit/1d056ca36aca7d9de8977b7a96783ed7a8d05367)) + +## [10.0.0-103](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-102...v10.0.0-103) (2026-04-18) + +## [10.0.0-102](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-101...v10.0.0-102) (2026-04-18) + +## [10.0.0-101](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-100...v10.0.0-101) (2026-04-18) + + +### Features + +* batch upload feedback csv ([#609](https://github.com/b0ink/doubtfire-api/issues/609)) ([5baa171](https://github.com/b0ink/doubtfire-api/commit/5baa171baff4343161e13e79613b81eb985acc29)) +* query pinned tasks in task explorer ([7a10685](https://github.com/b0ink/doubtfire-api/commit/7a10685138a99eac0719aa2a37f8f883167edc45)) + +## [10.0.0-100](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-99...v10.0.0-100) (2026-04-15) + + +### Features + +* add enforce feedback unit setting ([4d1c6ee](https://github.com/b0ink/doubtfire-api/commit/4d1c6ee8d227b1d4b0ac57bbeea30f20f47e8b71)) + +## [10.0.0-99](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-98...v10.0.0-99) (2026-04-13) + + +### Features + +* ensure tutorials are reassigned before deleting a unit role ([0bba827](https://github.com/b0ink/doubtfire-api/commit/0bba8279da40a8674f802b961302b84560855e76)) + +## [10.0.0-98](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-97...v10.0.0-98) (2026-03-27) + + +### Features + +* move tii and active record secrets to rails credentials ([#601](https://github.com/b0ink/doubtfire-api/issues/601)) ([4357f5c](https://github.com/b0ink/doubtfire-api/commit/4357f5c17db2d937aa0ab573567d45f41cadcf5b)) + +## [10.0.0-97](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-96...v10.0.0-97) (2026-03-27) + + +### Features + +* prefer encrypted rails credentials over env vars ([#600](https://github.com/b0ink/doubtfire-api/issues/600)) ([b1946bd](https://github.com/b0ink/doubtfire-api/commit/b1946bd8105e70e4baff1e0547b1b728d8a731c8)) + +## [10.0.0-96](https://github.com/b0ink/doubtfire-api/compare/v10.0.0-95...v10.0.0-96) (2026-03-26) + + +### Features + +* allow admin users to view any unit analytics ([#597](https://github.com/b0ink/doubtfire-api/issues/597)) ([eb2195a](https://github.com/b0ink/doubtfire-api/commit/eb2195a18d148264eb0570cdfc3ecfa6b577e1f9)) +* pause days awaiting feedback during breaks ([#599](https://github.com/b0ink/doubtfire-api/issues/599)) ([0aa2208](https://github.com/b0ink/doubtfire-api/commit/0aa220863329083b1fe7d53fc2a4f9ee9a3f9386)) + +## [10.0.0-95](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-94...v10.0.0-95) (2026-03-24) + + +### Bug Fixes + +* normalise test to ensure its consistent ([#594](https://github.com/b0ink/doubtfire-deploy/issues/594)) ([a607486](https://github.com/b0ink/doubtfire-deploy/commit/a607486e9953e28b2bd6b04e9a135a4190b87c6e)) + ## [10.0.0-94](https://github.com/b0ink/doubtfire-deploy/compare/v10.0.0-93...v10.0.0-94) (2026-03-23) diff --git a/Gemfile b/Gemfile index 90b4021f63..2c245b856c 100644 --- a/Gemfile +++ b/Gemfile @@ -52,6 +52,7 @@ gem 'puma' gem 'bootsnap', require: false gem 'csv' +gem 'uri', '>= 1.1.1' # CVE-2025-61594 # Extend irb for better output gem 'hirb' @@ -102,6 +103,7 @@ gem 'net-smtp', require: false gem 'tca_client' # Async jobs +gem 'ice_cube' gem 'sidekiq' gem 'sidekiq-cron' gem 'sidekiq-status' @@ -120,3 +122,6 @@ gem 'pdf-reader' gem 'oauth2' gem "sys-filesystem" + +gem "sentry-rails" +gem "sentry-ruby" diff --git a/Gemfile.lock b/Gemfile.lock index 6bb903f2a6..e1e2ad5f87 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -456,6 +456,13 @@ GEM ruby2_keywords (0.0.5) rubyzip (2.4.1) securerandom (0.4.1) + sentry-rails (6.5.0) + railties (>= 5.2.0) + sentry-ruby (~> 6.5.0) + sentry-ruby (6.5.0) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + logger set (1.1.1) sexp_processor (4.17.3) shellwords (0.2.2) @@ -539,7 +546,7 @@ GEM unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) - uri (1.0.3) + uri (1.1.1) useragent (0.16.11) version_gem (1.1.6) warden (1.2.9) @@ -582,6 +589,7 @@ DEPENDENCIES grape-swagger-rails hirb icalendar + ice_cube json-jwt listen minitest @@ -613,6 +621,8 @@ DEPENDENCIES ruby-lsp ruby-saml rubyzip + sentry-rails + sentry-ruby shellwords sidekiq sidekiq-cron @@ -623,6 +633,7 @@ DEPENDENCIES sprockets-rails sys-filesystem tca_client + uri (>= 1.1.1) webmock RUBY VERSION diff --git a/README.md b/README.md index e1207a2a90..c81d68e9cf 100644 --- a/README.md +++ b/README.md @@ -1,157 +1,229 @@ -![Doubtfire Logo](https://github.com/doubtfire-lms/doubtfire-web/raw/6.2.x/src/assets/icons/android-chrome-192x192.png) +

+ OnTrack logo +

-# Doubtfire API [![test-doubtfire-api](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/push.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/push.yml) [![CodeQL](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/codeql.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/codeql.yml) [![RuboCop](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/rubocop.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/rubocop.yml) +# OnTrack API -Doubtfire is a feedback-driven learning support system. +[![test-doubtfire-api](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/push.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/push.yml) [![CodeQL](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/codeql.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/codeql.yml) [![RuboCop](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/rubocop.yml/badge.svg)](https://github.com/doubtfire-lms/doubtfire-api/actions/workflows/rubocop.yml) -## Table of Contents +OnTrack (formerly Doubtfire) is a feedback-driven learning support system. This repository contains +the Rails and Grape API used by OnTrack. -- [Doubtfire API](#doubtfire-api--) - - [Table of Contents](#table-of-contents) - - [Getting started](#getting-started) - - [Clone Repository](#clone-repository) - - [Install script](#install-script) - - [Manual install](#manual-install) - - [Environment variables](#environment-variables) - - [Get it up and running!](#get-it-up-and-running) -- [Running Rake Tasks](#running-rake-tasks) -- [Testing](#testing) -- [Contributing](#contributing) -- [License](#license) +## Development and deployment -## Getting started +Use the +[doubtfire-deploy](https://github.com/doubtfire-lms/doubtfire-deploy) +repository for the supported development environment and the deployment guide. +It checks out this API as a submodule and provides the database, Redis, PDF +services, web app, and required configuration in one place. -See [Doubtfire Deploy](https://github.com/doubtfire-lms/doubtfire-deploy) for instructions on deploying, and contributing, to the Doubtfire project. +When the development environment is running, the API documentation is available +at . ## Environment variables -Doubtfire requires multiple environment variables that help define settings about the Doubtfire instance running. Whilst these will default to other values, you may want to override them in production. - -| Key | Description | Default | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | -| `DF_AUTH_METHOD` | The authentication method you would like Doubtfire to use. Possible values are `database` for standard authentication with the database, `ldap` | `database` | -| | for [LDAP](https://www.freebsd.org/doc/en/articles/ldap-auth/), `aaf` for [AAF Rapid Connect](https://rapid.aaf.edu.au/), or `SAML2` for [SAML2.0 auth](https://en.wikipedia.org/wiki/SAML_2.0). | | -| `DF_STUDENT_WORK_DIR` | The directory to store uploaded student work for processing. | `student_work` | -| `DF_ARCHIVE_DIR` | The directory to move archived unit files to, and access from. | `DF_STUDENT_WORK_DIR/archive` | -| `DF_INSTITUTION_NAME` | The name of your institution running Doubtfire. | _Doubtfire University_ | -| `DF_INSTITUTION_EMAIL_DOMAIN` | The email domain from which emails are sent to and from in your institution. | `doubtfire.com` | -| `DF_INSTITUTION_HOST` | The host running the Doubtfire instance. | `localhost:3000` | -| `DF_COOKIE_DOMAIN` | The domain to be associated with secure cookies. | Attempts to read from host | -| `DF_INSTITUTION_PRODUCT_NAME` | The name of the product (i.e. Doubtfire) at your institution. | _Doubtfire_ | -| `DF_INSTITUTION_HAS_LOGO` | Set to true (or 1) if there is an associated institution logo to be included in the header. | false | -| `DF_INSTITUTION_LOGO_URL` | The url of the logo to include in the header if there is a logo. | /assets/images/institution-logo.png | -| `DF_INSTITUTION_LOGO_LINK_URL` | The url used for the hyperlink associated with clicking the logo. | / | -| `DF_SECRET_KEY_BASE` | The Rails secret key. | Default key provided. | -| `DF_SECRET_KEY_ATTR` | The secret key to encrypt certain database fields. | Default key provided. | -| `DF_SECRET_KEY_DEVISE` | The secret key provided to Devise. | Default key provided. | -| `DF_SECRET_KEY_MOSS` | The secret key provided to [Moss](http://theory.stanford.edu/~aiken/moss/) for plagiarism detection. This value will need to be set to run `rake submission:check_plagiarism` (otherwise you **won't** need it). You will need to register for a Moss account to use this. | No default. | -| `DF_INSTITUTION_PRIVACY` | A statement related to the need for students to submit their own work, and that this work may be uploaded to 3rd parties for the purpose of plagiarism detection. | Default statement provided | -| `DF_INSTITUTION_PLAGIARISM` | A statement clarifying the terms plagiarism and collusion. | Default statement provided | -| `DF_INSTITUTION_SETTINGS_RB` | The path of the institution specific settings rb code - used to map student imports from institutional exports to a format understood by Doubtfire. | No default | -| `DF_FFMPEG_PATH` | The path of to the ffmpeg binary for audio processing. | ffmpeg | -| `DF_REDIS_CACHE_URL` | The redis URL for rails used for development and production, ignored in the test env. | `redis://localhost:6379/0` | -| `DF_REDIS_SIDEKIQ_URL` | The redis URL for sidekiq. A working redis server is **mandatory** for sidekiq in all environments. | `redis://localhost:6379/1` | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | -| **Turn It In Integration** | | | -| `TII_ENABLED` | Whether or not Turn It In integration is enabled. | 0 / false | -| `TII_INDEX_SUBMISSIONS` | Whether or not to index submissions in Turn It In. Should be set to 1 or true in production environments | 0 / false | -| `TII_REGISTER_WEBHOOK` | Whether or not to register a webhook with Turn It In. Should be set to 1 or true in production environments | 0 / false | -| `TCA_API_KEY` | The API key for Turn It In integration, acquire from the Turn It In administration interface. | No default | -| `TCA_HOST` | The host for the Turn It In integration, eg: https://institution.turnitin.com | No default | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | -| **D2L Integration** | | | -| `D2L_ENABLED` | Whether or not D2L integration is enabled. | 0 / false | -| `D2L_CLIENT_ID` | The client ID for D2L integration - from the oauth registration in D2L | No default | -| `D2L_CLIENT_SECRET` | The client secret for D2L integration - from the oauth registration in D2L | No default | -| `D2L_REDIRECT_URI` | The redirect URI for D2L integration. Must redirect to https://host/api/d2l/callback which must match the oauth registration in D2L | No default | -| `D2L_API_HOST` | The specific institutional URL for the D2L server, eg: https://d2l.institution.edu | No default | -| `D2L_OAUTH_SITE` | The location of the D2L authentication server. | `https://auth.brightspace.com` | -| `D2L_OAUTH_SITE_AUTHORIZE_URL` | The URL to authorize the D2L integration. | `/oauth2/auth` | -| `D2L_OAUTH_SITE_TOKEN_URL` | The URL to get the token for the D2L integration. | `/core/connect/token` | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------ | -| **Latex Configuration** | | | -| `LATEX_CONTAINER_NAME` | The name of the container housing the latex image. Used when pdfs are generated. | No default | -| `LATEX_BUILD_PATH` | The path to the latex build script within the container. | /texlive/shell/latex_build.sh | - -If you have chosen to use AAF Rapid Connect authentication, then you will also need to provide the following: - -| Key | Description | Default | -| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | -| `DF_AAF_ISSUER_URL` | The URL of the AFF issuer, either `https://rapid.test.aaf.edu.au` for testing or `https://rapid.aaf.edu.au` for production. | `https://rapid.test.aaf.edu.au` | -| `DF_AAF_AUDIENCE_URL` | The URL of the AAF registered application. | No default - required | -| `DF_AAF_CALLBACK_URL` | The secure endpoint within your application that AAF Rapid Connect should POST responses to. It **must end with `/api/auth/jwt`** to access the Doubtfire JWT authentication endpoint. | No default - required | -| `DF_AAF_UNIQUE_URL` | The unique URL provided by AAF Rapid Connect used for redirection out of Doubtfire. | No default - required | -| `DF_AAF_IDENTITY_PROVIDER_URL` | The URL of the AAF-registered identity provider. | No default - required | -| `DF_AAF_AUTH_SIGNOUT_URL` | The URL to redirect to on sign out in order to log out of AAF Rapid Connect. | No default - required | -| `DF_SECRET_KEY_AAF` | The secret used to register your application with AAF. | `secretsecret12345` | - -If you are authenticating using SAML2, then you will also need to provide the following: - -| Key | Description | Default | -| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | -| `DF_SAML_METADATA_URL` | The URL to getch the SAML metadata. Either the url or a file path to the meta data must be provided. Where the url is provided it will be used. | No default | -| `DF_SAML_METADATA_FILE_PATH` | The path to the metadata xml file. When the url is not set, this path will be used to get the metadata for the saml settings. | No default | -| `DF_SAML_CONSUMER_SERVICE_URL` | The URL of the SAML application. | No default - required | -| `DF_SAML_IDP_TARGET_URL` | The IDP SAML login URL, (e.g., "https://login.microsoftonline.com/xxxx/saml2") - OnTrack will redirect to this for login. | No default - required | -| `DF_SAML_IDP_SIGNOUT_URL` | The IDP SAML logout URL, (e.g., "https://login.microsoftonline.com/xxxx/saml2") - OnTrack will redirect to this for logout. | the SAML login url | -| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | -| `DF_AAF_CALLBACK_URL` | The secure endpoint within your application that AAF Rapid Connect should POST responses to. It **must end with `/api/auth/jwt`** to access the Doubtfire JWT authentication endpoint. | No default - required | -| `DF_AAF_UNIQUE_URL` | The unique URL provided by AAF Rapid Connect used for redirection out of Doubtfire. | No default - required | -| `DF_AAF_IDENTITY_PROVIDER_URL` | The URL of the AAF-registered identity provider. | No default - required | -| `DF_AAF_AUTH_SIGNOUT_URL` | The URL to redirect to on sign out in order to log out of AAF Rapid Connect. | No default - required | -| `DF_SECRET_KEY_AAF` | The secret used to register your application with AAF. | `secretsecret12345` | - -You may choose to keep your environment variables inside a `.env` file using key-value pairs: - -``` -DF_INSTITUTION_HOST=doubtfire.unifoo.edu.au -DF_INSTITUTION_NAME="University of Foo" -``` - -You can also keep multiple `.env` files for different environments, e.g.: `.env.production` is different to `.env.develoment`. Doubtfire uses the [dotenv](https://github.com/bkeepers/dotenv) gem to make this happen. - -### Get it up and running! - -Once you've installed using either in install script or the manual install steps. - -``` -$ bundle exec rails s -``` - -You should see all the Doubtfire endpoints at **[http://localhost:3000/api/docs/](http://localhost:3000/api/docs/)**, which means the API is running. - -# Running Rake Tasks - -You can perform developer-specific tasks using `rake`. For a list of all tasks, execute in the root directory: - -``` -rake --tasks -``` - -# Testing - -Our aim with testing Doubtfire is to migrate to a [Test-Driven Development](https://en.wikipedia.org/wiki/Test-driven_development) -strategy, testing all new models and API endpoints (although we plan on writing -more tests for _existing_ models and API endpoints). If you are writing a new -API endpoint or model, we strongly suggest you include unit tests in the -appropriate folders (see below). - -To run unit tests, execute: - -```bash -$ rake test +The API supports the environment variables below. The production deployment +template is maintained in +[`doubtfire-deploy`](https://github.com/doubtfire-lms/doubtfire-deploy/blob/main/production/api/.env.production). +An unset variable uses its application default where one exists; an empty value +may override that default with an empty string. + +### Application and database + +| Variable | Purpose | Default | +| --------------------------- | ----------------------------------------- | -------------------------- | +| `RAILS_ENV` | Rails runtime environment. | `development` | +| `DF_LOG_TO_STDOUT` | Send Rails logs to standard output. | `false` | +| `DF_PRODUCTION_DB_ADAPTER` | Production database adapter. | Deployment-specific | +| `DF_PRODUCTION_DB_HOST` | Production database host. | Deployment-specific | +| `DF_PRODUCTION_DB_DATABASE` | Production database name. | Deployment-specific | +| `DF_PRODUCTION_DB_USERNAME` | Production database username. | Deployment-specific | +| `DF_PRODUCTION_DB_PASSWORD` | Production database password. | Deployment-specific | +| `DF_REDIS_CACHE_URL` | Redis connection used by the Rails cache. | `redis://localhost:6379/0` | +| `DF_REDIS_SIDEKIQ_URL` | Redis connection used by Sidekiq. | `redis://localhost:6379/1` | + +### Storage and processing + +| Variable | Purpose | Default | +| ------------------------------------- | ---------------------------------------------------------------- | ------------------------------- | +| `DF_STUDENT_WORK_DIR` | Directory containing uploaded student work. | `student_work` | +| `DF_ARCHIVE_DIR` | Directory containing archived student work. | `DF_STUDENT_WORK_DIR/archive` | +| `DF_ARCHIVE_UNITS` | Enable automatic unit archiving. | `false` | +| `DF_UNIT_ARCHIVE_PERIOD` | Years to retain units before archiving. | `2` | +| `DF_MAX_PDF_GEN_PROCESSES` | Maximum concurrent PDF-generation processes. | `2` | +| `DF_MAX_FILE_SIZE` | Maximum uploaded file size in bytes. | `10000000` | +| `DF_ZIP_ENTRY_LIMIT` | Maximum number of entries in an uploaded ZIP. | `1000` | +| `DF_ZIP_COMPRESSION_RATIO_LIMIT` | Maximum permitted ZIP compression ratio. | `100` | +| `DF_ZIP_UNCOMPRESSED_SIZE_MULTIPLIER` | Maximum expanded ZIP size as a multiple of `DF_MAX_FILE_SIZE`. | `10` | +| `DF_AUDITOR_UNIT_ACCESS_YEARS` | Number of years of units visible to auditors. | `2` | +| `DF_IMPORT_STUDENTS_WEEKS_BEFPRE` | Weeks before a teaching period that student imports are allowed. | `1` | +| `DF_FFMPEG_PATH` | Path to FFmpeg for audio processing. | `ffmpeg` | +| `LATEX_CONTAINER_NAME` | Container used for LaTeX PDF generation. | Unset | +| `LATEX_BUILD_PATH` | LaTeX build script inside that container. | `/texlive/shell/latex_build.sh` | + +### Institution + +| Variable | Purpose | Default | +| ------------------------------ | --------------------------------------------- | ------------------------------------- | +| `DF_INSTITUTION_NAME` | Institution display name. | Institution configuration | +| `DF_INSTITUTION_EMAIL_DOMAIN` | Institution email domain. | Institution configuration | +| `DF_INSTITUTION_HOST` | Public application URL. | Institution configuration | +| `DF_COOKIE_DOMAIN` | Domain assigned to secure cookies. | Derived from `DF_INSTITUTION_HOST` | +| `DF_INSTITUTION_PRODUCT_NAME` | Product name shown to users. | Institution configuration | +| `DF_INSTITUTION_HAS_LOGO` | Enable an institution logo. | `false` | +| `DF_INSTITUTION_LOGO_URL` | Institution logo URL. | `/assets/images/institution-logo.png` | +| `DF_INSTITUTION_LOGO_LINK_URL` | Destination opened from the institution logo. | `/` | +| `DF_INSTITUTION_PRIVACY` | Submission privacy and authorship statement. | Institution configuration | +| `DF_INSTITUTION_PLAGIARISM` | Plagiarism and collusion statement. | Institution configuration | +| `DF_INSTITUTION_SETTINGS_RB` | Institution-specific Ruby configuration file. | Unset | + +### Authentication and encryption + +Rails credentials take precedence over the corresponding environment fallback +for `DF_SECRET_KEY_BASE`, `DF_SECRET_KEY_ATTR`, `DF_SECRET_KEY_DEVISE`, +`DF_SECRET_KEY_AAF`, `DF_SECRET_KEY_MOSS`, and `LTI_SHARED_API_SECRET`. + +| Variable | Purpose | Default | +| ----------------------------------- | ------------------------------------------------------------ | -------------------------------- | +| `DF_AUTH_METHOD` | Authentication method: `database`, `ldap`, `aaf`, or `saml`. | `database` | +| `DF_ACCESS_TOKEN_EXPIRY_SECONDS` | Access-token lifetime in seconds. | `7200` | +| `DF_REFRESH_TOKEN_EXPIRY_SECONDS` | Refresh-token lifetime in seconds. | `604800` | +| `DF_SECRET_KEY_BASE` | Rails secret key base fallback. | Required in production | +| `DF_SECRET_KEY_ATTR` | Legacy encrypted-attribute key fallback. | Required in production | +| `DF_SECRET_KEY_DEVISE` | Devise secret fallback. | Required in production | +| `DF_SECRET_KEY_MOSS` | MOSS integration secret fallback. | Unset | +| `DF_ENCRYPTION_PRIMARY_KEY` | Active Record Encryption primary key. | Required when encryption is used | +| `DF_ENCRYPTION_DETERMINISTIC_KEY` | Active Record Encryption deterministic key. | Required when encryption is used | +| `DF_ENCRYPTION_KEY_DERIVATION_SALT` | Active Record Encryption derivation salt. | Required when encryption is used | + +#### AAF Rapid Connect + +| Variable | Purpose | Default | +| ------------------------------ | ----------------------------------------- | ------------------------------- | +| `DF_AAF_ISSUER_URL` | AAF issuer URL. | `https://rapid.test.aaf.edu.au` | +| `DF_AAF_AUDIENCE_URL` | Registered application URL. | Required for AAF | +| `DF_AAF_CALLBACK_URL` | API JWT callback URL. | Required for AAF | +| `DF_AAF_IDENTITY_PROVIDER_URL` | Registered identity-provider URL. | Required for AAF | +| `DF_AAF_UNIQUE_URL` | Rapid Connect authentication-request URL. | Required for AAF | +| `DF_AAF_AUTH_SIGNOUT_URL` | URL used after sign-out. | Unset | +| `DF_SECRET_KEY_AAF` | AAF shared-secret fallback. | Required for AAF in production | + +#### SAML + +| Variable | Purpose | Default | +| ----------------------------------------- | ----------------------------------------------------------- | ---------------------- | +| `DF_SAML_METADATA_URL` | Identity-provider metadata URL. | Unset | +| `DF_SAML_METADATA_FILE_PATH` | Local identity-provider metadata file. | Unset | +| `DF_SAML_CONSUMER_SERVICE_URL` | SAML assertion consumer URL. | Required for SAML | +| `DF_SAML_SP_ENTITY_ID` | Service-provider entity ID. | Required for SAML | +| `DF_SAML_IDP_TARGET_URL` | Identity-provider login URL. | Required for SAML | +| `DF_SAML_IDP_SIGNOUT_URL` | Identity-provider logout URL. | Unset | +| `DF_SAML_IDP_CERT` | Identity-provider certificate when metadata is unavailable. | Conditionally required | +| `DF_SAML_IDP_SAML_NAME_IDENTIFIER_FORMAT` | SAML NameID format. | Email address | + +#### LDAP + +| Variable | Purpose | Default | +| --------------------------- | ---------------------------------------------------- | -------------------- | +| `DF_LDAP_HOST` | LDAP server host. | Required for LDAP | +| `DF_LDAP_PORT` | LDAP server port. | LDAP library default | +| `DF_LDAP_ATTRIBUTE` | LDAP attribute used as the login identifier. | Required for LDAP | +| `DF_LDAP_BASE` | LDAP search base. | Required for LDAP | +| `DF_LDAP_SSL` | Enable an encrypted LDAP connection. | `false` | +| `DF_LDAP_USE_ADMIN_TO_BIND` | Bind with an administrator account before searching. | `false` | +| `DF_LDAP_ADMIN_USER` | LDAP administrator bind user. | Unset | +| `DF_LDAP_ADMIN_PWD` | LDAP administrator bind password. | Unset | + +#### D2L + +| Variable | Purpose | Default | +| ------------------------------ | ----------------------------------------------------- | ------------------------------ | +| `D2L_ENABLED` | Enable D2L integration. | `false` | +| `D2L_CLIENT_ID` | D2L OAuth client ID. | Unset | +| `D2L_CLIENT_SECRET` | D2L OAuth client secret. | Unset | +| `D2L_REDIRECT_URI` | D2L OAuth callback URL ending in `/api/d2l/callback`. | Unset | +| `D2L_API_HOST` | Institution D2L API host. | Unset | +| `D2L_OAUTH_SITE` | D2L authorization server. | `https://auth.brightspace.com` | +| `D2L_OAUTH_SITE_AUTHORIZE_URL` | D2L authorization path. | `/oauth2/auth` | +| `D2L_OAUTH_SITE_TOKEN_URL` | D2L token path. | `/core/connect/token` | + +### Email + +| Variable | Purpose | Default | +| ---------------------------- | ---------------------------------------- | ------------------- | +| `DF_MAIL_PERFORM_DELIVERIES` | Enable outgoing email delivery. | Disabled | +| `DF_MAIL_DELIVERY_METHOD` | Action Mailer delivery method. | Deployment-specific | +| `DF_SMTP_ADDRESS` | SMTP server address. | Unset | +| `DF_SMTP_PORT` | SMTP server port. | Unset | +| `DF_SMTP_DOMAIN` | SMTP HELO domain. | Unset | +| `DF_SMTP_USERNAME` | SMTP username. | Unset | +| `DF_SMTP_PASSWORD` | SMTP password. | Unset | +| `DF_SMTP_AUTHENTICATION` | SMTP authentication method. | Unset | +| `DF_EMAIL_ERRORS_TO` | Address receiving PDF-generation errors. | Unset | + +### Integrations and background services + +| Variable | Purpose | Default | +| -------------------------------- | --------------------------------------------------------------- | ------- | +| `TII_ENABLED` | Enable Turnitin integration. | `false` | +| `TII_INDEX_SUBMISSIONS` | Index submissions in Turnitin. | `false` | +| `TII_REGISTER_WEBHOOK` | Register the Turnitin webhook. | `false` | +| `TCA_API_KEY` | Turnitin Core API key. | Unset | +| `TCA_HOST` | Turnitin institution host. | Unset | +| `DF_JPLAG_MIN_TOKENS` | Minimum matching-token threshold used by JPlag. | `-1` | +| `DF_JPLAG_SKIP_CLUSTER_CHECK` | Skip JPlag cluster calculation. | `false` | +| `DF_JPLAG_MAX_SHOWN_COMPARISONS` | Maximum comparisons retained in a JPlag report; `-1` means all. | `2500` | +| `LTI_ENABLED` | Enable LTI routes and authentication. | `false` | +| `LTI_SHARED_API_SECRET` | Shared secret between the Rails and LTI APIs. | Unset | +| `MODERATION_SCORE_FACTOR` | Multiplier applied to moderation score changes. | `1.0` | + +### Overseer and container access + +| Variable | Purpose | Default | +| ---------------------------------------------------- | ------------------------------------------------------------- | --------------------------------- | +| `OVERSEER_ENABLED` | Enable Overseer assessment services. | `false` | +| `OVERSEER_WORKDIR_VOLUME_MOUNT` | Host directory used for isolated Overseer work. | Required when Overseer is enabled | +| `OVERSEER_FALLBACK_VOLUME_CONTAINER` | Shared-container fallback when a host mount cannot be used. | Unset | +| `OVERSEER_STUDENT_NOTIFICATION_GRACE_PERIOD_MINUTES` | Delay before notifying students of unread failed assessments. | `30` | +| `DISK_SPACE_ENDPOINT_ENABLED` | Expose host storage availability to Overseer administrators. | `false` | +| `DOCKER_REGISTRY_URL` | Registry used for Overseer images. | Unset | +| `DOCKER_PROXY_URL` | Docker proxy or registry login endpoint. | Unset | +| `DOCKER_USER` | Docker registry username. | Unset | +| `DOCKER_TOKEN` | Docker registry access token. | Unset | + +### Error reporting + +| Variable | Purpose | Default | +| -------------------- | -------------------------------------------- | ----------------- | +| `SENTRY_DSN` | Sentry project data-source name. | Unset | +| `SENTRY_ENVIRONMENT` | Environment label attached to Sentry events. | Rails environment | + +## Testing + +OnTrack aims to follow a +[Test-Driven Development](https://en.wikipedia.org/wiki/Test-driven_development) +approach. New or changed models and API endpoints should be accompanied by +tests that describe and verify their expected behaviour. + +Run API commands inside the `doubtfire-deploy` Dev Container: + +```sh +cd /workspace/doubtfire-api +rails test + +# Running an individual test +rails test test/api/settings_test:14 # test_get_config_details ``` -Unit tests are located in the `test` directory, where **model** tests are under -the `model` subdirectory and **API** tests are under the `api` subdirectory. - -Any **helpers** should be included in the `helpers` subdirectory and helper -modules should be written under the `TestHelpers` module. +Tests are grouped under `test/models`, `test/api`, and `test/helpers`. Shared +test helpers should be placed in `test/helpers`, with helper modules defined +under the `TestHelpers` namespace. List the available maintenance and +development tasks with `rails --tasks`. -# Contributing +## Contributing -Refer to CONTRIBUTING.md +See [CONTRIBUTING.md](CONTRIBUTING.md). -# License +## License -Licensed under GNU Affero General Public License (AGPL) v3 +Licensed under the GNU Affero General Public License (AGPL) v3. diff --git a/app/api/api_root.rb b/app/api/api_root.rb index e36e21226e..694f279cdd 100644 --- a/app/api/api_root.rb +++ b/app/api/api_root.rb @@ -10,9 +10,6 @@ class ApiRoot < Grape::API format :json before do - header['Access-Control-Allow-Origin'] = '*' - header['Access-Control-Request-Method'] = '*' - Thread.current.thread_variable_set(:ip, request.ip) end @@ -47,6 +44,7 @@ class ApiRoot < Grape::API message = "Sorry... something went wrong with your request." status = 500 end + Sentry.capture_exception(e) Rack::Response.new({ error: message }.to_json, status, { 'Content-type' => 'text/error' }) end @@ -59,6 +57,7 @@ class ApiRoot < Grape::API mount AuthenticationApi mount BreaksApi mount DiscussionCommentApi + mount EngagementsApi mount ExtensionCommentsApi mount ScormExtensionCommentsApi mount GroupSetsApi @@ -79,6 +78,7 @@ class ApiRoot < Grape::API mount SidekiqApi mount LtiApi if Doubtfire::Application.config.lti_enabled mount TaskPrerequisitesApi + mount CommunicationRulesApi mount Tii::TurnItInApi mount Tii::TurnItInHooksApi @@ -93,6 +93,7 @@ class ApiRoot < Grape::API mount TutorialStreamsApi mount TutorialEnrolmentsApi mount UnitRolesApi + mount UnitContentsApi mount UnitsApi mount TutorNotesApi @@ -116,6 +117,7 @@ class ApiRoot < Grape::API AuthenticationHelpers.add_auth_to ActivityTypesAuthenticatedApi AuthenticationHelpers.add_auth_to BreaksApi AuthenticationHelpers.add_auth_to DiscussionCommentApi + AuthenticationHelpers.add_auth_to EngagementsApi AuthenticationHelpers.add_auth_to ExtensionCommentsApi AuthenticationHelpers.add_auth_to ScormExtensionCommentsApi AuthenticationHelpers.add_auth_to GroupSetsApi @@ -134,6 +136,7 @@ class ApiRoot < Grape::API AuthenticationHelpers.add_auth_to SidekiqApi AuthenticationHelpers.add_auth_to LtiApi if Doubtfire::Application.config.lti_enabled AuthenticationHelpers.add_auth_to TaskPrerequisitesApi + AuthenticationHelpers.add_auth_to CommunicationRulesApi AuthenticationHelpers.add_auth_to Tii::TurnItInApi AuthenticationHelpers.add_auth_to Tii::TiiGroupAttachmentApi @@ -159,7 +162,7 @@ class ApiRoot < Grape::API add_swagger_documentation \ base_path: nil, - doc_version: 'v10.0.0', + doc_version: 'v11.0.0', hide_documentation_path: true, info: { title: 'Doubtfire API Documentation', diff --git a/app/api/authentication_api.rb b/app/api/authentication_api.rb index f2b525af3a..e7f446278b 100644 --- a/app/api/authentication_api.rb +++ b/app/api/authentication_api.rb @@ -77,10 +77,12 @@ class AuthenticationApi < Grape::API token&.destroy! token = user.generate_authentication_token! + user.record_sign_in! # Return user details present :user, user, with: Entities::UserEntity present :auth_token, token.authentication_token + present :auth_token_expiry, token.auth_token_expiry set_refresh_cookie_in_response(remember) end end @@ -162,8 +164,10 @@ class AuthenticationApi < Grape::API requires :SAMLResponse, type: String, desc: 'SAML logout response data.' end post '/auth/saml_logout' do - response = OneLogin::RubySaml::Logoutresponse.new(params[:SAMLResponse], allowed_clock_drift: 1.second, - settings: AuthenticationHelpers.saml_settings) + response = OneLogin::RubySaml::Logoutresponse.new( + params[:SAMLResponse], + AuthenticationHelpers.saml_settings + ) # Check if the SAML response is valid - if not log an error unless response.is_valid? @@ -370,12 +374,14 @@ class AuthenticationApi < Grape::API # Invalidate the token and regenrate a new one token.destroy! token = user.generate_authentication_token! + user.record_sign_in! logger.info "Login #{params[:username]} from #{request.ip}" # Respond user details with new auth token present :user, user, with: Entities::UserEntity present :auth_token, token.authentication_token + present :auth_token_expiry, token.auth_token_expiry set_refresh_cookie_in_response(params[:remember]) end end @@ -479,12 +485,27 @@ class AuthenticationApi < Grape::API end end + desc 'Get unit content authentication token' + get '/auth/content' do + if authenticated?(:general) + token = current_user.auth_tokens.find_by(token_type: :content) + if token.nil? || token.auth_token_expiry <= Time.zone.now + token&.destroy + token = current_user.generate_content_authentication_token! + end + + present :content_auth_token, token.authentication_token + end + end + desc 'Get access token from the refresh token cookie' params do optional :delete_auth_token, type: Boolean, desc: 'Delete the auth token if also provided', default: true end post '/auth/access-token' do if authenticated_via_refresh_token? + current_user.record_access! + # Check if we have a auth token as well if params[:delete_auth_token] user_param, auth_param = get_user_and_token_from(:header) @@ -499,8 +520,10 @@ class AuthenticationApi < Grape::API end end # Return user details + token = current_user.generate_authentication_token!(token_type: :general, force_new: false) present :user, current_user, with: Entities::UserEntity - present :auth_token, current_user.generate_authentication_token!(token_type: :general, force_new: false).authentication_token + present :auth_token, token.authentication_token + present :auth_token_expiry, token.auth_token_expiry else present nil end diff --git a/app/api/communication_rules_api.rb b/app/api/communication_rules_api.rb new file mode 100644 index 0000000000..5ea54835bd --- /dev/null +++ b/app/api/communication_rules_api.rb @@ -0,0 +1,771 @@ +require 'grape' +require 'entities/communication_set_entity' +require 'entities/communication_rule_entity' +require 'entities/communication_condition_entity' +require 'entities/communication_action_entity' +require 'entities/communication_set_schedule_entity' +require 'entities/sidekiq_job_entity' + +class CommunicationRulesApi < Grape::API + helpers AuthenticationHelpers + helpers AuthorisationHelpers + helpers SidekiqHelper + helpers do + def permitted_schedule_params(raw_schedule) + ActionController::Parameters.new(raw_schedule).permit( + :id, + :name, + :active, + :anchor_week, + :anchor_day, + :hour, + :minute, + :timezone, + :recurrence, + :interval, + :repeat_count, + :until_at + ) + end + + def permitted_condition_params(raw_condition) + ActionController::Parameters.new(raw_condition).permit( + :type, + :operator, + :target_grade, + :task_definition_id, + :task_status_count, + :task_target_grade, + :last_sign_in_at, + :activity_days, + :spec_con_days, + :tutorial_id, + :tutorial_stream_id, + :campus_id, + :submitted_portfolio, + task_statuses: [] + ).to_h.compact + end + + def schedule_params_from_request + communication_set_params = params[:communication_set] || params['communication_set'] || {} + communication_set_params[:schedules] || communication_set_params['schedules'] + end + + def sync_set_schedules!(communication_set, raw_schedules) + schedules = Array(raw_schedules).map { |schedule| permitted_schedule_params(schedule).to_h } + + keep_ids = schedules.filter_map { |schedule| schedule['id'] || schedule[:id] } + + communication_set.transaction do + communication_set.communication_set_schedules.where.not(id: keep_ids).destroy_all + + schedules.each do |schedule_attrs| + schedule_id = schedule_attrs.delete('id') || schedule_attrs.delete(:id) + + if schedule_id.present? + communication_set.communication_set_schedules.find(schedule_id).update!(schedule_attrs) + else + communication_set.communication_set_schedules.create!(schedule_attrs) + end + end + end + end + end + + before do + authenticated? + + unit = Unit.find(params[:unit_id]) + unless authorise? current_user, unit, :mannage_communications + error!({ error: 'Not authorised to manage unit communications' }, 403) + end + end + + desc 'Get communication sets for a unit' + params do + requires :unit_id, type: Integer + end + get '/units/:unit_id/communication_sets' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + present unit.communication_sets.includes(:communication_set_schedules, communication_rules: [:communication_conditions, :communication_actions]), + with: Entities::CommunicationSetEntity + end + + desc 'Get a communication set for a unit with preview data' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + get '/units/:unit_id/communication_sets/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_students + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + communication_set = unit.communication_sets + .includes(:communication_set_schedules, communication_rules: [:communication_conditions, :communication_actions]) + .find(params[:id]) + + previews = communication_set.preview_allocations_by_rule + + present( + id: communication_set.id, + unit_id: communication_set.unit_id, + name: communication_set.name, + active: communication_set.active, + schedules: Entities::CommunicationSetScheduleEntity.represent(communication_set.communication_set_schedules), + rules: Entities::CommunicationRuleEntity.represent(communication_set.communication_rules), + previews: communication_set.communication_rules.map do |rule| + { + target_rule_id: rule.id, + allocations: previews.fetch(rule.id, []).map do |allocation| + { + rule_id: allocation[:rule].id, + rule_name: allocation[:rule].name, + position: allocation[:rule].position, + students: allocation[:projects].map do |project| + { + first_name: project.user&.first_name, + last_name: project.user&.last_name, + preferred_name: project.user&.nickname, + username: project.user&.username, + student_id: project.user&.student_id, + full_name: [project.user&.first_name, project.user&.last_name].compact.join(' '), + target_grade: project.target_grade, + spec_con_days: project.spec_con_days, + has_portfolio: project.portfolio_exists?, + last_sign_in_at: project.user&.last_sign_in_at, + last_viewed_at: project.last_viewed_at, + campus: project.campus&.name + } + end + } + end + } + end + ) + end + + desc 'Create a communication set for a unit' + params do + requires :unit_id, type: Integer + requires :communication_set, type: Hash do + requires :name, type: String + optional :active, type: Boolean + optional :schedules, type: Array do + optional :name, type: String + optional :active, type: Boolean + optional :anchor_week, type: Integer + optional :anchor_day, type: String + optional :hour, type: Integer + optional :minute, type: Integer + optional :timezone, type: String + optional :recurrence, type: String + optional :interval, type: Integer + optional :repeat_count, type: Integer + optional :until_at, type: DateTime + end + end + end + post '/units/:unit_id/communication_sets' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + set_params = ActionController::Parameters.new(params) + .require(:communication_set) + .permit(:name, :active) + + communication_set = unit.communication_sets.create!(set_params) + sync_set_schedules!(communication_set, schedule_params_from_request) + communication_set.reload + present communication_set, with: Entities::CommunicationSetEntity + end + + desc 'Update a communication set' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + requires :communication_set, type: Hash do + optional :name, type: String + optional :active, type: Boolean + optional :schedules, type: Array do + optional :id, type: Integer + optional :name, type: String + optional :active, type: Boolean + optional :anchor_week, type: Integer + optional :anchor_day, type: String + optional :hour, type: Integer + optional :minute, type: Integer + optional :timezone, type: String + optional :recurrence, type: String + optional :interval, type: Integer + optional :repeat_count, type: Integer + optional :until_at, type: DateTime + end + end + end + put '/units/:unit_id/communication_sets/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + communication_set = unit.communication_sets.find(params[:id]) + set_params = ActionController::Parameters.new(params) + .require(:communication_set) + .permit(:name, :active) + + communication_set.update!(set_params) + sync_set_schedules!(communication_set, schedule_params_from_request) if schedule_params_from_request.present? || (params[:communication_set] || params['communication_set'] || {}).key?(:schedules) || (params[:communication_set] || params['communication_set'] || {}).key?('schedules') + communication_set.reload + present communication_set, with: Entities::CommunicationSetEntity + end + + desc 'Delete a communication set' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_sets/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + unit.communication_sets.find(params[:id]).destroy! + status 204 + end + + desc 'Execute a communication set' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + post '/units/:unit_id/communication_sets/:id/execute' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to execute unit communications' }, 403) + end + + communication_set = unit.communication_sets.find(params[:id]) + job_id = ExecuteCommunicationSetJob.perform_async(communication_set.id) + job = setup_job(job_id) + + present job, with: Entities::SidekiqJobEntity + end + + desc 'Get schedules for a communication set' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + end + get '/units/:unit_id/communication_sets/:communication_set_id/schedules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + present communication_set.communication_set_schedules.order(:id), + with: Entities::CommunicationSetScheduleEntity + end + + desc 'Get a communication schedule' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :id, type: Integer + end + get '/units/:unit_id/communication_sets/:communication_set_id/schedules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + schedule = communication_set.communication_set_schedules.find(params[:id]) + present schedule, with: Entities::CommunicationSetScheduleEntity + end + + desc 'Create a communication schedule' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :communication_set_schedule, type: Hash do + requires :name, type: String + optional :active, type: Boolean + requires :anchor_week, type: Integer + requires :anchor_day, type: String + optional :hour, type: Integer + optional :minute, type: Integer + optional :timezone, type: String + optional :recurrence, type: String + optional :interval, type: Integer + optional :repeat_count, type: Integer + optional :until_at, type: DateTime + end + end + post '/units/:unit_id/communication_sets/:communication_set_id/schedules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + schedule_params = permitted_schedule_params(params[:communication_set_schedule]).to_h + schedule = communication_set.communication_set_schedules.create!(schedule_params) + schedule.reload + present schedule, with: Entities::CommunicationSetScheduleEntity + end + + desc 'Update a communication schedule' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :id, type: Integer + requires :communication_set_schedule, type: Hash do + optional :name, type: String + optional :active, type: Boolean + optional :anchor_week, type: Integer + optional :anchor_day, type: String + optional :hour, type: Integer + optional :minute, type: Integer + optional :timezone, type: String + optional :recurrence, type: String + optional :interval, type: Integer + optional :repeat_count, type: Integer + optional :until_at, type: DateTime + end + end + put '/units/:unit_id/communication_sets/:communication_set_id/schedules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + schedule = communication_set.communication_set_schedules.find(params[:id]) + schedule_params = permitted_schedule_params(params[:communication_set_schedule]).to_h + schedule.update!(schedule_params) + schedule.reload + present schedule, with: Entities::CommunicationSetScheduleEntity + end + + desc 'Delete a communication schedule' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_sets/:communication_set_id/schedules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update communication schedules' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + communication_set.communication_set_schedules.find(params[:id]).destroy! + status 204 + end + + desc 'Get communication rules for a unit' + params do + requires :unit_id, type: Integer + end + get '/units/:unit_id/communication_rules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + present unit.communication_rules.includes(:communication_conditions, :communication_actions), + with: Entities::CommunicationRuleEntity + end + + desc 'Get communication rules for a set' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + end + get '/units/:unit_id/communication_sets/:communication_set_id/rules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + + present communication_set.communication_rules.includes(:communication_conditions, :communication_actions), + with: Entities::CommunicationRuleEntity + end + + desc 'Create a communication rule for a communication set' + params do + requires :unit_id, type: Integer + requires :communication_set_id, type: Integer + requires :communication_rule, type: Hash do + requires :name, type: String + requires :operator, type: String + optional :position, type: Integer + optional :active, type: Boolean + optional :send_log_to_convenors, type: Boolean + end + end + post '/units/:unit_id/communication_sets/:communication_set_id/rules' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + communication_set = unit.communication_sets.find(params[:communication_set_id]) + rule_params = ActionController::Parameters.new(params) + .require(:communication_rule) + .permit(:name, :operator, :position, :active, :send_log_to_convenors) + + rule_params[:position] = communication_set.communication_rules.count if rule_params[:position].nil? + rule = communication_set.communication_rules.create!(rule_params) + present rule, with: Entities::CommunicationRuleEntity + end + + desc 'Update a communication rule' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + requires :communication_rule, type: Hash do + optional :name, type: String + optional :operator, type: String + optional :position, type: Integer + optional :active, type: Boolean + optional :send_log_to_convenors, type: Boolean + end + end + put '/units/:unit_id/communication_rules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:id]) + rule_params = ActionController::Parameters.new(params) + .require(:communication_rule) + .permit(:name, :operator, :position, :active, :send_log_to_convenors) + + rule.update!(rule_params) + present rule, with: Entities::CommunicationRuleEntity + end + + desc 'Execute a communication rule within its communication set' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + post '/units/:unit_id/communication_rules/:id/execute' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to execute unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:id]) + job_id = ExecuteCommunicationSetJob.perform_async(rule.communication_set_id, rule.id) + job = setup_job(job_id) + + present job, with: Entities::SidekiqJobEntity + end + + desc 'Preview projects matched by a communication rule' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + post '/units/:unit_id/communication_rules/:id/preview' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_students + error!({ error: 'Not authorised to preview unit communications' }, 403) + end + + # rule = unit.communication_rules.find(params[:id]) + # job_id = CommunicationRuleJob.perform_async(rule.id) + # job = setup_job(job_id) + + # present job, with: Entities::SidekiqJobEntity + # rule = unit.communication_rules.find(params[:id]) + + rule = unit.communication_rules.find(params[:id]) + allocations = rule.communication_set.preview_allocations_for_rule(rule) + + present( + target_rule_id: rule.id, + allocations: allocations.map do |allocation| + { + rule_id: allocation[:rule].id, + rule_name: allocation[:rule].name, + position: allocation[:rule].position, + students: allocation[:projects].map do |project| + { + first_name: project.user&.first_name, + last_name: project.user&.last_name, + preferred_name: project.user&.nickname, + username: project.user&.username, + student_id: project.user&.student_id, + full_name: [project.user&.first_name, project.user&.last_name].compact.join(' '), + target_grade: project.target_grade, + has_portfolio: project.portfolio_exists?, + last_sign_in_at: project.user&.last_sign_in_at, + last_viewed_at: project.last_viewed_at, + campus: project.campus&.name + } + end + } + end + ) + end + + desc 'Get communication conditions for a rule' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + end + get '/units/:unit_id/communication_rules/:communication_rule_id/conditions' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + + present rule.communication_conditions, with: Entities::CommunicationConditionEntity + end + + desc 'Delete a communication rule' + params do + requires :unit_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_rules/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:id]) + rule.destroy! + status 204 + end + + desc 'Create a communication condition' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :communication_condition, type: Hash do + requires :type, type: String + requires :operator, type: String + optional :target_grade, type: Integer + optional :task_definition_id, type: Integer + optional :task_statuses, type: Array[String] + optional :task_status_count, type: Integer + optional :task_target_grade, type: Integer + optional :last_sign_in_at, type: DateTime + optional :activity_days, type: Integer + optional :spec_con_days, type: Integer + optional :tutorial_id, type: Integer + optional :tutorial_stream_id, type: Integer + optional :campus_id, type: Integer + optional :submitted_portfolio, type: Boolean + end + end + post '/units/:unit_id/communication_rules/:communication_rule_id/conditions' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + condition = rule.communication_conditions.create!(permitted_condition_params(params[:communication_condition])) + present condition, with: Entities::CommunicationConditionEntity + end + + desc 'Update a communication condition' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :id, type: Integer + requires :communication_condition, type: Hash do + optional :type, type: String + optional :operator, type: String + optional :target_grade, type: Integer + optional :task_definition_id, type: Integer + optional :task_statuses, type: Array[String] + optional :task_status_count, type: Integer + optional :task_target_grade, type: Integer + optional :last_sign_in_at, type: DateTime + optional :activity_days, type: Integer + optional :spec_con_days, type: Integer + optional :tutorial_id, type: Integer + optional :tutorial_stream_id, type: Integer + optional :campus_id, type: Integer + optional :submitted_portfolio, type: Boolean + end + end + put '/units/:unit_id/communication_rules/:communication_rule_id/conditions/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + condition = rule.communication_conditions.find(params[:id]) + condition.update!(permitted_condition_params(params[:communication_condition])) + present condition, with: Entities::CommunicationConditionEntity + end + + desc 'Get communication actions for a rule' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + end + get '/units/:unit_id/communication_rules/:communication_rule_id/actions' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :get_unit + error!({ error: 'Not authorised to get unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + + present rule.communication_actions, with: Entities::CommunicationActionEntity + end + + desc 'Delete a communication condition' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_rules/:communication_rule_id/conditions/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + rule.communication_conditions.find(params[:id]).destroy! + status 204 + end + + desc 'Create a communication action' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :communication_action, type: Hash do + requires :type, type: String + optional :subject, type: String + optional :body, type: String + optional :email_tutors, type: Boolean + optional :email_convenors, type: Boolean + optional :target_grade, type: Integer + optional :task_definition_id, type: Integer + end + end + post '/units/:unit_id/communication_rules/:communication_rule_id/actions' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + action_params = ActionController::Parameters.new(params) + .require(:communication_action) + .permit( + :type, + :subject, + :body, + :email_tutors, + :email_convenors, + :target_grade, + :task_definition_id + ) + + action = rule.communication_actions.create!(action_params) + present action, with: Entities::CommunicationActionEntity + end + + desc 'Update a communication action' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :id, type: Integer + requires :communication_action, type: Hash do + optional :type, type: String + optional :subject, type: String + optional :body, type: String + optional :email_tutors, type: Boolean + optional :email_convenors, type: Boolean + optional :target_grade, type: Integer + optional :task_definition_id, type: Integer + end + end + put '/units/:unit_id/communication_rules/:communication_rule_id/actions/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + action = rule.communication_actions.find(params[:id]) + action_params = ActionController::Parameters.new(params) + .require(:communication_action) + .permit(:type, :subject, :body, :email_tutors, :email_convenors, :target_grade, :task_definition_id) + + action.update!(action_params) + present action, with: Entities::CommunicationActionEntity + end + + desc 'Delete a communication action' + params do + requires :unit_id, type: Integer + requires :communication_rule_id, type: Integer + requires :id, type: Integer + end + delete '/units/:unit_id/communication_rules/:communication_rule_id/actions/:id' do + unit = Unit.find(params[:unit_id]) + + unless authorise? current_user, unit, :update + error!({ error: 'Not authorised to update unit communications' }, 403) + end + + rule = unit.communication_rules.find(params[:communication_rule_id]) + rule.communication_actions.find(params[:id]).destroy! + status 204 + end +end diff --git a/app/api/discussion_prompts_api.rb b/app/api/discussion_prompts_api.rb index 77e9aa4987..34b11641be 100644 --- a/app/api/discussion_prompts_api.rb +++ b/app/api/discussion_prompts_api.rb @@ -108,7 +108,7 @@ class DiscussionPromptsApi < Grape::API error!({ error: 'You do not have permission to access this project' }, 403) end - tasks_to_discuss = project.tasks.where(task_status: [TaskStatus.discuss, TaskStatus.attention_required, TaskStatus.demonstrate]) + tasks_to_discuss = project.tasks.where(task_status: [TaskStatus.discuss, TaskStatus.rediscuss, TaskStatus.attention_required, TaskStatus.demonstrate]) task_definition_ids = tasks_to_discuss.pluck(:task_definition_id) result = DiscussionPrompt diff --git a/app/api/engagements_api.rb b/app/api/engagements_api.rb new file mode 100644 index 0000000000..6d474b95fb --- /dev/null +++ b/app/api/engagements_api.rb @@ -0,0 +1,288 @@ +require 'grape' + +class EngagementsApi < Grape::API + helpers AuthenticationHelpers + helpers AuthorisationHelpers + helpers FileStreamHelper + + before do + authenticated? + end + + helpers do + def engagements_for(project) + Engagement + .left_joins(:engagement_projects) + .where( + 'engagements.project_id = :project_id OR engagement_projects.project_id = :project_id', + project_id: project.id + ) + .distinct + end + + def engagement_for(project) + engagements_for(project).find(params[:id]) + end + + def validate_evidence!(attachment, evidence_url, remove_evidence: false) + if remove_evidence && (attachment.present? || evidence_url.present?) + error!({ error: 'Cannot remove evidence and provide replacement evidence together.' }, 400) + end + + if attachment.present? && evidence_url.present? + error!({ error: 'Provide either an attachment or an evidence URL, not both.' }, 400) + end + + return nil if attachment.blank? + + error!({ error: 'Attachment is empty.' }, 400) if File.size?(attachment['tempfile'].path).blank? + if File.size?(attachment['tempfile'].path) >= 30_000_000 + error!({ error: 'Attachment exceeds the maximum attachment size of 30MB.' }, 400) + end + + image_result = FileHelper.accept_file(attachment, 'engagement evidence image', 'image') + return 'image' if image_result[:accepted] + + pdf_result = FileHelper.accept_file(attachment, 'engagement evidence PDF', 'document') + return 'pdf' if pdf_result[:accepted] + + error!({ error: "File is not an acceptable image or PDF: #{pdf_result[:msg]}" }, 400) + end + end + + desc 'Get engagements for a project' + get '/projects/:project_id/engagements' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to view these engagements.' }, 403) unless authorise?(current_user, project, :get_engagements) + + engagements = engagements_for(project) + .includes(:user, :engagement_comments, project: :user, additional_projects: :user) + .order(:occurred_at, :created_at) + present engagements, with: Entities::EngagementEntity + end + + desc 'Get an engagement for a project' + get '/projects/:project_id/engagements/:id' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to view this engagement.' }, 403) unless authorise?(current_user, project, :get_engagements) + + engagement = engagements_for(project) + .includes(:user, project: :user, additional_projects: :user, engagement_comments: :user) + .find(params[:id]) + present engagement, with: Entities::EngagementDetailEntity + end + + desc 'Record an automatic class discussion engagement' + params do + requires :task_status_updates, type: Array do + requires :task_definition_id, type: Integer + requires :from_status, type: String + requires :to_status, type: String + end + end + post '/projects/:project_id/engagements/class_discussion' do + project = Project.find(params[:project_id]) + unless authorise?(current_user, project, :create_engagement) + error!({ error: 'You do not have permission to record a class discussion.' }, 403) + end + + task_status_updates = params[:task_status_updates].map do |update| + task_definition = project.unit.task_definitions.find(update[:task_definition_id]) + from_status = TaskStatus.status_for_name(update[:from_status]) + to_status = TaskStatus.status_for_name(update[:to_status]) + error!({ error: 'Invalid task status supplied for class discussion.' }, 422) unless from_status && to_status + + { + task: task_definition.abbreviation, + from: from_status.name, + to: to_status.name + } + end + + engagement = EngagementTracker.record_class_discussion( + user: current_user, + project: project, + task_status_updates: task_status_updates + ) + + present({ recorded: engagement.present? }, with: Grape::Presenters::Presenter) + end + + desc 'Create an engagement for a project' + params do + requires :engagement_type, type: String + requires :note, type: String + requires :occurred_at, type: DateTime + optional :project_ids, type: Array[Integer] + optional :evidence_url, type: String + optional :attachment, type: File + end + post '/projects/:project_id/engagements' do + project = Project.find(params[:project_id]) + project_ids = [project.id, *params.fetch(:project_ids, [])].uniq + projects = Project.where(id: project_ids).to_a + + unless projects.size == project_ids.size + error!({ error: 'One or more selected students could not be found.' }, 404) + end + unless projects.all? { |recipient| recipient.unit_id == project.unit_id } + error!({ error: 'All students must belong to the same unit.' }, 422) + end + unless projects.all? { |recipient| authorise?(current_user, recipient, :create_engagement) } + error!( + { error: 'You do not have permission to create an engagement for one or more selected students.' }, + 403 + ) + end + + attachment = params[:attachment] + attachment_type = validate_evidence!(attachment, params[:evidence_url]) + + engagement = nil + Engagement.transaction do + engagement = project.engagements.create!( + user: current_user, + engagement_type: params[:engagement_type], + note: params[:note], + occurred_at: params[:occurred_at], + evidence_url: attachment.present? ? nil : params[:evidence_url] + ) + engagement.additional_projects = projects.reject { |recipient| recipient.id == project.id } + end + + engagement.replace_attachment(attachment, attachment_type) if attachment.present? + present engagement, with: Entities::EngagementDetailEntity + rescue StandardError + engagement&.destroy + raise + end + + desc 'Update an engagement for a project' + params do + optional :engagement_type, type: String + optional :note, type: String + optional :occurred_at, type: DateTime + optional :evidence_url, type: String + optional :attachment, type: File + optional :remove_evidence, type: Boolean, default: false + end + put '/projects/:project_id/engagements/:id' do + project = Project.find(params[:project_id]) + engagement = engagement_for(project) + + can_edit = authorise?(current_user, project, :edit_engagement) && engagement.user_id == current_user.id + error!({ error: 'You do not have permission to edit this engagement.' }, 403) unless can_edit + + attachment = params[:attachment] + evidence_url = params.key?(:evidence_url) ? params[:evidence_url] : nil + attachment_type = validate_evidence!( + attachment, + evidence_url, + remove_evidence: params[:remove_evidence] + ) + + attributes = {} + attributes[:engagement_type] = params[:engagement_type] if params.key?(:engagement_type) + attributes[:note] = params[:note] if params.key?(:note) + attributes[:occurred_at] = params[:occurred_at] if params.key?(:occurred_at) + + if params[:remove_evidence] + engagement.remove_attachment + engagement.evidence_url = nil + elsif attachment.present? + engagement.assign_attributes(attributes) + engagement.replace_attachment(attachment, attachment_type) + attributes = {} + elsif params.key?(:evidence_url) + engagement.remove_attachment + engagement.evidence_url = evidence_url + end + + engagement.update!(attributes) + present engagement.reload, with: Entities::EngagementDetailEntity + end + + desc 'Delete an engagement for a project' + delete '/projects/:project_id/engagements/:id' do + project = Project.find(params[:project_id]) + engagement = engagement_for(project) + error!({ error: 'You do not have permission to delete this engagement.' }, 403) unless authorise?(current_user, project.unit, :delete_engagement) + + engagement.destroy! + present engagement.destroyed?, with: Grape::Presenters::Presenter + end + + desc 'Get evidence attached to an engagement' + params do + optional :as_attachment, type: Boolean, default: false + end + get '/projects/:project_id/engagements/:id/attachment' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to view this evidence.' }, 403) unless authorise?(current_user, project, :get_engagements) + + engagement = engagement_for(project) + error!({ error: 'No attachment for this engagement.' }, 404) unless engagement.attachment? + error!({ error: 'File missing.' }, 404) unless File.exist?(engagement.attachment_path) + + content_type engagement.attachment_mime_type + env['api.format'] = :binary + if params[:as_attachment] + header['Content-Disposition'] = "attachment; filename=#{engagement.attachment_file_name}" + end + + stream_file engagement.attachment_path + end + + desc 'Add a comment to an engagement' + params do + requires :comment, type: String + optional :reply_to_id, type: Integer + end + post '/projects/:project_id/engagements/:id/comments' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to comment on this engagement.' }, 403) unless authorise?(current_user, project, :comment_engagement) + + engagement = engagement_for(project) + reply_to = engagement.engagement_comments.find(params[:reply_to_id]) if params[:reply_to_id].present? + comment = engagement.engagement_comments.create!( + user: current_user, + comment: params[:comment], + reply_to: reply_to + ) + present comment, with: Entities::EngagementCommentEntity + end + + desc 'Update an engagement comment' + params do + requires :comment, type: String + end + put '/projects/:project_id/engagements/:id/comments/:comment_id' do + project = Project.find(params[:project_id]) + error!({ error: 'You do not have permission to comment on this engagement.' }, 403) unless authorise?(current_user, project, :comment_engagement) + + engagement = engagement_for(project) + comment = engagement.engagement_comments.find(params[:comment_id]) + error!({ error: 'You can only edit your own comments.' }, 403) unless comment.user_id == current_user.id + if comment.created_at < 10.minutes.ago + error!({ error: 'Comments can only be edited within 10 minutes of being created.' }, 403) + end + + comment.update!(comment: params[:comment]) + present comment, with: Entities::EngagementCommentEntity + end + + desc 'Delete an engagement comment' + delete '/projects/:project_id/engagements/:id/comments/:comment_id' do + project = Project.find(params[:project_id]) + engagement = engagement_for(project) + comment = engagement.engagement_comments.find(params[:comment_id]) + + can_delete_own = comment.user_id == current_user.id && + authorise?(current_user, project, :comment_engagement) + can_delete_any = authorise?(current_user, project.unit, :delete_engagement) + error!({ error: 'You do not have permission to delete this comment.' }, 403) unless can_delete_own || can_delete_any + + comment.destroy! + present comment.destroyed?, with: Grape::Presenters::Presenter + end +end diff --git a/app/api/entities/communication_action_entity.rb b/app/api/entities/communication_action_entity.rb new file mode 100644 index 0000000000..f44ecc2cc9 --- /dev/null +++ b/app/api/entities/communication_action_entity.rb @@ -0,0 +1,13 @@ +module Entities + class CommunicationActionEntity < Grape::Entity + expose :id + expose :type + expose :communication_rule_id + expose :subject + expose :body + expose :email_tutors + expose :email_convenors + expose :target_grade + expose :task_definition_id + end +end diff --git a/app/api/entities/communication_condition_entity.rb b/app/api/entities/communication_condition_entity.rb new file mode 100644 index 0000000000..32000932e9 --- /dev/null +++ b/app/api/entities/communication_condition_entity.rb @@ -0,0 +1,20 @@ +module Entities + class CommunicationConditionEntity < Grape::Entity + expose :id + expose :type + expose :communication_id, as: :communication_rule_id + expose :operator + expose :target_grade + expose :task_definition_id + expose :task_statuses + expose :task_status_count + expose :task_target_grade + expose :last_sign_in_at + expose :activity_days + expose :spec_con_days + expose :tutorial_id + expose :tutorial_stream_id + expose :campus_id + expose :submitted_portfolio + end +end diff --git a/app/api/entities/communication_rule_entity.rb b/app/api/entities/communication_rule_entity.rb new file mode 100644 index 0000000000..97f56da5a6 --- /dev/null +++ b/app/api/entities/communication_rule_entity.rb @@ -0,0 +1,17 @@ +module Entities + class CommunicationRuleEntity < Grape::Entity + expose :id + expose :communication_set_id + expose :name + expose :operator + expose :position + expose :active + expose :send_log_to_convenors + expose :communication_conditions, + as: :conditions, + using: Entities::CommunicationConditionEntity + expose :communication_actions, + as: :actions, + using: Entities::CommunicationActionEntity + end +end diff --git a/app/api/entities/communication_set_entity.rb b/app/api/entities/communication_set_entity.rb new file mode 100644 index 0000000000..a9b6029b9d --- /dev/null +++ b/app/api/entities/communication_set_entity.rb @@ -0,0 +1,17 @@ +require 'entities/communication_rule_entity' +require 'entities/communication_set_schedule_entity' + +module Entities + class CommunicationSetEntity < Grape::Entity + expose :id + expose :unit_id + expose :name + expose :active + expose :communication_set_schedules, + as: :schedules, + using: Entities::CommunicationSetScheduleEntity + expose :communication_rules, + as: :rules, + using: Entities::CommunicationRuleEntity + end +end diff --git a/app/api/entities/communication_set_schedule_entity.rb b/app/api/entities/communication_set_schedule_entity.rb new file mode 100644 index 0000000000..1c1573b024 --- /dev/null +++ b/app/api/entities/communication_set_schedule_entity.rb @@ -0,0 +1,21 @@ +module Entities + class CommunicationSetScheduleEntity < Grape::Entity + expose :id + expose :communication_set_id + expose :name + expose :active + expose :anchor_week + expose :anchor_day + expose :hour + expose :minute + expose :timezone + expose :recurrence + expose :interval + expose :repeat_count + expose :until_at + expose :ice_cube_schedule + expose :next_run_at + expose :last_run_at + expose :last_enqueued_at + end +end diff --git a/app/api/entities/engagement_comment_entity.rb b/app/api/entities/engagement_comment_entity.rb new file mode 100644 index 0000000000..4abf5deb8a --- /dev/null +++ b/app/api/entities/engagement_comment_entity.rb @@ -0,0 +1,10 @@ +module Entities + class EngagementCommentEntity < Grape::Entity + expose :id + expose :comment + expose :reply_to_id + expose :user, using: Entities::Minimal::MinimalUserEntity + expose :created_at + expose :updated_at + end +end diff --git a/app/api/entities/engagement_detail_entity.rb b/app/api/entities/engagement_detail_entity.rb new file mode 100644 index 0000000000..d0bf9fc591 --- /dev/null +++ b/app/api/entities/engagement_detail_entity.rb @@ -0,0 +1,7 @@ +module Entities + class EngagementDetailEntity < EngagementEntity + expose :engagement_comments, + as: :comments, + using: Entities::EngagementCommentEntity + end +end diff --git a/app/api/entities/engagement_entity.rb b/app/api/entities/engagement_entity.rb new file mode 100644 index 0000000000..0841439c00 --- /dev/null +++ b/app/api/entities/engagement_entity.rb @@ -0,0 +1,24 @@ +module Entities + class EngagementEntity < Grape::Entity + expose :id + expose :project_id + expose :engagement_type + expose :note + expose :occurred_at + expose :evidence_url + expose :content_type + expose :has_attachment do |engagement, _options| + engagement.attachment? + end + expose :attachment_file_name, if: ->(engagement, _) { engagement.attachment? } + expose :user, using: Entities::Minimal::MinimalUserEntity + expose :students, using: Entities::Minimal::MinimalUserEntity do |engagement| + [engagement.project.user, *engagement.additional_projects.map(&:user)] + end + expose :comment_count do |engagement| + engagement.engagement_comments.size + end + expose :created_at + expose :updated_at + end +end diff --git a/app/api/entities/minimal/minimal_unit_entity.rb b/app/api/entities/minimal/minimal_unit_entity.rb index bbead2cd81..7e71b425b3 100644 --- a/app/api/entities/minimal/minimal_unit_entity.rb +++ b/app/api/entities/minimal/minimal_unit_entity.rb @@ -20,6 +20,10 @@ class MinimalUnitEntity < Grape::Entity end expose :active + expose :grade_values + expose :grade_definitions + expose :discuss_timeout_enabled, unless: :summary_only + expose :discuss_timeout_expire_days, unless: :summary_only end end end diff --git a/app/api/entities/overseer_assessment_entity.rb b/app/api/entities/overseer_assessment_entity.rb index e44d84d4ee..4e893a5d0b 100644 --- a/app/api/entities/overseer_assessment_entity.rb +++ b/app/api/entities/overseer_assessment_entity.rb @@ -2,6 +2,7 @@ module Entities class OverseerAssessmentEntity < Grape::Entity expose :id expose :task_id + expose :submission_history_id expose :submission_timestamp expose :result_task_status expose :status diff --git a/app/api/entities/submission_history_entity.rb b/app/api/entities/submission_history_entity.rb new file mode 100644 index 0000000000..d33eee7193 --- /dev/null +++ b/app/api/entities/submission_history_entity.rb @@ -0,0 +1,13 @@ +module Entities + class SubmissionHistoryEntity < Grape::Entity + expose :id + expose :task_id + expose :submission_timestamp + expose :created_at + expose :has_submission_files?, as: :has_submission_files + + expose :overseer_assessment_id do |history| + history.overseer_assessment&.id + end + end +end diff --git a/app/api/entities/task_definition_entity.rb b/app/api/entities/task_definition_entity.rb index 04c15e74d0..9ea759bafc 100644 --- a/app/api/entities/task_definition_entity.rb +++ b/app/api/entities/task_definition_entity.rb @@ -1,3 +1,5 @@ +require 'entities/unit_content_link_entity' + module Entities class TaskDefinitionEntity < Grape::Entity format_with(:date_only) do |date| @@ -19,16 +21,10 @@ def staff?(my_role) expose :target_date expose :due_date expose :start_date - # expose :p_target_date, expose_nil: false - expose :c_target_date, expose_nil: false - expose :d_target_date, expose_nil: false - expose :hd_target_date, expose_nil: false - - expose :c_start_date, expose_nil: false - expose :d_start_date, expose_nil: false - expose :hd_start_date, expose_nil: false end + expose :grade_due_date_overrides, as: :grade_due_dates, expose_nil: false + expose :upload_requirements, expose_nil: false do |task_definition, options| if staff?(options[:my_role]) task_definition.upload_requirements @@ -49,6 +45,10 @@ def staff?(my_role) expose :has_task_assessment_resources?, as: :has_task_assessment_resources, if: ->(unit, options) { staff?(options[:my_role]) } expose :has_task_assessment_script?, as: :has_task_assessment_script, if: ->(unit, options) { staff?(options[:my_role]) } expose :has_scorm_data?, as: :has_scorm_data + expose :has_content_link?, as: :has_content_link + expose :content_link, using: UnitContentLinkEntity, expose_nil: false + expose :has_task_resource_link?, as: :has_task_resource_link + expose :task_resource_link, using: UnitContentLinkEntity, expose_nil: false expose :scorm_enabled expose :scorm_allow_review expose :scorm_bypass_test diff --git a/app/api/entities/task_entity.rb b/app/api/entities/task_entity.rb index 1de8fb6b39..93d6190b6f 100644 --- a/app/api/entities/task_entity.rb +++ b/app/api/entities/task_entity.rb @@ -18,6 +18,9 @@ class TaskEntity < Grape::Entity expose :target_start_date, expose_nil: false end + expose :moved_to_discuss_at, expose_nil: false + expose :discuss_timeout_expiry_at, expose_nil: false + expose :extensions expose :scorm_extensions diff --git a/app/api/entities/teaching_period_entity.rb b/app/api/entities/teaching_period_entity.rb index 67a9b570e6..772e2b3e89 100644 --- a/app/api/entities/teaching_period_entity.rb +++ b/app/api/entities/teaching_period_entity.rb @@ -9,7 +9,7 @@ class TeachingPeriodEntity < Grape::Entity expose :active do |teaching_period, options| object.active_until > DateTime.now end - expose :breaks, if: :full_details, using: Entities::BreakEntity + expose :breaks, if: :include_breaks, using: Entities::BreakEntity expose :units, if: :full_details do |teaching_period, options| Entities::UnitEntity.represent teaching_period.units, summary_only: true, user: options[:user] end diff --git a/app/api/entities/unit_content_link_entity.rb b/app/api/entities/unit_content_link_entity.rb new file mode 100644 index 0000000000..6e548428ad --- /dev/null +++ b/app/api/entities/unit_content_link_entity.rb @@ -0,0 +1,13 @@ +require 'entities/unit_content_site_entity' + +module Entities + class UnitContentLinkEntity < Grape::Entity + expose :id + expose :unit_id + expose :unit_content_site_id + expose :context_type + expose :context_key + expose :route + expose :unit_content_site, as: :site, using: Entities::UnitContentSiteEntity + end +end diff --git a/app/api/entities/unit_content_site_entity.rb b/app/api/entities/unit_content_site_entity.rb new file mode 100644 index 0000000000..dc03e4f45c --- /dev/null +++ b/app/api/entities/unit_content_site_entity.rb @@ -0,0 +1,14 @@ +module Entities + class UnitContentSiteEntity < Grape::Entity + expose :id + expose :unit_id + expose :name + expose :original_filename + expose :root_dir + expose :root_dir_options + expose :file_paths, if: ->(_site, options) { options[:include_file_paths] } + expose :is_main + expose :created_at + expose :updated_at + end +end diff --git a/app/api/entities/unit_entity.rb b/app/api/entities/unit_entity.rb index 4743f67511..c820f2e362 100644 --- a/app/api/entities/unit_entity.rb +++ b/app/api/entities/unit_entity.rb @@ -1,3 +1,5 @@ +require 'entities/unit_content_link_entity' + module Entities class UnitEntity < Grape::Entity format_with(:date_only) do |date| @@ -34,7 +36,18 @@ def can_read_unit_config?(my_role) expose :portfolio_auto_generation_date, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) }, expose_nil: false end + expose :current_unit_week do |unit| + unit.week_number(Time.current) + end + expose :active + expose :grade_values + expose :grade_definitions + expose :has_main_content_site?, as: :has_main_content_site, unless: :summary_only + expose :unit_content_links, + as: :content_links, + using: UnitContentLinkEntity, + unless: :summary_only expose :overseer_image_id, unless: :summary_only, if: lambda { |unit, options| can_read_unit_config?(options[:my_role]) } expose :assessment_enabled, unless: :summary_only @@ -67,5 +80,11 @@ def can_read_unit_config?(my_role) expose :feedback_warning_threshold_days, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) } expose :feedback_overflow_threshold_days, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) } + + expose :discuss_timeout_enabled, unless: :summary_only + expose :discuss_timeout_warning_days, unless: :summary_only, if: lambda { |unit, options| is_staff?(options[:my_role]) } + expose :discuss_timeout_expire_days, unless: :summary_only + + expose :enforce_feedback_before_discussed_in_class, if: lambda { |unit, options| is_staff?(options[:my_role]) } end end diff --git a/app/api/marking_sessions_api.rb b/app/api/marking_sessions_api.rb index 54edc83e7f..36d16af2f5 100644 --- a/app/api/marking_sessions_api.rb +++ b/app/api/marking_sessions_api.rb @@ -27,7 +27,7 @@ class MarkingSessionsApi < Grape::API end unit_role = unit.unit_role_for(current_user) - unless unit_role + unless unit_role || current_user.role == Role.admin error!({ error: "You are not authorised to view marking sessions for this unit" }, 403) end @@ -51,7 +51,7 @@ class MarkingSessionsApi < Grape::API .where(unit: unit) .where(start_time: start_date..end_date) - sessions = sessions.where(user_id: current_user.id) if unit_role.role != Role.convenor + sessions = sessions.where(user_id: current_user.id) if unit_role && unit_role&.role != Role.convenor present sessions, with: Entities::MarkingSessionEntity end diff --git a/app/api/overseer_steps_api.rb b/app/api/overseer_steps_api.rb index 018f3cc547..5327b08bfd 100644 --- a/app/api/overseer_steps_api.rb +++ b/app/api/overseer_steps_api.rb @@ -14,7 +14,7 @@ class OverseerStepsApi < Grape::API requires :overseer_step, type: Hash do requires :name, type: String optional :description, type: String - optional :display_name, type: String + requires :display_name, type: String optional :display_description, type: String optional :run_command, type: String optional :timeout, type: Integer @@ -86,7 +86,9 @@ class OverseerStepsApi < Grape::API if result.nil? error!({ error: 'No overseer step added' }, 403) else - present result, with: Entities::OverseerStepEntity + present result, + with: Entities::OverseerStepEntity, + my_role: task_definition.unit.role_for(current_user) end end @@ -175,7 +177,9 @@ class OverseerStepsApi < Grape::API overseer_step.update!(overseer_step_params) - present overseer_step, with: Entities::OverseerStepEntity + present overseer_step, + with: Entities::OverseerStepEntity, + my_role: task_definition.unit.role_for(current_user) end desc 'Delete an overseer step' diff --git a/app/api/projects_api.rb b/app/api/projects_api.rb index fb88a683a1..61a66c1fdd 100644 --- a/app/api/projects_api.rb +++ b/app/api/projects_api.rb @@ -23,11 +23,22 @@ class ProjectsApi < Grape::API desc 'Get project' params do requires :id, type: Integer, desc: 'The id of the project to get' + optional :record_attendance, type: Boolean, default: false, + desc: 'Record attendance when fetching the project during an enrolled tutorial' end get '/projects/:id' do project = Project.eager_load(:unit, :user).find(params[:id]) if authorise? current_user, project, :get + if params[:record_attendance] + unless authorise?(current_user, project, :create_engagement) + error!({ error: 'You do not have permission to record attendance for this project.' }, 403) + end + + EngagementTracker.record_attendance(user: current_user, project: project) + end + + project.update!(last_viewed_at: Time.current) if project.user_id == current_user.id present project, with: Entities::ProjectEntity, user: current_user, for_student: true, in_project: true else error!({ error: "Couldn't find Project with id=#{params[:id]}" }, 403) @@ -98,6 +109,10 @@ class ProjectsApi < Grape::API error!({ error: "You do not have permissions to change this student" }, 403) end + unless project.unit.grade_value?(params[:target_grade]) + error!({ error: 'Target grade is not enabled for this unit' }, 422) + end + project.target_grade = params[:target_grade] project.save elsif !params[:submitted_grade].nil? @@ -107,6 +122,9 @@ class ProjectsApi < Grape::API if project.portfolio_exists? error!({ error: "You cannot change your submitted grade after portfolio submission" }, 403) end + unless project.unit.grade_value?(params[:submitted_grade]) + error!({ error: 'Submitted grade is not enabled for this unit' }, 422) + end project.submitted_grade = params[:submitted_grade] project.save @@ -126,7 +144,6 @@ class ProjectsApi < Grape::API if params[:old_grade] != project.grade error!({ error: 'Existing project grade does not match current grade. Refresh project and try again.' }, 403) end - for_student = false project.grade = params[:grade] @@ -215,4 +232,17 @@ class ProjectsApi < Grape::API present portfolio_tasks.map(&:id) end + desc 'Get IDs of tasks that are still processing a PDF ' + get '/projects/:id/tasks_processing' do + project = Project.find(params[:id]) + + unless authorise? current_user, project, :get + error!({ error: "Couldn't find Project with id=#{params[:id]}" }, 403) + end + + portfolio_tasks = project.tasks_processing_pdf + + present portfolio_tasks.map(&:id) + end + end diff --git a/app/api/submission/batch_task_api.rb b/app/api/submission/batch_task_api.rb index c53d721cdd..1ab664dabd 100644 --- a/app/api/submission/batch_task_api.rb +++ b/app/api/submission/batch_task_api.rb @@ -5,6 +5,7 @@ class BatchTaskApi < Grape::API helpers GenerateHelpers helpers AuthenticationHelpers helpers AuthorisationHelpers + helpers SidekiqHelper before do authenticated? @@ -61,5 +62,38 @@ class BatchTaskApi < Grape::API # present unit.upload_batch_task_zip_or_csv(current_user, params[:file]), with: Grape::Presenters::Presenter # end # post + + desc 'Upload a batch feedback CSV or zip package for a selected task definition.' + params do + requires :file, type: File, desc: 'Batch feedback csv or zip upload' + requires :unit_id, type: Integer, desc: 'Unit ID to upload marked submissions to.' + requires :task_definition_id, type: Integer, desc: 'Task definition ID the uploaded CSV relates to.' + end + post '/submission/batch_feedback_csv/' do + unit = Unit.find(params[:unit_id]) + unit.task_definitions.find(params[:task_definition_id]) + + unless authorise? current_user, unit, :provide_bulk_feedback + error!({ error: 'Not authorised to batch upload feedback csv' }, 401) + end + + error!({ error: "No file uploaded" }, 403) if params[:file].blank? + + import_dir = Rails.root.join(FileHelper.tmp_file_dir, 'batch-feedback') + FileUtils.mkdir_p(import_dir) + + extension = File.extname(params[:file][:filename].to_s) + extension = '.upload' if extension.blank? + file_name = File.join( + import_dir, + "batch-feedback-#{unit.id}-#{params[:task_definition_id]}-#{Process.pid}-#{Thread.current.object_id}-#{current_user.id}#{extension}" + ) + + FileUtils.cp(params[:file][:tempfile].path, file_name) + + job_id = ImportBatchFeedbackJob.perform_async(unit.id, current_user.id, params[:task_definition_id], file_name) + job = setup_job(job_id) + present job, with: Entities::SidekiqJobEntity + end end end diff --git a/app/api/submission/generate_helpers.rb b/app/api/submission/generate_helpers.rb index 1d74ac443e..c78564e4a2 100644 --- a/app/api/submission/generate_helpers.rb +++ b/app/api/submission/generate_helpers.rb @@ -18,7 +18,7 @@ def scoop_files(params, upload_reqs) files[key][:id] = files[key]['name'] files[key][:name] = detail['name'] - files[key][:type] = detail['type'] + files[key][:type] = detail['type'] == 'archive' ? 'zip' : detail['type'] end # File didn't get assigned an id above, then reject it since there was a mismatch diff --git a/app/api/submission/portfolio_api.rb b/app/api/submission/portfolio_api.rb index 611616813e..aec64c65e1 100644 --- a/app/api/submission/portfolio_api.rb +++ b/app/api/submission/portfolio_api.rb @@ -55,6 +55,10 @@ class PortfolioApi < Grape::API # Remove file or portfolio? if params[:idx].nil? && params[:name].nil? && params[:kind].nil? + project.update!({ + portfolio_submission_date: nil, + portfolio_production_date: nil + }) project.remove_portfolio # returns details of file elsif !(params[:idx].nil? || params[:name].nil? || params[:kind].nil?) idx = params[:idx] diff --git a/app/api/submission/portfolio_evidence_api.rb b/app/api/submission/portfolio_evidence_api.rb index d17d340a27..8a6d36fe84 100644 --- a/app/api/submission/portfolio_evidence_api.rb +++ b/app/api/submission/portfolio_evidence_api.rb @@ -22,6 +22,7 @@ def self.logger ready_for_feedback: 1, assess_in_portfolio: 1, discuss: 2, + rediscuss: 2, attention_required: 0, demonstrate: 2, complete: 3 @@ -67,6 +68,8 @@ def self.logger 'completed' when TaskStatus.discuss 'discussed' + when TaskStatus.rediscuss + 'rediscussed' when TaskStatus.demonstrate 'demonstrated' when TaskStatus.ready_for_feedback @@ -182,6 +185,48 @@ def self.logger present result, with: Entities::OverseerAssessmentEntity end + desc 'Get all retained submission histories for a task' + get '/projects/:id/task_def_id/:task_definition_id/submission_histories' do + project = Project.find(params[:id]) + task_definition = project.unit.task_definitions.find(params[:task_definition_id]) + + unless authorise? current_user, project, :get_submission + error!({ error: "Not authorised to get submission history for task '#{task_definition.name}'" }, 401) + end + + task = project.task_for_task_definition(task_definition) + unless task + error!({ error: 'A submission for this task definition has never been created' }, 404) + end + + present task.submission_histories.order(submission_timestamp: :desc), + with: Entities::SubmissionHistoryEntity + end + + desc 'Download a retained submission history archive' + get '/projects/:id/task_def_id/:task_definition_id/submission_histories/:history_id/files' do + project = Project.find(params[:id]) + task_definition = project.unit.task_definitions.find(params[:task_definition_id]) + + unless authorise? current_user, project.unit, :provide_feedback + error!({ error: "Not authorised to get submission history for task '#{task_definition.name}'" }, 401) + end + + task = project.task_for_task_definition(task_definition) + history = task&.submission_histories&.find_by(id: params[:history_id]) + error!({ error: 'Submission history was not found' }, 404) unless history + error!({ error: 'Submission history files are not available' }, 404) unless history.has_submission_files? + + filename = "#{project.student.username}-#{task_definition.abbreviation}-#{history.submission_timestamp}.zip" + + content_type 'application/octet-stream' + header['Content-Disposition'] = "attachment; filename=#{filename}" + submission_zip_data = history.submission_zip_data + header['Content-Length'] = submission_zip_data.bytesize.to_s + env['api.format'] = :binary + body submission_zip_data + end + desc 'Trigger an overseer assessment to run again' put '/projects/:id/task_def_id/:task_definition_id/overseer_assessment/:oa_id/trigger' do project = Project.find(params[:id]) @@ -280,12 +325,12 @@ def self.logger error!({ error: 'A submission for this task definition have never been created' }, 401) end - oa = task.overseer_assessments.find_by(submission_timestamp: params[:timestamp]) - unless oa - error!({ error: "No overseer assessment found for timestamp '#{params[:timestamp]}'" }, 404) + history = task.submission_histories.find_by(submission_timestamp: params[:timestamp]) + unless history + error!({ error: "No submission history found for timestamp '#{params[:timestamp]}'" }, 404) end - unless oa.has_submission_files? + unless history.has_submission_files? error!({ error: "No submission files are available for timestamp '#{params[:timestamp]}'" }, 404) end @@ -294,7 +339,10 @@ def self.logger content_type 'application/octet-stream' header['Content-Disposition'] = "attachment; filename=#{filename}" - stream_file oa.submission_zip_file_name + submission_zip_data = history.submission_zip_data + header['Content-Length'] = submission_zip_data.bytesize.to_s + env['api.format'] = :binary + body submission_zip_data end desc 'Get the result of the submission of a task made last' diff --git a/app/api/task_comments_api.rb b/app/api/task_comments_api.rb index 51eaf2f5da..cfe2500a87 100644 --- a/app/api/task_comments_api.rb +++ b/app/api/task_comments_api.rb @@ -178,6 +178,21 @@ class TaskCommentsApi < Grape::API error!({ error: 'Not authorised to delete this comment' }, 403) end + # Comments that don't reveal a delete button + protected_comment_types = [ + AssessmentComment, + ExtensionComment, + ScormComment, + TaskCheckedInComment, + TaskDiscussedComment, + TaskFeedbackReviewRequestComment, + TaskStatusComment + ] + + if protected_comment_types.any? { |comment_type| task_comment.is_a?(comment_type) } + error!({ error: 'This comment type cannot be deleted' }, 403) + end + task_comment.destroy SessionTracker.record_assessment_activity( @@ -191,6 +206,54 @@ class TaskCommentsApi < Grape::API present false end + desc 'Edit a comment' + params do + requires :comment, type: String, desc: 'The updated comment text' + end + put '/projects/:project_id/task_def_id/:task_definition_id/comments/:id' do + project = Project.find(params[:project_id]) + task_definition = project.unit.task_definitions.find(params[:task_definition_id]) + + unless authorise? current_user, project, :make_submission + error!({ error: 'Not authorised to edit this comment' }, 403) + end + + task = project.task_for_task_definition(task_definition) + task_comment = task.all_comments.find(params[:id]) + + unless task_comment.user == current_user + error!({ error: 'You can only edit your own comments' }, 403) + end + + unless task_comment.content_type.blank? || task_comment.content_type == 'text' + error!({ error: 'Only text comments can be edited' }, 403) + end + + if task_comment.created_at < 10.minutes.ago + error!({ error: 'Comments can only be edited within 10 minutes of being created' }, 403) + end + + if params[:comment].blank? + error!({ error: 'Comment text is empty, unable to update comment' }, 403) + end + + task_comment.comment = params[:comment] + + unless task_comment.save + error!({ error: task_comment.errors.full_messages.to_sentence.presence || 'Unable to update comment' }, 403) + end + + SessionTracker.record_assessment_activity( + action: 'edit-comment', + user: current_user, + project: project, + ip_address: request.ip, + task: task + ) + + present task_comment.serialize(current_user), with: Grape::Presenters::Presenter + end + desc 'Mark a comment as unread' post '/projects/:project_id/task_def_id/:task_definition_id/comments/:id' do project = Project.find(params[:project_id]) diff --git a/app/api/task_definitions_api.rb b/app/api/task_definitions_api.rb index 8da40c69ad..c50fa07b4b 100644 --- a/app/api/task_definitions_api.rb +++ b/app/api/task_definitions_api.rb @@ -1,4 +1,5 @@ require 'grape' +require 'mime/types' class TaskDefinitionsApi < Grape::API helpers AuthenticationHelpers @@ -86,6 +87,10 @@ class TaskDefinitionsApi < Grape::API task_params[:unit_id] = unit.id task_params[:upload_requirements] = params[:task_def][:upload_requirements].present? ? JSON.parse(params[:task_def][:upload_requirements]) : [] + unless unit.grade_value?(task_params[:target_grade]) + error!({ error: 'Target grade is not enabled for this unit' }, 422) + end + task_def = TaskDefinition.new(task_params) # Set the tutorial stream @@ -139,13 +144,11 @@ class TaskDefinitionsApi < Grape::API optional :requires_discussion, type: Boolean, desc: 'Whether task must be discussed in class before it can be signed off as complete' optional :use_resources_for_jplag_base_code, type: Boolean, desc: 'Include the common base code from task resources for JPlag comparisons' optional :lock_assessments_to_tutorial_stream, type: Boolean, desc: 'Only allow tutors in this tutorial stream to assess this task' - # optional :p_target_date, type: Date, desc: 'Pass due date override' - optional :c_target_date, type: Date, desc: 'Credit due date override' - optional :d_target_date, type: Date, desc: 'Distinction due date override' - optional :hd_target_date, type: Date, desc: 'High Distinction due date override' - optional :c_start_date, type: Date, desc: 'Credit start date override' - optional :d_start_date, type: Date, desc: 'Distinction start date override' - optional :hd_start_date, type: Date, desc: 'High Distinction start date override' + optional :grade_due_dates, type: Array do + requires :target_grade, type: Integer + optional :target_due_date, type: Date + optional :start_date, type: Date + end end end put '/units/:unit_id/task_definitions/:id' do @@ -157,14 +160,7 @@ class TaskDefinitionsApi < Grape::API end # strip these out so TaskDefinition#update! never sees them - grade_due_overrides = params[:task_def].slice( - 'p_target_date', 'c_target_date', 'd_target_date', 'hd_target_date', - 'p_start_date', 'c_start_date', 'd_start_date', 'hd_start_date' - ) - params[:task_def].except!( - 'p_target_date', 'c_target_date', 'd_target_date', 'hd_target_date', - 'p_start_date', 'c_start_date', 'd_start_date', 'hd_start_date' - ) + grade_due_date_rows = params[:task_def].delete('grade_due_dates') task_params = ActionController::Parameters.new(params) .require(:task_def) @@ -212,6 +208,10 @@ class TaskDefinitionsApi < Grape::API end end + if task_params.key?(:target_grade) && !unit.grade_value?(task_params[:target_grade]) + error!({ error: 'Target grade is not enabled for this unit' }, 422) + end + # Bulk update task definition with permitted parameters task_def.update!(task_params) @@ -239,29 +239,32 @@ class TaskDefinitionsApi < Grape::API end end - grade_number = { 'c' => 1, 'd' => 2, 'hd' => 3 } - field_map = { 'target_date' => :target_due_date, 'start_date' => :start_date } - - grade_due_overrides.each do |key, date| - next if date.blank? - - # if task_def.start_date > date - # error!({ error: 'Target date cannot be earlier than start date' }, 400) - # end - + if grade_due_date_rows.present? unless unit.allow_flexible_dates error!({ error: 'This unit must have Allow Flexible Dates enabled to modify target dates per grade' }, 403) end - grade_key, kind = key.to_s.split('_', 2) # e.g. "c", "target_date" - next unless grade_number.key?(grade_key) - next unless field_map.key?(kind) + grade_due_date_rows.each do |row_params| + target_grade = row_params[:target_grade] || row_params['target_grade'] + next if target_grade.to_i.zero? + + unless unit.grade_value?(target_grade) + error!({ error: "Target grade #{target_grade} is not enabled for this unit" }, 422) + end - row = TaskDefinitionGradeDueDate.find_or_initialize_by( - task_definition: task_def, - target_grade: grade_number[grade_key] - ) - row.update!(field_map[kind] => date) + target_due_date = row_params[:target_due_date] || row_params['target_due_date'] + start_date = row_params[:start_date] || row_params['start_date'] + row = TaskDefinitionGradeDueDate.find_or_initialize_by( + task_definition: task_def, + target_grade: target_grade + ) + + if target_due_date.blank? && start_date.blank? + row.destroy if row.persisted? + else + row.update!(target_due_date: target_due_date, start_date: start_date) + end + end end present task_def, with: Entities::TaskDefinitionEntity, my_role: unit.role_for(current_user) @@ -319,7 +322,8 @@ class TaskDefinitionsApi < Grape::API end task_def.destroy - task_def.destroyed? + error!({ error: task_def.errors.full_messages.last }, 403) unless task_def.destroyed? + true end desc 'Upload the task sheet for a given task' @@ -559,6 +563,7 @@ class TaskDefinitionsApi < Grape::API .joins("LEFT JOIN task_comments ON task_comments.task_id = tasks.id AND (task_comments.type IS NULL OR task_comments.type <> 'TaskStatusComment')") .joins("LEFT OUTER JOIN (#{subquery}) as sq ON sq.project_id = projects.id") .joins('LEFT OUTER JOIN task_similarities ON tasks.id = task_similarities.task_id') + .joins("LEFT JOIN task_pins ON task_pins.task_id = tasks.id AND task_pins.user_id = #{current_user.id}") .select( 'sq.tutorial_stream_id as tutorial_stream_id', 'sq.tutorial_id as tutorial_id', @@ -572,7 +577,8 @@ class TaskDefinitionsApi < Grape::API 'grade', 'quality_pts', "SUM(case when task_comments.date_extension_assessed IS NULL AND task_comments.type = 'ExtensionComment' AND NOT task_comments.id IS NULL THEN 1 ELSE 0 END) > 0 as has_extensions", - 'SUM(case when task_similarities.flagged then 1 else 0 end) as similar_to_count' + 'SUM(case when task_similarities.flagged then 1 else 0 end) as similar_to_count', + 'COUNT(distinct task_pins.task_id) != 0 as pinned' ) .where('task_definition_id = :id', id: params[:task_def_id]) .group( @@ -603,7 +609,8 @@ class TaskDefinitionsApi < Grape::API similarity_flag: t.similar_to_count > 0, grade: t.grade, quality_pts: t.quality_pts, - has_extensions: t.has_extensions + has_extensions: t.has_extensions, + pinned: t.pinned } end @@ -653,7 +660,14 @@ class TaskDefinitionsApi < Grape::API error!({ error: 'Not authorised to download task details of unit' }, 403) end - if task_def.has_task_resources? + resource = task_def.linked_task_resource + + if resource + path = resource[:path] + filename = File.basename(resource[:filename]).gsub(/[\r\n"]/, '_') + content_type MIME::Types.type_for(filename).first&.content_type || 'application/octet-stream' + header['Content-Disposition'] = "attachment; filename=\"#{filename}\"" + elsif task_def.has_uploaded_task_resources? path = task_def.task_resources content_type 'application/octet-stream' header['Content-Disposition'] = "attachment; filename=#{task_def.abbreviation}-resources.zip" diff --git a/app/api/tasks_api.rb b/app/api/tasks_api.rb index ac3a636fdd..acb4e02290 100644 --- a/app/api/tasks_api.rb +++ b/app/api/tasks_api.rb @@ -158,6 +158,7 @@ class TasksApi < Grape::API optional :grade, type: Integer, desc: 'Grade value if task is a graded task (required if task definition is a graded task)' optional :quality_pts, type: Integer, desc: 'Quality points value if task has quality assessment' optional :discussed, type: Boolean, desc: 'Mark task as discussed' + optional :trigger_recursive_fix, desc: 'If marking fix and resubmit, recursively update preqreuisite submissions to fix' end put '/projects/:id/task_def_id/:task_definition_id' do project = Project.find(params[:id]) @@ -168,10 +169,8 @@ class TasksApi < Grape::API # check the user can put this task if authorise? current_user, project, :make_submission task = project.task_for_task_definition(task_definition) - - if !params[:discussed].nil? && authorise?(current_user, project, :assess) - task.add_discussed_comment(current_user) - end + mark_as_discussed = params[:discussed] == true && authorise?(current_user, project, :assess) + needs_discussed_comment = mark_as_discussed && !task.has_discussed_in_class_comment? # if trigger supplied... unless params[:trigger].nil? @@ -198,15 +197,33 @@ class TasksApi < Grape::API error!({ error: 'This task can only be assessed in portfolio.' }, 403) end - if task.task_definition.requires_discussion && params[:trigger] == 'complete' && !task.has_discussed_in_class_comment? + if task.task_definition.requires_discussion && params[:trigger] == 'complete' && + !task.has_discussed_in_class_comment? && !mark_as_discussed error!({ error: 'This task must be discussed in class before it can be marked complete.' }, 403) end logger.info "#{current_user.username} assessing task #{task.id} to #{params[:trigger]}" - result = task.trigger_transition(trigger: params[:trigger], by_user: current_user, quality: params[:quality_pts]) + begin + task.discussion_confirmed_for_transition = mark_as_discussed + result = task.trigger_transition( + trigger: params[:trigger], + by_user: current_user, + quality: params[:quality_pts], + recursive_fix: params[:trigger_recursive_fix], + check_feedback: true + ) + ensure + task.discussion_confirmed_for_transition = false + end + + if result.nil? && task.errors.any? + error!({ error: task.errors.full_messages.to_sentence }, 403) + end if result.nil? && task.task_definition.restrict_status_updates error!({ error: 'This task can only be updated by your tutor.' }, 403) end + task.add_discussed_comment(current_user) if result && needs_discussed_comment + SessionTracker.record_assessment_activity( action: "assessing", user: current_user, @@ -216,6 +233,8 @@ class TasksApi < Grape::API ) end + task.add_discussed_comment(current_user) if params[:trigger].nil? && needs_discussed_comment + # if grade was supplied unless grade.nil? # try to grade the task @@ -506,19 +525,30 @@ class TasksApi < Grape::API error!({ error: "This task has already been claimed by another tutor" }, 409) end - inactive_claim = task.overflow_task_claim - inactive_claim&.destroy! + claimed_at = Time.zone.now + original_tutor = task.tutor + + ActiveRecord::Base.transaction do + task.overflow_task_claim&.destroy! - task_claim = OverflowTaskClaim.create!({ - task: task, - claimed_by_unit_role_id: my_unit_role.id - }) + OverflowTaskClaim.create!( + task: task, + claimed_by_unit_role: my_unit_role + ) - unless task_claim.valid? - error!({ error: "Failed to claim task" }, 400) + OverflowTaskClaimLog.create!( + unit: unit, + task: task, + claimed_by_unit_role: my_unit_role, + claimed_by_user: current_user, + original_tutor_user: original_tutor, + student_user: project.student, + days_awaiting_feedback: task.days_awaiting_feedback(claimed_at), + claimed_at: claimed_at + ) end - logger.info "Overflow task claim: {\"user_id\": #{current_user.id},\"task_id\": #{task.id}, \"timestamp\": \"#{Time.zone.now}\", \"original_tutor_user_id\": #{task.tutor ? task.tutor.id : -1}}" + logger.info "Overflow task claim: {\"user_id\": #{current_user.id},\"task_id\": #{task.id}, \"timestamp\": \"#{claimed_at}\", \"original_tutor_user_id\": #{original_tutor ? original_tutor.id : -1}}" true end diff --git a/app/api/teaching_periods_public_api.rb b/app/api/teaching_periods_public_api.rb index 9f17860541..704527fd96 100644 --- a/app/api/teaching_periods_public_api.rb +++ b/app/api/teaching_periods_public_api.rb @@ -4,12 +4,12 @@ class TeachingPeriodsPublicApi < Grape::API desc "Get a teaching period's details" get '/teaching_periods/:id' do teaching_period = TeachingPeriod.find(params[:id]) - present teaching_period, with: Entities::TeachingPeriodEntity, full_details: true, user: current_user + present teaching_period, with: Entities::TeachingPeriodEntity, full_details: true, include_breaks: true, user: current_user end desc 'Get all the Teaching Periods' get '/teaching_periods' do teaching_periods = TeachingPeriod.all - present teaching_periods, with: Entities::TeachingPeriodEntity + present teaching_periods, with: Entities::TeachingPeriodEntity, include_breaks: true end end diff --git a/app/api/unit_contents_api.rb b/app/api/unit_contents_api.rb new file mode 100644 index 0000000000..aee17fbd45 --- /dev/null +++ b/app/api/unit_contents_api.rb @@ -0,0 +1,355 @@ +require 'grape' +require 'entities/unit_content_link_entity' +require 'entities/unit_content_site_entity' +require 'mime/types' +require 'uri' + +class UnitContentsApi < Grape::API + helpers AuthenticationHelpers + helpers AuthorisationHelpers + helpers FileStreamHelper + helpers MimeCheckHelpers + + helpers do + def unit_content_link_for_route(unit, content_route) + normalized_route = "/#{content_route.to_s.gsub(%r{\A/+|/+\z}, '')}" + normalized_route = '/' if normalized_route.blank? + + unit.unit_content_links + .where.not(context_type: 'task_definition_resource') + .find_by(route: normalized_route) + end + + def authorise_unit_content_management!(unit) + return if authorise?(current_user, unit, :manage_unit_content) || + authorise?(current_user, User, :admin_units) + + error!({ error: 'Not authorised to manage unit content' }, 403) + end + + def unit_content_reference_url( + unit_id, + site_id, + reference, + current_path, + username, + content_token + ) + return reference if reference.blank? || reference.match?(%r{\A(?:[a-z][a-z0-9+.-]*:|//|#)}i) + + reference_path = reference.split(/[?#]/, 2).first + resolved_path = + if reference_path.start_with?('/') + reference_path + else + File.expand_path(reference_path, "/#{File.dirname(current_path)}") + end + + query = Rack::Utils.build_query( + content_route: resolved_path, + content_site_id: site_id, + username: username, + content_token: content_token + ) + fragment = reference.include?('#') ? "##{reference.split('#', 2).last}" : '' + + "/api/units/#{unit_id}/content?#{query}#{fragment}" + end + + def rewrite_unit_content_response( + contents, + content_type, + unit_id, + site_id, + current_path, + username, + content_token + ) + rewrite_reference = lambda do |reference| + unit_content_reference_url( + unit_id, + site_id, + reference, + current_path, + username, + content_token + ) + end + + case content_type + when 'text/html' + contents = contents.gsub( + /(<(?:iframe|img|link|script|source|video|audio)\b[^>]*?\b(?:href|poster|src)=)(["'])([^"']+)\2/i + ) do + "#{Regexp.last_match(1)}#{Regexp.last_match(2)}" \ + "#{rewrite_reference.call(Regexp.last_match(3))}#{Regexp.last_match(2)}" + end + contents.gsub(/\bsrcset=(["'])([^"']+)\1/i) do + quote = Regexp.last_match(1) + srcset = Regexp.last_match(2).split(',').map do |entry| + reference, descriptor = entry.strip.split(/\s+/, 2) + [rewrite_reference.call(reference), descriptor].compact.join(' ') + end.join(', ') + + "srcset=#{quote}#{srcset}#{quote}" + end + when 'text/css' + contents.gsub(/url\((["']?)([^"')]+)\1\)/i) do + quote = Regexp.last_match(1) + "url(#{quote}#{rewrite_reference.call(Regexp.last_match(2))}#{quote})" + end + when 'text/javascript', 'application/javascript' + contents = contents.gsub( + /\b((?:import|export)(?:\s*[^"']*?\s*from\s*)?\s*)(["'])([^"']+)\2/ + ) do + "#{Regexp.last_match(1)}#{Regexp.last_match(2)}" \ + "#{rewrite_reference.call(Regexp.last_match(3))}#{Regexp.last_match(2)}" + end + contents.gsub(/\b(import\s*\(\s*)(["'])([^"']+)\2(\s*\))/) do + "#{Regexp.last_match(1)}#{Regexp.last_match(2)}#{rewrite_reference.call(Regexp.last_match(3))}" \ + "#{Regexp.last_match(2)}#{Regexp.last_match(4)}" + end + else + contents + end + end + end + + before do + if request.path.match?(%r{/units/\d+/content\z}) + authenticated?(:content) + else + authenticated? + end + end + + desc 'Get a unit content route' + params do + optional :content_route, type: String, desc: 'The content route being loaded' + optional :content_site_id, type: Integer, desc: 'Specific content site to load' + requires :username, type: String, desc: 'Username associated with the scoped content token' + requires :content_token, type: String, desc: 'Scoped content authentication token' + end + get '/units/:id/content' do + unit = Unit.find(params[:id]) + + unless authorise?(current_user, unit, :get_unit) || authorise?(current_user, User, :admin_units) + error!({ error: "Couldn't find Unit with id=#{params[:id]}" }, 403) + end + + link = nil + site = if params[:content_site_id].present? + unit.unit_content_sites.find(params[:content_site_id]) + else + link = unit_content_link_for_route(unit, params[:content_route]) + link&.unit_content_site || + unit.unit_content_sites.find_by(is_main: true) + end + + error!({ error: 'Unit content archive is not configured' }, 404) unless site + + error!({ error: 'Unit content archive is not available' }, 404) unless File.exist?(site.archive_path) + + content_route = URI::DEFAULT_PARSER.unescape(params[:content_route].presence || '/') + route_parts = content_route.split('/').reject(&:blank?) + error!({ error: 'Invalid unit content route' }, 422) if route_parts.any? { |part| ['.', '..'].include?(part) } + + root_parts = site.root_dir.to_s.split('/').reject(&:blank?) + requested_entry_path = (root_parts + route_parts).join('/') + archive_entry_paths = [ + requested_entry_path, + (root_parts + route_parts + ['index.html']).join('/') + ].uniq + archive_entry = nil + file_contents = nil + + Zip::File.open(site.archive_path) do |zip_file| + archive_entry = archive_entry_paths.filter_map { |path| zip_file.find_entry(path) }.find(&:file?) + file_contents = archive_entry.get_input_stream.read if archive_entry + end + + error!({ error: "Unit content route '#{content_route}' is not available" }, 404) unless archive_entry + + response_content_type = MIME::Types.type_for(archive_entry.name).first&.content_type + response_content_type ||= 'application/octet-stream' + root_prefix = root_parts.join('/') + current_path = archive_entry.name.delete_prefix("#{root_prefix}/") + file_contents = rewrite_unit_content_response( + file_contents, + response_content_type, + unit.id, + site.id, + current_path, + params[:username], + params[:content_token] + ) + + content_type response_content_type + header['Content-Disposition'] = "inline; filename=#{File.basename(archive_entry.name)}" + header['X-Content-Site-Id'] = site.id.to_s + header['X-Content-Route'] = link&.route || content_route + header['X-Content-Root-Dir'] = site.root_dir + header['Access-Control-Expose-Headers'] = + 'Content-Disposition,X-Content-Site-Id,X-Content-Route,X-Content-Root-Dir' + header['Cache-Control'] = 'no-cache, no-store, must-revalidate' + header['Referrer-Policy'] = 'strict-origin' + env['api.format'] = :binary + + body file_contents + rescue Zip::Error + error!({ error: 'Unit content archive is invalid' }, 422) + end + + desc 'List unit content sites' + get '/units/:id/content/sites' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + present unit.unit_content_sites.order(created_at: :desc), + with: Entities::UnitContentSiteEntity, + include_file_paths: true + end + + desc 'Download a unit content site archive' + get '/units/:id/content/sites/:site_id/archive' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + site = unit.unit_content_sites.find(params[:site_id]) + filename = FileHelper.sanitized_filename(site.original_filename) + + content_type 'application/zip' + header['Content-Disposition'] = "attachment; filename=\"#{filename}\"" + stream_file site.archive_path + end + + desc 'Upload a unit content site archive' + params do + requires :file, type: File, desc: 'The static content site zip' + optional :name, type: String, desc: 'Display name for the uploaded site' + end + post '/units/:id/content/sites' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + file = params[:file] + check_mime_against_list! file[:tempfile].path, + 'zip', + ['application/zip', + 'multipart/x-gzip', + 'multipart/x-zip', + 'application/x-gzip', + 'application/octet-stream'] + + site = UnitContentSite.store_upload!(unit, file, name: params[:name]) + present site, with: Entities::UnitContentSiteEntity, include_file_paths: true + end + + desc 'Delete a unit content site' + delete '/units/:id/content/sites/:site_id' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + unit.unit_content_sites.find(params[:site_id]).destroy! + true + end + + desc 'Update a unit content site' + params do + optional :file, type: File, desc: 'Replacement static content site zip' + optional :name, type: String, desc: 'Display name for the uploaded site' + optional :root_dir, type: String, desc: 'Folder within the zip to serve as the site root' + optional :is_main, type: Boolean, desc: 'Use this as the default site for unit content' + end + put '/units/:id/content/sites/:site_id' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + site = unit.unit_content_sites.find(params[:site_id]) + update_params = declared(params, include_missing: false).slice(:name, :root_dir, :is_main) + file = params[:file] + root_dir = update_params[:root_dir] + + if file.present? + check_mime_against_list! file[:tempfile].path, + 'zip', + ['application/zip', + 'multipart/x-gzip', + 'multipart/x-zip', + 'application/x-gzip', + 'application/octet-stream'] + end + + root_dir_options = + file.present? ? UnitContentSite.root_dir_options_for(file[:tempfile].path) : site.root_dir_options + + if root_dir.present? && !root_dir_options.include?(root_dir) + error!({ error: 'Root directory is not available in this content site archive' }, 422) + end + + if update_params[:is_main] + unit.unit_content_sites.where.not(id: site.id).find_each do |content_site| + content_site.update!(is_main: false) + end + end + + if file.present? + site.replace_upload!(file, root_dir: root_dir) + update_params.except!(:root_dir) + end + + site.update!(update_params) + present site, with: Entities::UnitContentSiteEntity, include_file_paths: true + end + + desc 'List unit content links' + get '/units/:id/content/links' do + unit = Unit.find(params[:id]) + + unless authorise?(current_user, unit, :get_unit) || authorise?(current_user, User, :admin_units) + error!({ error: "Couldn't find Unit with id=#{params[:id]}" }, 403) + end + + present unit.unit_content_links.includes(:unit_content_site).order(:context_type, :context_key), + with: Entities::UnitContentLinkEntity + end + + desc 'Replace unit content links' + params do + requires :links, type: Array do + requires :context_type, type: String + requires :context_key, type: String + requires :unit_content_site_id, type: Integer + optional :route, type: String + end + end + put '/units/:id/content/links' do + unit = Unit.find(params[:id]) + authorise_unit_content_management!(unit) + + submitted_contexts = params[:links].map do |link_params| + [link_params[:context_type], link_params[:context_key]] + end + + unit.unit_content_links + .where(context_type: %w[grade grade_overview task_definition task_definition_resource]) + .find_each do |link| + link.destroy! unless submitted_contexts.include?([link.context_type, link.context_key]) + end + + links = params[:links].map do |link_params| + site = unit.unit_content_sites.find(link_params[:unit_content_site_id]) + link = unit.unit_content_links.find_or_initialize_by( + context_type: link_params[:context_type], + context_key: link_params[:context_key] + ) + + link.unit_content_site = site + link.route = link_params[:route].presence || '/' + link.save! + link + end + + present links, with: Entities::UnitContentLinkEntity + end +end diff --git a/app/api/unit_roles_api.rb b/app/api/unit_roles_api.rb index ff0d9f9004..9b03699c05 100644 --- a/app/api/unit_roles_api.rb +++ b/app/api/unit_roles_api.rb @@ -25,6 +25,10 @@ class UnitRolesApi < Grape::API end desc 'Delete a unit role' + params do + requires :id, type: Integer, desc: 'The id of the unit role to delete' + optional :reassign_to_unit_role_id, type: Integer, desc: 'The unit role to reassign tutorials to before deletion' + end delete '/unit_roles/:id' do unit_role = UnitRole.find(params[:id]) @@ -32,7 +36,28 @@ class UnitRolesApi < Grape::API error!({ error: "You do not have permission to perform this action" }, 403) end - unit_role.destroy! + tutorials = unit_role.unit.tutorials.where(unit_role_id: unit_role.id) + + if tutorials.exists? + if params[:reassign_to_unit_role_id].blank? + error!({ error: 'Unable to delete this unit role while tutorials are assigned without providing a reassignment target' }, 400) + end + + reassignment_role = unit_role.unit.staff.find_by(id: params[:reassign_to_unit_role_id]) + + if reassignment_role.nil? || reassignment_role.id == unit_role.id + error!({ error: 'Unable to delete this unit role with the provided reassignment target' }, 400) + end + + ActiveRecord::Base.transaction do + tutorials.find_each do |tutorial| + tutorial.update!(unit_role: reassignment_role) + end + unit_role.destroy! + end + else + unit_role.destroy! + end end desc 'Employ a user as a teaching role in a unit' diff --git a/app/api/units_api.rb b/app/api/units_api.rb index a20c8a3d7a..89e0105eee 100644 --- a/app/api/units_api.rb +++ b/app/api/units_api.rb @@ -46,6 +46,7 @@ class UnitsApi < Grape::API { unit_roles: [:role, :user] }, { task_definitions: :tutorial_stream }, :learning_outcomes, + { unit_content_links: :unit_content_site }, { tutorial_streams: :activity_type }, { tutorials: [:tutor, :tutorial_stream] }, :tutorial_enrolments, @@ -92,6 +93,16 @@ class UnitsApi < Grape::API optional :assessment_enabled, type: Boolean optional :feedback_warning_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its highlighted in the tutors inbox' optional :feedback_overflow_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its added to overflow marking' + optional :discuss_timeout_enabled, type: Boolean, desc: 'Move stale Discuss tasks back to Fix and Resubmit after a warning period' + optional :discuss_timeout_warning_days, type: Integer, desc: 'Number of days in Discuss before warning the student' + optional :discuss_timeout_expire_days, type: Integer, desc: 'Number of days in Discuss before moving the task to Fix and Resubmit' + optional :enforce_feedback_before_discussed_in_class, type: Boolean, desc: 'Require feedback to be completed before tasks can be marked discussed in class' + optional :grade_definitions, type: Array do + requires :id, type: String + requires :value, type: Integer + requires :label, type: String + requires :abbreviation, type: String + end mutually_exclusive :teaching_period_id, :start_date mutually_exclusive :teaching_period_id, :end_date @@ -126,7 +137,12 @@ class UnitsApi < Grape::API :overseer_image_id, :assessment_enabled, :feedback_warning_threshold_days, - :feedback_overflow_threshold_days + :feedback_overflow_threshold_days, + :discuss_timeout_enabled, + :discuss_timeout_warning_days, + :discuss_timeout_expire_days, + :enforce_feedback_before_discussed_in_class, + grade_definitions: [:id, :value, :label, :abbreviation] ) if unit.teaching_period_id.present? && (unit_parameters.key?(:start_date) || unit_parameters['teaching_period_id'] == -1) @@ -174,6 +190,16 @@ class UnitsApi < Grape::API optional :allow_student_change_tutorial, type: Boolean, desc: 'Can turn on/off student ability to change tutorials', default: true optional :feedback_warning_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its highlighted in the tutors inbox' optional :feedback_overflow_threshold_days, type: Integer, desc: 'Number of days since a submission without feedback before its added to overflow marking' + optional :discuss_timeout_enabled, type: Boolean, desc: 'Move stale Discuss tasks back to Fix and Resubmit after a warning period', default: false + optional :discuss_timeout_warning_days, type: Integer, desc: 'Number of days in Discuss before warning the student', default: 7 + optional :discuss_timeout_expire_days, type: Integer, desc: 'Number of days in Discuss before moving the task to Fix and Resubmit', default: 14 + optional :enforce_feedback_before_discussed_in_class, type: Boolean, desc: 'Require feedback to be completed before tasks can be marked discussed in class', default: false + optional :grade_definitions, type: Array do + requires :id, type: String + requires :value, type: Integer + requires :label, type: String + requires :abbreviation, type: String + end mutually_exclusive :teaching_period_id, :start_date mutually_exclusive :teaching_period_id, :end_date @@ -205,7 +231,12 @@ class UnitsApi < Grape::API :portfolio_auto_generation_date, :allow_student_change_tutorial, :feedback_warning_threshold_days, - :feedback_overflow_threshold_days + :feedback_overflow_threshold_days, + :discuss_timeout_enabled, + :discuss_timeout_warning_days, + :discuss_timeout_expire_days, + :enforce_feedback_before_discussed_in_class, + grade_definitions: [:id, :value, :label, :abbreviation] ) # Ensure the user is authorised to convene units @@ -416,6 +447,36 @@ class UnitsApi < Grape::API present job, with: Entities::SidekiqJobEntity end + desc 'Upload CSV of staff notes for students in a unit' + params do + requires :file, type: File, desc: 'CSV upload file.' + end + post '/csv/units/:id/staff_notes' do + unit = Unit.find(params[:id]) + unless authorise? current_user, unit, :upload_staff_notes_csv + error!({ error: "Not authorised to upload staff notes to #{unit.code}" }, 403) + end + + if params[:file].blank? + error!({ error: "No file uploaded" }, 403) + end + + ensure_csv!(params[:file][:tempfile]) + + import_csv_dir = Rails.root.join(FileHelper.tmp_file_dir, 'csv') + file_name = File.join(import_csv_dir, "import-staff-notes-csv-#{unit.id}-#{Process.pid}-#{Thread.current.object_id}-#{current_user.id}.csv") + FileUtils.mkdir_p(import_csv_dir) + + csv = CSV.read(params[:file][:tempfile], headers: true) + CSV.open(file_name, "w", write_headers: true, headers: csv.headers) do |out| + csv.each { |row| out << row } + end + + job_id = ImportStaffNotesCsvJob.perform_async(unit.id, current_user.id, file_name) + job = setup_job(job_id) + present job, with: Entities::SidekiqJobEntity + end + desc 'Upload CSV with the students to un-enrol from the unit' params do requires :file, type: File, desc: 'CSV upload file.' @@ -532,6 +593,18 @@ class UnitsApi < Grape::API present job, with: Entities::SidekiqJobEntity end + desc 'Download CSV of overflow task claims in this unit' + get '/csv/units/:id/overflow_task_claims' do + unit = Unit.find(params[:id]) + unless authorise? current_user, unit, :download_overflow_stats + error!({ error: "Not authorised to download overflow task claim stats for #{unit.code}" }, 403) + end + + job_id = DownloadOverflowTaskClaimsCsvJob.perform_async(unit.id) + job = setup_job(job_id) + present job, with: Entities::SidekiqJobEntity + end + desc 'Download CSV of all student tasks in this unit' get '/csv/units/:id/task_completion' do unit = Unit.find(params[:id]) @@ -574,6 +647,61 @@ class UnitsApi < Grape::API present unit.student_task_completion_stats, with: Grape::Presenters::Presenter end + desc 'Get historical task completion snapshots' + params do + optional :start_date, type: Date, desc: 'Include snapshots captured on or after this date' + optional :end_date, type: Date, desc: 'Include snapshots captured on or before this date' + optional :limit, type: Integer, desc: 'Maximum number of snapshots to return', default: 365 + end + get '/units/:id/stats/task_completion_snapshots' do + unit = Unit.find(params[:id]) + unless authorise? current_user, unit, :download_stats + error!({ error: "Not authorised to download stats of student tasks in #{unit.code}" }, 403) + end + + snapshots = unit.task_completion_snapshots.order(snapshot_timestamp: :desc) + if params[:start_date].present? + start_timestamp = params[:start_date].in_time_zone.beginning_of_day.to_i + snapshots = snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) >= ?', start_timestamp) + end + if params[:end_date].present? + end_timestamp = params[:end_date].in_time_zone.end_of_day.to_i + snapshots = snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) <= ?', end_timestamp) + end + snapshots = snapshots.limit([params[:limit].to_i, 365].min) + + present snapshots.map { |snapshot| + stats = snapshot.load_stats + + { + snapshot_date: snapshot.snapshot_date, + snapshot_timestamp: snapshot.snapshot_timestamp, + stats: stats + } + }, with: Grape::Presenters::Presenter + end + + desc 'Capture task completion snapshot immediately for this unit' + post '/units/:id/stats/task_completion_snapshots/capture' do + unit = Unit.find(params[:id]) + unless authorise? current_user, unit, :capture_task_completion_snapshot + error!({ error: "Not authorised to capture stats of student tasks in #{unit.code}" }, 403) + end + + # Check if a snapshot was captured within the past 30 minutes + recent_snapshot = unit.task_completion_snapshots.where('CAST(snapshot_timestamp AS UNSIGNED) > ?', 30.minutes.ago.to_i).order(snapshot_timestamp: :desc).first + if recent_snapshot.present? + recent_snapshot_time = recent_snapshot.snapshot_time + remaining_seconds = [(recent_snapshot_time + 30.minutes - Time.zone.now).ceil, 0].max + remaining_minutes = [(remaining_seconds / 60.0).ceil, 1].max + error!({ error: "A snapshot was captured at #{recent_snapshot_time.strftime('%H:%M')}. Please wait #{remaining_minutes} more minute(s) before capturing another snapshot." }, 429) + end + + job_id = AggregateTaskCompletionStatsJob.perform_async(unit.id) + job = setup_job(job_id) + present job, with: Entities::SidekiqJobEntity + end + desc 'Download stats related to the number of tasks assessed by each tutor' get '/csv/units/:id/tutor_assessments' do unit = Unit.find(params[:id]) diff --git a/app/api/webcal_public_api.rb b/app/api/webcal_public_api.rb index a2bc091e9f..4c18aebc1f 100644 --- a/app/api/webcal_public_api.rb +++ b/app/api/webcal_public_api.rb @@ -14,9 +14,10 @@ class WebcalPublicApi < Grape::API webcal = Webcal.find_by!(guid: params[:guid]) # Serve the iCalendar with the correct MIME type. + env['api.format'] = :txt content_type 'text/calendar' - # Seve ical. - present webcal.to_ical.to_ical + # Serve ical. + webcal.to_ical.to_ical end end diff --git a/app/helpers/authentication_helpers.rb b/app/helpers/authentication_helpers.rb index d43c635c09..1b59bfffba 100644 --- a/app/helpers/authentication_helpers.rb +++ b/app/helpers/authentication_helpers.rb @@ -98,6 +98,7 @@ def authenticated?(token_type = :general) user_param, auth_param = get_user_and_token_from(:cookie) else user_param, auth_param = get_user_and_token_from(:header) + auth_param = params['content_token'] if token_type == :content end case user_auth_token_type(user_param, auth_param, token_type) @@ -221,7 +222,7 @@ def set_refresh_cookie_in_response(remember) # Generate a new token when the old one is absent or getting close to expiring if token.nil? || token.auth_token_expiry <= Time.zone.now - 12.hours - token = current_user.generate_authentication_token!(token_type: :refresh_token, expiry: Time.zone.now + 1.week) + token = current_user.generate_authentication_token!(token_type: :refresh_token) end domain = Doubtfire::Application.config.institution[:cookie_domain] diff --git a/app/helpers/authorisation_helpers.rb b/app/helpers/authorisation_helpers.rb index b27fd59024..0b25682df6 100644 --- a/app/helpers/authorisation_helpers.rb +++ b/app/helpers/authorisation_helpers.rb @@ -26,7 +26,9 @@ def get_permission_hash(role, perm_hash, _other) :get_staff_note, :get_members, :get_groups, - :get_discussion_prompt + :get_discussion_prompt, + :get_engagements, + :get_tutor_times ].freeze # @@ -40,6 +42,13 @@ def get_permission_hash(role, perm_hash, _other) def authorise?(user, object, action, perm_get_fn = method(:get_permission_hash), other = nil) # Can pass in instance or class obj_class = object.class == Class ? object : object.class + perm_hash = obj_class.permissions + + # System administrator permissions take precedence over contextual roles. + if user.has_admin_capability? + system_perms = perm_get_fn.call(user.role.to_sym, perm_hash, other) + return true if system_perms&.include?(action) + end role_obj = object.role_for(user) @@ -58,7 +67,6 @@ def authorise?(user, object, action, perm_get_fn = method(:get_permission_hash), end role = role_obj.to_sym - perm_hash = obj_class.permissions perms = perm_get_fn.call(role, perm_hash, other) # No permissions, default to false authorise, else check if the action diff --git a/app/helpers/file_helper.rb b/app/helpers/file_helper.rb index 1f589f08a8..0953907850 100644 --- a/app/helpers/file_helper.rb +++ b/app/helpers/file_helper.rb @@ -6,14 +6,28 @@ require 'open3' require 'shellwords' require 'pdf-reader' +require 'zlib' +require 'rubygems/package' module FileHelper extend LogHelper extend TimeoutHelper extend MimeCheckHelpers + ZIP_NESTED_ARCHIVE_EXTENSIONS = %w[ + .7z .bz2 .ear .gz .jar .rar .tar .tar.bz2 .tar.gz .tar.xz .tbz .tbz2 .tgz .txz .war .xz .zip + ].freeze + WORD_DOCUMENT_EXTENSION = '.docx' + WORD_DOCUMENT_MIME_TYPE = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + COMPOUND_FILE_SIGNATURE = "\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b.freeze + WORD_DOCUMENT_ENCRYPTION_STREAM_NAMES = %w[EncryptedPackage EncryptionInfo].map do |name| + name.encode(Encoding::UTF_16LE).b.freeze + end.freeze + + class DocumentConversionError < StandardError; end + def known_extension?(extn) - allow_extensions = %w(pdf ps csv xls xlsx pas cpp c cs csv h hpp java py js html coffee scss yaml yml xml json ts r rb rmd rnw rhtml rpres tex vb sql txt md jack hack asm hdl tst out cmp vm sh bat dat ipynb css png bmp tiff tif jpeg jpg gif zip gz tar wav ogg mp3 mp4 webm aac pcm aiff flac wma alac pml vue) + allow_extensions = %w(pdf ps docx csv xls xlsx pas cpp c cs csv h hpp java py js html coffee scss yaml yml xml json ts r rb rmd rnw rhtml rpres tex vb sql txt md jack hack asm hdl tst out cmp vm sh bat dat ipynb css png bmp tiff tif jpeg jpg gif zip gz tgz tar wav ogg mp3 mp4 webm aac pcm aiff flac wma alac pml vue) # Allow empty or nil extensions for blobs otherwise check that it matches the allowed list extn.blank? || allow_extensions.include?(extn) @@ -24,6 +38,8 @@ def known_extension?(extn) # - file is passed the file uploaded to Doubtfire (a hash with all relevant data about the file) # def accept_file(file, name, kind) + word_document = kind == 'document' && word_document?(file[:filename] || file['tempfile'].path) + case kind when 'image' mime_allow_list = ['image/png', 'image/gif', 'image/bmp', 'image/tiff', 'image/jpeg', 'image/x-ms-bmp'] @@ -33,7 +49,21 @@ def accept_file(file, name, kind) 'text/x-yaml', 'application/xml', 'text/x-typescript', 'text/x-vhdl', 'text/x-asm', 'text/x-jack', 'application/x-httpd-php', 'application/tst', 'text/x-cmp', 'text/x-vm', 'application/x-sh', 'application/x-bat', 'application/dat', 'application/x-wine-extension-ini'] when 'document' - mime_allow_list = [ 'application/pdf' ] + mime_allow_list = if word_document + [WORD_DOCUMENT_MIME_TYPE] + else + ['application/pdf'] + end + when 'zip', 'archive' + mime_allow_list = [ + 'application/zip', + 'application/x-zip', + 'application/x-zip-compressed', + 'multipart/x-zip', + 'application/x-tar', + 'application/gzip', + 'application/x-gzip' + ] when 'audio' mime_allow_list = ['audio/', 'video/webm', 'application/ogg', 'application/octet-stream'] when 'comment_attachment' @@ -54,6 +84,24 @@ def accept_file(file, name, kind) } end + if word_document && !word_document_conversion_configured? + msg = 'Word documents are currently not supported. Please export your document to PDF.' + logger.error 'Word document upload rejected because conversion is not configured' + return { + accepted: false, + msg: msg + } + end + + if word_document && encrypted_word_document?(file['tempfile'].path) + msg = 'Word document is encrypted or password protected. Remove the password protection and upload it again.' + logger.debug 'Word document is encrypted or password protected' + return { + accepted: false, + msg: msg + } + end + mime_check = mime_in_list?(file['tempfile'].path, mime_allow_list) unless mime_check msg = 'invalid file MIME type, file is likely corrupted.' @@ -64,8 +112,9 @@ def accept_file(file, name, kind) } end - # Extra checks for PDF documents - if kind == 'document' + # Extra checks for PDF documents. DOCX files are converted and validated as + # PDFs when the submission is processed asynchronously. + if kind == 'document' && !word_document pdf_validation_result = validate_pdf(file['tempfile'].path) if pdf_validation_result[:encrypted] @@ -87,6 +136,18 @@ def accept_file(file, name, kind) end end + if %w[zip archive].include?(kind) + zip_validation_result = validate_zip_upload(file['tempfile'].path, File.basename(file[:filename].to_s)) + + unless zip_validation_result[:valid] + logger.debug "Zip file is invalid: #{zip_validation_result[:msg]}" + return { + accepted: false, + msg: zip_validation_result[:msg] + } + end + end + logger.debug 'Uploaded file is accepted' # All checks are done @@ -96,6 +157,119 @@ def accept_file(file, name, kind) } end + def word_document?(path) + File.extname(path.to_s).casecmp(WORD_DOCUMENT_EXTENSION).zero? + end + + def word_document_conversion_configured? + config = Doubtfire::Application.config + config.gotenberg_image.present? && + (config.gotenberg_workdir_volume_mount.present? || config.gotenberg_fallback_volume_container.present?) + end + + # Password-protected OOXML files are stored in an OLE compound file rather + # than the ZIP container used by normal DOCX files. Confirm the compound-file + # signature and both encryption stream names to avoid treating any malformed + # or incorrectly named DOCX as encrypted. + def encrypted_word_document?(path) + longest_stream_name = WORD_DOCUMENT_ENCRYPTION_STREAM_NAMES.map(&:bytesize).max + matched_stream_names = Array.new(WORD_DOCUMENT_ENCRYPTION_STREAM_NAMES.length, false) + + File.open(path, 'rb') do |file| + return false unless file.read(COMPOUND_FILE_SIGNATURE.bytesize) == COMPOUND_FILE_SIGNATURE + + buffer = ''.b + while (chunk = file.read(16 * 1024)) + buffer << chunk + WORD_DOCUMENT_ENCRYPTION_STREAM_NAMES.each_with_index do |stream_name, index| + matched_stream_names[index] ||= buffer.include?(stream_name) + end + return true if matched_stream_names.all? + + buffer = buffer.byteslice(-(longest_stream_name - 1), longest_stream_name - 1) || ''.b + end + end + + false + rescue Errno::ENOENT, Errno::EACCES, IOError + false + end + + def convert_word_document_to_pdf(source_path, destination_path, work_id: SecureRandom.uuid) + work_id = work_id.to_s + unless work_id.match?(/\A[A-Za-z0-9_-]+\z/) + raise DocumentConversionError, 'Invalid Gotenberg work id.' + end + + work_root = Rails.root.join("tmp/gotenberg") + work_dir = work_root.join(work_id) + input_path = work_dir.join('input.docx') + output_path = work_dir.join('output.pdf') + + FileUtils.mkdir_p(work_dir) + FileUtils.chmod(0o777, work_dir) + FileUtils.cp(source_path, input_path) + + stdout, stderr, status = run_word_document_conversion(work_id) + unless status.success? + details = [stdout, stderr].compact.join("\n").strip.first(1_000) + message = "Word document conversion failed with exit status #{status.exitstatus}." + message = "#{message} #{details}" if details.present? + raise DocumentConversionError, message + end + + unless File.exist?(output_path) && validate_pdf(output_path)[:valid] + raise DocumentConversionError, 'Gotenberg did not produce a valid PDF for the Word document.' + end + + FileUtils.mv(output_path, destination_path) + destination_path + rescue SystemCallError => e + raise DocumentConversionError, "Word document conversion failed: #{e.message}" + ensure + FileUtils.rm_rf(work_dir) if defined?(work_dir) && work_dir + end + + def run_word_document_conversion(work_id) + Open3.capture3(*word_document_conversion_command(work_id)) + end + + def word_document_conversion_command(work_id) + config = Doubtfire::Application.config + raise DocumentConversionError, 'GOTENBERG_IMAGE is not configured.' if config.gotenberg_image.blank? + + timeout_seconds = config.word_document_conversion_timeout_seconds.to_i + timeout_seconds = 120 unless timeout_seconds.positive? + + [ + 'docker', 'run', '--rm', + '--cpus', '1', + '--network', 'none', + *gotenberg_volume_arguments(work_id), + '--name', "gotenberg-word-#{work_id}", + '--env', "WORD_DOCUMENT_CONVERSION_TIMEOUT_SECONDS=#{timeout_seconds}", + '--entrypoint', config.word_document_build_path, + config.gotenberg_image, + work_id + ] + end + + def gotenberg_volume_arguments(work_id) + config = Doubtfire::Application.config + mount = config.gotenberg_workdir_volume_mount + + if mount.present? + host_work_dir = File.join(mount, work_id) + container_work_dir = "/workdir/gotenberg/#{work_id}" + ['--volume', "#{host_work_dir}:#{container_work_dir}"] + elsif config.gotenberg_fallback_volume_container.present? + ['--volumes-from', config.gotenberg_fallback_volume_container] + else + raise DocumentConversionError, + 'Set GOTENBERG_WORKDIR_VOLUME_MOUNT or GOTENBERG_FALLBACK_VOLUME_CONTAINER.' + end + end + # # Sanitize the passed in paths, and ensure each part is valid # Will kill things like ../ etc or spaces in paths @@ -276,6 +450,25 @@ def unit_portfolio_dir(unit, create: true, archived: true) dst end + def unit_analytics_dir(unit, create: true, archived: true) + dst = unit_work_root(unit, archived: archived) + dst << 'analytics/' + + FileUtils.mkdir_p(dst) if create + dst + end + + def unit_task_status_snapshot_path(unit, create: true, archived: true) + analytics_dir = unit_analytics_dir(unit, create: create, archived: archived) + FileUtils.mkdir_p(analytics_dir) if create + File.join(analytics_dir, 'task-status-snapshots.zip') + end + + def snapshot_csv_filename(snapshot_timestamp) + return nil if snapshot_timestamp.blank? + "#{sanitized_filename(snapshot_timestamp.to_s)}.csv" + end + # # Generates a path for storing student portfolios # @@ -293,19 +486,45 @@ def student_portfolio_path(unit, username, create: true, archived: true) File.join(student_portfolio_dir(unit, username, create: create, archived: archived), FileHelper.sanitized_filename("#{username}-portfolio.pdf")) end - def task_jplag_report_dir(unit) - file_server = Doubtfire::Application.config.jplag_report_dir - "#{file_server}/#{unit.code}-#{unit.id}/" # trust the server config and passed in type for paths + def root_jplag_report_dir(archived: false) + file_server = if archived + archive_root + else + student_work_root + end + + "#{file_server}/jplag/results/" + end + + def unit_jplag_report_dir(unit, create: false, archived: true) + dst = if (unit.archived && archived) || (archived == :force) + File.join(root_jplag_report_dir(archived: true), sanitized_path("#{unit.code}-#{unit.id}")) + else + File.join(root_jplag_report_dir(archived: false), sanitized_path("#{unit.code}-#{unit.id}")) + end + + FileUtils.mkdir_p dst if create + "#{dst}/" + end + + def task_jplag_report_dir(unit, create: false, archived: true) + unit_jplag_report_dir(unit, create: create, archived: archived) end def task_jplag_report_path(unit, task) - File.join(task_jplag_report_dir(unit), FileHelper.sanitized_filename("#{task.abbreviation}-result.jplag")) + File.join(unit_jplag_report_dir(unit), FileHelper.sanitized_filename("#{task.abbreviation}-result.jplag")) end def comment_attachment_path(task_comment, attachment_extension) "#{File.join(student_work_dir(:comment, task_comment.task), "#{task_comment.id.to_s}#{attachment_extension}")}" end + def engagement_attachment_path(engagement, attachment_extension) + dir = File.join(project_work_root(engagement.project), 'engagement') + FileUtils.mkdir_p(dir) + File.join(dir, "#{engagement.id}#{attachment_extension}") + end + def comment_prompt_path(task_comment, attachment_extension, count) "#{File.join(student_work_dir(:discussion, task_comment.task), "#{task_comment.id.to_s}_#{count.to_s}#{attachment_extension}")}" end @@ -317,7 +536,7 @@ def comment_reply_prompt_path(discussion_comment, attachment_extension) def compress_image_to_dest(source, dest, delete_frames = false) exec = "convert -quiet \ \"#{source}\" \ - #{delete_frames ? '-delete 1--1' : ''} -strip -density 72 -quality 85% -resize 2048x2048\\> -resize 48x48\\< \ + #{delete_frames ? '-delete 1--1' : ''} -auto-orient -strip -density 72 -quality 85% -resize 2048x2048\\> -resize 48x48\\< \ \"#{dest}\" >>/dev/null 2>>/dev/null" system_try_within 40, 'compressing image using convert', exec @@ -383,6 +602,193 @@ def compress_pdf(path, max_size: 2_500_000, timeout_seconds: 30) FileUtils.rm_f tmp_file end + def zip_path_safe?(path) + return false if path.blank? + + clean_path = path.tr('\\', '/') + return false if clean_path.start_with?('/') || clean_path.include?("\0") + + clean_path.sub!(%r{\A\./+}, '') + clean_path.split('/').none? { |part| part.blank? || part == '.' || part == '..' } + end + + def zip_entry_limit + limit = Doubtfire::Application.config.zip_entry_limit.to_i + limit.positive? ? limit : 1_000 + end + + def zip_compression_ratio_limit + limit = Doubtfire::Application.config.zip_compression_ratio_limit.to_i + limit.positive? ? limit : 100 + end + + def zip_uncompressed_size_multiplier + multiplier = Doubtfire::Application.config.zip_uncompressed_size_multiplier.to_i + multiplier.positive? ? multiplier : 10 + end + + def zip_nested_archive?(path) + clean_path = path.to_s.downcase + ZIP_NESTED_ARCHIVE_EXTENSIONS.any? { |extension| clean_path.end_with?(extension) } + end + + def validate_zip_upload_entry!(name, size, zip_stats, _max_file_size, max_uncompressed_size) + raise 'Zip contains a file with an unsafe path.' unless zip_path_safe?(name) + raise 'Zip contains another archive file. Nested archives are not allowed.' if zip_nested_archive?(name) + + zip_stats[:entries] += 1 + zip_stats[:total_uncompressed_size] += size.to_i + + raise "Zip contains too many files. Limit is #{zip_entry_limit} files." if zip_stats[:entries] > zip_entry_limit + # raise "Zip contains a file larger than the #{max_file_size / 1_000_000}MB file limit." if size.to_i > max_file_size + if zip_stats[:total_uncompressed_size] > max_uncompressed_size + raise "Zip expands beyond the #{max_uncompressed_size / 1_000_000}MB uncompressed size limit." + end + end + + def validate_zip_file(path, max_file_size, max_uncompressed_size) + stats = { entries: 0, total_uncompressed_size: 0 } + + Zip::File.open(path) do |zip_file| + zip_file.each do |entry| + raise 'Encrypted zip entries are not supported.' if entry.respond_to?(:encrypted?) && entry.encrypted? + raise 'Zip contains an unsupported link entry.' if entry.respond_to?(:ftype) && entry.ftype == :symlink + next if entry.directory? + + validate_zip_upload_entry!(entry.name, entry.size, stats, max_file_size, max_uncompressed_size) + end + end + + stats + end + + def validate_tar_file(io, max_file_size, max_uncompressed_size) + stats = { entries: 0, total_uncompressed_size: 0 } + + Gem::Package::TarReader.new(io) do |tar| + tar.each do |entry| + next if entry.directory? + raise 'Zip contains an unsupported non-file entry.' unless entry.file? + + validate_zip_upload_entry!(entry.full_name, entry.header.size, stats, max_file_size, max_uncompressed_size) + end + end + + stats + end + + def validate_zip_upload(path, filename) + max_file_size = Doubtfire::Application.config.max_file_size.to_i + max_file_size = 10_000_000 if max_file_size <= 0 + max_uncompressed_size = max_file_size * zip_uncompressed_size_multiplier + return { valid: false, msg: "Zip exceeds the #{max_file_size / 1_000_000}MB file limit." } if File.size(path) > max_file_size + + begin + stats = + if filename.downcase.end_with?('.zip') + validate_zip_file(path, max_file_size, max_uncompressed_size) + elsif filename.downcase.end_with?('.tar') + File.open(path, 'rb') { |file| validate_tar_file(file, max_file_size, max_uncompressed_size) } + elsif filename.downcase.end_with?('.tar.gz', '.tgz') + Zlib::GzipReader.open(path) { |gzip| validate_tar_file(gzip, max_file_size, max_uncompressed_size) } + else + return { valid: false, msg: 'Unsupported zip format. Use .zip, .tar, .tar.gz, or .tgz.' } + end + + return { valid: false, msg: 'Zip must contain at least one file.' } if stats[:entries].zero? + + compressed_size = [File.size(path), 1].max + if stats[:total_uncompressed_size] / compressed_size > zip_compression_ratio_limit + return { valid: false, msg: "Zip compression ratio is too high. Limit is #{zip_compression_ratio_limit}:1." } + end + + { valid: true, msg: 'success' } + rescue Zip::Error, Zlib::Error, Gem::Package::TarInvalidError, EOFError + { valid: false, msg: 'Zip file is corrupted or not a supported zip.' } + rescue StandardError => e + { valid: false, msg: e.message } + end + end + + def zip_tree_add_path(tree, path) + clean_path = path.to_s.tr('\\', '/').sub(%r{\A\./+}, '').sub(%r{/+\z}, '') + return if clean_path.blank? + + parts = clean_path.split('/').reject(&:blank?) + node = tree + + parts.each_with_index do |part, index| + key = index == parts.length - 1 ? part : "#{part}/" + node[key] ||= {} + node = node[key] + end + end + + def zip_tree_walk(node, prefix = '', lines = []) + sorted_entries = node.keys.sort_by { |key| [key.end_with?('/') ? 0 : 1, key.downcase] } + + sorted_entries.each_with_index do |name, index| + last = index == sorted_entries.length - 1 + connector = '↳ ' + lines << "#{prefix}#{connector}#{name}" + zip_tree_walk(node[name], "#{prefix} ", lines) if node[name].any? + end + + lines + end + + def zip_file_tree(path, filename, display_limit: 200) + tree = {} + entries = 0 + + read_entry = lambda do |entry_name| + return unless zip_path_safe?(entry_name) + + entries += 1 + zip_tree_add_path(tree, entry_name) + end + + if filename.downcase.end_with?('.zip') + Zip::File.open(path) do |zip_file| + zip_file.each do |entry| + next if entry.directory? + + read_entry.call(entry.name) + end + end + elsif filename.downcase.end_with?('.tar') + File.open(path, 'rb') do |file| + Gem::Package::TarReader.new(file) do |tar| + tar.each do |entry| + next if entry.directory? + + read_entry.call(entry.full_name) + end + end + end + elsif filename.downcase.end_with?('.tar.gz', '.tgz', '.gz') + Zlib::GzipReader.open(path) do |gzip| + Gem::Package::TarReader.new(gzip) do |tar| + tar.each do |entry| + next if entry.directory? + + read_entry.call(entry.full_name) + end + end + end + end + + all_lines = zip_tree_walk(tree) + lines = all_lines.first(display_limit) + { lines: lines, entries: entries, tree_lines: all_lines.length, truncated: all_lines.length > display_limit } + rescue Zip::Error, Zlib::Error, Gem::Package::TarInvalidError, EOFError => e + logger.debug "Could not read zip file tree for #{filename}: #{e.message}" + { lines: [], entries: 0, truncated: false, error: true } + rescue StandardError => e + logger.debug "Could not read zip file tree for #{filename}: #{e.message}" + { lines: [], entries: 0, truncated: false, error: true } + end + def pages_in_pdf(path) exec = "qpdf --show-npages #{path}" @@ -765,6 +1171,13 @@ def line_wrap(path, width: 160) end # Export functions as module functions module_function :accept_file + module_function :word_document? + module_function :word_document_conversion_configured? + module_function :encrypted_word_document? + module_function :convert_word_document_to_pdf + module_function :run_word_document_conversion + module_function :word_document_conversion_command + module_function :gotenberg_volume_arguments module_function :sanitized_path module_function :sanitized_filename module_function :task_file_dir_for_unit @@ -778,11 +1191,15 @@ def line_wrap(path, width: 160) module_function :unit_dir module_function :root_portfolio_dir module_function :unit_portfolio_dir + module_function :unit_analytics_dir + module_function :unit_task_status_snapshot_path + module_function :snapshot_csv_filename module_function :unit_work_root module_function :project_work_root module_function :student_portfolio_dir module_function :student_portfolio_path module_function :comment_attachment_path + module_function :engagement_attachment_path module_function :comment_prompt_path module_function :comment_reply_prompt_path module_function :compress_image_to_dest @@ -790,6 +1207,18 @@ def line_wrap(path, width: 160) module_function :qpdf module_function :move_files module_function :validate_pdf + module_function :zip_path_safe? + module_function :zip_entry_limit + module_function :zip_compression_ratio_limit + module_function :zip_uncompressed_size_multiplier + module_function :zip_nested_archive? + module_function :validate_zip_upload_entry! + module_function :validate_zip_file + module_function :validate_tar_file + module_function :validate_zip_upload + module_function :zip_tree_add_path + module_function :zip_tree_walk + module_function :zip_file_tree module_function :copy_pdf module_function :read_file_to_str module_function :path_to_plagarism_html @@ -815,6 +1244,8 @@ def line_wrap(path, width: 160) module_function :known_extension? module_function :pages_in_pdf module_function :line_wrap + module_function :root_jplag_report_dir + module_function :unit_jplag_report_dir module_function :task_jplag_report_dir module_function :task_jplag_report_path end diff --git a/app/helpers/file_stream_helper.rb b/app/helpers/file_stream_helper.rb index 1644b3fe4b..8317b9d393 100644 --- a/app/helpers/file_stream_helper.rb +++ b/app/helpers/file_stream_helper.rb @@ -1,4 +1,9 @@ module FileStreamHelper + def expose_headers(*headers_to_expose) + exposed_headers = header['Access-Control-Expose-Headers'].to_s.split(',').map(&:strip) + header['Access-Control-Expose-Headers'] = (exposed_headers + headers_to_expose).compact_blank.uniq.join(',') + end + # Extract part of the contents so that is can be streamed to the client # file_path is the path to the file to be streamed # this will set the headers and return the content @@ -33,7 +38,7 @@ def stream_file(file_path) begin_point = 0 end_point = 10_485_760 else - header['Access-Control-Expose-Headers'] = 'Content-Disposition' if header.key?('Content-Disposition') + expose_headers('Content-Disposition') if header.key?('Content-Disposition') sendfile file_path return @@ -41,7 +46,7 @@ def stream_file(file_path) # Return the requested content content_length = [end_point - begin_point + 1, 0].max # Ensure we don't attempt to read a negative length - header['Access-Control-Expose-Headers'] = header.key?('Content-Disposition') ? 'Content-Disposition,Content-Range,Accept-Ranges' : 'Content-Range,Accept-Ranges' + expose_headers(('Content-Disposition' if header.key?('Content-Disposition')), 'Content-Range', 'Accept-Ranges') header['Content-Range'] = "bytes #{begin_point}-#{end_point}/#{file_size}" header['Content-Length'] = content_length.to_s header['Accept-Ranges'] = 'bytes' @@ -52,5 +57,6 @@ def stream_file(file_path) body = File.binread(file_path, content_length, begin_point) end + module_function :expose_headers module_function :stream_file end diff --git a/app/helpers/turn_it_in.rb b/app/helpers/turn_it_in.rb index 7bf73e4f7a..c2c0e765ef 100644 --- a/app/helpers/turn_it_in.rb +++ b/app/helpers/turn_it_in.rb @@ -31,7 +31,7 @@ def self.load_config(config) # Setup authorization TCAClient.configure do |tii_config| # Configure API key authorization: api_key - tii_config.api_key['api_key'] = ENV.fetch('TCA_API_KEY', nil) + tii_config.api_key['api_key'] = Doubtfire::Application.fetch_credential_or_env(:tii, :api_key, env_key: 'TCA_API_KEY') # Uncomment the following line to set a prefix for the API key, e.g. 'Bearer' (defaults to nil) tii_config.api_key_prefix['api_key'] = 'Bearer' tii_config.host = ENV.fetch('TCA_HOST', nil) @@ -84,6 +84,7 @@ def self.handle_tii_error(action, error) @@delay_call_until = DateTime.now + 1.minute when 403 # forbidden, issue with authentication... notify admin begin + Sentry.capture_exception(error) if defined?(Sentry) ErrorLogMailer.error_message('TII Credentials', "TII Error: #{error.message}", error).deliver rescue StandardError => e Rails.logger.error "Failed to send error email: #{e}" diff --git a/app/mailers/communications_mailer.rb b/app/mailers/communications_mailer.rb new file mode 100644 index 0000000000..95034342d9 --- /dev/null +++ b/app/mailers/communications_mailer.rb @@ -0,0 +1,37 @@ +class CommunicationsMailer < ApplicationMailer + def communication_email(to:, from:, subject:, body:, recipient:, sender:, unit:, rule:) + @recipient = recipient + @sender = sender + @unit = unit + @rule = rule + @body = body.to_s + @body_paragraphs = @body.split(/\r?\n/).map(&:strip).reject(&:blank?) + + @doubtfire_host = Doubtfire::Application.config.institution[:host] + @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] + @unsubscribe_url = "#{@doubtfire_host}/edit_profile" + + mail(to: to, from: from, subject: subject) + end + + def action_log_email(payload) + @recipient = payload[:recipient] + @sender = payload[:sender] + @unit = payload[:unit] + @rule = payload[:rule] + @body = payload[:body].to_s + @body_paragraphs = @body.split(/\r?\n/).map(&:strip).reject(&:blank?) + @affected_students_count = payload[:affected_students_count] + + @doubtfire_host = Doubtfire::Application.config.institution[:host] + @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] + @unsubscribe_url = "#{@doubtfire_host}/edit_profile" + + attachments[payload[:csv_filename]] = { + mime_type: 'text/csv', + content: payload[:csv_content] + } + + mail(to: payload[:to], from: payload[:from], subject: payload[:subject]) + end +end diff --git a/app/mailers/error_log_mailer.rb b/app/mailers/error_log_mailer.rb index e08607fd31..578a6f6fc3 100644 --- a/app/mailers/error_log_mailer.rb +++ b/app/mailers/error_log_mailer.rb @@ -8,7 +8,8 @@ def error_message(subject, message, exception) end @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] - @error_log = "#{message}\n\n#{exception.message}\n\n#{exception.backtrace.join("\n")}" + backtrace = exception.backtrace&.join("\n") || 'No backtrace available' + @error_log = "#{message}\n\n#{exception.message}\n\n#{backtrace}" mail(to: email, from: email, subject: "#{@doubtfire_product_name} Error Log - #{subject}") end diff --git a/app/mailers/notifications_mailer.rb b/app/mailers/notifications_mailer.rb index 748d5bd029..f4b0d255ad 100644 --- a/app/mailers/notifications_mailer.rb +++ b/app/mailers/notifications_mailer.rb @@ -1,4 +1,7 @@ class NotificationsMailer < ApplicationMailer + layout 'discussion_deadline_mailer', + only: %i[discussion_deadline_approaching discussion_deadline_missed] + def add_general @doubtfire_host = Doubtfire::Application.config.institution[:host] @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] @@ -61,7 +64,7 @@ def weekly_student_summary(project, summary_stats, did_revert_to_pass) @student_engagements = @engagements.select { |e| [TaskStatus.not_started.name, TaskStatus.need_help.name, TaskStatus.working_on_it.name, TaskStatus.ready_for_feedback.name].include? e.engagement }.count - @staff_engagements = @engagements.select { |e| [TaskStatus.complete.name, TaskStatus.feedback_exceeded.name, TaskStatus.redo.name, TaskStatus.discuss.name, TaskStatus.attention_required.name, TaskStatus.demonstrate.name, TaskStatus.fail.name].include? e.engagement }.count + @staff_engagements = @engagements.select { |e| [TaskStatus.complete.name, TaskStatus.feedback_exceeded.name, TaskStatus.redo.name, TaskStatus.discuss.name, TaskStatus.rediscuss.name, TaskStatus.attention_required.name, TaskStatus.demonstrate.name, TaskStatus.fail.name].include? e.engagement }.count @task_states = project.tasks.joins(:task_status).select("count(tasks.id) as number, task_statuses.name as status").group("task_statuses.name") @@ -80,6 +83,27 @@ def weekly_student_summary(project, summary_stats, did_revert_to_pass) mail(to: email_with_name, from: tutor_email, subject: subject) end + def discussion_deadline_approaching(task, sender, expiry_date) + add_discussion_deadline_details(task, sender) + @deadline = task.unit.formatted_discuss_timeout_date(expiry_date) + + mail( + to: %("#{@student.name}" <#{@student.email}>), + from: %("#{@sender.name}" <#{@sender.email}>), + subject: "#{@unit.code}: Discussion deadline approaching for #{@task.task_definition.abbreviation}" + ) + end + + def discussion_deadline_missed(task, sender) + add_discussion_deadline_details(task, sender) + + mail( + to: %("#{@student.name}" <#{@student.email}>), + from: %("#{@sender.name}" <#{@sender.email}>), + subject: "#{@unit.code}: Discussion deadline missed for #{@task.task_definition.abbreviation}" + ) + end + def top_task_desc(tt) "#{tt[:task_definition].abbreviation} - #{tt[:task_definition].name} #{"- which you need to discuss with your tutor" if tt[:status] == :discuss}" end @@ -112,4 +136,16 @@ def this_these(num) helper_method :were_was helper_method :are_is helper_method :this_these + + private + + def add_discussion_deadline_details(task, sender) + add_general + @task = task + @project = task.project + @unit = task.unit + @student = @project.student + @sender = sender + @task_url = "#{@doubtfire_host}/projects/#{@project.id}/dashboard/#{@task.task_definition.abbreviation}" + end end diff --git a/app/mailers/portfolio_evidence_mailer.rb b/app/mailers/portfolio_evidence_mailer.rb index 7b2f56d817..743503c25b 100644 --- a/app/mailers/portfolio_evidence_mailer.rb +++ b/app/mailers/portfolio_evidence_mailer.rb @@ -17,7 +17,7 @@ def task_pdf_failed(project, tasks) email_with_name = %("#{@student.name}" <#{@student.email}>) tutor_email = %("#{@tutor.name}" <#{@tutor.email}>) - subject = "#{project.unit.name}: Task PDFs ready to view" + subject = "#{project.unit.code} #{project.unit.name}: Task submission processing failed" mail(to: email_with_name, from: tutor_email, subject: subject) end @@ -54,6 +54,22 @@ def task_feedback_ready(project, tasks) mail(to: email_with_name, from: tutor_email, subject: subject) end + def overseer_assessment_failed(project, tasks) + return nil if project.nil? || tasks.nil? || tasks.empty? + + add_general + @student = project.student + @project = project + @tasks = tasks.sort_by { |t| t.task_definition.abbreviation } + @tutor = project.main_convenor_user + return nil if @tutor.nil? || @student.nil? + + email_with_name = %("#{@student.name}" <#{@student.email}>) + tutor_email = %("#{@tutor.name}" <#{@tutor.email}>) + subject = "#{project.unit.code} #{project.unit.name}: Automated feedback needs your attention" + mail(to: email_with_name, from: tutor_email, subject: subject) + end + def portfolio_ready(project) return nil if project.nil? diff --git a/app/middleware/sentry_tunnel_middleware.rb b/app/middleware/sentry_tunnel_middleware.rb new file mode 100644 index 0000000000..8556130117 --- /dev/null +++ b/app/middleware/sentry_tunnel_middleware.rb @@ -0,0 +1,65 @@ +require 'rest-client' +require 'uri' + +class SentryTunnelMiddleware + PATH = '/api/client-reports'.freeze + + def initialize(app) + @app = app + end + + def call(env) + return @app.call(env) unless env['REQUEST_METHOD'] == 'POST' && env['PATH_INFO'] == PATH + + forward_envelope(env) + [204, {}, []] + end + + private + + def forward_envelope(env) + envelope_url = sentry_envelope_url + return if envelope_url.blank? + + body = env['rack.input'].read + return if body.blank? + + RestClient::Request.execute( + method: :post, + url: envelope_url, + payload: body, + headers: sentry_headers(env), + timeout: 5, + open_timeout: 2 + ) + rescue RestClient::ExceptionWithResponse => e + Rails.logger.warn "Unable to forward Sentry envelope: #{e.class} #{e.response&.code}" + rescue RestClient::Exception, SocketError, Timeout::Error => e + Rails.logger.warn "Unable to forward Sentry envelope: #{e.class}" + ensure + env['rack.input'].rewind if env['rack.input'].respond_to?(:rewind) + end + + def sentry_headers(env) + headers = { + content_type: env['CONTENT_TYPE'].presence || 'application/x-sentry-envelope' + } + + headers[:content_encoding] = env['HTTP_CONTENT_ENCODING'] if env['HTTP_CONTENT_ENCODING'].present? + headers + end + + def sentry_envelope_url + dsn = ENV.fetch('SENTRY_DSN', nil) + return nil if dsn.blank? + + uri = URI.parse(dsn) + project_id = uri.path.delete_prefix('/') + public_key = uri.user + return nil if project_id.blank? || public_key.blank? + + "#{uri.scheme}://#{uri.host}/api/#{project_id}/envelope/?sentry_key=#{URI.encode_www_form_component(public_key)}" + rescue URI::InvalidURIError + nil + end +end diff --git a/app/models/auth_token.rb b/app/models/auth_token.rb index e494567de3..35e4079bac 100644 --- a/app/models/auth_token.rb +++ b/app/models/auth_token.rb @@ -10,7 +10,8 @@ class AuthToken < ApplicationRecord general: 0, login: 1, scorm: 2, - refresh_token: 3 + refresh_token: 3, + content: 4 } def self.generate(user, remember, expiry_time = Time.zone.now + 2.hours, token_type = :general) diff --git a/app/models/break.rb b/app/models/break.rb index f857fd12ba..ed1c76b1ad 100644 --- a/app/models/break.rb +++ b/app/models/break.rb @@ -1,6 +1,9 @@ class Break < ApplicationRecord belongs_to :teaching_period, optional: false + after_save :refresh_teaching_period_communication_schedule_caches + after_destroy :refresh_teaching_period_communication_schedule_caches + validates :start_date, presence: true validates :number_of_weeks, presence: true validates :teaching_period_id, presence: true @@ -46,4 +49,10 @@ def monday_after_break def end_date start_date + duration end + + private + + def refresh_teaching_period_communication_schedule_caches + teaching_period.refresh_communication_schedule_caches + end end diff --git a/app/models/comments/discuss_timeout_comment.rb b/app/models/comments/discuss_timeout_comment.rb new file mode 100644 index 0000000000..a45b8aadd6 --- /dev/null +++ b/app/models/comments/discuss_timeout_comment.rb @@ -0,0 +1,12 @@ +class DiscussTimeoutComment < TaskComment + WARNING_CONTENT_TYPE = 'discuss_timeout_warning'.freeze + EXPIRED_CONTENT_TYPE = 'discuss_timeout_expired'.freeze + + def self.warning + WARNING_CONTENT_TYPE + end + + def self.expired + EXPIRED_CONTENT_TYPE + end +end diff --git a/app/models/communication/campus_condition.rb b/app/models/communication/campus_condition.rb new file mode 100644 index 0000000000..07e2bc0096 --- /dev/null +++ b/app/models/communication/campus_condition.rb @@ -0,0 +1,4 @@ +class CampusCondition < CommunicationCondition + validates :campus, presence: true + validates :operator, inclusion: { in: ENROLMENT_OPERATORS } +end diff --git a/app/models/communication/change_target_grade_action.rb b/app/models/communication/change_target_grade_action.rb new file mode 100644 index 0000000000..efcfc2cb8e --- /dev/null +++ b/app/models/communication/change_target_grade_action.rb @@ -0,0 +1,12 @@ +class ChangeTargetGradeAction < CommunicationAction + validates :target_grade, presence: true + validate :target_grade_enabled_for_unit + + private + + def target_grade_enabled_for_unit + return if target_grade.nil? || communication_rule&.unit&.grade_value?(target_grade) + + errors.add(:target_grade, 'is not enabled for this unit') + end +end diff --git a/app/models/communication/communication_action.rb b/app/models/communication/communication_action.rb new file mode 100644 index 0000000000..af287b6fa2 --- /dev/null +++ b/app/models/communication/communication_action.rb @@ -0,0 +1,13 @@ +class CommunicationAction < ApplicationRecord + VALID_TYPES = %w[ + EmailStudentAction + EmailStaffAction + ChangeTargetGradeAction + TaskCommentAction + ].freeze + + belongs_to :communication_rule, class_name: 'CommunicationRule' + belongs_to :task_definition, optional: true + + validates :type, presence: true, inclusion: { in: VALID_TYPES } +end diff --git a/app/models/communication/communication_condition.rb b/app/models/communication/communication_condition.rb new file mode 100644 index 0000000000..fb9e8a73ee --- /dev/null +++ b/app/models/communication/communication_condition.rb @@ -0,0 +1,96 @@ +class CommunicationCondition < ApplicationRecord + VALID_TYPES = %w[ + TargetGradeCondition + TaskDefinitionStatusCondition + TaskStatusCountCondition + LoginStatusCondition + UnitViewedStatusCondition + SpecConCondition + TutorialEnrolmentCondition + TutorialStreamEnrolmentCondition + CampusCondition + PortfolioSubmittedCondition + ].freeze + + GRADE_OPERATORS = %w[ + greater_than + greater_than_or_equal_to + less_than + less_than_or_equal_to + equal_to + not_equal_to + ].freeze + + EQUALITY_OPERATORS = %w[equal_to not_equal_to].freeze + ACTIVITY_OPERATORS = %w[more_than within_last].freeze + ENROLMENT_OPERATORS = %w[enrolled_in not_enrolled_in].freeze + TASK_STATUS_KEYS = %w[ + not_started + complete + need_help + working_on_it + fix_and_resubmit + feedback_exceeded + redo + discuss + ready_for_feedback + demonstrate + fail + time_exceeded + assess_in_portfolio + attention_required + rediscuss + ].freeze + + belongs_to :communication, + class_name: 'CommunicationRule', + inverse_of: :communication_conditions + + belongs_to :task_definition, optional: true + belongs_to :tutorial, optional: true + belongs_to :tutorial_stream, optional: true + belongs_to :campus, optional: true + + attribute :task_statuses, :json, default: -> { [] } + + validates :type, presence: true, inclusion: { in: VALID_TYPES } + validates :operator, presence: true + before_validation :normalize_task_statuses + + def task_statuses_must_be_present + unless task_statuses.is_a?(Array) && task_statuses.any?(&:present?) + errors.add(:task_statuses, 'must include at least one task status') + return + end + + invalid_statuses = task_statuses.reject { |status| TASK_STATUS_KEYS.include?(status) } + return if invalid_statuses.empty? + + errors.add(:task_statuses, "contains invalid task statuses: #{invalid_statuses.join(', ')}") + end + + private + + def normalize_task_statuses + parsed_statuses = + case task_statuses + when nil + nil + when String + begin + JSON.parse(task_statuses) + rescue JSON::ParserError + [task_statuses] + end + when Array + task_statuses + else + Array(task_statuses) + end + + self.task_statuses = + parsed_statuses&.filter_map do |status| + status.is_a?(String) ? status.strip.presence : status.presence + end + end +end diff --git a/app/models/communication/communication_rule.rb b/app/models/communication/communication_rule.rb new file mode 100644 index 0000000000..a528b413fa --- /dev/null +++ b/app/models/communication/communication_rule.rb @@ -0,0 +1,144 @@ +class CommunicationRule < ApplicationRecord + LOGICAL_OPERATORS = %w[and or].freeze + + belongs_to :communication_set, class_name: 'CommunicationSet' + delegate :unit, to: :communication_set + + has_many :communication_conditions, + class_name: 'CommunicationCondition', + foreign_key: :communication_id, + inverse_of: :communication, + dependent: :destroy + has_many :communication_actions, class_name: 'CommunicationAction', dependent: :destroy + + validates :name, presence: true + validates :operator, presence: true, inclusion: { in: LOGICAL_OPERATORS } + validates :position, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + def matching_projects(projects = nil) + projects ||= communication_set.eligible_projects + return projects if communication_conditions.empty? + + evaluated_at = Time.current + projects.select do |project| + matches = communication_conditions.map { |condition| condition_match?(project, condition, evaluated_at) } + + operator == 'or' ? matches.any? : matches.all? + end + end + + private + + def condition_match?(project, condition, evaluated_at) + case condition.type + when 'TargetGradeCondition' + target_grade_condition_match?(project, condition) + when 'TaskDefinitionStatusCondition' + task_definition_status_condition_match?(project, condition) + when 'TaskStatusCountCondition' + task_status_count_condition_match?(project, condition) + when 'LoginStatusCondition' + activity_condition_match?(project.user&.last_sign_in_at, condition, evaluated_at) + when 'UnitViewedStatusCondition' + activity_condition_match?(project.last_viewed_at, condition, evaluated_at) + when 'SpecConCondition' + spec_con_condition_match?(project, condition) + when 'TutorialEnrolmentCondition' + tutorial_enrolment_condition_match?(project, condition) + when 'TutorialStreamEnrolmentCondition' + tutorial_stream_enrolment_condition_match?(project, condition) + when 'CampusCondition' + campus_condition_match?(project, condition) + when 'PortfolioSubmittedCondition' + portfolio_submitted_condition_match?(project, condition) + else + false + end + end + + def target_grade_condition_match?(project, condition) + project_target_grade = project.target_grade + return false if project_target_grade.nil? + + compare_value(project_target_grade, condition.target_grade, condition.operator) + end + + def task_definition_status_condition_match?(project, condition) + task = project.tasks.find { |t| t.task_definition_id == condition.task_definition_id } + status = task&.task_status&.status_key&.to_s || 'not_started' + statuses = condition.task_statuses || [] + + case condition.operator + when 'equal_to' then statuses.include?(status) + when 'not_equal_to' then !statuses.include?(status) + else false + end + end + + def task_status_count_condition_match?(project, condition) + statuses = condition.task_statuses || [] + relevant_task_definitions = project.unit.task_definitions.select do |task_definition| + task_definition.target_grade == condition.task_target_grade + end + + count = relevant_task_definitions.count do |task_definition| + task = project.tasks.find { |project_task| project_task.task_definition_id == task_definition.id } + task_status = task&.task_status&.status_key&.to_s || 'not_started' + + statuses.include?(task_status) + end + + compare_value(count, condition.task_status_count, condition.operator) + end + + def activity_condition_match?(last_activity_at, condition, evaluated_at) + return false if condition.activity_days.blank? + + threshold = evaluated_at - condition.activity_days.days + case condition.operator + when 'more_than' then last_activity_at.nil? || last_activity_at < threshold + when 'within_last' then last_activity_at.present? && last_activity_at >= threshold + else false + end + end + + def spec_con_condition_match?(project, condition) + compare_value(project.spec_con_days, condition.spec_con_days, condition.operator) + end + + def tutorial_enrolment_condition_match?(project, condition) + enrolled = project.tutorial_enrolments.any? { |enrolment| enrolment.tutorial_id == condition.tutorial_id } + + condition.operator == 'not_enrolled_in' ? !enrolled : enrolled + end + + def tutorial_stream_enrolment_condition_match?(project, condition) + enrolled = project.tutorial_enrolments.any? do |enrolment| + enrolment.tutorial&.tutorial_stream_id == condition.tutorial_stream_id + end + + condition.operator == 'not_enrolled_in' ? !enrolled : enrolled + end + + def campus_condition_match?(project, condition) + enrolled = project.campus_id == condition.campus_id + + condition.operator == 'not_enrolled_in' ? !enrolled : enrolled + end + + def portfolio_submitted_condition_match?(project, condition) + project.portfolio_exists? == condition.submitted_portfolio + end + + def compare_value(left, right, operator) + case operator + when 'greater_than' then left > right + when 'greater_than_or_equal_to' then left >= right + when 'less_than' then left < right + when 'less_than_or_equal_to' then left <= right + when 'equal_to' then left == right + when 'not_equal_to' then left != right + else false + end + end +end diff --git a/app/models/communication/communication_set.rb b/app/models/communication/communication_set.rb new file mode 100644 index 0000000000..9d9a818996 --- /dev/null +++ b/app/models/communication/communication_set.rb @@ -0,0 +1,113 @@ +class CommunicationSet < ApplicationRecord + belongs_to :unit + + has_many :communication_set_schedules, + class_name: 'CommunicationSetSchedule', + inverse_of: :communication_set, + dependent: :destroy + + has_many :communication_rules, + -> { order(:position) }, + class_name: 'CommunicationRule', + inverse_of: :communication_set, + dependent: :destroy + + validates :name, presence: true + + def eligible_projects + unit.projects + .where(enrolled: true) + .includes(:user, :campus, { tasks: [:task_status, :task_definition] }, { tutorial_enrolments: :tutorial }) + .to_a + end + + def preview_projects_for_rule(target_rule) + preview_allocations_for_rule(target_rule) + .find { |allocation| allocation[:rule].id == target_rule.id } + &.fetch(:projects, []) || [] + end + + def preview_allocations_by_rule + communication_rules.each_with_object({}) do |rule, allocations_by_rule| + allocations_by_rule[rule.id] = preview_allocations_for_rule(rule) + end + end + + def preview_allocations_for_rule(target_rule) + remaining_projects = eligible_projects + allocations = [] + + communication_rules.each do |rule| + matched_projects = rule.matching_projects(remaining_projects) + allocations << { rule: rule, projects: matched_projects } + return allocations if rule.id == target_rule.id + + remaining_projects -= matched_projects + end + + allocations + end + + def copy_to(other_unit) + new_set = dup + new_set.unit = other_unit + new_set.save! + + communication_set_schedules.each do |schedule| + new_schedule = schedule.dup + new_schedule.communication_set = new_set + new_schedule.ice_cube_schedule = nil + new_schedule.next_run_at = nil + new_schedule.last_run_at = nil + new_schedule.last_enqueued_at = nil + new_schedule.save! + end + + communication_rules.each do |rule| + new_rule = rule.dup + new_rule.communication_set = new_set + new_rule.save! + + rule.communication_conditions.each do |condition| + new_condition = condition.dup + new_condition.communication = new_rule + new_condition.task_definition = matching_task_definition(other_unit, condition) + new_condition.tutorial_stream = matching_tutorial_stream(other_unit, condition) + new_condition.tutorial = matching_tutorial(other_unit, condition) + new_condition.save! + end + + rule.communication_actions.each do |action| + new_action = action.dup + new_action.communication_rule = new_rule + new_action.task_definition = matching_task_definition(other_unit, action) + new_action.save! + end + end + + new_set + end + + private + + def matching_task_definition(unit, condition) + return nil if condition.task_definition.blank? + + unit.task_definitions.find_by(abbreviation: condition.task_definition.abbreviation) + end + + def matching_tutorial_stream(unit, condition) + return nil if condition.tutorial_stream.blank? + + unit.tutorial_streams.find_by(abbreviation: condition.tutorial_stream.abbreviation) + end + + def matching_tutorial(unit, condition) + return nil if condition.tutorial.blank? + + unit.tutorials.find_by( + abbreviation: condition.tutorial.abbreviation, + campus_id: condition.tutorial.campus_id + ) + end +end diff --git a/app/models/communication/communication_set_schedule.rb b/app/models/communication/communication_set_schedule.rb new file mode 100644 index 0000000000..26f0a41628 --- /dev/null +++ b/app/models/communication/communication_set_schedule.rb @@ -0,0 +1,175 @@ +class CommunicationSetSchedule < ApplicationRecord + RECURRENCES = %w[none daily weekly monthly].freeze + VALID_DAYS = Date::DAYNAMES.freeze + DAY_ABBREVIATIONS = Date::ABBR_DAYNAMES.freeze + + belongs_to :communication_set, class_name: 'CommunicationSet', inverse_of: :communication_set_schedules + delegate :unit, to: :communication_set + + validates :name, presence: true + validates :anchor_week, presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :anchor_day, presence: true + validates :hour, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than: 24 } + validates :minute, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than: 60 } + validates :interval, presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :recurrence, presence: true, inclusion: { in: RECURRENCES } + validates :repeat_count, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true + validates :active, inclusion: { in: [true, false] } + validate :anchor_day_supported + validate :timezone_supported + validate :anchor_date_resolves + + before_validation :normalize_anchor_day + before_validation :default_timezone + before_validation :sync_schedule_cache + + scope :active, -> { where(active: true) } + scope :due, ->(time = Time.zone.now) { where('next_run_at IS NOT NULL AND next_run_at <= ?', time) } + scope :with_active_unit, -> { joins(communication_set: :unit).where(units: { active: true }) } + + def resolved_anchor_date + return nil if unit.blank? || anchor_week.blank? || anchor_day.blank? + + unit.date_for_week_and_day(anchor_week, anchor_day_abbreviation) + end + + def resolved_start_at + date = resolved_anchor_date + return nil if date.nil? + + timezone_object.local(date.year, date.month, date.day, hour, minute) + end + + def build_ice_cube_schedule + start_at = resolved_start_at + return nil if start_at.nil? + + schedule = IceCube::Schedule.new(start_at) + rule = recurrence_rule + schedule.add_recurrence_rule(rule) if rule + schedule + end + + def next_occurrence_after(time = Time.zone.now) + schedule = build_ice_cube_schedule + return nil if schedule.nil? + + time_for_schedule = time.in_time_zone(timezone_object) + occurrence = if recurrence == 'none' + occurrence = schedule.start_time + occurrence >= time_for_schedule ? occurrence : nil + else + schedule.next_occurrence(time_for_schedule) + end + + return nil if occurrence.blank? + + occurrence_within_unit_dates?(occurrence) ? occurrence : nil + end + + def due?(time = Time.zone.now) + active_for_scheduling? && next_run_at.present? && next_run_at <= time + end + + def refresh_next_run_at!(from_time = Time.zone.now) + update!(next_run_at: active_for_scheduling? ? next_occurrence_after(from_time) : nil) + end + + def anchor_day_abbreviation + return nil if anchor_day.blank? + + day = anchor_day.to_s.strip + return day if DAY_ABBREVIATIONS.include?(day) + + index = VALID_DAYS.index(day.titlecase) + index.nil? ? nil : DAY_ABBREVIATIONS[index] + end + + private + + def recurrence_rule + case recurrence + when 'daily' + IceCube::Rule.daily(interval) + when 'weekly' + IceCube::Rule.weekly(interval) + when 'monthly' + IceCube::Rule.monthly(interval) + end&.tap do |rule| + rule.count(repeat_count) if repeat_count.present? + rule.until(until_at.in_time_zone(timezone_object)) if until_at.present? + end + end + + def timezone_object + ActiveSupport::TimeZone[timezone.presence || Time.zone.name] || Time.zone + end + + def normalize_anchor_day + return if anchor_day.blank? + + full_day = + if VALID_DAYS.include?(anchor_day.to_s.titlecase) + anchor_day.to_s.titlecase + else + day_index = DAY_ABBREVIATIONS.index(anchor_day.to_s.titlecase) + day_index.nil? ? anchor_day : VALID_DAYS[day_index] + end + + self.anchor_day = full_day + end + + def default_timezone + self.timezone = timezone.presence || Time.zone.name + end + + def sync_schedule_cache + schedule = build_ice_cube_schedule + self.ice_cube_schedule = schedule.present? ? JSON.generate(schedule.to_hash) : nil + self.next_run_at = next_occurrence_after(Time.zone.now) if active_for_scheduling? && should_refresh_next_run_at? + self.next_run_at = nil unless active_for_scheduling? + end + + def active_for_scheduling? + active? && unit&.active? + end + + def occurrence_within_unit_dates?(occurrence) + return true if unit&.end_date.blank? + + occurrence.to_date <= unit.end_date + end + + def should_refresh_next_run_at? + will_save_change_to_anchor_week? || + will_save_change_to_anchor_day? || + will_save_change_to_hour? || + will_save_change_to_minute? || + will_save_change_to_timezone? || + will_save_change_to_recurrence? || + will_save_change_to_interval? || + will_save_change_to_repeat_count? || + will_save_change_to_until_at? || + will_save_change_to_active? || + next_run_at.blank? + end + + def anchor_day_supported + return if anchor_day.blank? || VALID_DAYS.include?(anchor_day.to_s.titlecase) + + errors.add(:anchor_day, 'must be a valid day name') + end + + def timezone_supported + return if timezone.blank? || ActiveSupport::TimeZone[timezone].present? + + errors.add(:timezone, 'must be a valid timezone') + end + + def anchor_date_resolves + return if unit.blank? || anchor_week.blank? || anchor_day.blank? + return if resolved_anchor_date.present? + + errors.add(:base, 'schedule anchor could not be resolved for this unit') + end +end diff --git a/app/models/communication/email_staff_action.rb b/app/models/communication/email_staff_action.rb new file mode 100644 index 0000000000..fc1c2d36f3 --- /dev/null +++ b/app/models/communication/email_staff_action.rb @@ -0,0 +1,15 @@ +class EmailStaffAction < CommunicationAction + validates :subject, presence: true + validates :body, presence: true + validates :email_tutors, inclusion: { in: [true, false] } + validates :email_convenors, inclusion: { in: [true, false] } + validate :staff_recipient? + + private + + def staff_recipient? + return if email_tutors || email_convenors + + errors.add(:base, 'must email tutors or convenors') + end +end diff --git a/app/models/communication/email_student_action.rb b/app/models/communication/email_student_action.rb new file mode 100644 index 0000000000..fdd0ecb17d --- /dev/null +++ b/app/models/communication/email_student_action.rb @@ -0,0 +1,4 @@ +class EmailStudentAction < CommunicationAction + validates :subject, presence: true + validates :body, presence: true +end diff --git a/app/models/communication/login_status_condition.rb b/app/models/communication/login_status_condition.rb new file mode 100644 index 0000000000..cf6cbebea5 --- /dev/null +++ b/app/models/communication/login_status_condition.rb @@ -0,0 +1,6 @@ +class LoginStatusCondition < CommunicationCondition + validates :activity_days, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 1 } + validates :operator, inclusion: { in: ACTIVITY_OPERATORS } +end diff --git a/app/models/communication/portfolio_submitted_condition.rb b/app/models/communication/portfolio_submitted_condition.rb new file mode 100644 index 0000000000..ea63f16e14 --- /dev/null +++ b/app/models/communication/portfolio_submitted_condition.rb @@ -0,0 +1,4 @@ +class PortfolioSubmittedCondition < CommunicationCondition + validates :submitted_portfolio, inclusion: { in: [true, false] } + validates :operator, inclusion: { in: EQUALITY_OPERATORS } +end diff --git a/app/models/communication/spec_con_condition.rb b/app/models/communication/spec_con_condition.rb new file mode 100644 index 0000000000..cebeb8565f --- /dev/null +++ b/app/models/communication/spec_con_condition.rb @@ -0,0 +1,4 @@ +class SpecConCondition < CommunicationCondition + validates :spec_con_days, presence: true, numericality: { only_integer: true } + validates :operator, inclusion: { in: GRADE_OPERATORS } +end diff --git a/app/models/communication/target_grade_condition.rb b/app/models/communication/target_grade_condition.rb new file mode 100644 index 0000000000..77f195a473 --- /dev/null +++ b/app/models/communication/target_grade_condition.rb @@ -0,0 +1,13 @@ +class TargetGradeCondition < CommunicationCondition + validates :target_grade, presence: true + validates :operator, inclusion: { in: GRADE_OPERATORS } + validate :target_grade_enabled_for_unit + + private + + def target_grade_enabled_for_unit + return if target_grade.nil? || communication&.unit&.grade_value?(target_grade) + + errors.add(:target_grade, 'is not enabled for this unit') + end +end diff --git a/app/models/communication/task_comment_action.rb b/app/models/communication/task_comment_action.rb new file mode 100644 index 0000000000..d42165b51e --- /dev/null +++ b/app/models/communication/task_comment_action.rb @@ -0,0 +1,4 @@ +class TaskCommentAction < CommunicationAction + validates :task_definition, presence: true + validates :body, presence: true +end diff --git a/app/models/communication/task_definition_status_condition.rb b/app/models/communication/task_definition_status_condition.rb new file mode 100644 index 0000000000..83ab40bdec --- /dev/null +++ b/app/models/communication/task_definition_status_condition.rb @@ -0,0 +1,5 @@ +class TaskDefinitionStatusCondition < CommunicationCondition + validates :task_definition, presence: true + validates :operator, inclusion: { in: EQUALITY_OPERATORS } + validate :task_statuses_must_be_present +end diff --git a/app/models/communication/task_status_count_condition.rb b/app/models/communication/task_status_count_condition.rb new file mode 100644 index 0000000000..42659a930e --- /dev/null +++ b/app/models/communication/task_status_count_condition.rb @@ -0,0 +1,15 @@ +class TaskStatusCountCondition < CommunicationCondition + validates :task_status_count, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :task_target_grade, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :operator, inclusion: { in: GRADE_OPERATORS } + validate :task_statuses_must_be_present + validate :task_target_grade_enabled_for_unit + + private + + def task_target_grade_enabled_for_unit + return if task_target_grade.nil? || communication&.unit&.grade_value?(task_target_grade) + + errors.add(:task_target_grade, 'is not enabled for this unit') + end +end diff --git a/app/models/communication/tutorial_enrolment_condition.rb b/app/models/communication/tutorial_enrolment_condition.rb new file mode 100644 index 0000000000..024ed79ce1 --- /dev/null +++ b/app/models/communication/tutorial_enrolment_condition.rb @@ -0,0 +1,4 @@ +class TutorialEnrolmentCondition < CommunicationCondition + validates :tutorial, presence: true + validates :operator, inclusion: { in: ENROLMENT_OPERATORS } +end diff --git a/app/models/communication/tutorial_stream_enrolment_condition.rb b/app/models/communication/tutorial_stream_enrolment_condition.rb new file mode 100644 index 0000000000..e2153037fa --- /dev/null +++ b/app/models/communication/tutorial_stream_enrolment_condition.rb @@ -0,0 +1,4 @@ +class TutorialStreamEnrolmentCondition < CommunicationCondition + validates :tutorial_stream, presence: true + validates :operator, inclusion: { in: ENROLMENT_OPERATORS } +end diff --git a/app/models/communication/unit_viewed_status_condition.rb b/app/models/communication/unit_viewed_status_condition.rb new file mode 100644 index 0000000000..9ce4152885 --- /dev/null +++ b/app/models/communication/unit_viewed_status_condition.rb @@ -0,0 +1,6 @@ +class UnitViewedStatusCondition < CommunicationCondition + validates :activity_days, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 1 } + validates :operator, inclusion: { in: ACTIVITY_OPERATORS } +end diff --git a/app/models/engagement.rb b/app/models/engagement.rb new file mode 100644 index 0000000000..8d16954ea6 --- /dev/null +++ b/app/models/engagement.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require 'uri' + +class Engagement < ApplicationRecord + include FileHelper + include MimeCheckHelpers + + belongs_to :project, optional: false, inverse_of: :engagements + belongs_to :user, optional: false, inverse_of: :engagements + + has_many :engagement_projects, dependent: :destroy, inverse_of: :engagement + has_many :additional_projects, through: :engagement_projects, source: :project + has_many :engagement_comments, + -> { order(:created_at) }, + dependent: :destroy, + inverse_of: :engagement + + validates :engagement_type, presence: true, length: { maximum: 255 } + validates :note, presence: true, length: { maximum: 4095 } + validates :occurred_at, presence: true + validates :evidence_url, length: { maximum: 2048, allow_blank: true } + validates :content_type, inclusion: { in: %w[image pdf], allow_nil: true } + validate :valid_evidence_url + validate :single_evidence_source + validate :consistent_attachment_metadata + + before_validation :normalise_text + before_destroy :delete_attachment + + def attachment? + content_type.present? && attachment_extension.present? + end + + def attachment_path + FileHelper.engagement_attachment_path(self, attachment_extension) + end + + def attachment_file_name + "engagement-#{id}#{attachment_extension}" + end + + def attachment_mime_type + mime_type(attachment_path) + end + + def replace_attachment(file_upload, attachment_type) + delete_attachment + self.evidence_url = nil + self.content_type = attachment_type + + if attachment_type == 'image' + self.attachment_extension = + if mime_type(file_upload['tempfile'].path).starts_with?('image/gif') + '.gif' + else + '.jpg' + end + save! + image_saved = FileHelper.compress_image_to_dest(file_upload['tempfile'].path, attachment_path) + raise 'Failed to save engagement image attachment' unless image_saved && File.exist?(attachment_path) + else + self.attachment_extension = '.pdf' + save! + FileHelper.compress_pdf(file_upload['tempfile'].path) + FileUtils.mv(file_upload['tempfile'].path, attachment_path) + raise 'Failed to save engagement PDF attachment' unless File.exist?(attachment_path) + end + + file_upload['tempfile'].unlink if File.exist?(file_upload['tempfile'].path) + true + end + + def remove_attachment + delete_attachment + self.content_type = nil + self.attachment_extension = nil + end + + private + + def normalise_text + self.engagement_type = engagement_type&.strip + self.note = note&.strip + self.evidence_url = evidence_url&.strip.presence + end + + def valid_evidence_url + return if evidence_url.blank? + + uri = URI.parse(evidence_url) + return if uri.is_a?(URI::HTTP) && uri.host.present? + + errors.add(:evidence_url, 'must be a valid HTTP or HTTPS URL') + rescue URI::InvalidURIError + errors.add(:evidence_url, 'must be a valid HTTP or HTTPS URL') + end + + def single_evidence_source + return unless evidence_url.present? && (content_type.present? || attachment_extension.present?) + + errors.add(:base, 'An engagement can have either an evidence URL or an attachment, not both') + end + + def consistent_attachment_metadata + return if content_type.present? == attachment_extension.present? + + errors.add(:base, 'Attachment content type and extension must both be present') + end + + def delete_attachment + FileUtils.rm_f(attachment_path) if attachment_extension.present? + end +end diff --git a/app/models/engagement_comment.rb b/app/models/engagement_comment.rb new file mode 100644 index 0000000000..d02b4fca89 --- /dev/null +++ b/app/models/engagement_comment.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class EngagementComment < ApplicationRecord + belongs_to :engagement, optional: false, inverse_of: :engagement_comments + belongs_to :user, optional: false, inverse_of: :engagement_comments + belongs_to :reply_to, class_name: 'EngagementComment', optional: true + has_many :replies, + class_name: 'EngagementComment', + foreign_key: :reply_to_id, + dependent: :nullify, + inverse_of: :reply_to + + validates :comment, presence: true, length: { maximum: 4095 } + validate :reply_belongs_to_engagement + + before_validation do + self.comment = comment&.strip + end + + private + + def reply_belongs_to_engagement + return if reply_to.nil? || reply_to.engagement_id == engagement_id + + errors.add(:reply_to, 'must belong to the same engagement') + end +end diff --git a/app/models/engagement_project.rb b/app/models/engagement_project.rb new file mode 100644 index 0000000000..0c86fbb334 --- /dev/null +++ b/app/models/engagement_project.rb @@ -0,0 +1,6 @@ +class EngagementProject < ApplicationRecord + belongs_to :engagement, optional: false, inverse_of: :engagement_projects + belongs_to :project, optional: false, inverse_of: :engagement_projects + + validates :project_id, uniqueness: { scope: :engagement_id } +end diff --git a/app/models/feedback/feedback_chip.rb b/app/models/feedback/feedback_chip.rb index deabbeccff..e39208f95d 100644 --- a/app/models/feedback/feedback_chip.rb +++ b/app/models/feedback/feedback_chip.rb @@ -196,15 +196,23 @@ def self.context_for_csv(row, context_type, context) # Find unit - which contains task definition if present unit_code = row['unit_code'] if unit_code.present? - unit = Unit.find_by(code: unit_code) + unit = if context_type == 'TaskDefinition' + context.unit + elsif context_type == 'LearningOutcome' && context.context_type == 'TaskDefinition' + context.context.unit + elsif context_type == 'LearningOutcome' + context.context + else + context + end if unit.nil? return { success: false, message: "Unit #{unit_code} not found" } - elsif (context_type == 'Unit' && context.id != unit.id) || (context_type == 'TaskDefinition' && context.unit_id != unit.id) - return { success: false, message: "Is not for unit #{context.code}" } + elsif unit_code != unit.code + return { success: false, message: "Unit #{unit_code} does not match unit #{unit.code}" } end elsif %w[Unit TaskDefinition].include? context_type # the unit code is not present, but the context is a unit or task definition - return { success: false, message: "Is not for unit #{context.code}" } + return { success: false, message: "Is not for unit #{context_type == 'Unit' ? context.code : context.unit.code}" } end task_abbreviation = row['task_abbreviation'] @@ -230,16 +238,8 @@ def self.context_for_csv(row, context_type, context) return { success: false, message: 'Missing learning outcome abbreviation' } end - search_context = if context_type == 'LearningOutcome' - context.context - else - context - end - + search_context = task_definition || (context_type == 'LearningOutcome' ? context.context : context) learning_outcome = LearningOutcome.find_by(abbreviation: learning_outcome_abbreviation, context: search_context) - if learning_outcome.nil? && task_definition.present? && context_type == 'Unit' - learning_outcome = LearningOutcome.find_by(abbreviation: learning_outcome_abbreviation, context: task_definition) - end if learning_outcome.nil? return { success: false, message: "Learning outcome #{learning_outcome_abbreviation} not found" } diff --git a/app/models/overflow_task_claim.rb b/app/models/overflow_task_claim.rb index 2559ce5964..986d055b6d 100644 --- a/app/models/overflow_task_claim.rb +++ b/app/models/overflow_task_claim.rb @@ -1,4 +1,5 @@ class OverflowTaskClaim < ApplicationRecord belongs_to :task + belongs_to :claimed_by_unit_role, class_name: 'UnitRole' end diff --git a/app/models/overflow_task_claim_log.rb b/app/models/overflow_task_claim_log.rb new file mode 100644 index 0000000000..9b4b606115 --- /dev/null +++ b/app/models/overflow_task_claim_log.rb @@ -0,0 +1,8 @@ +class OverflowTaskClaimLog < ApplicationRecord + belongs_to :unit + belongs_to :task, optional: true + belongs_to :claimed_by_unit_role, class_name: 'UnitRole', optional: true + belongs_to :claimed_by_user, class_name: 'User', optional: true + belongs_to :original_tutor_user, class_name: 'User', optional: true + belongs_to :student_user, class_name: 'User', optional: true +end diff --git a/app/models/overseer_assessment.rb b/app/models/overseer_assessment.rb index 84a2077dd0..d01a3210ed 100644 --- a/app/models/overseer_assessment.rb +++ b/app/models/overseer_assessment.rb @@ -1,6 +1,7 @@ # rubocop:disable Rails/Output class OverseerAssessment < ApplicationRecord belongs_to :task, optional: false + belongs_to :submission_history, optional: false has_one :project, through: :task has_many :assessment_comments, as: :commentable, dependent: :destroy @@ -11,22 +12,79 @@ class OverseerAssessment < ApplicationRecord validates :submission_timestamp, presence: true validates :submission_timestamp, uniqueness: { scope: :task_id } + validates :submission_history_id, uniqueness: true + validate :submission_history_matches_task enum :status, { pre_queued: 0, passed: 1, failed: 2 } - after_destroy :delete_associated_files + def submission_history_matches_task + return if submission_history.nil? || task.nil? || submission_history.task_id == task_id + + errors.add(:submission_history, 'must belong to the same task') + end + + def self.student_notification_grace_period + Doubtfire::Application.config.overseer_student_notification_grace_period + end + + scope :awaiting_student_failure_notification, lambda { |grace_period: student_notification_grace_period| + notification_cutoff = grace_period.ago + + joins(task: { project: :user }) + .joins(<<~SQL.squish) + INNER JOIN task_comments assessment_comments + ON assessment_comments.commentable_type = 'OverseerAssessment' + AND assessment_comments.commentable_id = overseer_assessments.id + AND assessment_comments.type = 'AssessmentComment' + SQL + .joins(<<~SQL.squish) + LEFT JOIN comments_read_receipts student_read_receipts + ON student_read_receipts.task_comment_id = assessment_comments.id + AND student_read_receipts.user_id = projects.user_id + SQL + .where(status: statuses[:failed], student_notified_at: nil) + .where(users: { receive_task_notifications: true }) + .where('overseer_assessments.updated_at <= ?', notification_cutoff) + .where('student_read_receipts.id IS NULL') + .where(<<~SQL.squish) + assessment_comments.id = ( + SELECT latest_comment.id + FROM task_comments latest_comment + WHERE latest_comment.commentable_type = 'OverseerAssessment' + AND latest_comment.commentable_id = overseer_assessments.id + AND latest_comment.type = 'AssessmentComment' + ORDER BY latest_comment.created_at DESC, latest_comment.id DESC + LIMIT 1 + ) + SQL + .where(<<~SQL.squish) + NOT EXISTS ( + SELECT 1 + FROM overseer_assessments newer_assessments + WHERE newer_assessments.task_id = overseer_assessments.task_id + AND ( + newer_assessments.created_at > overseer_assessments.created_at OR + ( + newer_assessments.created_at = overseer_assessments.created_at AND + newer_assessments.id > overseer_assessments.id + ) + ) + ) + SQL + } # TODO: track how many tests ran, and how many tests total at the time # TODO: we might not have an overseerStepResult because a new test was added later # Creates an OverseerAssessment object for a new submission - def self.create_for(task, test_submission) + def self.create_for(submission_history, test_submission) # Create only if: # unit's assessment is enabled && # task's assessment is enabled && # task definition has an assessment resources zip file && # task has a student submission + task = submission_history.task task_definition = task.task_definition unit = task_definition.unit @@ -40,50 +98,21 @@ def self.create_for(task, test_submission) return nil if docker_image_name_tag.nil? || docker_image_name_tag.strip.empty? - result = OverseerAssessment.create!( + OverseerAssessment.create!( task: task, + submission_history: submission_history, status: :pre_queued, - submission_timestamp: Time.now.utc.to_i + submission_timestamp: submission_history.submission_timestamp ) - - # Create the submission folder and give access - FileUtils.mkdir_p result.output_path - result.grant_access_to_submission - - result.copy_latest_files_to_submission - - result end - def has_submission_files? - File.exist? submission_zip_file_name - end - - def submission_zip_file_name - "#{output_path}/submission.zip" - end + delegate :has_submission_files?, + :submission_zip_file_name, + :output_path, + to: :submission_history - def grant_access_to_submission - # TODO: Use FACL instead in future. - `chmod o+w #{output_path}` - end - - def copy_latest_files_to_submission - zip_file_path = submission_zip_file_name - - if task.has_new_files? - puts "Copying new files to submission at: #{zip_file_path}" - # Generate a zip file for this particular submission with timestamp value and put it here - task.compress_new_to_done zip_file_path: zip_file_path, rm_task_dir: false, rename_files: true - else - puts "Copying done file to submission at: #{zip_file_path}" - task.copy_done_to zip_file_path - end - end - - # Path to where the submission and output are stored - includes the submission when it is to be processed - def output_path - FileHelper.task_submission_identifier_path_with_timestamp(:done, task, submission_timestamp) + def latest_assessment_comment + assessment_comments.order(created_at: :desc, id: :desc).first end def add_assessment_comment(text = 'Automated Assessment Started') @@ -156,7 +185,7 @@ def send_to_overseer(test_submission: false) return { error: "This assessment is no longer setup for automated feedback. Automated feedback is turned off at either the unit or task level, or the task does not have the scripts needed to automate assessment." } end - unless File.exist? submission_zip_file_name + unless has_submission_files? puts "ERROR: Student submission history zip file doesn't exist #{submission_zip_file_name}. Unable to send - OverseerAssessment #{id}" return { error: "We no longer have the files associated with this submission. Please test a later submission, or upload your work again." } end @@ -259,10 +288,6 @@ def update_from_output(work_dir_path) self.save! end - def delete_associated_files - FileUtils.rm_rf output_path - end - def base64?(value) value.is_a?(String) && Base64.strict_encode64(Base64.decode64(value)) == value end diff --git a/app/models/overseer_step.rb b/app/models/overseer_step.rb index 6273f0db06..33da31aa5b 100644 --- a/app/models/overseer_step.rb +++ b/app/models/overseer_step.rb @@ -1,6 +1,7 @@ class OverseerStep < ApplicationRecord belongs_to :task_definition, optional: false + validates :name, :display_name, :step_type, presence: true validates :timeout, presence: true, numericality: { greater_than_or_equal_to: 1, less_than_or_equal_to: 300, message: 'must be between 1 and 300' } validates :sort_order, presence: true, numericality: { greater_than_or_equal_to: 0 } end diff --git a/app/models/pdf_generation/project_compile_portfolio_module.rb b/app/models/pdf_generation/project_compile_portfolio_module.rb index f5e2c8bba5..eab5a5292b 100644 --- a/app/models/pdf_generation/project_compile_portfolio_module.rb +++ b/app/models/pdf_generation/project_compile_portfolio_module.rb @@ -42,6 +42,9 @@ class ProjectAppController < ApplicationController :base_path, :image_path, :learning_summary_report, + :submission_date, + :formatted_submission_date, + :formatted_submission_time, :ordered_tasks, :portfolio_tasks, :task_defs, @@ -55,6 +58,8 @@ def init(project, is_retry) @student = project.student @project = project @learning_summary_report = project.learning_summary_report_path + @submission_date = project.portfolio_submission_date + @formatted_submission_date, @formatted_submission_time = format_submission_date(project) @files = project.portfolio_files(ensure_valid: true, force_ascii: is_retry) @base_path = project.portfolio_temp_path @image_path = Rails.root.join('public/assets/images') @@ -66,13 +71,36 @@ def init(project, is_retry) @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] @is_retry = is_retry @include_pax = !is_retry - @work_id = "portfolio-#{project.id}-#{Time.now.to_i}-#{Process.pid}-#{Thread.current.object_id}#{'-retry' if is_retry}" + @work_id = FileHelper.sanitized_path( + "portfolio-#{Time.current.strftime('%Y%m%d-%H%M')}-#{project.student.username}-#{project.id}-#{Process.pid}#{'-retry' if is_retry}" + ) end def make_pdf logger.debug 'Running make_pdf: (portfolio)' generate_pdf(template: '/portfolio/portfolio_pdf') end + + private + + def format_submission_date(project) + return [nil, nil] if @submission_date.blank? + + campus_timezone = project.campus&.timezone.presence + + submission_time = + if campus_timezone.present? + @submission_date.in_time_zone(campus_timezone) + else + @submission_date.to_time.getlocal + end + + timezone_label = campus_timezone || ENV['TZ'].presence || submission_time.zone || Time.zone.name + [ + submission_time.strftime('%d %b %Y'), + "#{submission_time.strftime('%I:%M %p')} #{timezone_label}" + ] + end end # A custom error to capture the log message from the latex error @@ -87,9 +115,7 @@ def initialize(log_message) # Create the portfolio for this project def create_portfolio - self.compile_portfolio = false - save! - + logger.info "Creating portfolio for #{user.username} in #{unit.code}" begin pac = ProjectAppController.new pac.init(self, false) @@ -116,8 +142,13 @@ def create_portfolio logger.info "Created portfolio at #{portfolio_path} - #{log_details}" self.portfolio_production_date = Time.zone.now - save + self.compile_portfolio = false + save! + true rescue StandardError => e + self.compile_portfolio = false + save! + logger.error "Failed to convert portfolio to PDF - #{log_details} -\nError: #{e.message}" log_file = e.message.scan(%r{/.*\.log}).first @@ -174,6 +205,19 @@ def portfolio_tasks end end + # Return the tasks that are currently being processed + def tasks_processing_pdf + # Get assigned tasks that should be included in the portfolio + tasks = self.tasks.joins(:task_definition).order('task_definitions.target_date, task_definitions.abbreviation') + + # Select tasks that should have a PDF submission, but is currently being processed + tasks.select do |task| + !task.has_pdf && + task.processing_pdf? && + task.task_definition.upload_requirements.present? + end + end + # # Return the path to the student's learning summary report. # This returns nil if there is no learning summary report. @@ -260,7 +304,7 @@ def portfolio_files(ensure_valid: false, force_ascii: false) result = [] Dir.chdir(portfolio_tmp_dir) - files = Dir.glob('*').select { |f| (f =~ /^\d{3}-(cover|document|code|image)/) == 0 } + files = Dir.glob('*').select { |f| (f =~ /^\d{3}-(cover|document|code|image|zip|archive)/) == 0 } files.each do |file| parts = file.split('-') idx = parts[0].to_i diff --git a/app/models/project.rb b/app/models/project.rb index 1444730f9b..785f9b39b9 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -33,6 +33,9 @@ class Project < ApplicationRecord has_many :session_activities, dependent: :destroy has_many :staff_notes, dependent: :destroy + has_many :engagements, dependent: :destroy, inverse_of: :project + has_many :engagement_projects, dependent: :destroy + has_many :shared_engagements, through: :engagement_projects, source: :engagement # Callbacks - methods called are private before_destroy :can_destroy? @@ -59,7 +62,9 @@ def self.permissions :make_submission, :get_submission, :change, - :reprocess_submission + :reprocess_submission, + :get_engagements, + :comment_engagement ] # What can tutors do with projects? tutor_role_permissions = [ @@ -74,7 +79,11 @@ def self.permissions :get_staff_note, :create_staff_note, :reprocess_submission, - :get_discussion_prompt + :get_discussion_prompt, + :get_engagements, + :create_engagement, + :edit_engagement, + :comment_engagement ] # What can admins do with projects? @@ -82,7 +91,8 @@ def self.permissions :get, :get_submission, :reprocess_submission, - :get_discussion_prompt + :get_discussion_prompt, + :get_engagements ] # What can auditors do with projects? @@ -91,7 +101,8 @@ def self.permissions :get_submission, :get_staff_note, :reprocess_submission, - :get_discussion_prompt + :get_discussion_prompt, + :get_engagements ] # What can nil users do with projects? @@ -261,16 +272,7 @@ def active? # Get a string representation of the Target Grade # def target_grade_desc - case target_grade - when 1 - 'Credit' - when 2 - 'Distinction' - when 3 - 'High Distinction' - else - 'Pass' - end + unit.grade_label(target_grade) end def reference_date @@ -278,6 +280,8 @@ def reference_date end def task_details_for_shallow_serializer(user) + teaching_breaks = unit.teaching_period&.breaks.to_a + tasks .joins(:task_status) .joins("LEFT JOIN task_comments ON task_comments.task_id = tasks.id AND (task_comments.type IS NULL OR task_comments.type <> 'TaskStatusComment')") @@ -309,6 +313,8 @@ def task_details_for_shallow_serializer(user) extensions: t.extensions, scorm_extensions: t.scorm_extensions, due_date: t.due_date, + moved_to_discuss_at: t.moved_to_discuss_at, + discuss_timeout_expiry_at: t.discuss_timeout_expiry_at(teaching_breaks: teaching_breaks), submission_date: t.submission_date, completion_date: t.completion_date, target_start_date: t.target_start_date, @@ -361,7 +367,7 @@ def top_tasks # overdue_tasks = task_states.select { |ts| to_target.call(ts) < Time.zone.today } - for i in GradeHelper::RANGE + for i in unit.grade_values graded_tasks = overdue_tasks.select { |ts| ts[:task_definition].target_grade == i } graded_tasks.each do |ts| @@ -377,7 +383,7 @@ def top_tasks # soon_tasks = task_states.select { |ts| to_target.call(ts) >= Time.zone.today && to_target.call(ts) < Time.zone.today + 7.days } - for i in GradeHelper::RANGE + for i in unit.grade_values graded_tasks = soon_tasks.select { |ts| ts[:task_definition].target_grade == i } graded_tasks.each do |ts| @@ -392,7 +398,7 @@ def top_tasks # ahead_tasks = task_states.select { |ts| to_target.call(ts) >= Time.zone.today + 7.days } - for i in GradeHelper::RANGE + for i in unit.grade_values graded_tasks = ahead_tasks.select { |ts| ts[:task_definition].target_grade == i } graded_tasks.each do |ts| @@ -506,7 +512,7 @@ def self.create_task_stats_from(total_task_counts, project_task_counts, target_g red_pct = ((project_task_counts.fail_count + project_task_counts.feedback_exceeded_count + project_task_counts.time_exceeded_count) / total_task_counts[target_grade]).signif(2) orange_pct = ((project_task_counts.redo_count + project_task_counts.need_help_count + project_task_counts.fix_and_resubmit_count) / total_task_counts[target_grade]).signif(2) - green_pct = ((project_task_counts.discuss_count + project_task_counts.demonstrate_count + project_task_counts.complete_count) / total_task_counts[target_grade]).signif(2) + green_pct = ((project_task_counts.discuss_count + project_task_counts.rediscuss_count + project_task_counts.demonstrate_count + project_task_counts.complete_count) / total_task_counts[target_grade]).signif(2) blue_pct = (project_task_counts.ready_for_feedback_count / total_task_counts[target_grade]).signif(2) grey_pct = (1 - red_pct - orange_pct - green_pct - blue_pct).signif(2) @@ -545,19 +551,14 @@ def revert_overdue_tasks # task_stats field def update_task_stats # generate SQL for columns that count the number of tasks per grade - count_by_grade = (GradeHelper::RANGE).map { |grade_id| "SUM(CASE WHEN target_grade <= #{grade_id} THEN 1 ELSE 0 END) AS count_#{grade_id}" } + count_by_grade = unit.grade_values.map { |grade_id| "SUM(CASE WHEN target_grade <= #{grade_id} THEN 1 ELSE 0 END) AS count_#{grade_id}" } # Get the count of the total number of tasks less than each target grade task_count = unit .task_definitions .select(*count_by_grade) # create columns for each grade .map do |r| # map to array - [ - r['count_0'].to_f || 0.0, - r['count_1'].to_f || 0.0, - r['count_2'].to_f || 0.0, - r['count_3'].to_f || 0.0 - ] + unit.grade_values.index_with { |grade_id| r["count_#{grade_id}"].to_f || 0.0 } end .first # there is only one row returned... diff --git a/app/models/session_activity.rb b/app/models/session_activity.rb index 22c135923d..18e6a55a65 100644 --- a/app/models/session_activity.rb +++ b/app/models/session_activity.rb @@ -4,6 +4,6 @@ class SessionActivity < ApplicationRecord belongs_to :task, optional: true belongs_to :task_definition, optional: true - VALID_ACTIONS = %w[inbox GET PUT assessing add-comment get-comments delete-comment get-comment-attachment mark-comment-unread get-submission-details get-submission-files].freeze + VALID_ACTIONS = %w[inbox GET PUT assessing add-comment edit-comment get-comments delete-comment get-comment-attachment mark-comment-unread get-submission-details get-submission-files].freeze validates :action, presence: true, inclusion: { in: VALID_ACTIONS } end diff --git a/app/models/similarity/unit_similarity_module.rb b/app/models/similarity/unit_similarity_module.rb index c003c36b56..041040d711 100644 --- a/app/models/similarity/unit_similarity_module.rb +++ b/app/models/similarity/unit_similarity_module.rb @@ -105,10 +105,10 @@ def check_jplag_similarity(force: false) # need pwd to restore after cding into submission folder (so the files do not have full path) pwd = FileUtils.pwd + completed_all_checks = true # making temp directory for unit - jplag root_work_dir = Rails.root.join("tmp", "jplag", "#{code}-#{id}") - unit_code = "#{code}-#{id}" begin logger.info "Checking plagiarsm for unit #{code} - #{name} (id=#{id})" @@ -138,20 +138,27 @@ def check_jplag_similarity(force: false) FileUtils.mkdir_p(tasks_dir) # There are new tasks, check these with JPLAG - run_jplag_on_done_files(td, tasks_dir, tasks_with_files, unit_code) - report_path = "#{Doubtfire::Application.config.jplag_report_dir}/#{unit_code}/#{td.abbreviation}-result.jplag" - warn_pct = td.plagiarism_warn_pct || 50 - logger.debug "Warn PCT: #{warn_pct}" + report_path = FileHelper.task_jplag_report_path(self, td) + begin + run_jplag_on_done_files(td, tasks_dir, tasks_with_files, report_path) + warn_pct = td.plagiarism_warn_pct || 50 + logger.debug "Warn PCT: #{warn_pct}" - # Remove any existing plagiarism links that are below the threshold, in case it has been updated since the last analysis - JplagTaskSimilarity.joins(:task) - .where("pct < ? AND tasks.task_definition_id = ?", warn_pct, td.id) - .delete_all + # Remove any existing plagiarism links that are below the threshold, in case it has been updated since the last analysis + JplagTaskSimilarity.joins(:task) + .where("pct < ? AND tasks.task_definition_id = ?", warn_pct, td.id) + .delete_all - process_jplag_plagiarism_report(report_path, warn_pct, td.group_set) + process_jplag_plagiarism_report(report_path, warn_pct, td.group_set) + rescue StandardError => e + completed_all_checks = false + logger.error "Failed to check JPlag similarity for task #{td.name} (id=#{td.id}). Error: #{e.message}" + end + end + if completed_all_checks + self.last_plagarism_scan = Time.zone.now + save! end - self.last_plagarism_scan = Time.zone.now - save! ensure FileUtils.chdir(pwd) if FileUtils.pwd != pwd end @@ -230,29 +237,36 @@ def update_moss_plagiarism_stats # end # JPLAG Function - extracts "done" files for each task and packages them into a directory for JPLAG to run on - def run_jplag_on_done_files(task_definition, tasks_dir, tasks_with_files, unit_code) + def run_jplag_on_done_files(task_definition, tasks_dir, tasks_with_files, report_path) similarity_pct = task_definition.plagiarism_warn_pct return if similarity_pct.nil? # Check if the directory exists and create it if it doesn't - results_dir = "/jplag/results/#{unit_code}" + results_dir = File.dirname(report_path) system("docker exec -i jplag sh -c 'if [ ! -d \"#{results_dir}\" ]; then mkdir -p \"#{results_dir}\"; fi'") || raise('Failed to create JPlag results directory') # Remove existing result file if it exists - result_file = "#{results_dir}/#{task_definition.abbreviation}-result.jplag" - system("docker exec -i jplag sh -c 'if [ -f \"#{result_file}\" ]; then rm \"#{result_file}\"; fi'") || raise('Failed to remove previous JPlag report') + system("docker exec -i jplag sh -c 'if [ -f \"#{report_path}\" ]; then rm \"#{report_path}\"; fi'") || raise('Failed to remove previous JPlag report') # Extract task resources for base code use_base_code = false if task_definition.has_task_resources? && task_definition.use_resources_for_jplag_base_code use_base_code = true - path = task_definition.task_resources - - Zip::File.open(path) do |zip_file| - zip_file.each do |entry| - dest = File.join(tasks_dir, 'base', entry.name) - FileUtils.mkdir_p(File.dirname(dest)) - entry.extract(dest) { true } + linked_resource = task_definition.linked_task_resource + + if linked_resource && !task_definition.task_resource_zip?(linked_resource) + base_dir = File.join(tasks_dir, 'base') + FileUtils.mkdir_p(base_dir) + FileUtils.cp(linked_resource[:path], File.join(base_dir, linked_resource[:filename])) + else + path = linked_resource ? linked_resource[:path] : task_definition.task_resources + + Zip::File.open(path) do |zip_file| + zip_file.each do |entry| + dest = File.join(tasks_dir, 'base', entry.name) + FileUtils.mkdir_p(File.dirname(dest)) + entry.extract(dest) { true } + end end end end @@ -305,8 +319,26 @@ def run_jplag_on_done_files(task_definition, tasks_dir, tasks_with_files, unit_c skip_cluster_check = Doubtfire::Application.config.jplag_skip_cluster_check skip_cluster_string = skip_cluster_check ? '--cluster-skip' : '' + max_shown_comparisons = Doubtfire::Application.config.jplag_max_shown_comparisons + max_shown_comparisons = 2500 if max_shown_comparisons.nil? + # Run JPLAG on the extracted files. JPlag container should already be in the /jplag/ workdir. - docker_command = "docker exec -i jplag java -jar jplag-jar-with-dependencies.jar --skip-version-check #{tasks_dir_split}/submissions #{base_code_string} -l #{file_lang} --similarity-threshold=#{similarity_threshold} #{min_token_string} #{skip_cluster_string} -M RUN -r #{results_dir}/#{task_definition.abbreviation}-result --overwrite" + docker_command = [ + "docker exec -i jplag", + "java -jar jplag-jar-with-dependencies.jar", + "--skip-version-check", + "#{tasks_dir_split}/submissions", + base_code_string, + "-l #{file_lang}", + "--similarity-threshold=#{similarity_threshold}", + "--shown-comparisons=#{max_shown_comparisons}", + min_token_string, + skip_cluster_string, + "-M RUN", + "-r #{report_path.delete_suffix('.jplag')}", + "--overwrite" + ].join(" ") + logger.debug "Executing command: #{docker_command}" system(docker_command) diff --git a/app/models/submission_history.rb b/app/models/submission_history.rb new file mode 100644 index 0000000000..d1f1faecc9 --- /dev/null +++ b/app/models/submission_history.rb @@ -0,0 +1,204 @@ +require 'zip' +require 'stringio' + +class SubmissionHistory < ApplicationRecord + belongs_to :task, optional: false + has_one :overseer_assessment, dependent: :destroy + + validates :submission_timestamp, presence: true, uniqueness: { scope: :task_id } + + after_destroy :delete_associated_files + + def self.enabled_requirements(task) + task.upload_requirements.each_index.select do |index| + task.upload_requirements[index]['submission_history'] == true + end + end + + def self.create_archive!(task, submission_timestamp) + if exists?(task: task, submission_timestamp: submission_timestamp.to_s) + raise ActiveRecord::RecordNotUnique, 'Submission history already exists for this task and timestamp' + end + + enabled_indexes = enabled_requirements(task) + raise 'No upload requirements are enabled for submission history' if enabled_indexes.empty? + + source_path = FileHelper.zip_file_path_for_done_task(task) + raise "Submission file not found: #{source_path}" unless File.exist?(source_path) + + history = new(task: task, submission_timestamp: submission_timestamp.to_s) + FileUtils.mkdir_p(history.output_path) + + temporary_history = "#{history.archive_file_name}.tmp-#{SecureRandom.hex(8)}" + copied_files = 0 + archive_updated = false + begin + Zip::File.open(temporary_history, create: true) do |destination| + copy_archive_entries(history.archive_file_name, destination) if File.exist?(history.archive_file_name) + + Zip::File.open(source_path) do |source| + source.each do |entry| + next if entry.name_is_directory? + + file_name = entry.name.split('/').last + next unless file_name&.match?(/^\d{3}-(?:document|code|image|zip|archive)/) + upload_index = file_name.to_i + next unless enabled_indexes.include?(upload_index) + + preview_path = document_preview_path(task, upload_index) + if file_name.match?(/^\d{3}-document.*\.docx\z/i) && File.exist?(preview_path) + preview_entry_name = File.join(history.entry_prefix, entry.name.sub(/\.docx\z/i, '.pdf')) + destination.get_output_stream(preview_entry_name) do |output| + File.open(preview_path, 'rb') { |input| IO.copy_stream(input, output) } + end + else + destination.get_output_stream(File.join(history.entry_prefix, entry.name)) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + copied_files += 1 + end + end + end + + raise 'No selected submission files were found in the completed submission' if copied_files.zero? + + FileUtils.mv(temporary_history, history.archive_file_name) + archive_updated = true + system('chmod', 'o+w', history.output_path) + history.save! + history + ensure + FileUtils.rm_f(temporary_history) + history.delete_associated_files if archive_updated && !history.persisted? + if !history.persisted? && Dir.exist?(history.output_path) && Dir.empty?(history.output_path) + FileUtils.rm_rf(history.output_path) + end + end + end + + def output_path + FileHelper.task_submission_identifier_path(:done, task) + end + + def archive_file_name + File.join(output_path, 'history.zip') + end + + def entry_prefix + "#{FileHelper.sanitized_path(submission_timestamp.to_s)}/" + end + + def submission_entry_prefix + File.join(entry_prefix, task.id.to_s, '/') + end + + # Kept for Overseer compatibility; submissions are entries within this archive. + def submission_zip_file_name + archive_file_name + end + + def submission_zip_data + buffer = Zip::OutputStream.write_buffer do |output| + Zip::File.open(archive_file_name) do |archive| + submission_entries(archive).each do |entry| + output.put_next_entry(entry.name.delete_prefix(entry_prefix)) + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + end + + buffer.string + end + + def has_submission_files? # rubocop:disable Naming/PredicateName + return false unless File.exist?(archive_file_name) + + Zip::File.open(archive_file_name) { |archive| submission_entries(archive).any? } + rescue Zip::Error + false + end + + def delete_associated_files + return unless File.exist?(archive_file_name) + + temporary_history = "#{archive_file_name}.tmp-#{SecureRandom.hex(8)}" + + Zip::File.open(temporary_history, create: true) do |destination| + self.class.copy_archive_entries(archive_file_name, destination, excluding_prefix: entry_prefix) + end + + if Zip::File.open(temporary_history) { |archive| archive.entries.empty? } + FileUtils.rm_f(archive_file_name) + FileUtils.rm_f(temporary_history) + FileUtils.rm_rf(output_path) if Dir.empty?(output_path) + else + FileUtils.mv(temporary_history, archive_file_name) + end + ensure + FileUtils.rm_f(temporary_history) if temporary_history + end + + def submission_entries(archive) + archive.entries.reject(&:name_is_directory?).select do |entry| + entry.name.start_with?(submission_entry_prefix) + end + end + + def self.copy_archive_entries(source_path, destination, excluding_prefix: nil) + Zip::File.open(source_path) do |source| + source.each do |entry| + next if excluding_prefix && entry.name.start_with?(excluding_prefix) + + if entry.name_is_directory? + destination.mkdir(entry.name) unless destination.find_entry(entry.name) + else + destination.get_output_stream(entry.name) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + end + end + end + + def self.pending_marker_path(task) + File.join(FileHelper.task_submission_identifier_path(:pending, task), 'submission-history') + end + + def self.document_preview_dir(task) + File.join(FileHelper.task_submission_identifier_path(:pending, task), 'document-previews') + end + + def self.document_preview_path(task, upload_index) + File.join(document_preview_dir(task), "#{upload_index.to_s.rjust(3, '0')}-document.pdf") + end + + def self.stage_document_preview!(task, upload_index, source_path) + raise "Converted document preview was not found: #{source_path}" unless File.exist?(source_path) + + FileUtils.mkdir_p(document_preview_dir(task)) + FileUtils.cp(source_path, document_preview_path(task, upload_index)) + end + + def self.clear_document_previews(task) + FileUtils.rm_rf(document_preview_dir(task)) + end + + def self.mark_pending(task) + marker_path = pending_marker_path(task) + FileUtils.mkdir_p(File.dirname(marker_path)) + FileUtils.touch(marker_path) + end + + def self.clear_pending(task) + FileUtils.rm_f(pending_marker_path(task)) + clear_document_previews(task) + + pending_dir = FileHelper.task_submission_identifier_path(:pending, task) + FileUtils.rm_rf(pending_dir) if Dir.exist?(pending_dir) && Dir.empty?(pending_dir) + end + + def self.pending?(task) + File.exist?(pending_marker_path(task)) + end +end diff --git a/app/models/task.rb b/app/models/task.rb index ee3e76564c..9a535b5a13 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -6,6 +6,8 @@ class Task < ApplicationRecord include ApplicationHelper include GradeHelper + attr_accessor :discussion_confirmed_for_transition + # # Permissions around task data # @@ -134,6 +136,7 @@ def specific_permission_hash(role, perm_hash, _other) has_many :reverse_moss_similarities, class_name: 'MossTaskSimilarity', dependent: :destroy, inverse_of: :other_task, foreign_key: 'other_task_id' has_many :task_engagements, dependent: :destroy has_many :task_submissions, dependent: :destroy + has_many :submission_histories, dependent: :destroy has_many :overseer_assessments, dependent: :destroy has_many :tii_submissions, dependent: :destroy has_many :test_attempts, dependent: :destroy @@ -146,6 +149,7 @@ def specific_permission_hash(role, perm_hash, _other) delegate :target_date, to: :task_definition delegate :update_task_stats, to: :project + before_save :set_discuss_timeout_tracking, if: :will_save_change_to_task_status_id? after_update :update_task_stats, if: :saved_change_to_task_status_id? # TODO: consider moving to async task validates :task_definition_id, uniqueness: { scope: :project, @@ -169,7 +173,8 @@ def prevent_complete_if_assess_in_portfolio_only end def prevent_complete_if_requires_discussion - if task_definition&.requires_discussion && task_status == TaskStatus.complete && !has_discussed_in_class_comment? + if task_definition&.requires_discussion && task_status == TaskStatus.complete && + !has_discussed_in_class_comment? && !discussion_confirmed_for_transition errors.add(:task_status, "cannot be 'complete' until task has been discussed in class") end end @@ -259,6 +264,12 @@ def comments_for_user(user) ) end + def set_discuss_timeout_tracking + self.moved_to_discuss_at = task_status_id == TaskStatus.discuss.id ? Time.zone.now : nil + self.notified_discuss_warning_at = nil + self.notified_discuss_expiry_at = nil + end + def current_task_similarities task_similarities.where(dismissed: false) end @@ -408,16 +419,90 @@ def due_date return extension_date end + def local_due_date + if unit.allow_flexible_dates + return target_due_date if target_due_date.present? + + grade_target_date = task_definition.grade_target_date(project.target_grade) + return grade_target_date if grade_target_date.present? + end + + due_date + end + + def local_start_date + if unit.allow_flexible_dates + return target_start_date if target_start_date.present? + + grade_start_date = task_definition.grade_start_date(project.target_grade) + return grade_start_date if grade_start_date.present? + end + + return task_definition.start_date + extensions.weeks if extensions.negative? + + task_definition.start_date + end + + def days_awaiting_feedback(now_time = Time.zone.now) + return 0 if submission_date.blank? + + submission_time = submission_date.to_f + current_time = now_time.to_f + return 0 if current_time <= submission_time + + teaching_breaks = unit&.teaching_period&.breaks || [] + paused_seconds = break_overlap_seconds(submission_time, current_time, teaching_breaks) + + ([0, current_time - submission_time - paused_seconds].max / 1.day).floor + end + + def discuss_timeout_elapsed_days(now_time = Time.zone.now, teaching_breaks: nil) + return 0 if moved_to_discuss_at.blank? + + discussion_time = moved_to_discuss_at.to_f + current_time = now_time.to_f + return 0 if current_time <= discussion_time + + teaching_breaks ||= unit&.teaching_period&.breaks || [] + paused_seconds = break_overlap_seconds(discussion_time, current_time, teaching_breaks) + + ([0, current_time - discussion_time - paused_seconds].max / 1.day).floor + end + + def discuss_timeout_expiry_at(timeout_days = unit.discuss_timeout_expire_days, teaching_breaks: nil) + return nil if moved_to_discuss_at.blank? + + deadline = moved_to_discuss_at + timeout_days.days + teaching_breaks ||= unit&.teaching_period&.breaks || [] + + teaching_breaks.sort_by(&:start_date).each do |teaching_break| + break_start = teaching_break.start_date + break_end = break_start + teaching_break.number_of_weeks.to_i.weeks + next if break_end <= moved_to_discuss_at || break_start >= deadline + + deadline += break_end - [break_start, moved_to_discuss_at].max + end + + deadline + end + + # Excludes any breaks that would otherwise "pause" feedback + def calendar_days_awaiting_feedback(now_time = Time.zone.now) + return 0 if submission_date.blank? + + [0, (now_time.to_date - submission_date.to_date).to_i].max + end + def complete? status == :complete end def discuss_or_demonstrate? - [:discuss, :demonstrate].include?(status) + [:discuss, :rediscuss, :demonstrate].include?(status) end def discuss? - status == :discuss + [:discuss, :rediscuss].include?(status) end def demonstrate? @@ -437,7 +522,7 @@ def ready_for_feedback? end def ready_or_complete? - [:complete, :discuss, :demonstrate, :ready_for_feedback, :assess_in_portfolio].include? status + [:complete, :discuss, :rediscuss, :demonstrate, :ready_for_feedback, :assess_in_portfolio].include? status end def submitted_status? @@ -507,6 +592,25 @@ def active_overflow_task_claim claim end + def transition_assignment_allowed?(by_user, system_transition) + return true if system_transition + + if task_definition.lock_assessments_to_tutorial_stream + unit_role = unit.unit_role_for(by_user) + return false unless task_definition.tutorial_stream.tutorials.any? { |tutorial| tutorial.unit_role == unit_role } + end + + claim = active_overflow_task_claim + return true if claim.blank? + + unit_role = unit.unit_role_for(by_user) + unit_role.nil? || unit_role.id == claim.claimed_by_unit_role_id + end + + def transition_feedback_check_required?(check_feedback, system_transition) + check_feedback && !system_transition + end + def group return nil unless group_task? @@ -522,11 +626,13 @@ def ensured_group_submission group.create_submission self, '', group.projects.map { |proj| { project: proj, pct: 100 / group.projects.count } } end - def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: false, quality: 1) + def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: false, quality: 1, recursive_fix: false, + check_feedback: false, system_transition: false) # # Ensure that assessor is allowed to update the task in the indicated way # role = role_for(by_user) + role = :tutor if system_transition && by_user.present? return nil if role.nil? @@ -541,20 +647,7 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: # Protect closed states from student changes return nil if [:student, :group_member].include?(role) && task_submission_closed? - if task_definition.lock_assessments_to_tutorial_stream - unit_role = unit.unit_role_for(by_user) - tutorial_stream = task_definition.tutorial_stream - tutorials = tutorial_stream.tutorials - return nil unless tutorials.any? { |t| t.unit_role == unit_role } - end - - # Check to see if another tutor has claimed this task from overflow - if active_overflow_task_claim - unit_role = unit.unit_role_for(by_user) - if unit_role && unit_role.id != active_overflow_task_claim.claimed_by_unit_role_id - return nil - end - end + return nil unless transition_assignment_allowed?(by_user, system_transition) # # State transitions based upon the trigger # @@ -572,10 +665,28 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: else # Only tutors can perform these actions if role == :tutor - if status == TaskStatus.complete && task_definition.requires_discussion && !has_discussed_in_class_comment? + if status == TaskStatus.complete && task_definition.requires_discussion && + !has_discussed_in_class_comment? && !discussion_confirmed_for_transition + return nil + end + + if status == TaskStatus.rediscuss && task_status != TaskStatus.discuss return nil end + if transition_feedback_check_required?(check_feedback, system_transition) + if status == TaskStatus.complete && !has_manual_feedback_since_first_ready_for_feedback? + errors.add(:task_status, "cannot be moved to '#{status.name}' until feedback has been given") + return nil + end + + if [TaskStatus.fix_and_resubmit, TaskStatus.redo].include?(status) && + !has_recent_manual_feedback_from_tutor?(by_user) + errors.add(:task_status, "cannot be moved to '#{status.name}' until feedback has been given") + return nil + end + end + if task_definition.assess_in_portfolio_only # Block assess_in_portfolio_only tasks from being signed off as complete if status == TaskStatus.complete @@ -585,7 +696,7 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: # Can only be graded if task_def is not assess_in_portfolio_only if task_definition.max_quality_pts > 0 case status - when TaskStatus.complete, TaskStatus.discuss, TaskStatus.demonstrate, TaskStatus.attention_required + when TaskStatus.complete, TaskStatus.discuss, TaskStatus.rediscuss, TaskStatus.demonstrate, TaskStatus.attention_required update(quality_pts: quality) end end @@ -594,7 +705,7 @@ def trigger_transition(trigger: '', by_user: nil, bulk: false, group_transition: lc = comments.last # Prevent duplicate status comments during feedback unless lc && lc.user == by_user && lc.comment == status.name && (lc.content_type != 'status' || lc.task_status == status) - assess status, by_user + assess status, by_user, Time.zone.now, recursive_fix # Add a status comment for new assessments - only recorded on submitter's task in groups add_status_comment(by_user, status) @@ -620,6 +731,30 @@ def has_discussed_in_class_comment? comments.where(content_type: 'discussed_in_class').exists? end + def has_manual_feedback_since_first_ready_for_feedback? + first_ready_for_feedback_at = comments + .where(content_type: 'status', task_status_id: TaskStatus.ready_for_feedback.id) + .order(:created_at) + .pick(:created_at) + + feedback_comments = comments + .where(content_type: %w[text audio image pdf discussion]) + .where(user_id: unit.staff.select(:user_id)) + + feedback_comments = feedback_comments.where('created_at >= ?', first_ready_for_feedback_at) if first_ready_for_feedback_at + + feedback_comments.where.not("COALESCE(comment, '') LIKE ?", '**Automated Message:%').exists? + end + + def has_recent_manual_feedback_from_tutor?(tutor) + comments + .where(content_type: %w[text audio image pdf discussion]) + .where(user: tutor) + .where('created_at >= ?', 10.minutes.ago) + .where.not("COALESCE(comment, '') LIKE ?", '**Automated Message:%') + .exists? + end + def grade_desc grade_for(grade) end @@ -633,13 +768,9 @@ def grade_task(new_grade, ui = nil, grading_group = false) raise message end - grade_map = { - 'f' => -1, - 'p' => 0, - 'c' => 1, - 'd' => 2, - 'hd' => 3 - } + grade_map = unit.grade_definitions.to_h do |definition| + [definition['abbreviation'].downcase, definition['value']] + end if task_definition.is_graded if new_grade.nil? raise_error.call("No grade was supplied for a graded task (task id #{id})") @@ -651,13 +782,16 @@ def grade_task(new_grade, ui = nil, grading_group = false) if new_grade.is_a?(String) if grade_map.keys.include?(new_grade.downcase) # convert string representation to integer representation - new_grade = grade_map[new_grade] + new_grade = grade_map[new_grade.downcase] else - raise_error.call("New grade supplied to task is not a valid string - expects one of {f|p|c|d|hd} (task id #{id})") + raise_error.call("New grade supplied to task is not a valid abbreviation (task id #{id})") end end - unless new_grade.is_a?(Integer) && grade_map.values.include?(new_grade.to_i) - raise_error.call("New grade supplied to task is not a valid integer - expects one of {-1|0|1|2|3} (task id #{id})") + unless new_grade.is_a?(Integer) + raise_error.call("New grade supplied to task is not a valid integer (task id #{id})") + end + unless unit.assessment_grade_value?(new_grade) + raise_error.call("Grade is not enabled for this unit (task id #{id})") end # propagate new grade to all OTHER group members if group_task? && !grading_group @@ -674,7 +808,7 @@ def grade_task(new_grade, ui = nil, grading_group = false) end end - def assess(task_status, assessor, assess_date = Time.zone.now) + def assess(task_status, assessor, assess_date = Time.zone.now, recursive_fix = false) # Set the task's status to the assessment outcome status # and flag it as no longer awaiting signoff self.task_status = task_status @@ -697,7 +831,7 @@ def assess(task_status, assessor, assess_date = Time.zone.now) # Grant an extension on fix if due date is within 1 week case task_status - when TaskStatus.fix_and_resubmit, TaskStatus.discuss, TaskStatus.demonstrate + when TaskStatus.fix_and_resubmit, TaskStatus.discuss, TaskStatus.rediscuss, TaskStatus.demonstrate if to_same_day_anywhere_on_earth(due_date) < Time.zone.now + 7.days && can_apply_for_extension? && unit.extension_weeks_on_resubmit_request > 0 grant_extension(assessor, unit.extension_weeks_on_resubmit_request) end @@ -725,7 +859,7 @@ def assess(task_status, assessor, assess_date = Time.zone.now) end end - if task_status == TaskStatus.fix_and_resubmit + if task_status == TaskStatus.fix_and_resubmit && recursive_fix # Look for other submitted tasks from this student that has this task as a prerequisite # If they are ready for feedback, automatically assess them to fix and resubmit dependents = TaskPrerequisite.where(prerequisite_id: task_definition.id) @@ -738,7 +872,7 @@ def assess(task_status, assessor, assess_date = Time.zone.now) next unless task.task_status == TaskStatus.ready_for_feedback # Since we are calling this assess method again, we recursively check for more dependent tasks that need to be updated - task.assess(TaskStatus.fix_and_resubmit, assessor, assess_date) + task.assess(TaskStatus.fix_and_resubmit, assessor, assess_date, recursive_fix) task.add_status_comment(assessor, TaskStatus.fix_and_resubmit) task.add_text_comment(assessor, "**Automated comment**: A prerequisite task was updated to Fix and Resubmit, so this task was updated as well. You may need to review and update the prerequisite before resubmitting.") end @@ -901,6 +1035,20 @@ def add_discussed_comment(current_user) discussed end + def add_discuss_timeout_comment(current_user, content_type, text) + return nil unless individual_task_or_submitter_of_group_task? + + comment = DiscussTimeoutComment.create + comment.task = self + comment.user = current_user + comment.comment = text + comment.content_type = content_type + comment.recipient = project.student + comment.save! + + comment + end + def add_checked_in_comment(current_user) discussed = TaskCheckedInComment.create discussed.task = self @@ -1082,7 +1230,7 @@ def compress_new_to_done(task_dir: student_work_dir(:new, false), zip_file_path: FileUtils.rm("#{task_dir}#{img}") unless dest_file == "#{task_dir}#{img}" end - input_files = Dir.entries(task_dir).select { |f| (f =~ /^\d{3}.(cover|document|code|image)/) == 0 } + input_files = Dir.entries(task_dir).select { |f| (f =~ /^\d{3}.(cover|document|code|image|zip|archive)/) == 0 } if input_files.length != task_definition.number_of_uploaded_files logger.error "Error processing task #{log_details} - missing files expected #{task_definition.number_of_uploaded_files} got #{input_files.length}" @@ -1270,6 +1418,7 @@ class TaskAppController < ApplicationController attr_accessor :base_path attr_accessor :image_path attr_accessor :include_pax + attr_accessor :submitted_files_url def init(task, is_retry) @task = task @@ -1279,7 +1428,13 @@ def init(task, is_retry) @institution_name = Doubtfire::Application.config.institution[:name] @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name] @include_pax = !is_retry - @work_id = "task-#{task.id}-#{Time.now.to_i}-#{Process.pid}-#{Thread.current.object_id}#{'-retry' if is_retry}" + @work_id = FileHelper.sanitized_path( + "task-#{Time.current.strftime('%Y%m%d-%H%M')}-#{task.project.student.username}-#{task.task_definition.abbreviation}-#{task.id}-#{Process.pid}#{'-retry' if is_retry}" + ) + host = Doubtfire::Application.config.institution[:host].to_s + host = "http://#{host}" unless host.match?(%r{\Ahttps?://}) + host = host.sub(%r{/*\z}, '') + @submitted_files_url = "#{host}/projects/#{task.project.id}/task_def_id/#{task.task_definition.id}/submission_files/download" end def make_pdf @@ -1393,6 +1548,8 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end begin + converted_word_documents = convert_word_documents_to_pdf + tac = TaskAppController.new tac.init(self, false) @@ -1430,7 +1587,7 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end end - raise LatexError.new(log_message), 'Failed to convert your submission to PDF. Check code files submitted for invalid characters, that documents are valid pdfs, and that images are valid.' + raise LatexError.new(log_message), 'Failed to convert your submission to PDF. Check code files submitted for invalid characters, that documents are valid PDFs or DOCX files, images are valid, and zip files are valid.' end end @@ -1451,9 +1608,11 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end end + stage_word_document_previews(converted_word_documents) save return true rescue => e + SubmissionHistory.clear_document_previews(self) trigger_transition trigger: 'fix', by_user: project.tutor_for(task_definition) add_text_comment project.tutor_for(task_definition), "**Automated Comment**: Something went wrong with your submission. Check the files and resubmit this task. #{e.message}" raise e @@ -1465,6 +1624,38 @@ def convert_submission_to_pdf(source_folder: FileHelper.student_work_dir(:new), end end + def convert_word_documents_to_pdf + in_process_dir = student_work_dir(:in_process, false) + return [] unless Dir.exist?(in_process_dir) + + converted_documents = [] + + Dir.glob(File.join(in_process_dir, '*-document.*')).each do |source_path| + next unless FileHelper.word_document?(source_path) + + upload_index = File.basename(source_path).to_i + destination_path = source_path.sub(/\.docx\z/i, '.pdf') + FileHelper.convert_word_document_to_pdf( + source_path, + destination_path, + work_id: "task-#{id}-#{SecureRandom.uuid}" + ) + FileUtils.rm_f(source_path) + converted_documents << { upload_index: upload_index, path: destination_path } + end + + converted_documents + end + + def stage_word_document_previews(converted_documents) + converted_documents.each do |document| + requirement = upload_requirements[document[:upload_index]] + next unless requirement&.dig('submission_history') == true + + SubmissionHistory.stage_document_preview!(self, document[:upload_index], document[:path]) + end + end + # # The student has uploaded new work... # @@ -1506,11 +1697,21 @@ def create_submission_and_trigger_state_change(user, propagate = true, contribut # Checks to make sure that the files match what we expect # def accept_submission(current_user, files, ui, contributions, trigger, alignments, accepted_tii_eula: false, test_submission: false) + submission_lock_target.with_lock do # Ensure there is not a submission already in process if processing_pdf? ui.error!({ 'error' => 'A submission is already being processed. Please wait for the current submission process to complete.' }, 403) end + if SubmissionHistory.pending?(self) + ui.error!({ 'error' => 'Submission history is still being created. Please wait before submitting again.' }, 403) + end + + if !test_submission && (overseer_enabled? || task_definition.assessment_enabled) && + overseer_assessments.where(status: OverseerAssessment.statuses[:pre_queued]).exists? + ui.error!({ 'error' => 'A submission is already waiting for automated feedback. Please wait for the current Overseer job to complete before submitting again.' }, 403) + end + # Ensure all of the files are present if files.nil? || files.length != task_definition.number_of_uploaded_files ui.error!({ 'error' => 'Some files are missing from the submission upload' }, 403) @@ -1605,6 +1806,11 @@ def accept_submission(current_user, files, ui, contributions, trigger, alignment # Trigger processing of new submission - async AcceptSubmissionJob.perform_async(id, current_user.id, accepted_tii_eula, test_submission) + end + end + + def submission_lock_target + group_task? ? group : self end # The name that should be used for the uploaded file (based on index of upload requirements) @@ -1695,6 +1901,21 @@ def mark_as_moderated(moderation_type: :random_sample) private + def break_overlap_seconds(start_time, end_time, teaching_breaks) + teaching_breaks.sum do |teaching_break| + break_start = teaching_break.start_date.to_f + break_duration = teaching_break.number_of_weeks.to_i.weeks + break_end = break_start + break_duration + + next 0 unless break_start.finite? && break_duration.positive? + + overlap_start = [start_time, break_start].max + overlap_end = [end_time, break_end].min + + [0, overlap_end - overlap_start].max + end + end + def delete_associated_files if group_submission && group_submission.tasks.count <= 1 group_submission.destroy diff --git a/app/models/task_completion_snapshot.rb b/app/models/task_completion_snapshot.rb new file mode 100644 index 0000000000..ee6ae30a00 --- /dev/null +++ b/app/models/task_completion_snapshot.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +require 'csv' +require 'zip' + +class TaskCompletionSnapshot < ApplicationRecord + include FileHelper + + belongs_to :unit + + validates :snapshot_timestamp, presence: true + validates :snapshot_timestamp, uniqueness: { scope: :unit_id } + + after_destroy :delete_snapshot_file + + def snapshot_file_path + return nil if unit.blank? + FileHelper.unit_task_status_snapshot_path(unit, create: true) + end + + def snapshot_contents + file_path = snapshot_file_path + return nil if file_path.blank? + if File.exist?(file_path) + return read_csv_from_zip(file_path, snapshot_timestamp) + end + nil + rescue Zip::Error + nil + end + + def snapshot_date + return nil if snapshot_timestamp.blank? + + snapshot_time.to_date + end + + def snapshot_time + return nil if snapshot_timestamp.blank? + + Time.zone.at(snapshot_timestamp.to_i) + end + + def load_stats + snapshot_contents = self.snapshot_contents + + return {} if snapshot_contents.blank? + + parse_csv_stats(snapshot_contents) + rescue CSV::MalformedCSVError + {} + end + + def store_stats!(payload) + file_path = snapshot_file_path + raise 'Cannot store stats without a unit' if file_path.blank? + + FileUtils.mkdir_p(File.dirname(file_path)) + + csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp) + raise 'Cannot store stats without a valid snapshot timestamp' if csv_filename.blank? + + tmp_path = "#{file_path}.tmp" + + # Read existing zip entries (if file exists) + existing_entries = {} + if File.exist?(file_path) + Zip::File.open(file_path) do |zip_file| + zip_file.each do |entry| + next if entry.directory? + existing_entries[entry.name] = entry.get_input_stream.read + end + end + end + + # Update or add the current snapshot entry + existing_entries[csv_filename] = payload.to_s + + # Write the zip file with all entries + Zip::OutputStream.open(tmp_path) do |zip| + existing_entries.each do |filename, content| + zip.put_next_entry(filename) + zip.write(content) + end + end + + FileUtils.mv(tmp_path, file_path) + ensure + FileUtils.rm_f(tmp_path) if defined?(tmp_path) && tmp_path + end + + private + + def parse_csv_stats(csv_text) + csv = CSV.parse(csv_text, headers: true) + return {} if csv.empty? + + stream_headers = unit.tutorial_streams.pluck(:abbreviation) + stream_headers = ['Tutorial'] if stream_headers.empty? + task_definitions = unit.task_definitions_by_grade + + stats = Hash.new { |hash, key| hash[key] = Hash.new { |tutorial_hash, tutorial_key| tutorial_hash[tutorial_key] = Hash.new { |task_hash, task_key| task_hash[task_key] = Hash.new(0) } } } + + csv.each do |row| + campus_abbreviation = row['Campus'].to_s.strip + next if campus_abbreviation.blank? + + campus_name = Campus.find_by(abbreviation: campus_abbreviation)&.name || campus_abbreviation + + stream_headers.each do |stream_header| + tutorial_name = row[stream_header].to_s.strip + next if tutorial_name.blank? + + task_definitions.each do |task_definition| + status_value = row[task_definition.abbreviation].to_s.strip + status_key = TaskStatus.id_to_key(status_value.to_i) || :not_started + stats[campus_name][tutorial_name][task_definition.abbreviation][status_key.to_s] += 1 + end + end + end + + stats + end + + def read_csv_from_zip(zip_path, snapshot_timestamp) + csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp) + Zip::File.open(zip_path) do |zip_file| + entry = zip_file.find_entry(csv_filename) + return nil if entry.nil? + + entry.get_input_stream.read + end + end + + def delete_snapshot_file + return if snapshot_timestamp.blank? + + file_path = snapshot_file_path + return if file_path.blank? || !File.exist?(file_path) + + csv_filename = FileHelper.snapshot_csv_filename(snapshot_timestamp) + return if csv_filename.blank? + + tmp_path = "#{file_path}.tmp" + + begin + # Read existing zip entries excluding the one we want to delete + remaining_entries = {} + Zip::File.open(file_path) do |zip_file| + zip_file.each do |entry| + next if entry.directory? + next if entry.name == csv_filename + remaining_entries[entry.name] = entry.get_input_stream.read + end + end + + if remaining_entries.empty? + # If no entries left, just delete the zip file + FileUtils.rm_f(file_path) + else + # Write the zip file with remaining entries + Zip::OutputStream.open(tmp_path) do |zip| + remaining_entries.each do |filename, content| + zip.put_next_entry(filename) + zip.write(content) + end + end + FileUtils.mv(tmp_path, file_path) + end + rescue StandardError => e + # If anything goes wrong with zip operations, just clean up and log + logger.error("Error managing snapshot zip file: #{e.message}") + FileUtils.rm_f(tmp_path) + end + end +end diff --git a/app/models/task_definition.rb b/app/models/task_definition.rb index 8d2fa7c5cd..afa22855ee 100644 --- a/app/models/task_definition.rb +++ b/app/models/task_definition.rb @@ -55,6 +55,8 @@ def self.permissions delegate :role_for, to: :unit + before_validation :normalize_upload_requirement_keys + before_destroy :ensure_not_used_as_prerequisite, prepend: true before_destroy :delete_associated_files after_update :move_files_on_abbreviation_change, if: :saved_change_to_abbreviation? @@ -93,10 +95,12 @@ def self.permissions validates :name, uniqueness: { scope: :unit_id } # task definition names within a unit must be unique validates :abbreviation, uniqueness: { scope: :unit_id } # task definition names within a unit must be unique - validates :target_grade, inclusion: { in: GradeHelper::RANGE, message: '%{value} is not a valid target grade' } + validates :target_grade, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validate :target_grade_enabled_for_unit validates :max_quality_pts, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 100, message: 'must be between 0 and 100' } validate :upload_requirements, :check_upload_requirements_format + validate :submission_history_required_for_overseer validates :description, length: { maximum: 4095, allow_blank: true } @@ -113,36 +117,49 @@ def self.permissions include TaskDefinitionTiiModule include TaskDefinitionSimilarityModule - # def p_target_date - # due_date - # end + def grade_due_date_overrides + grade_due_dates.map do |override| + { + target_grade: override.target_grade, + target_due_date: override.target_due_date, + start_date: override.start_date + } + end + end - # Per-grade target date overrides + def content_link + return @content_link if defined?(@content_link) - def c_target_date - grade_due_dates.find { |g| g.target_grade == 1 }&.target_due_date + @content_link = unit.unit_content_links.find do |link| + link.context_type == 'task_definition' && link.context_key == abbreviation + end || unit.unit_content_links.find_by(context_type: 'task_definition', context_key: abbreviation) end - def d_target_date - grade_due_dates.find { |g| g.target_grade == 2 }&.target_due_date + def has_content_link? + content_link.present? end - def hd_target_date - grade_due_dates.find { |g| g.target_grade == 3 }&.target_due_date - end + def task_resource_link + return @task_resource_link if defined?(@task_resource_link) - # Per-grade start date overrides + @task_resource_link = unit.unit_content_links.find do |link| + link.context_type == 'task_definition_resource' && link.context_key == abbreviation + end || unit.unit_content_links.find_by( + context_type: 'task_definition_resource', + context_key: abbreviation + ) + end - def c_start_date - grade_due_dates.find { |g| g.target_grade == 1 }&.start_date + def has_task_resource_link? + task_resource_link.present? && task_resource_link.unit_content_site.file?(task_resource_link.route) end - def d_start_date - grade_due_dates.find { |g| g.target_grade == 2 }&.start_date + def grade_target_date(target_grade) + grade_due_dates.find { |g| g.target_grade == target_grade.to_i }&.target_due_date end - def hd_start_date - grade_due_dates.find { |g| g.target_grade == 3 }&.start_date + def grade_start_date(target_grade) + grade_due_dates.find { |g| g.target_grade == target_grade.to_i }&.start_date end def unit_must_be_same @@ -169,6 +186,8 @@ def cant_disable_aip_only_if_aip_tasks_exist def check_existing_prerequisites prereqs = TaskPrerequisite.where(task_definition_id: id) prereqs.each do |dp| + next if dp.prerequisite.nil? + if target_grade < dp.prerequisite.target_grade errors.add(:target_grade, "cannot be lower than prerequisite #{dp.prerequisite.abbreviation}'s target grade") end @@ -176,12 +195,22 @@ def check_existing_prerequisites dependents = TaskPrerequisite.where(prerequisite_id: id) dependents.each do |pr| + next if pr.task_definition.nil? + if target_grade > pr.task_definition.target_grade errors.add(:target_grade, "cannot exceed the target grade #{pr.task_definition.abbreviation} because this is a prerequisite") end end end + def ensure_not_used_as_prerequisite + return if destroyed_by_association&.name == :task_definitions + return unless TaskPrerequisite.exists?(prerequisite_id: id) + + errors.add(:base, "Cannot delete task definition while it is used as a prerequisite. Remove the prerequisite links first.") + throw :abort + end + # In the rollover process, copy this definition into another unit # Copy this task into the other unit def copy_to(other_unit) @@ -216,7 +245,7 @@ def copy_to(other_unit) FileUtils.cp(task_sheet, new_td.task_sheet()) end - if has_task_resources? + if has_uploaded_task_resources? # Copy the task resources, and trigger tii integration if needed new_td.add_task_resources(task_resources, copy: true) end @@ -226,6 +255,11 @@ def copy_to(other_unit) end new_td.save! + overseer_steps.find_each do |step| + new_td.overseer_steps.create!( + step.attributes.except('id', 'task_definition_id', 'created_at', 'updated_at') + ) + end new_td end @@ -278,6 +312,10 @@ def reset_overdue_tasks def move_files_on_abbreviation_change old_abbr = saved_change_to_abbreviation[0] # 0 is original abbreviation + unit.unit_content_links + .where(context_type: %w[task_definition task_definition_resource], context_key: old_abbr) + .find_each { |link| link.update!(context_key: abbreviation) } + if File.exist? task_sheet_with_abbreviation(old_abbr, false) FileUtils.mv(task_sheet_with_abbreviation(old_abbr), task_sheet()) end @@ -335,8 +373,10 @@ def check_upload_requirements_format return end - # Check keys only contain key, type, name, tii_check, and tii_pct - unless req.keys.excluding('key', 'type', 'name', 'tii_check', 'tii_pct').empty? + req['type'] = 'zip' if req['type'] == 'archive' + + # Check keys only contain supported upload requirement settings + unless req.keys.excluding('key', 'type', 'name', 'tii_check', 'tii_pct', 'submission_history').empty? errors.add(:upload_requirements, "has additional values for item #{i + 1} --> #{req.keys.join(' ')}.") end @@ -345,8 +385,8 @@ def check_upload_requirements_format errors.add(:upload_requirements, "the name for item #{i + 1} does not seem to be a valid filename --> #{req['name']}.") end - # Check the type is either document or image or code - unless %w(document image code).include? req['type'] + # Check the type is either document, image, code, or zip + unless %w(document image code zip).include? req['type'] errors.add(:upload_requirements, "the type for item #{i + 1} is not valid --> #{req['type']}.") end @@ -360,10 +400,29 @@ def check_upload_requirements_format errors.add(:upload_requirements, "the tii_pct for item #{i + 1} is not a non-negative number --> #{req['tii_pct']}.") end + unless req['submission_history'].blank? || [true, false].include?(req['submission_history']) + errors.add(:upload_requirements, "the submission_history for item #{i + 1} is not a boolean --> #{req['submission_history']}.") + end + i += 1 end end + def normalize_upload_requirement_keys + return unless upload_requirements.is_a?(Array) + return unless upload_requirements.all? { |requirement| requirement.is_a?(Hash) && requirement.key?('key') } + + normalized_requirements = self.class.normalize_upload_requirement_keys(upload_requirements) + self.upload_requirements = normalized_requirements unless normalized_requirements == upload_requirements + end + + def submission_history_required_for_overseer + return unless assessment_enabled? + return if upload_requirements&.any? { |requirement| requirement['submission_history'] == true } + + errors.add(:upload_requirements, 'must include at least one file in submission history when Overseer is enabled') + end + def number_of_uploaded_files upload_requirements.length end @@ -510,7 +569,7 @@ def propogate_date_changes date_diff def to_csv_row TaskDefinition.csv_columns - .reject { |col| [:start_week, :start_day, :target_week, :target_day, :due_week, :due_day, :upload_requirements, :group_set, :tutorial_stream, :assess_in_portfolio_only, :task_prerequisites, :discussion_prompts].include? col} + .reject { |col| [:start_week, :start_day, :target_week, :target_day, :due_week, :due_day, :upload_requirements, :group_set, :tutorial_stream, :assess_in_portfolio_only, :task_prerequisites, :discussion_prompts, :overseer_steps].include? col} .map { |column| attributes[column.to_s] } + [ group_set.nil? ? "" : group_set.name, @@ -535,6 +594,30 @@ def to_csv_row content: prompt.content, priority: prompt.priority } + end.to_json, + overseer_steps.map do |step| + { + name: step.name, + description: step.description, + display_name: step.display_name, + display_description: step.display_description, + run_command: step.run_command, + timeout: step.timeout, + sort_order: step.sort_order, + step_type: step.step_type, + partial_output_diff: step.partial_output_diff, + stdin_input_file: step.stdin_input_file, + expected_output_file: step.expected_output_file, + feedback_message: step.feedback_message, + status_on_success: TaskStatus.find_by(id: step.status_on_success_id)&.status_key, + status_on_failure: TaskStatus.find_by(id: step.status_on_failure_id)&.status_key, + halt_on_success: step.halt_on_success, + halt_on_failure: step.halt_on_failure, + show_expected_output: step.show_expected_output, + show_stdin: step.show_stdin, + show_stdout: step.show_stdout, + enabled: step.enabled + } end.to_json ] # [target_date.strftime('%d-%m-%Y')] + @@ -545,7 +628,11 @@ def self.csv_columns [:name, :abbreviation, :description, :weighting, :target_grade, :restrict_status_updates, :max_quality_pts, :is_graded, :plagiarism_warn_pct, :scorm_enabled, :scorm_allow_review, :scorm_bypass_test, :scorm_time_delay_enabled, :scorm_attempt_limit, :group_set, :upload_requirements, :start_week, :start_day, :target_week, :target_day, - :due_week, :due_day, :tutorial_stream, :assess_in_portfolio_only, :task_prerequisites, :discussion_prompts] + :due_week, :due_day, :tutorial_stream, :assess_in_portfolio_only, :task_prerequisites, :discussion_prompts, :overseer_steps] + end + + def self.required_csv_columns + csv_columns - [:overseer_steps] end def self.task_def_for_csv_row(unit, row) @@ -611,6 +698,7 @@ def self.task_def_for_csv_row(unit, row) end import_discussion_prompts_from_csv_row(result, row) + import_overseer_steps_from_csv_row(result, row) result.assess_in_portfolio_only = %w(Yes y Y yes true TRUE 1).include? "#{row[:assess_in_portfolio_only]}".strip @@ -658,11 +746,52 @@ def self.import_discussion_prompts_from_csv_row(task_definition, row) end end + def self.import_overseer_steps_from_csv_row(task_definition, row) + task_definition.overseer_steps.destroy_all + return if row[:overseer_steps].blank? + + JSON.parse(row[:overseer_steps]).each do |step| + OverseerStep.create!( + task_definition: task_definition, + name: step['name'], + description: step['description'], + display_name: step['display_name'], + display_description: step['display_description'], + run_command: step['run_command'], + timeout: step['timeout'], + sort_order: step['sort_order'], + step_type: step['step_type'], + partial_output_diff: step['partial_output_diff'], + stdin_input_file: step['stdin_input_file'], + expected_output_file: step['expected_output_file'], + feedback_message: step['feedback_message'], + status_on_success_id: status_id_from_csv(step['status_on_success']), + status_on_failure_id: status_id_from_csv(step['status_on_failure']), + halt_on_success: step['halt_on_success'], + halt_on_failure: step['halt_on_failure'], + show_expected_output: step['show_expected_output'], + show_stdin: step['show_stdin'], + show_stdout: step['show_stdout'], + enabled: step.key?('enabled') ? step['enabled'] : true + ) + end + end + + def self.status_id_from_csv(value) + return nil if value.blank? + + TaskStatus.status_for_name(value)&.id || TaskStatus.find_by(id: value.to_i)&.id + end + def is_group_task? !group_set.nil? end def has_task_resources? + has_task_resource_link? || has_uploaded_task_resources? + end + + def has_uploaded_task_resources? File.exist? task_resources(false) end @@ -739,10 +868,10 @@ def add_task_resources(file, copy: false) end def remove_task_resources() - if has_task_resources? + if has_uploaded_task_resources? FileUtils.rm task_resources - tii_group_attachments.destroy_all if tii_checks? + tii_group_attachments.destroy_all if tii_checks? && !has_task_resource_link? end end @@ -840,12 +969,18 @@ def related_tasks_with_files(consolidate_groups = true) # Read a file from the task definition resources. # - # @param filename [String] The name of the file to read from the zipfile. + # @param filename [String] The linked filename or path within the resource zip. # @return [String] The contents of the file, or nil if the file does not exist. def read_file_from_resources(filename) - return nil unless has_task_resources? + linked_resource = linked_task_resource + return nil unless linked_resource || has_uploaded_task_resources? - Zip::File.open(task_resources) do |zip_file| + if linked_resource && !task_resource_zip?(linked_resource) + return filename == linked_resource[:filename] ? File.binread(linked_resource[:path]) : nil + end + + resource_path = linked_resource ? linked_resource[:path] : task_resources + Zip::File.open(resource_path) do |zip_file| entry = zip_file.glob(filename).first return entry.get_input_stream.read if entry end @@ -853,8 +988,24 @@ def read_file_from_resources(filename) nil end + def linked_task_resource + return unless has_task_resource_link? + + task_resource_link.unit_content_site.extract_file(task_resource_link.route) + end + + def task_resource_zip?(resource) + resource.present? && File.extname(resource[:filename]).casecmp('.zip').zero? + end + private + def target_grade_enabled_for_unit + return if unit.nil? || target_grade.nil? || unit.grade_value?(target_grade) + + errors.add(:target_grade, 'is not enabled for this unit') + end + def delete_associated_files() remove_task_sheet() remove_task_resources() diff --git a/app/models/task_status.rb b/app/models/task_status.rb index b4dbac8cd4..bc6eec9b99 100644 --- a/app/models/task_status.rb +++ b/app/models/task_status.rb @@ -72,6 +72,10 @@ def self.attention_required TaskStatus.find(14) end + def self.rediscuss + TaskStatus.find(15) + end + class << self # Provide access to the count from the database via a new db_count method alias_method :db_count, :count @@ -84,7 +88,7 @@ class << self # Keep this hard coded! Saves cache load time. # Important: count must equal the largest id in the database def self.count - 14 + 15 end def self.status_for_name(name) @@ -103,6 +107,8 @@ def self.status_for_name(name) TaskStatus.working_on_it when 'discuss', 'd' TaskStatus.discuss + when 'rediscuss', 're-discuss', 're discuss' + TaskStatus.rediscuss when 'demonstrate', 'demo' TaskStatus.demonstrate when 'ready for feedback', 'ready_for_feedback', 'ready to mark', 'ready_to_mark', 'rtm', 'rff' @@ -142,6 +148,7 @@ def self.id_to_key(id) when 12 then :time_exceeded when 13 then :assess_in_portfolio when 14 then :attention_required + when 15 then :rediscuss else :not_started end end @@ -161,6 +168,7 @@ def status_key return :time_exceeded if self == TaskStatus.time_exceeded return :assess_in_portfolio if self == TaskStatus.assess_in_portfolio return :attention_required if self == TaskStatus.attention_required + return :rediscuss if self == TaskStatus.rediscuss return :not_started end diff --git a/app/models/teaching_period.rb b/app/models/teaching_period.rb index eaf5d6b2c9..6d55ea0e48 100644 --- a/app/models/teaching_period.rb +++ b/app/models/teaching_period.rb @@ -18,6 +18,7 @@ class TeachingPeriod < ApplicationRecord validate :validate_end_date_after_start_date, :validate_active_until_after_end_date after_update :propogate_date_changes + after_update :refresh_communication_schedule_caches, if: :saved_change_to_teaching_dates? # Public methods @@ -115,7 +116,10 @@ def date_for_week_and_day(week, day) start_day_num = start_date.wday - result = week_start + (day_num - start_day_num).days + day_offset = day_num - start_day_num + day_offset += 7 if day_offset.negative? + + result = week_start + day_offset.days for a_break in breaks do if result >= a_break.start_date && result < a_break.end_date @@ -132,6 +136,13 @@ def future_teaching_periods TeachingPeriod.where("start_date > :end_date", end_date: end_date) end + def refresh_communication_schedule_caches + CommunicationSetSchedule + .joins(communication_set: :unit) + .where(units: { teaching_period_id: id }) + .find_each(&:refresh_next_run_at!) + end + private def can_destroy? @@ -160,4 +171,8 @@ def propogate_date_changes u.update(start_date: self.start_date, end_date: self.end_date) end end + + def saved_change_to_teaching_dates? + saved_change_to_start_date? || saved_change_to_end_date? + end end diff --git a/app/models/turn_it_in/task_definition_tii_module.rb b/app/models/turn_it_in/task_definition_tii_module.rb index 8e9d9e3097..0f7340b933 100644 --- a/app/models/turn_it_in/task_definition_tii_module.rb +++ b/app/models/turn_it_in/task_definition_tii_module.rb @@ -37,10 +37,21 @@ def send_group_attachments_to_tii return if tii_group_id.blank? return unless has_task_resources? + linked_resource = linked_task_resource + if linked_resource && !task_resource_zip?(linked_resource) + filename = linked_resource[:filename] + return unless filename.downcase.end_with?('.doc', '.docx') + return if filename.include?('__MACOSX') || File.size(linked_resource[:path]) < 50 + + TiiGroupAttachment.find_or_create_from_task_definition(self, filename) + return + end + count = 0 # loop through files in the task resources zip file - Zip::File.open(task_resources) do |zip_file| + resource_path = linked_resource ? linked_resource[:path] : task_resources + Zip::File.open(resource_path) do |zip_file| zip_file.each do |entry| next unless entry.file? next unless entry.name.end_with?('.doc', '.docx') diff --git a/app/models/unit.rb b/app/models/unit.rb index d31d84fb78..2ebe9c58de 100644 --- a/app/models/unit.rb +++ b/app/models/unit.rb @@ -6,8 +6,17 @@ require 'moss_ruby' require 'csv_helper' require 'grade_helper' +require 'securerandom' class Unit < ApplicationRecord + DEFAULT_GRADE_DEFINITIONS = [ + { 'id' => 'fail', 'value' => -1, 'label' => 'Fail', 'abbreviation' => 'F' }, + { 'id' => 'pass', 'value' => 0, 'label' => 'Pass', 'abbreviation' => 'P' }, + { 'id' => 'credit', 'value' => 1, 'label' => 'Credit', 'abbreviation' => 'C' }, + { 'id' => 'distinction', 'value' => 2, 'label' => 'Distinction', 'abbreviation' => 'D' }, + { 'id' => 'high-distinction', 'value' => 3, 'label' => 'High Distinction', 'abbreviation' => 'HD' } + ].freeze + include ApplicationHelper include FileHelper include MimeCheckHelpers @@ -52,8 +61,10 @@ def self.permissions :add_tutorial, :add_task_def, :provide_feedback, + :provide_bulk_feedback, :change_project_enrolment, :download_stats, + :download_overflow_stats, :download_grades, :download_jplag_report, :rollover_unit, @@ -66,7 +77,12 @@ def self.permissions :get_tutor_times_summary, :get_marking_sessions, :upload_grades_csv, - :get_staff_notes + :upload_staff_notes_csv, + :get_staff_notes, + :capture_task_completion_snapshot, + :manage_unit_content, + :mannage_communications, + :delete_engagement ] # What can admin do with units? @@ -82,6 +98,7 @@ def self.permissions :add_tutorial, :add_task_def, :download_stats, + :download_overflow_stats, :download_unit_csv, :download_grades, :exceed_capacity, @@ -91,7 +108,11 @@ def self.permissions :grant_spec_con, :download_jplag_report, :get_marking_sessions, - :get_staff_notes + :get_staff_notes, + :upload_staff_notes_csv, + :get_tutor_times, + :manage_unit_content, + :mannage_communications, ] # What can auditors do with units? @@ -144,6 +165,7 @@ def role_for(user) after_update :move_files_on_code_change, if: :saved_change_to_code? after_update :propogate_date_changes_to_tasks, if: :saved_change_to_start_date? after_update :update_overdue_tasks_aip, if: :saved_change_to_mark_late_submissions_as_assess_in_portfolio? + after_update :refresh_communication_schedule_caches, if: :saved_change_to_communication_schedule_inputs? # Model associations. # When a Unit is destroyed, any TaskDefinitions, Tutorials, and ProjectConvenor instances will also be destroyed. @@ -155,6 +177,12 @@ def role_for(user) has_many :unit_roles, dependent: :destroy, inverse_of: :unit has_many :learning_outcomes, as: :context, dependent: :destroy # inverse_of: :unit has_many :marking_sessions, dependent: :destroy + has_many :task_completion_snapshots, dependent: :destroy, inverse_of: :unit + has_many :communication_sets, class_name: 'CommunicationSet', dependent: :destroy + has_many :communication_rules, through: :communication_sets, class_name: 'CommunicationRule' + has_many :communication_set_schedules, through: :communication_sets, class_name: 'CommunicationSetSchedule' + has_many :unit_content_sites, dependent: :destroy + has_many :unit_content_links, dependent: :destroy has_many :comments, through: :projects has_many :tasks, through: :projects @@ -194,7 +222,13 @@ def role_for(user) validates :feedback_overflow_threshold_days, numericality: { greater_than_or_equal_to: 0 } + validates :discuss_timeout_warning_days, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :discuss_timeout_expire_days, + numericality: { only_integer: true, greater_than_or_equal_to: 1 } + validate :warning_not_greater_than_overflow + validate :discuss_timeout_warning_before_expiry validate :validate_end_date_after_start_date validate :ensure_teaching_period_dates_match, if: :has_teaching_period? @@ -205,6 +239,8 @@ def role_for(user) validate :autogen_date_within_unit_active_period, if: -> { start_date_changed? || end_date_changed? || teaching_period_id_changed? || portfolio_auto_generation_date_changed? } validate :cant_disable_aip_only_if_aip_tasks_exist + validate :grade_definitions_are_valid + validate :configured_grades_preserve_used_values, if: :will_save_change_to_grade_values? scope :current, -> { current_for_date(Time.zone.now) } scope :current_for_date, ->(date) { where('start_date <= ? AND end_date >= ?', date, date) } @@ -228,6 +264,117 @@ def warning_not_greater_than_overflow ) end + def discuss_timeout_warning_before_expiry + return unless discuss_timeout_enabled + return if discuss_timeout_warning_days < discuss_timeout_expire_days + + errors.add(:discuss_timeout_warning_days, 'must be less than the expiry days') + end + + def self.notify_discuss_timeouts! + set_active.find_each(&:notify_discuss_timeouts!) + end + + def notify_discuss_timeouts! + return 0 unless discuss_timeout_enabled + + teaching_breaks = teaching_period&.breaks.to_a + discuss_timeout_tasks.find_each.sum do |task| + notify_discuss_timeout_for(task, teaching_breaks: teaching_breaks) + end + end + + def discuss_timeout_tasks + tasks + .includes(:project, :task_definition) + .where(task_status_id: TaskStatus.discuss.id) + .where.not(moved_to_discuss_at: nil) + .where('moved_to_discuss_at <= ?', discuss_timeout_warning_days.days.ago) + end + + def notify_discuss_timeout_for(task, teaching_breaks: nil, now_time: Time.zone.now) + return 0 if task.moved_to_discuss_at.blank? + + actor = task.project.tutor_for(task.task_definition) || main_convenor&.user + return 0 if actor.blank? + + elapsed_days = task.discuss_timeout_elapsed_days(now_time, teaching_breaks: teaching_breaks) + if elapsed_days >= discuss_timeout_expire_days + expire_discuss_timeout_task(task, actor) + elsif elapsed_days >= discuss_timeout_warning_days + warn_discuss_timeout_task(task, actor, teaching_breaks: teaching_breaks) + else + 0 + end + end + + def warn_discuss_timeout_task(task, actor, teaching_breaks: nil) + return 0 if task.notified_discuss_warning_at.present? + + expiry_date = discuss_timeout_expiry_date(task, teaching_breaks: teaching_breaks) + created_comment = false + Task.transaction do + comment = task.add_discuss_timeout_comment( + actor, + DiscussTimeoutComment.warning, + "You must discuss this task with your tutor before #{formatted_discuss_timeout_date(expiry_date)}. If it has not been discussed by then, it will move to Fix and Resubmit, and you will need to resubmit your work." + ) + raise ActiveRecord::Rollback if comment.blank? + + task.update!(notified_discuss_warning_at: Time.zone.now) + queue_discuss_timeout_email(task, actor, :approaching, expiry_date) + created_comment = true + end + + created_comment ? 1 : 0 + end + + def expire_discuss_timeout_task(task, actor) + return 0 if task.notified_discuss_expiry_at.present? + + created_comment = false + Task.transaction do + task.update!(notified_discuss_expiry_at: Time.zone.now) + unless task.trigger_transition(trigger: 'fix', by_user: actor, system_transition: true) + raise ActiveRecord::Rollback + end + + comment = task.add_discuss_timeout_comment( + actor, + DiscussTimeoutComment.expired, + "This task moved to Fix and Resubmit because it was not discussed by the deadline. Review any feedback and resubmit it when you are ready." + ) + unless comment + raise ActiveRecord::Rollback + end + + queue_discuss_timeout_email(task, actor, :missed) + created_comment = true + end + + created_comment ? 1 : 0 + end + + def discuss_timeout_expiry_date(task, teaching_breaks: nil) + task + .discuss_timeout_expiry_at(discuss_timeout_expire_days, teaching_breaks: teaching_breaks) + &.to_date + end + + def formatted_discuss_timeout_date(date) + result = "the #{date.day.ordinalize} of #{Date::MONTHNAMES[date.month]}" + return result if date.year == Time.zone.today.year + + "#{result} #{date.year}" + end + + def queue_discuss_timeout_email(task, actor, type, expiry_date = nil) + return unless send_notifications + return unless task.project.student.receive_feedback_notifications + + SendDiscussTimeoutEmailJob.perform_async(task.id, actor.id, type.to_s, expiry_date&.iso8601) + end + def detailed_name "#{name} #{teaching_period.present? ? teaching_period.detailed_name : start_date.strftime('%Y-%m-%d')}" end @@ -236,6 +383,14 @@ def active_projects projects.where(enrolled: true) end + def refresh_communication_schedule_caches + communication_set_schedules.find_each(&:refresh_next_run_at!) + end + + def saved_change_to_communication_schedule_inputs? + saved_change_to_active? || saved_change_to_start_date? || saved_change_to_end_date? + end + def ordered_task_definitions task_definitions.order('start_date ASC, abbreviation ASC') end @@ -294,6 +449,58 @@ def has_teaching_period? self.teaching_period.present? end + def has_main_content_site? + unit_content_sites.exists?(is_main: true) + end + + def grade_values + grade_definitions.filter_map { |definition| definition['value'] unless definition['value'] == -1 } + end + + def grade_definitions + normalize_grade_definitions(self[:grade_values]) + end + + def grade_value?(value) + grade_values.include?(value.to_i) + end + + def assessment_grade_value?(value) + grade_definitions.any? { |definition| definition['value'] == value.to_i } + end + + def grade_definition(value) + return nil if value.nil? + + grade_definitions.find { |definition| definition['value'] == value.to_i } + end + + def grade_label(value) + grade_definition(value)&.fetch('label', nil) || GradeHelper.grade_for(value) + end + + def grade_abbreviation(value) + grade_definition(value)&.fetch('abbreviation', nil) || GradeHelper.short_grade_for(value) + end + + def grade_definitions=(definitions) + normalized = normalize_grade_definitions(definitions, sort: false) + fail_definition = normalized.find { |definition| definition['value'] == -1 } + target_definitions = normalized.reject { |definition| definition['value'] == -1 } + + fail_definition['value'] = -1 + target_definitions.each_with_index { |definition, index| definition['value'] = index } + self[:grade_values] = grade_values_for_column([fail_definition, *target_definitions]) + end + + def grade_values_for_column(definitions) + if self.class.type_for_attribute('grade_values').is_a?(ActiveRecord::Type::Json) + definitions + else + definitions.to_json + end + end + def ensure_teaching_period_dates_match if self[:start_date] != teaching_period.start_date errors.add(:start_date, "should match teaching period date") @@ -308,6 +515,7 @@ def ensure_main_convenor_is_appropriate errors.add(:main_convenor, "must be a staff member from unit") unless id == main_convenor.unit_id errors.add(:main_convenor, "must be configured to administer unit") unless main_convenor.is_convenor? + errors.add(:main_convenor, "cannot be observer only") if main_convenor.observer_only? errors.add(:main_convenor, "must be capable of administering units - ensure user has appropriate permissions (contact admin staff to update)") unless main_convenor_user.has_convenor_capability? end @@ -320,6 +528,79 @@ def cant_disable_aip_only_if_aip_tasks_exist end end + def grade_definitions_are_valid + definitions = grade_definitions + values = definitions.map { |definition| definition['value'] } + target_values = values.reject { |value| value == -1 } + + errors.add(:grade_definitions, 'must include a failure grade and at least one target grade') if definitions.length < 2 + errors.add(:grade_definitions, 'must include index -1 exactly once') unless values.count(-1) == 1 + errors.add(:grade_definitions, 'target grade indexes must be unique non-negative integers') unless target_values.all? { |value| value >= 0 } && target_values.uniq.length == target_values.length + errors.add(:grade_definitions, 'must use unique identifiers') unless definitions.map { |definition| definition['id'] }.uniq.length == definitions.length + errors.add(:grade_definitions, 'must use unique labels') unless definitions.map { |definition| definition['label'].downcase }.uniq.length == definitions.length + errors.add(:grade_definitions, 'must use unique abbreviations') unless definitions.map { |definition| definition['abbreviation'].downcase }.uniq.length == definitions.length + + definitions.each do |definition| + errors.add(:grade_definitions, 'labels must be present and no longer than 50 characters') unless definition['label'].present? && definition['label'].length <= 50 + errors.add(:grade_definitions, 'abbreviations must be present and no longer than 10 characters') unless definition['abbreviation'].present? && definition['abbreviation'].length <= 10 + end + end + + def configured_grades_preserve_used_values + used_values = task_definitions.distinct.pluck(:target_grade) + used_values |= projects.distinct.pluck(:target_grade, :submitted_grade).flatten.compact + used_values |= tasks.distinct.pluck(:grade).compact + used_values |= communication_rules.joins(:communication_conditions) + .pluck('communication_conditions.target_grade', 'communication_conditions.task_target_grade') + .flatten + .compact + used_values |= communication_rules.joins(:communication_actions) + .pluck('communication_actions.target_grade') + .compact + previous_definitions = normalize_grade_definitions(attribute_in_database('grade_values')) + current_by_id = grade_definitions.index_by { |definition| definition['id'] } + + changed_values = used_values.select do |value| + previous_definition = previous_definitions.find { |definition| definition['value'] == value } + current_definition = current_by_id[previous_definition&.fetch('id', nil)] + current_definition.nil? || current_definition['value'] != value + end + + errors.add(:grade_definitions, "cannot remove or reorder grades currently in use at indexes: #{changed_values.uniq.sort.join(', ')}") if changed_values.any? + end + + def normalize_grade_definitions(raw_definitions, sort: true) + raw_definitions = DEFAULT_GRADE_DEFINITIONS if raw_definitions.blank? + raw_definitions = JSON.parse(raw_definitions) if raw_definitions.is_a?(String) + + definitions = Array(raw_definitions).map do |definition| + definition = definition.to_h if definition.respond_to?(:to_h) + if definition.is_a?(Hash) + definition = definition.stringify_keys + { + 'id' => definition['id'].presence || SecureRandom.uuid, + 'value' => definition['value'].to_i, + 'label' => definition['label'].to_s.strip, + 'abbreviation' => definition['abbreviation'].to_s.strip.upcase + } + else + value = definition.to_i + default = DEFAULT_GRADE_DEFINITIONS.find { |item| item['value'] == value } + default&.dup || { + 'id' => "grade-#{value}", + 'value' => value, + 'label' => "Grade #{value}", + 'abbreviation' => "G#{value}" + } + end + end + + unless definitions.any? { |definition| definition['value'] == -1 } + definitions.unshift(DEFAULT_GRADE_DEFINITIONS.first.dup) + end + + sort ? definitions.sort_by { |definition| definition['value'] } : definitions + end def validate_end_date_after_start_date if end_date.present? && start_date.present? && end_date < start_date @@ -433,23 +714,26 @@ def rollover(teaching_period, start_date, end_date, new_code) end end + communication_sets.each do |communication_set| + communication_set.copy_to(new_unit) + end + # Now duplicate all feedback chips chip_mapping = {} outcome_mapping.each do |source_outcome, new_outcome| - source_outcome.feedback_chips.each do |chip| + source_outcome.feedback_chips.find_each do |chip| new_chip = chip.dup - new_outcome.feedback_chips << new_chip - new_chip.learning_outcome_id = new_outcome.id - new_chip.parent_chip_id = nil + new_chip.learning_outcome = new_outcome + new_chip.parent_chip = nil new_chip.save! - chip_mapping[chip] = new_chip + chip_mapping[chip.id] = new_chip end source_outcome.feedback_chips.where.not(parent_chip_id: nil).find_each do |old_chip| - child_chip = chip_mapping[old_chip] - parent_chip = chip_mapping[old_chip.parent_chip] - child_chip.update(parent_chip_id: parent_chip.id) + child_chip = chip_mapping.fetch(old_chip.id) + parent_chip = chip_mapping.fetch(old_chip.parent_chip_id) + child_chip.update!(parent_chip: parent_chip) end end @@ -1548,15 +1832,19 @@ def date_for_week_and_day(week, day) start_day_num = start_date.wday - start_date + week.weeks + (day_num - start_day_num).days + start_date + (week - 1).weeks + (day_num - start_day_num).days end end def week_number(date) + return nil if date.nil? || start_date.nil? + if teaching_period.present? teaching_period.week_number(date) else - ((date - start_date) / 1.week).floor + 1 + target_date = date.to_date + unit_start_date = start_date.to_date + ((target_date - unit_start_date).to_i / 7).floor + 1 end end @@ -1573,23 +1861,28 @@ def import_tasks_from_csv(file) headers: true, header_converters: [->(i) { i.nil? ? '' : i }, :downcase, ->(hdr) { hdr.strip.tr(' ', '_').to_sym unless hdr.nil? }], converters: [->(body) { body&.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') }]).each do |row| - next if row[0] =~ /^(Task Name)|(name)/ # Skip header + next if ['Task Name', 'name'].include?(row[0].to_s.strip) # Skip header rows begin - missing = missing_headers(row, TaskDefinition.csv_columns) + missing = missing_headers(row, TaskDefinition.required_csv_columns) if missing.count > 0 errors << { row: row, message: "Missing headers: #{missing.join(', ')}" } next end + existing_task_definition = task_definitions.find_by(abbreviation: row[:abbreviation]&.strip) + existing_task_definition ||= task_definitions.find_by(name: row[:name]&.strip) + existing_task_definition&.task_prerequisites&.destroy_all + task_definition, new_task, message = TaskDefinition.task_def_for_csv_row(self, row) - prerequisites_by_task[task_definition.abbreviation] = JSON.parse(row[:task_prerequisites]) unless row[:task_prerequisites].nil? if task_definition.nil? errors << { row: row, message: message } next end + prerequisites_by_task[task_definition.abbreviation] = JSON.parse(row[:task_prerequisites]) unless row[:task_prerequisites].nil? + success << { row: row, message: message } rescue Exception => e errors << { row: row, message: e.message } @@ -1611,6 +1904,14 @@ def import_tasks_from_csv(file) prerequisite_td = task_definitions.find_by(abbreviation: abbreviation) task_status_id = prerequisite['task_status_id'].to_i + if prerequisite_td.nil? + errors << { + row: "TaskDef '#{task_abbreviation}' prerequisites: #{prerequisites_list}", + message: "Unable to find prerequisite task definition with abbreviation #{abbreviation}." + } + next + end + TaskPrerequisite.create!({ task_definition_id: td.id, prerequisite: prerequisite_td, @@ -1645,7 +1946,7 @@ def tutorial_stream_abbr def times_tasks_have_been_assessed CSV.generate() do |csv| # Add headers - csv << ([ + csv << [ 'Tutorial', 'Tutor', 'Student Username', @@ -1661,7 +1962,7 @@ def times_tasks_have_been_assessed 'demonstrate', 'ready_for_feedback', 'discussed_in_class' - ]) + ] tasks .joins("LEFT JOIN task_engagements ON task_engagements.task_id = tasks.id") @@ -1715,7 +2016,7 @@ def times_tasks_have_been_assessed def days_awaiting_feedback_by_tutorial_csv CSV.generate() do |csv| # Add headers - csv << ([ + csv << [ 'Tutorial', 'Tutor', 'Username', @@ -1723,64 +2024,80 @@ def days_awaiting_feedback_by_tutorial_csv 'Project ID', 'Task Definition', 'Task ID', - 'Days Awaiting Feedback' - ]) + 'Days Awaiting Feedback', + 'Days Awaiting Feedback (Incl. Breaks)' + ] # Add data - tasks - .joins(:task_definition) - .joins('INNER JOIN users ON users.id = projects.user_id') - .joins("LEFT OUTER JOIN (#{tutorial_enrolment_subquery}) as sq ON sq.project_id = projects.id AND (sq.tutorial_stream_id = task_definitions.tutorial_stream_id OR sq.tutorial_stream_id IS NULL)") - .joins('LEFT JOIN tutorials ON tutorials.id = sq.tutorial_id') - .select( - 'users.username AS username', - 'users.first_name AS first_name', - 'users.last_name AS last_name', - 'tasks.id as task_id', - 'task_definitions.abbreviation as task_abbr', - 'tasks.project_id as project_id', - 'DATEDIFF(CURDATE(),submission_date) AS days_since_submission', - 'tutorial_id', - 'tutorials.unit_role_id as unit_role_id', - 'tutorials.abbreviation AS tutorial_abbreviation' - ) - .group('tasks.id', 'task_definitions.abbreviation', 'tasks.project_id', 'tutorial_id', 'unit_role_id', 'submission_date') - .order('unit_role_id', 'days_since_submission DESC') - .where(projects: { enrolled: true }) - .where(task_status: TaskStatus.ready_for_feedback) - .each do |row| - csv << ([ + rows = tasks + .includes(project: { unit: { teaching_period: :breaks } }) + .joins(:task_definition) + .joins('INNER JOIN users ON users.id = projects.user_id') + .joins("LEFT OUTER JOIN (#{tutorial_enrolment_subquery}) as sq ON sq.project_id = projects.id AND (sq.tutorial_stream_id = task_definitions.tutorial_stream_id OR sq.tutorial_stream_id IS NULL)") + .joins('LEFT JOIN tutorials ON tutorials.id = sq.tutorial_id') + .select( + 'users.username AS username', + 'users.first_name AS first_name', + 'users.last_name AS last_name', + 'tasks.id as task_id', + 'task_definitions.abbreviation as task_abbr', + 'tasks.project_id as project_id', + 'tasks.submission_date', + 'tutorial_id', + 'tutorials.unit_role_id as unit_role_id', + 'tutorials.abbreviation AS tutorial_abbreviation' + ) + .group('tasks.id', 'task_definitions.abbreviation', 'tasks.project_id', 'tutorial_id', 'unit_role_id', 'tasks.submission_date') + .where(projects: { enrolled: true }) + .where(task_status: TaskStatus.ready_for_feedback) + .to_a + + unit_roles_by_id = UnitRole.includes(:user).where(id: rows.map(&:unit_role_id).compact.uniq).index_by(&:id) + + rows + .sort_by { |row| [row.unit_role_id || Float::INFINITY, -row.days_awaiting_feedback] } + .each do |row| + csv << [ row['tutorial_abbreviation'], - row['unit_role_id'].present? ? UnitRole.find(row['unit_role_id']).user.name : '', + row.unit_role_id.present? ? unit_roles_by_id[row.unit_role_id]&.user&.name.to_s : '', row['username'], "#{row['first_name']} #{row['last_name']}", row['project_id'], row['task_abbr'], row['task_id'], - row['days_since_submission'] - ]) + row.days_awaiting_feedback, + row.calendar_days_awaiting_feedback + ] end end end def task_completion_csv + task_completion_csv_generator() + end + + def task_completion_csv_generator(task_status_uses_id: false) task_def_by_grade = task_definitions_by_grade streams = tutorial_streams grp_sets = group_sets + base_headers = [ + 'Student ID', + 'Username', + 'Student Name', + ] + base_headers << 'Campus' + base_headers.push( + 'Target Grade', + 'Email', + 'Portfolio', + 'Grade', + 'Rationale', + 'Assessor', + ) CSV.generate() do |csv| # Add header row - csv << ([ - 'Student ID', - 'Username', - 'Student Name', - 'Target Grade', - 'Email', - 'Portfolio', - 'Grade', - 'Rationale', - 'Assessor', - ] + + csv << (base_headers + (streams.count > 0 ? streams.map { |t| t.abbreviation } : ['Tutorial']) + grp_sets.map(&:name) + task_def_by_grade.map do |task_definition| @@ -1795,18 +2112,25 @@ def task_completion_csv # Get the details to fetch for each task definition... td_select = task_def_by_grade.map do |td| result = [] - result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN (CASE WHEN task_statuses.name IS NULL THEN 'Not Started' ELSE task_statuses.name END) ELSE NULL END) AS status_#{td.id}" + if task_status_uses_id + result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN (CASE WHEN tasks.task_status_id IS NULL THEN #{TaskStatus.not_started.id} ELSE tasks.task_status_id END) ELSE NULL END) AS status_#{td.id}" + else + result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN (CASE WHEN task_statuses.name IS NULL THEN 'Not Started' ELSE task_statuses.name END) ELSE NULL END) AS status_#{td.id}" + end result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN tasks.grade ELSE NULL END) AS grade_#{td.id}" if td.is_graded? result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN tasks.quality_pts ELSE NULL END) AS stars_#{td.id}" if td.has_stars? result << "MAX(CASE WHEN tasks.task_definition_id = #{td.id} THEN tasks.contribution_pts ELSE NULL END) AS people_#{td.id}" if td.is_group_task? result end.flatten + group_table_alias = 'task_completion_groups' + # Query across all projects, joined to task's via definitions to ensure all definitions are covered active_projects .joins( :unit, 'INNER JOIN users ON projects.user_id = users.id', + 'LEFT OUTER JOIN campuses ON campuses.id = projects.campus_id', 'INNER JOIN task_definitions ON task_definitions.unit_id = units.id', 'LEFT OUTER JOIN tutorial_streams ON tutorial_streams.unit_id = units.id', 'LEFT OUTER JOIN tutorial_enrolments ON tutorial_enrolments.project_id = projects.id', @@ -1814,24 +2138,28 @@ def task_completion_csv 'LEFT OUTER JOIN tasks ON tasks.task_definition_id = task_definitions.id AND projects.id = tasks.project_id', 'LEFT OUTER JOIN task_statuses ON tasks.task_status_id = task_statuses.id', 'LEFT OUTER JOIN group_memberships ON group_memberships.project_id = projects.id AND group_memberships.active = TRUE', - 'LEFT OUTER JOIN groups ON groups.id = group_memberships.group_id' + "LEFT OUTER JOIN #{Group.quoted_table_name} #{group_table_alias} ON #{group_table_alias}.id = group_memberships.group_id" ).select( 'projects.id as project_id', 'users.student_id as student_id', 'users.username as username', 'users.first_name as first_name', 'projects.assessor_id as project_assessor', - 'users.last_name as last_name', 'projects.target_grade', 'users.email as email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale', + 'users.last_name as last_name', 'campuses.abbreviation as campus_abbreviation', 'projects.target_grade', 'users.email as email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale', *td_select, # Get tutorial for each stream in unit *streams.map { |s| "MAX(CASE WHEN tutorials.tutorial_stream_id = #{s.id} OR tutorials.tutorial_stream_id IS NULL THEN tutorials.abbreviation ELSE NULL END) AS tutorial_#{s.id}" }, # Get tutorial for case when no stream "MAX(CASE WHEN tutorial_streams.id IS NULL THEN tutorials.abbreviation ELSE NULL END) AS tutorial", - *grp_sets.map { |gs| "MAX(CASE WHEN groups.group_set_id = #{gs.id} THEN groups.name ELSE NULL END) AS grp_#{gs.id}" } + *grp_sets.map { |gs| "MAX(CASE WHEN #{group_table_alias}.group_set_id = #{gs.id} THEN #{group_table_alias}.name ELSE NULL END) AS grp_#{gs.id}" } ).group( - 'projects.id', 'student_id', 'username', 'first_name', 'last_name', 'target_grade', 'email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale' + 'projects.id', 'student_id', 'username', 'first_name', 'last_name', 'campus_abbreviation', 'target_grade', 'email', 'compile_portfolio', 'portfolio_production_date', 'grade', 'grade_rationale' ).each do |row| - csv << ([ + student_details = [ row['student_id'], row['username'], "#{row['first_name']} #{row['last_name']}", - GradeHelper.grade_for(row['target_grade']), + ] + student_details << row['campus_abbreviation'] + + csv << (student_details + [ + grade_label(row['target_grade']), row['email'], row['portfolio_production_date'].present? && !row['compile_portfolio'] && File.exist?(FileHelper.student_portfolio_path(self, row['username'], create: true)), row['grade'] > 0 ? row['grade'] : nil, @@ -1846,8 +2174,12 @@ def task_completion_csv end.flatten + grp_sets.map do |gs| row["grp_#{gs.id}"] end + task_def_by_grade.map do |td| - result = [row["status_#{td.id}"].nil? ? TaskStatus.not_started.name : row["status_#{td.id}"]] - result << GradeHelper.short_grade_for(row["grade_#{td.id}"]) if td.is_graded? + if task_status_uses_id + result = [row["status_#{td.id}"].nil? ? TaskStatus.not_started.id : row["status_#{td.id}"].to_i] + else + result = [row["status_#{td.id}"].nil? ? TaskStatus.not_started.name : row["status_#{td.id}"]] + end + result << grade_abbreviation(row["grade_#{td.id}"]) if td.is_graded? result << row["stars_#{td.id}"] if td.has_stars? result << row["people_#{td.id}"] if td.is_group_task? result @@ -1859,7 +2191,7 @@ def task_completion_csv def staff_notes_csv CSV.generate() do |csv| # Add headers - csv << ([ + csv << [ 'Student Username', 'Project ID', 'Student Name', @@ -1867,7 +2199,7 @@ def staff_notes_csv 'Created', 'Author Name', 'Author Username', - ]) + ] StaffNote.joins(project: :unit) .where(units: { id: id }) @@ -1886,6 +2218,97 @@ def staff_notes_csv end end + def import_staff_notes_from_csv(file, author_id, progress_callback: nil) + success = [] + errors = [] + ignored = [] + author = User.find(author_id) + + csv = CSV.new(File.read(file), headers: true, + header_converters: [->(i) { i.nil? ? '' : i }, :downcase, ->(hdr) { hdr.strip unless hdr.nil? }], + converters: [->(i) { i.nil? ? '' : i }, ->(body) { body&.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') }]) + unless csv.header_row? + errors << { row: [], message: "Header row missing" } + return { success: success, ignored: ignored, errors: errors } + end + + csv.shift + total_rows = csv.read.size + progress_callback&.call(message: "Importing staff notes", rows_processed: 0, total_rows: total_rows) + + CSV.foreach(file, headers: true, + header_converters: [->(i) { i.nil? ? '' : i }, :downcase, ->(hdr) { hdr.strip unless hdr.nil? }], + converters: [->(i) { i.nil? ? '' : i }, ->(body) { body&.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') }]).with_index(1) do |row, row_count| + progress_callback&.call(message: "Importing staff notes", rows_processed: row_count, total_rows: total_rows) + + begin + missing = missing_headers(row, ['comment']) + unless row.headers.include?('username') || row.headers.include?('login_id') + missing << 'username or login_id' + end + if missing.any? + errors << { row: row, message: "Missing headers: #{missing.join(', ')}" } + next + end + + username = row['username']&.strip + login_id = row['login_id']&.strip + comment = row['comment']&.strip + + if username.blank? && login_id.blank? + errors << { row: row, message: "Username or login_id is required" } + next + end + + if comment.blank? + errors << { row: row, message: "Comment is required" } + next + end + + username_user = User.find_by(username: username.downcase) if username.present? + login_id_user = User.find_by(login_id: login_id) if login_id.present? + + if username.present? && username_user.nil? + errors << { row: row, message: "Could not find user with username #{username}" } + next + end + + if login_id.present? && login_id_user.nil? + errors << { row: row, message: "Could not find user with login_id #{login_id}" } + next + end + + if username_user && login_id_user && username_user != login_id_user + errors << { row: row, message: "Username and login_id do not match the same user" } + next + end + + student = username_user || login_id_user + project = projects.find_by(user: student) + if project.nil? + errors << { row: row, message: "Student is not enrolled in unit" } + next + end + + if project.staff_notes.last&.note == comment + ignored << { row: row, message: "Staff note already exists" } + next + end + + project.add_staff_note(author, comment) + success << { row: row, message: "Staff note added for #{student.username}" } + rescue Exception => e + errors << { row: row, message: e.message } + end + end + + { + success: success, + ignored: ignored, + errors: errors + } + end + def get_portfolio_zip_filename(current_user) filename = FileHelper.sanitized_filename("portfolios-#{code}-#{current_user.username}") "#{FileHelper.tmp_file(filename)}.zip" @@ -1942,8 +2365,19 @@ def get_task_resources_zip end if td.has_task_resources? - dst_path = FileHelper.sanitized_filename(td.abbreviation.to_s) + '.zip' - zip.add(dst_path, td.task_resources) + linked_resource = td.linked_task_resource + + if linked_resource && !td.task_resource_zip?(linked_resource) + dst_path = File.join( + FileHelper.sanitized_filename(td.abbreviation.to_s), + FileHelper.sanitized_filename(linked_resource[:filename]) + ) + zip.add(dst_path, linked_resource[:path]) + else + dst_path = FileHelper.sanitized_filename(td.abbreviation.to_s) + '.zip' + resource_path = linked_resource ? linked_resource[:path] : td.task_resources + zip.add(dst_path, resource_path) + end end end end # zip @@ -2171,7 +2605,7 @@ def get_all_tasks_for(user, my_tutorials_only = false) # def tasks_awaiting_feedback(user) get_all_tasks_for(user) - .where('task_statuses.id IN (:ids)', ids: [TaskStatus.discuss, TaskStatus.attention_required, TaskStatus.redo, TaskStatus.demonstrate, TaskStatus.fix_and_resubmit]) + .where('task_statuses.id IN (:ids)', ids: [TaskStatus.discuss, TaskStatus.rediscuss, TaskStatus.attention_required, TaskStatus.redo, TaskStatus.demonstrate, TaskStatus.fix_and_resubmit]) .order('task_definition_id') end @@ -2465,7 +2899,7 @@ def student_task_completion_stats result[:tutorial][t.id] = _calculate_task_completion_stats(data.select { |r| r[:tutorial_id] == t.id }) end - for i in GradeHelper::RANGE do + for i in grade_values do result[:grade][i] = _calculate_task_completion_stats(data.select { |r| r[:grade] == i }) end @@ -2474,9 +2908,9 @@ def student_task_completion_stats def student_grades_csv CSV.generate do |row| - row << %w(unit_code username student_id target_grade submitted_grade portfolio_production_date has_portfolio spec_con_days grade rationale assessor assessor_id) + row << %w(unit_code username student_id target_grade submitted_grade portfolio_submission_date portfolio_production_date has_portfolio spec_con_days grade rationale assessor assessor_id) active_projects.each do |project| - row << [project.unit.code, project.student.username, project.student.student_id, project.target_grade, project.submitted_grade, project.portfolio_production_date, project.portfolio_exists?, project.spec_con_days, project.grade, project.grade_rationale, project.assessor&.name, project.assessor&.id] + row << [project.unit.code, project.student.username, project.student.student_id, project.target_grade, project.submitted_grade, project.portfolio_submission_date, project.portfolio_production_date, project.portfolio_exists?, project.spec_con_days, project.grade, project.grade_rationale, project.assessor&.name, project.assessor&.id] end end end @@ -2500,6 +2934,43 @@ def tutor_assessment_csv end end + def overflow_task_claims_csv + CSV.generate do |csv| + csv << [ + 'Tutor who claimed', + 'Claiming Unit Role ID', + 'Original Tutor', + 'Student Username', + 'Student ID', + 'Task ID', + 'Task Definition', + 'Days Awaiting Feedback', + 'Timestamp' + ] + + OverflowTaskClaimLog + .where(unit_id: id) + .includes(:claimed_by_user, :original_tutor_user, :student_user, task: :task_definition) + .order(:claimed_at) + .each do |claim| + task = claim.task + student = claim.student_user + + csv << [ + claim.claimed_by_user&.name, + claim.claimed_by_unit_role_id, + claim.original_tutor_user&.name, + student&.username, + student&.student_id, + claim.task_id, + task&.task_definition&.abbreviation, + claim.days_awaiting_feedback, + claim.claimed_at, + ] + end + end + end + #---------------------------------------------------------------------------- # Task updates from offline download/upload #---------------------------------------------------------------------------- @@ -2688,15 +3159,28 @@ def update_task_status_from_csv(user, csv_str, success, _ignored, errors) end begin - task.trigger_transition(trigger: task_entry['status'], by_user: user, quality: task_entry['new quality'].to_i) # saves task - task.grade_task(task_entry['new grade']) # try to grade task if need be + requested_status = TaskStatus.status_for_name(task_entry['status'].to_s) + raise "Unable to update task status to '#{task_entry['status']}'." if requested_status.nil? - if task_entry['new comment'].blank? - success << { row: task_entry, message: "Updated task #{task.task_definition.abbreviation} for #{owner_text}" } - else - task.add_text_comment user, task_entry['new comment'] - success << { row: task_entry, message: "Updated task #{task.task_definition.abbreviation} for #{owner_text}" } - success << { row: {}, message: "Added comment to #{task.task_definition.abbreviation} for #{owner_text}" } + status_changed = false + comment_added = false + + unless task.task_status == requested_status + task.trigger_transition(trigger: task_entry['status'], by_user: user, quality: task_entry['new quality'].to_i) # saves task + task.grade_task(task_entry['new grade']) # try to grade task if need be + status_changed = true + end + + new_comment = task_entry['new comment'].to_s.strip + last_comment = task.comments.last&.comment.to_s.strip + if new_comment.present? && last_comment != new_comment + task.add_text_comment user, new_comment + comment_added = true + end + + if status_changed || comment_added + success << { row: task_entry, message: "Updated task #{task.task_definition.abbreviation} for #{owner_text}" } if status_changed + success << { row: {}, message: "Added comment to #{task.task_definition.abbreviation} for #{owner_text}" } if comment_added end rescue Exception => e errors << { row: task_entry, message: e.message } @@ -2799,63 +3283,66 @@ def upload_batch_task_zip_or_csv(user, file) # Copy over the updated/marked files to the file system zip.each do |file| # Skip processing marking file - next if ['marks.csv', 'readme.txt'].include?(File.basename(file[:name])) + next if ['marks.csv', 'readme.txt'].include?(File.basename(file.name)) # Test filename pattern - if (/.*-\d+.pdf/i =~ File.basename(file[:name])) != 0 - if file[:name][-1] != '/' - ignored << { row: "File #{file[:name]}", message: 'Does not appear to be a task PDF.' } + if (/.*-\d+.pdf/i =~ File.basename(file.name)) != 0 + if file.name[-1] != '/' + ignored << { row: "File #{file.name}", message: 'Does not appear to be a task PDF.' } end next end - if (/\._.*/ =~ File.basename(file[:name])) == 0 - ignored << { row: "File #{file[:name]}", message: 'Does not appear to be a task PDF.' } + if (/\._.*/ =~ File.basename(file.name)) == 0 + ignored << { row: "File #{file.name}", message: 'Does not appear to be a task PDF.' } next end # Extract the id from the filename - task_id_from_filename = File.basename(file[:name], '.pdf').split('-').last + task_id_from_filename = File.basename(file.name, '.pdf').split('-').last task = Task.find_by(id: task_id_from_filename) if task.nil? - ignored << { row: "File #{file[:name]}", message: 'Unable to find associated task.' } + ignored << { row: "File #{file.name}", message: 'Unable to find associated task.' } next end # Ensure that this task's id is inside entry_data task_entry = entry_data.select { |t| t['task'] == task.task_definition.abbreviation.tr(',', '_') && t['username'] == task.project.user.username }.first if task_entry.nil? - # error!({"error" => "File #{file[:name]} has a mismatch of task id ##{task.id} (this task id does not exist in marks.csv)"}, 403) - errors << { row: "File #{file[:name]}", message: "Task id #{task.id} not in marks.csv" } + # error!({"error" => "File #{file.name} has a mismatch of task id ##{task.id} (this task id does not exist in marks.csv)"}, 403) + errors << { row: "File #{file.name}", message: "Task id #{task.id} not in marks.csv" } next end if task.unit != self - errors << { row: "File #{file[:name]}", message: 'This task does not relate to this unit.' } + errors << { row: "File #{file.name}", message: 'This task does not relate to this unit.' } next end # Can the user assess this task? unless AuthorisationHelpers.authorise? user, task, :put - errors << { row: "File #{file[:name]}", error: "You do not have permission to assess task with id #{task.id}" } + errors << { row: "File #{file.name}", error: "You do not have permission to assess task with id #{task.id}" } next end # Read into the task's final pdf path the new file - tmp_file = File.join(tmp_dir, File.basename(file[:name])) + tmp_file = File.join(tmp_dir, File.basename(file.name)) # get file out of zip... to tmp_file file.extract(tmp_file) { true } # copy tmp_file to dest - if FileHelper.copy_pdf(tmp_file, task.final_pdf_path) + destination_path = task.final_pdf_path(ignore_portfolio_evidence: true) + + if FileHelper.copy_pdf(tmp_file, destination_path) + task.update(portfolio_evidence: nil) if task.portfolio_evidence.present? if task.group.nil? - success << { row: "File #{file[:name]}", message: "Replace PDF of task #{task.task_definition.abbreviation} for #{task.student.name}" } + success << { row: "File #{file.name}", message: "Replace PDF of task #{task.task_definition.abbreviation} for #{task.student.name}" } else - success << { row: "File #{file[:name]}", message: "Replace PDF of group task #{task.task_definition.abbreviation} for #{task.group.name}" } + success << { row: "File #{file.name}", message: "Replace PDF of group task #{task.task_definition.abbreviation} for #{task.group.name}" } end FileUtils.rm tmp_file else - errors << { row: "File #{file[:name]}", message: 'The file does not appear to be a valid PDF.' } + errors << { row: "File #{file.name}", message: 'The file does not appear to be a valid PDF.' } next end end @@ -2872,6 +3359,327 @@ def upload_batch_task_zip_or_csv(user, file) } end + def batch_feedback_csv_required_headers + ['username', 'student id', 'status', 'comment'] + end + + def parse_batch_feedback_csv(csv_str, return_headers: false) + CSV.parse( + csv_str, + headers: true, + return_headers: return_headers, + header_converters: [->(body) { body&.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')&.downcase }], + converters: [->(body) { body&.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') }] + ) + end + + def find_project_for_batch_feedback_csv_entry(task_entry) + username = task_entry['username'].to_s.strip + student_id = task_entry['student id'].to_s.strip + + return [nil, 'Provide either Username or Student ID.'] if username.blank? && student_id.blank? + + username_project = if username.present? + projects.joins(:user).where('LOWER(users.username) = ?', username.downcase).first + end + student_id_project = if student_id.present? + projects.joins(:user).where(users: { student_id: student_id }).first + end + + if username.present? && username_project.nil? && student_id.blank? + return [nil, "Unable to find student with username '#{username}'."] + end + + if student_id.present? && student_id_project.nil? && username.blank? + return [nil, "Unable to find student with student ID '#{student_id}'."] + end + + if username_project.present? && student_id_project.present? && username_project != student_id_project + return [nil, "Username '#{username}' and student ID '#{student_id}' refer to different students."] + end + + project = username_project || student_id_project + return [nil, 'Unable to find student project for this row.'] if project.nil? + + [project, nil] + end + + def build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: nil, progress_callback: nil) + task_rows = [] + + csv_str.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') + csv_str.tr!("\r", "\n") + csv_str.gsub!("\n\n", "\n") + + entries = parse_batch_feedback_csv(csv_str, return_headers: true) + total_rows = 0 + + entries.each do |task_entry| + next if task_entry.header_row? + next if task_entry.to_hash.values.all? { |value| value.to_s.strip.blank? } + + total_rows += 1 + end + + rows_processed = 0 + + progress_callback&.call(message: 'Validating batch feedback rows', total_rows: total_rows, rows_processed: 0) + + entries.each do |task_entry| + if task_entry.header_row? + batch_feedback_csv_required_headers.each do |expect_header| + unless task_entry.to_hash.keys.include?(expect_header) + errors << { row: task_entry, message: "Missing header '#{expect_header}', ensure first row has header information." } + return nil + end + end + next + end + + next if task_entry.to_hash.values.all? { |value| value.to_s.strip.blank? } + + rows_processed += 1 + progress_callback&.call(message: 'Validating batch feedback rows', rows_processed: rows_processed) + + project, project_error = find_project_for_batch_feedback_csv_entry(task_entry) + if project_error.present? + errors << { row: task_entry, message: project_error } + next + end + + task = project.task_for_task_definition(task_definition) + if task.nil? + errors << { row: task_entry, message: "Unable to find task for #{task_definition.abbreviation}" } + next + end + + if task.group_task? + errors << { row: task_entry, message: 'Batch feedback upload does not support group tasks.' } + next + end + + status = task_entry['status'].to_s.strip + if status.blank? + errors << { row: task_entry, message: 'Status cannot be blank.' } + next + end + + if TaskStatus.status_for_name(status).nil? + errors << { row: task_entry, message: "Unable to update task status to '#{status}'." } + next + end + + student_entries = zip.nil? ? [] : batch_feedback_entries_for_project(zip, project) + pdf_entries = zip.nil? ? [] : batch_feedback_named_pdf_entries_for_project(zip, project) + + if zip.present? + if student_entries.any? && pdf_entries.empty? + expected_identifier = batch_feedback_primary_identifier_for_project(project) + errors << { + row: task_entry, + message: "Expected a PDF named #{expected_identifier}.pdf inside #{expected_identifier}'s folder." + } + next + end + + if pdf_entries.length > 1 + expected_identifier = batch_feedback_primary_identifier_for_project(project) + errors << { + row: task_entry, + message: "Found multiple PDFs named #{expected_identifier}.pdf inside #{expected_identifier}'s folder." + } + next + end + end + + task_rows << { + task: task, + project: project, + task_entry: task_entry, + pdf_entry: pdf_entries.first + } + end + + task_rows + end + + def write_batch_feedback_csv_file(task_rows) + file = Tempfile.new(["batch_feedback_#{id}_", '.csv']) + file.write(build_batch_feedback_legacy_marks_csv(task_rows)) + file.rewind + file + end + + def upload_batch_feedback_csv(user, task_definition, file, progress_callback: nil) + success = [] + errors = [] + ignored = [] + + type = mime_type(file["tempfile"].path) + + unless mime_in_list?(file["tempfile"].path, ['text/', 'text/plain', 'text/csv', 'application/zip', 'multipart/x-gzip', 'multipart/x-zip', 'application/x-gzip', 'application/octet-stream']) + errors << { row: {}, message: "File given is not a csv or zip file - detected #{type}" } + return { + success: success, + ignored: ignored, + errors: errors + } + end + + if type.start_with?('text/', 'text/plain', 'text/csv') + task_rows = build_batch_feedback_task_rows( + task_definition, + File.read(file["tempfile"].path), + errors, + progress_callback: progress_callback + ) + + if task_rows.blank? + return { + success: success, + ignored: ignored, + errors: errors + } + end + + converted_csv = write_batch_feedback_csv_file(task_rows) + progress_callback&.call(message: 'Applying batch feedback updates', total_rows: task_rows.count, rows_processed: 0) + result = upload_batch_task_zip_or_csv(user, { 'tempfile' => converted_csv }) + result[:errors] = errors + result[:errors] + result + else + upload_batch_feedback_zip(user, task_definition, file, progress_callback: progress_callback) + end + ensure + converted_csv.close! if defined?(converted_csv) && converted_csv.present? + end + + def batch_feedback_primary_identifier_for_project(project) + project.user.student_id.to_s.strip.presence || project.user.username.to_s.strip + end + + def batch_feedback_identifiers_for_project(project) + [ + project.user.student_id.to_s.strip.presence, + project.user.username.to_s.strip.presence + ].compact.uniq + end + + def batch_feedback_entries_for_identifier(zip, identifier) + zip.select do |entry| + path_parts = entry.name.split('/').reject(&:blank?) + next false if path_parts.empty? + + if entry.name_is_directory? + path_parts.any? { |part| part.casecmp(identifier).zero? } + else + path_parts[0...-1].any? { |part| part.casecmp(identifier).zero? } + end + end + end + + def batch_feedback_entries_for_project(zip, project) + batch_feedback_identifiers_for_project(project).flat_map do |identifier| + batch_feedback_entries_for_identifier(zip, identifier) + end.uniq + end + + def batch_feedback_named_pdf_entries_for_identifier(zip, identifier) + batch_feedback_entries_for_identifier(zip, identifier).select do |entry| + next false if entry.name_is_directory? + next false unless File.extname(entry.name).casecmp('.pdf').zero? + + File.basename(entry.name, '.pdf').casecmp(identifier).zero? + end + end + + def batch_feedback_named_pdf_entries_for_project(zip, project) + batch_feedback_identifiers_for_project(project).flat_map do |identifier| + batch_feedback_named_pdf_entries_for_identifier(zip, identifier) + end.uniq + end + + def build_batch_feedback_legacy_marks_csv(task_rows) + CSV.generate do |csv| + csv << check_mark_csv_headers.split(',') + + task_rows.each do |task_row| + task = task_row[:task] + project = task_row[:project] + task_entry = task_row[:task_entry] + tutorial = project.tutorial_for(task.task_definition)&.abbreviation.to_s + + csv << [ + project.user.username, + project.user.name, + tutorial, + task.task_definition.abbreviation, + task_entry['status'].to_s.strip, + '', + '', + task_entry['comment'].to_s.strip + ] + end + end + end + + def upload_batch_feedback_zip(user, task_definition, file, progress_callback: nil) + success = [] + errors = [] + ignored = [] + repacked_zip = Tempfile.new(["batch_feedback_#{id}_", '.zip']) + + Zip::File.open(file["tempfile"].path) do |zip| + marking_file = zip.glob('**/marks.csv').first + if marking_file.nil? + errors << { row: {}, message: 'No marks.csv contained in zip.' } + return { + success: success, + ignored: ignored, + errors: errors + } + end + + csv_str = marking_file.get_input_stream.read + csv_str.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') unless csv_str.nil? + + task_rows = build_batch_feedback_task_rows(task_definition, csv_str, errors, zip: zip, progress_callback: progress_callback) + + if task_rows.blank? + return { + success: success, + ignored: ignored, + errors: errors + } + end + + Zip::File.open(repacked_zip.path, Zip::File::CREATE) do |output_zip| + output_zip.get_output_stream('marks.csv') do |f| + f.write(build_batch_feedback_legacy_marks_csv(task_rows)) + end + + progress_callback&.call(message: 'Preparing PDF replacements', total_rows: task_rows.count, rows_processed: 0) + + task_rows.each_with_index do |task_row, index| + next if task_row[:pdf_entry].nil? + + output_name = "#{task_row[:task].task_definition.abbreviation}-#{task_row[:task].id}.pdf" + output_zip.get_output_stream(output_name) do |f| + f.write(task_row[:pdf_entry].get_input_stream.read) + end + + progress_callback&.call(message: 'Preparing PDF replacements', rows_processed: index + 1) + end + end + end + + result = upload_batch_task_zip_or_csv(user, { 'tempfile' => repacked_zip }) + result[:errors] = errors + result[:errors] + result + ensure + repacked_zip.close! if defined?(repacked_zip) && repacked_zip.present? + end + def send_weekly_status_emails(summary_stats) return unless send_notifications @@ -3026,6 +3834,15 @@ def move_files_to_archive FileUtils.mkdir_p(FileHelper.root_submission_history_dir(archived: true)) FileUtils.mv(original_submission_history_path, archive_submission_history_path) end + + # Move JPlag reports + archive_jplag_report_path = FileHelper.unit_jplag_report_dir(self, create: false, archived: :force) + original_jplag_report_path = FileHelper.unit_jplag_report_dir(self, create: false, archived: false) + + if File.exist?(original_jplag_report_path) && ! File.exist?(archive_jplag_report_path) + FileUtils.mkdir_p(FileHelper.root_jplag_report_dir(archived: true)) + FileUtils.mv(original_jplag_report_path, archive_jplag_report_path) + end end def get_tutor_times(start_date: nil, end_date: nil, timezone: nil, ignore_sessions_during_tutorials: false) @@ -3109,37 +3926,52 @@ def get_tutor_times_csv(start_date: nil, end_date: nil, timezone: nil, ignore_se CSV.generate() do |csv| # Add headers - csv << ([ + csv << [ 'User ID', 'Tutor', 'Total Minutes', 'Assessments', 'Comments', - ]) + ] summary.each do |row| - csv << ([ + csv << [ row[:user_id].to_s, row[:tutor_name], row[:total_minutes].to_s, row[:assessments_made].to_s, row[:comments_made].to_s, - ]) + ] end end end + def capture_task_complete_stats_snapshot!(snapshot_time: Time.zone.now) + snapshot_payload = task_completion_csv_generator(task_status_uses_id: true) + + timestamp = snapshot_time.to_i.to_s + + task_completion_snapshots + .find_or_initialize_by(snapshot_timestamp: timestamp) + .tap do |snapshot| + snapshot.save! + snapshot.store_stats!(snapshot_payload) + end + end + private def delete_associated_files unit_path = FileHelper.unit_dir(self, create: false) unit_portfolio_path = FileHelper.unit_portfolio_dir(self, create: false) submission_history_path = FileHelper.unit_submission_history_dir(self) + jplag_report_path = FileHelper.unit_jplag_report_dir(self, create: false) FileUtils.rm_rf unit_path FileUtils.rm_rf unit_portfolio_path FileUtils.rm_rf submission_history_path + FileUtils.rm_rf jplag_report_path FileUtils.cd FileHelper.student_work_dir end @@ -3178,6 +4010,26 @@ def move_files_on_code_change FileUtils.mv(old_dir, new_dir) unless File.exist?(new_dir) end + old_submission_history_dir = File.join( + FileHelper.root_submission_history_dir(archived: archived), + FileHelper.sanitized_path("#{saved_change_to_code[0]}-#{id}") + ) + + if File.exist? old_submission_history_dir + new_submission_history_dir = FileHelper.unit_submission_history_dir(self) + FileUtils.mv(old_submission_history_dir, new_submission_history_dir) unless File.exist? new_submission_history_dir + end + + old_jplag_report_dir = File.join( + FileHelper.root_jplag_report_dir(archived: archived), + FileHelper.sanitized_path("#{saved_change_to_code[0]}-#{id}") + ) + + if File.exist? old_jplag_report_dir + new_jplag_report_dir = FileHelper.unit_jplag_report_dir(self, create: false) + FileUtils.mv(old_jplag_report_dir, new_jplag_report_dir) unless File.exist? new_jplag_report_dir + end + # rubocop:disable Rails/SkipsModelValidations tasks.where('portfolio_evidence IS NOT NULL').update_all("portfolio_evidence = REPLACE(portfolio_evidence, '#{saved_change_to_code[0]}-#{id}', '#{code}-#{id}')") # rubocop:enable Rails/SkipsModelValidations diff --git a/app/models/unit_content_link.rb b/app/models/unit_content_link.rb new file mode 100644 index 0000000000..6115ca7676 --- /dev/null +++ b/app/models/unit_content_link.rb @@ -0,0 +1,27 @@ +class UnitContentLink < ApplicationRecord + CONTEXT_TYPES = %w[grade grade_overview task_definition task_definition_resource].freeze + + belongs_to :unit + belongs_to :unit_content_site + + validates :context_type, inclusion: { in: CONTEXT_TYPES } + validates :context_key, :route, presence: true + validates :context_key, uniqueness: { scope: [:unit_id, :context_type] } + validate :resource_path_must_exist + + before_validation :normalise_route + + private + + def normalise_route + self.route = "/#{route.to_s.gsub(%r{\A/+|/+\z}, '')}" + self.route = '/' if route.blank? + end + + def resource_path_must_exist + return unless context_type == 'task_definition_resource' + return if unit_content_site&.file?(route) + + errors.add(:route, 'must be a file in the selected content site') + end +end diff --git a/app/models/unit_content_site.rb b/app/models/unit_content_site.rb new file mode 100644 index 0000000000..3b1a02ecf8 --- /dev/null +++ b/app/models/unit_content_site.rb @@ -0,0 +1,180 @@ +require 'fileutils' +require 'digest/sha1' +require 'securerandom' +require 'set' +require 'zip' + +class UnitContentSite < ApplicationRecord + include FileHelper + + belongs_to :unit + has_many :unit_content_links, dependent: :destroy + + validates :name, :original_filename, :archive_path, presence: true + validates :name, uniqueness: { scope: :unit_id, case_sensitive: false } + validates :root_dir, presence: true + + after_destroy :delete_archive + + def self.archive_dir_for(unit) + File.join(FileHelper.unit_dir(unit), 'content_sites') + end + + def self.store_upload!(unit, file, name: nil) + original_filename = file[:filename] || file[:name] || 'content.zip' + site_name = name.presence || File.basename(original_filename, '.*') + archive_dir = archive_dir_for(unit) + FileUtils.mkdir_p archive_dir + + site = unit.unit_content_sites.create!( + name: site_name, + original_filename: original_filename, + root_dir: '/', + is_main: unit.unit_content_sites.none?, + archive_path: File.join( + archive_dir, + "#{SecureRandom.hex(8)}-#{FileHelper.sanitized_filename(original_filename)}" + ) + ) + + FileUtils.cp file[:tempfile].path, site.archive_path + site + end + + def replace_upload!(file, root_dir: nil) + original_archive_path = archive_path + replacement_original_filename = file[:filename] || file[:name] || original_filename + replacement_archive_path = File.join( + self.class.archive_dir_for(unit), + "#{SecureRandom.hex(8)}-#{FileHelper.sanitized_filename(replacement_original_filename)}" + ) + + FileUtils.cp file[:tempfile].path, replacement_archive_path + replacement_root_options = self.class.root_dir_options_for(replacement_archive_path) + replacement_root_dir = + root_dir.presence || + (replacement_root_options.include?(self.root_dir) ? self.root_dir : '/') + + update!( + original_filename: replacement_original_filename, + archive_path: replacement_archive_path, + root_dir: replacement_root_dir + ) + FileUtils.rm_f original_archive_path if original_archive_path.present? + self + rescue StandardError + FileUtils.rm_f replacement_archive_path if replacement_archive_path.present? + raise + end + + def self.root_dir_options_for(archive_path) + root_dir_options_from_entries(archive_entries_for(archive_path)) + rescue Zip::Error + ['/'] + end + + def self.archive_entries_for(archive_path) + entries = [] + + Zip::File.open(archive_path) do |zip| + zip.each do |entry| + entries << entry.name unless entry.directory? + end + end + + entries + end + + def self.root_dir_options_from_entries(entries) + dirs = entries.each_with_object(Set.new(['/'])) do |entry, paths| + parts = entry.split('/').reject(&:blank?) + next if parts.any? { |part| ignored_archive_path?(part) } + + parts[0...-1].each_index do |index| + paths << "/#{parts[0..index].join('/')}" + end + end + + dirs.to_a.sort + end + + def self.ignored_archive_path?(path) + path.start_with?('__MACOSX') || path == '.DS_Store' || path.start_with?('._') + end + + def root_dir_options + self.class.root_dir_options_for(archive_path) + end + + def file_paths + root_prefix = normalized_root_dir + + self.class.archive_entries_for(archive_path).filter_map do |entry| + parts = entry.split('/').reject(&:blank?) + next if parts.any? { |part| self.class.ignored_archive_path?(part) } + next unless root_prefix.blank? || entry.start_with?("#{root_prefix}/") + + relative_path = root_prefix.blank? ? entry : entry.delete_prefix("#{root_prefix}/") + "/#{relative_path}" if relative_path.present? + end.sort + rescue Zip::Error, Errno::ENOENT + [] + end + + def file?(path) + entry_name = archive_entry_name(path) + return false if entry_name.blank? || !File.exist?(archive_path) + + Zip::File.open(archive_path) do |zip| + entry = zip.find_entry(entry_name) + entry.present? && entry.file? + end + rescue Zip::Error + false + end + + def extract_file(path) + entry_name = archive_entry_name(path) + return nil if entry_name.blank? || !File.exist?(archive_path) + + Zip::File.open(archive_path) do |zip| + entry = zip.find_entry(entry_name) + return nil unless entry&.file? + + filename = File.basename(entry.name) + cache_key = Digest::SHA1.hexdigest("#{archive_path}:#{entry.name}:#{updated_at.to_f}") + extracted_path = FileHelper.tmp_file( + "unit-content-#{id}-#{cache_key}-#{FileHelper.sanitized_filename(filename)}" + ) + + unless File.exist?(extracted_path) && File.size(extracted_path) == entry.size + temporary_path = "#{extracted_path}.#{SecureRandom.hex(6)}.tmp" + entry.extract(temporary_path) { true } + FileUtils.mv(temporary_path, extracted_path) + end + + { path: extracted_path, filename: filename } + ensure + FileUtils.rm_f(temporary_path) if defined?(temporary_path) && temporary_path.present? + end + rescue Zip::Error + nil + end + + private + + def archive_entry_name(path) + relative_path = path.to_s.gsub(%r{\A/+|/+\z}, '') + return nil if relative_path.blank? + + [normalized_root_dir, relative_path].compact_blank.join('/') + end + + def normalized_root_dir + root_dir.to_s.gsub(%r{\A/+|/+\z}, '') + end + + def delete_archive + FileUtils.rm_f archive_path if archive_path.present? + end +end diff --git a/app/models/unit_role.rb b/app/models/unit_role.rb index 765bf117f8..68fc0f5e09 100644 --- a/app/models/unit_role.rb +++ b/app/models/unit_role.rb @@ -38,7 +38,9 @@ def tasks_awaiting_feedback end def oldest_task_awaiting_feedback - tasks_awaiting_feedback.order("submission_date ASC").first + tasks_awaiting_feedback + .includes(project: { unit: { teaching_period: :breaks } }) + .max_by(&:days_awaiting_feedback) end # @@ -134,7 +136,7 @@ def populate_summary_stats(summary_stats, tutorial_stream, tutorial, row) data[:engagements] = all_engagements data[:total_staff_engagements] = all_engagements.count - data[:staff_engagements] = weekly_engagements.where(engagement: [TaskStatus.complete.name, TaskStatus.feedback_exceeded.name, TaskStatus.redo.name, TaskStatus.discuss.name, TaskStatus.attention_required.name, TaskStatus.demonstrate.name, TaskStatus.fail.name]) + data[:staff_engagements] = weekly_engagements.where(engagement: [TaskStatus.complete.name, TaskStatus.feedback_exceeded.name, TaskStatus.redo.name, TaskStatus.discuss.name, TaskStatus.rediscuss.name, TaskStatus.attention_required.name, TaskStatus.demonstrate.name, TaskStatus.fail.name]) # Weekly task engagements for this tutorial data[:weekly_engagements_count] = weekly_engagements.count @@ -147,7 +149,11 @@ def populate_summary_stats(summary_stats, tutorial_stream, tutorial, row) .distinct if tutorial_tasks.count > 0 - data[:oldest_task_days] = (Time.zone.now - tutorial_tasks.order("submission_date ASC").first.submission_date.to_time).to_i / 1.day + oldest_task = tutorial_tasks + .includes(project: { unit: { teaching_period: :breaks } }) + .max_by(&:days_awaiting_feedback) + + data[:oldest_task_days] = oldest_task&.days_awaiting_feedback || 0 data[:tasks_awaiting_feedback_count] = tutorial_tasks.count else data[:oldest_task_days] = 0 @@ -258,7 +264,7 @@ def get_marking_sessions_csv(start_date: nil, end_date: nil, timezone: nil) CSV.generate do |csv| # Add headers - csv << ([ + csv << [ 'Start Date', 'Start Time', 'End Date', @@ -270,13 +276,13 @@ def get_marking_sessions_csv(start_date: nil, end_date: nil, timezone: nil) 'Comments Added', 'Assessments Made', 'During Tutorial' - ]) + ] result.each do |row| start_time = row[:start_time].in_time_zone(tz) end_time = row[:end_time].in_time_zone(tz) - csv << ([ + csv << [ start_time.strftime('%Y-%m-%d %A'), start_time.strftime('%H:%M'), end_time.strftime('%Y-%m-%d %A'), @@ -288,7 +294,7 @@ def get_marking_sessions_csv(start_date: nil, end_date: nil, timezone: nil) row[:comments_added], row[:assessments], row[:during_tutorial] ? 'TRUE' : 'FALSE' - ]) + ] end end diff --git a/app/models/user.rb b/app/models/user.rb index c360c140ff..67a50d1877 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -90,16 +90,39 @@ def authenticate?(data) end end + # Record a completed interactive sign in. Signing in is also an access event. + def record_sign_in! + now = Time.current + update!(last_sign_in_at: now, last_access_at: now) + end + + # Refresh-token exchanges provide a low-write indication that the user is + # still accessing OnTrack without updating the user on every API request. + def record_access! + update!(last_access_at: Time.current) + end + # # Force-generates a new authentication token, regardless of whether or not # it is actually expired # - def generate_authentication_token!(remember: false, expiry: Time.zone.now + 2.hours, token_type: :general, force_new: true) + def generate_authentication_token!(remember: false, expiry: nil, token_type: :general, force_new: true) # Ensure this user is saved... so it has an id self.save unless self.persisted? + expiry_duration = + if token_type.to_sym == :refresh_token + Doubtfire::Application.config.refresh_token_expiry + else + Doubtfire::Application.config.access_token_expiry + end + expiry ||= Time.zone.now + expiry_duration + + # Reuse tokens for up to 75% of their configured lifetime, then rotate early. + token_reuse_duration = expiry_duration * 0.75 + # Get a recent token, or create a new one token = self.auth_tokens.where(token_type: token_type).last unless force_new - if token.nil? || token.created_at <= Time.zone.now - 90.minutes + if token.nil? || token.auth_token_expiry <= Time.zone.now || token.created_at <= Time.zone.now - token_reuse_duration token = AuthToken.generate(self, remember, expiry, token_type) end @@ -120,6 +143,13 @@ def generate_scorm_authentication_token! generate_authentication_token!(token_type: :scorm) end + # + # Generate an authentication token for unit content asset retrieval + # + def generate_content_authentication_token! + generate_authentication_token!(token_type: :content) + end + # # Returns authentication of the user # @@ -143,6 +173,8 @@ def token_for_text?(a_token, token_type) belongs_to :role, optional: false # Foreign Key has_many :unit_roles, dependent: :destroy, inverse_of: :user has_many :projects, dependent: :restrict_with_exception, inverse_of: :user + has_many :engagements, dependent: :restrict_with_exception, inverse_of: :user + has_many :engagement_comments, dependent: :restrict_with_exception, inverse_of: :user has_many :auth_tokens, dependent: :destroy, inverse_of: :user has_many :user_oauth_tokens, dependent: :destroy, inverse_of: :user has_many :user_oauth_states, dependent: :destroy, inverse_of: :user diff --git a/app/models/webcal.rb b/app/models/webcal.rb index 06ce4f3f61..d0ca3c3e3b 100644 --- a/app/models/webcal.rb +++ b/app/models/webcal.rb @@ -13,6 +13,20 @@ def self.valid_time_units %w(W D H M) end + def reminder_trigger + return nil unless reminder? + + trigger = + case reminder_unit + when 'W', 'D' + "-P#{reminder_time}#{reminder_unit}" + when 'H', 'M' + "-PT#{reminder_time}#{reminder_unit}" + end + + Icalendar::Values::Duration.new(trigger, 'RELATED' => 'START') + end + # # Represents the presence of `reminder_time` and `reminder_unit`. # @@ -28,7 +42,7 @@ def reminder? def task_definitions TaskDefinition .joins(:unit, unit: :projects) - .includes(:unit, unit: :projects) + .includes(:grade_due_dates, unit: :projects) .where( projects: { user_id: user_id, enrolled: true }, units: { active: true } @@ -71,8 +85,15 @@ def to_ical(task_defs = task_definitions) ical.publish ical.prodid = Doubtfire::Application.config.institution[:product_name] - # load all of the tasks... uses the preloaded project - tasks = Task.where(task_definition: task_defs, project: task_defs.map { |t| t.unit.projects.first }.uniq) + projects = Project + .includes(:unit) + .where(user_id: user_id, enrolled: true, unit_id: task_defs.map(&:unit_id).uniq) + .index_by(&:unit_id) + + tasks = Task + .includes(:project, task_definition: :grade_due_dates) + .where(task_definition: task_defs, project: projects.values) + .index_by(&:task_definition_id) # Add iCalendar events for the specified definition. task_defs.each do |td| @@ -83,7 +104,7 @@ def to_ical(task_defs = task_definitions) ev_date_format = '%Y%m%d' ev_reminders = reminder? - ev_reminder_trigger = "-PT#{reminder_time}#{reminder_unit}" + ev_reminder_trigger = reminder_trigger # Add event for start date, if the user opted in. if include_start_dates @@ -91,7 +112,8 @@ def to_ical(task_defs = task_definitions) ev.uid = "S-#{td.id}" ev.summary = event_name_for_task_definition(td, 'start') ev.status = 'CONFIRMED' - ev.dtstart = ev.dtend = Icalendar::Values::Date.new(td.start_date.strftime(ev_date_format)) + start_date = Webcal.start_date_for_task_definition(td, tasks[td.id], projects[td.unit_id]) + ev.dtstart = ev.dtend = Icalendar::Values::Date.new(start_date.strftime(ev_date_format)) Webcal.add_metadata_to_ical_event(ev, td) @@ -110,7 +132,8 @@ def to_ical(task_defs = task_definitions) ev.uid = "E-#{td.id}" ev.summary = event_name_for_task_definition(td, 'end') ev.status = 'CONFIRMED' - ev.dtstart = ev.dtend = Icalendar::Values::Date.new(Webcal.end_date_for_task_definition(td, tasks).strftime(ev_date_format)) + end_date = Webcal.end_date_for_task_definition(td, tasks[td.id], projects[td.unit_id]) + ev.dtstart = ev.dtend = Icalendar::Values::Date.new(end_date.strftime(ev_date_format)) Webcal.add_metadata_to_ical_event(ev, td) @@ -137,9 +160,27 @@ def to_ical(task_defs = task_definitions) # # Returns the target/extended date for the specified task definition. # - def self.end_date_for_task_definition(task_def, tasks) - task = tasks.select { |t| t.task_definition_id == task_def.id }.first - task.present? ? task.due_date : task_def.target_date + def self.end_date_for_task_definition(task_def, task = nil, project = nil) + return task.local_due_date if task.present? + + flexible_grade_date_for_task_definition(task_def, project, :target_date) || task_def.target_date + end + + # + # Returns the start date for the specified task definition. + # + def self.start_date_for_task_definition(task_def, task = nil, project = nil) + return task.local_start_date if task.present? + + flexible_grade_date_for_task_definition(task_def, project, :start_date) || task_def.start_date + end + + def self.flexible_grade_date_for_task_definition(task_def, project, date_type) + return nil unless project&.unit&.allow_flexible_dates + + return task_def.grade_target_date(project.target_grade) if date_type == :target_date + + task_def.grade_start_date(project.target_grade) end # diff --git a/app/services/engagement_tracker.rb b/app/services/engagement_tracker.rb new file mode 100644 index 0000000000..85448dbf59 --- /dev/null +++ b/app/services/engagement_tracker.rb @@ -0,0 +1,103 @@ +class EngagementTracker + DEBOUNCE = 15.minutes + TUTORIAL_DURATION = 2.hours + + def self.record_attendance(user:, project:, occurred_at: Time.zone.now) + return unless during_enrolled_tutorial?(project, occurred_at) + + record_engagement( + user: user, + project: project, + engagement_type: 'Attendance', + note: 'Attended tutorial and QR scanned.', + occurred_at: occurred_at + ) + end + + def self.record_class_discussion(user:, project:, task_status_updates:, occurred_at: Time.zone.now) + timing = during_enrolled_tutorial?(project, occurred_at) ? 'during tutorial' : 'outside of tutorial' + note = if task_status_updates.any? + status_update_note(timing, task_status_updates) + else + "Class discussion #{timing}; no task statuses were updated." + end + + record_engagement( + user: user, + project: project, + engagement_type: 'Discussion', + note: note, + occurred_at: occurred_at + ) do |recent_engagement| + merge_status_updates!(recent_engagement, timing, task_status_updates) + end + end + + def self.record_engagement(user:, project:, engagement_type:, note:, occurred_at:) + project.with_lock do + recent_engagement = recently_recorded(project, engagement_type, occurred_at) + if recent_engagement + yield recent_engagement if block_given? + return + end + + project.engagements.create!( + user: user, + engagement_type: engagement_type, + note: note, + occurred_at: occurred_at + ) + end + end + + def self.recently_recorded(project, engagement_type, occurred_at) + project.engagements + .where('LOWER(engagement_type) = ?', engagement_type.downcase) + .where(occurred_at: (occurred_at - DEBOUNCE)..occurred_at) + .order(occurred_at: :desc) + .first + end + + def self.status_update_note(timing, task_status_updates) + ["Updated task statuses #{timing}.", *status_update_sentences(task_status_updates)].join(' ') + end + + def self.status_update_sentences(task_status_updates) + task_status_updates.map do |update| + "#{update[:task]}: #{update[:from]} → #{update[:to]}." + end + end + + def self.merge_status_updates!(engagement, timing, task_status_updates) + updates = status_update_sentences(task_status_updates) + return if updates.empty? + + if engagement.note.include?('no task statuses were updated') + engagement.update!(note: status_update_note(timing, task_status_updates)) + return + end + + new_updates = updates.reject { |update| engagement.note.include?(update) } + engagement.update!(note: "#{engagement.note} #{new_updates.join(' ')}") if new_updates.any? + end + + def self.during_enrolled_tutorial?(project, occurred_at) + project.tutorial_enrolments.includes(tutorial: :campus).any? do |enrolment| + tutorial = enrolment.tutorial + timezone = ActiveSupport::TimeZone[tutorial.campus&.timezone] || Time.zone + local_time = occurred_at.in_time_zone(timezone) + + next false unless tutorial.meeting_day == local_time.strftime('%A') + + tutorial_start = timezone.parse("#{local_time.to_date} #{tutorial.meeting_time}") + local_time >= tutorial_start && local_time < tutorial_start + TUTORIAL_DURATION + end + end + + private_class_method :record_engagement, + :recently_recorded, + :status_update_note, + :status_update_sentences, + :merge_status_updates!, + :during_enrolled_tutorial? +end diff --git a/app/sidekiq/accept_overseer_job.rb b/app/sidekiq/accept_overseer_job.rb index 5f176e2dfd..59ddd753fd 100644 --- a/app/sidekiq/accept_overseer_job.rb +++ b/app/sidekiq/accept_overseer_job.rb @@ -9,8 +9,7 @@ class AcceptOverseerJob include FileHelper sidekiq_options lock: :until_executed, - # TODO: should students be allowed to submit a new task submission when the previous overseer job has not started/completed? - lock_args_method: ->(args) { [args.first, 'overseer-assessment'] }, + lock_args_method: ->(args) { [args.first, args.last, 'overseer-assessment'] }, on_conflict: :reject, retry: 1 @@ -41,7 +40,7 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment work_dir = Rails.root.join("tmp", "overseer", work_dir_name) FileUtils.mkdir_p(work_dir) - extract_student_submission_files(task, submission, work_dir) + extract_student_submission_files(task, submission, work_dir, timestamp) extract_overseer_resource_files(assessment, work_dir) success_status = nil @@ -57,6 +56,11 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment assessment_pass = true + overseer_image = task_definition.overseer_image || + task.unit.overseer_image || + OverseerImage.find_by(tag: docker_image_name_tag) + ensure_docker_image_present(docker_image_name_tag, overseer_image) + active_overseer_steps.each do |step| result = run_overseer_step( step: step, @@ -96,7 +100,9 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment end else oa.update!(status: :failed) - unless failure_status.nil? + # preserve_status_on_failure = [TaskStatus.time_exceeded.id, TaskStatus.assess_in_portfolio.id].include?(task.task_status_id) + + unless failure_status.nil? # || preserve_status_on_failure # TODO: have an override status setting for the step? eg. if the task is overdue, let it remain overdue, otherwise use this task status task.update!(task_status: failure_status) task.add_status_comment(task.project.tutor_for(task.task_definition), failure_status) @@ -113,6 +119,18 @@ def perform(task_id, _output_path, docker_image_name_tag, submission, assessment raise e end + def ensure_docker_image_present(docker_image_name_tag, overseer_image) + _, _, inspect_status = Open3.capture3('docker', 'image', 'inspect', docker_image_name_tag) + return if inspect_status.success? + + raise "Docker image #{docker_image_name_tag} is not configured" if overseer_image.nil? + + overseer_image.pull_from_docker + return if overseer_image.success? + + raise "Unable to pull Docker image #{docker_image_name_tag}: #{overseer_image.pulled_image_text}" + end + def run_overseer_step(step:, work_dir:, work_dir_name:, task_id:, timestamp:, docker_image_name_tag:, overseer_assessment_id:) script_contents = step.run_command raise "Execution script is empty" if script_contents.blank? @@ -155,6 +173,7 @@ def run_overseer_step(step:, work_dir:, work_dir_name:, task_id:, timestamp:, do command = %( timeout #{timeout} docker run --rm -i \ + --pull never \ --cpus 1 \ --network none \ #{volume_mount} \ @@ -239,25 +258,32 @@ def run_overseer_step(step:, work_dir:, work_dir_name:, task_id:, timestamp:, do ) end - def extract_student_submission_files(task, submission, work_dir) - # Extract submission files, removing any parent folders - Zip::File.open(submission) do |zip_file| - zip_file.each do |entry| - next if entry.name_is_directory? + def extract_student_submission_files(task, submission, work_dir, timestamp) + # Submission files are stored directly under their timestamp in the task archive. + Zip::File.open(submission) do |history_zip| + prefix = "#{FileHelper.sanitized_path(timestamp.to_s)}/" + entries = history_zip.entries.reject(&:name_is_directory?).select { |entry| entry.name.start_with?(prefix) } + raise "Submission history entries not found for timestamp: #{timestamp}" if entries.empty? - parts = entry.name.split('/')[1..] - next unless parts.length >= 1 + extract_submission_entries(task, entries, work_dir, prefix) + end + end - file_name = parts.first - index = file_name.to_i + def extract_submission_entries(task, entries, work_dir, prefix) + # Extract submission files, removing any parent folders. + entries.each do |entry| + parts = entry.name.delete_prefix(prefix).split('/') + next unless parts.first == task.id.to_s && parts.length >= 2 - file = task.upload_requirements[index] - final_name = file['name'] + file_name = parts.second + index = file_name.to_i - dest_path = File.join(work_dir, final_name) - FileUtils.mkdir_p(File.dirname(dest_path)) - zip_file.extract(entry, dest_path) { true } - end + file = task.upload_requirements[index] + final_name = file['name'] + + dest_path = File.join(work_dir, final_name) + FileUtils.mkdir_p(File.dirname(dest_path)) + entry.extract(dest_path) { true } end end diff --git a/app/sidekiq/accept_submission_job.rb b/app/sidekiq/accept_submission_job.rb index 15a92d084e..eaaf9b380e 100644 --- a/app/sidekiq/accept_submission_job.rb +++ b/app/sidekiq/accept_submission_job.rb @@ -2,6 +2,11 @@ class AcceptSubmissionJob include Sidekiq::Job include LogHelper + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first] }, + on_conflict: :reject, + retry: false + def perform(task_id, user_id, accepted_tii_eula, test_submission) begin # Ensure cwd is valid... @@ -36,6 +41,16 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) begin # Notify system admin + if defined?(Sentry) + Sentry.capture_exception( + e, + extra: { + task_id: task.id, + task_definition_abbreviation: task.task_definition.abbreviation, + latex_log_message: e.respond_to?(:log_message) ? e.log_message.to_s.last(5000) : nil + } + ) + end mail = ErrorLogMailer.error_message('Accept Submission', "Failed to convert submission to PDF for task #{task.log_details}", e) mail.deliver if mail.present? rescue StandardError => e @@ -60,19 +75,24 @@ def perform(task_id, user_id, accepted_tii_eula, test_submission) task.send_documents_to_tii(user, accepted_tii_eula: accepted_tii_eula) end - if task.overseer_enabled? || test_submission - overseer_assessment = OverseerAssessment.create_for(task, test_submission) - if overseer_assessment.present? - logger.info "Launching Overseer assessment for task_def_id: #{task.task_definition.id} task_id: #{task.id}" - - overseer_assessment.send_to_overseer(test_submission: test_submission) - - else - logger.info "Overseer assessment for task_def_id: #{task.task_definition.id} task_id: #{task.id} was not performed #{overseer_assessment.inspect}" - end + if SubmissionHistory.enabled_requirements(task).any? + submission_timestamp = Time.now.utc.to_i + SubmissionHistory.mark_pending(task) + CreateSubmissionHistoryJob.perform_async(task.id, submission_timestamp, test_submission) + elsif task.overseer_enabled? || test_submission + logger.error "Overseer assessment was not performed because task definition #{task.task_definition.id} has no submission history files configured" end rescue StandardError => e # to raise error message to avoid unnecessary retry logger.error e + if defined?(Sentry) + Sentry.capture_exception( + e, + extra: { + task_id: task&.id, + task_definition_abbreviation: task&.task_definition&.abbreviation + } + ) + end task.clear_in_process end end diff --git a/app/sidekiq/aggregate_task_completion_stats_job.rb b/app/sidekiq/aggregate_task_completion_stats_job.rb new file mode 100644 index 0000000000..4bb375ebec --- /dev/null +++ b/app/sidekiq/aggregate_task_completion_stats_job.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class AggregateTaskCompletionStatsJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + include ApplicationHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first] }, + on_conflict: :reject, + retry: false + + def perform(unit_id = nil) + logger.info 'Starting task completion stats aggregation...' + + at(0) + total(1) + + if unit_id.present? + Unit.find(unit_id).capture_task_complete_stats_snapshot! + else + Unit.active_units.find_each(&:capture_task_complete_stats_snapshot!) + end + + at(1) + logger.info 'Completed task completion stats aggregation!' + rescue StandardError => e + logger.error e + raise e + end +end diff --git a/app/sidekiq/archive_old_units_job.rb b/app/sidekiq/archive_old_units_job.rb index 7ebb68e666..db4ce2f11b 100644 --- a/app/sidekiq/archive_old_units_job.rb +++ b/app/sidekiq/archive_old_units_job.rb @@ -17,6 +17,7 @@ def perform rescue StandardError => e begin # Notify system admin + Sentry.capture_exception(e) if defined?(Sentry) mail = ErrorLogMailer.error_message('Archive Units', "Failed to move old units to archive", e) mail.deliver if mail.present? diff --git a/app/sidekiq/communication_rule_job.rb b/app/sidekiq/communication_rule_job.rb new file mode 100644 index 0000000000..af8e5831fd --- /dev/null +++ b/app/sidekiq/communication_rule_job.rb @@ -0,0 +1,40 @@ +class CommunicationRuleJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + include ApplicationHelper + include FileHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first, args.last, 'communication-rule'] }, + on_conflict: :reject, + retry: 1 + + def perform(rule_id) + logger.info "Starting communication rule job..." + + at(0) + total(1) + + rule = CommunicationRule.find(rule_id) + + projects = rule.communication_set.preview_projects_for_rule(rule) + store( + result: projects.map do |project| + { + username: project.user&.username, + student_id: project.user&.student_id, + target_grade: project.target_grade, + last_sign_in_at: project.user&.last_sign_in_at, + last_viewed_at: project.last_viewed_at + } + end + ) + + logger.info "Completed communication job" + rescue StandardError => e + logger.error e + raise e + end + +end diff --git a/app/sidekiq/create_submission_history_job.rb b/app/sidekiq/create_submission_history_job.rb new file mode 100644 index 0000000000..9efedb25ff --- /dev/null +++ b/app/sidekiq/create_submission_history_job.rb @@ -0,0 +1,27 @@ +class CreateSubmissionHistoryJob + include Sidekiq::Job + include LogHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first, 'submission-history'] }, + on_conflict: :reject, + retry: false + + def perform(task_id, submission_timestamp, test_submission) + task = Task.find(task_id) + history = SubmissionHistory.create_archive!(task, submission_timestamp) + + return unless task.overseer_enabled? || test_submission + + assessment = OverseerAssessment.create_for(history, test_submission) + return if assessment.nil? + + assessment.update!(student_notified_at: Time.current) if test_submission + assessment.send_to_overseer(test_submission: test_submission) + rescue StandardError => e + logger.error e + Sentry.capture_exception(e, extra: { task_id: task_id }) if defined?(Sentry) + ensure + SubmissionHistory.clear_pending(task) if task + end +end diff --git a/app/sidekiq/download_overflow_task_claims_csv_job.rb b/app/sidekiq/download_overflow_task_claims_csv_job.rb new file mode 100644 index 0000000000..5fd7429b66 --- /dev/null +++ b/app/sidekiq/download_overflow_task_claims_csv_job.rb @@ -0,0 +1,33 @@ +require 'csv' + +class DownloadOverflowTaskClaimsCsvJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + include ApplicationHelper + include FileHelper + include MimeCheckHelpers + include CsvHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first] }, + on_conflict: :reject, + retry: false + + def perform(unit_id) + logger.info "Starting overflow task claims csv download..." + + at(0) + total(1) + + unit = Unit.find(unit_id) + csv = unit.overflow_task_claims_csv + + store(result: csv) + + logger.info "Completed overflow task claims csv download!" + rescue StandardError => e + logger.error e + raise e + end +end diff --git a/app/sidekiq/execute_communication_set_job.rb b/app/sidekiq/execute_communication_set_job.rb new file mode 100644 index 0000000000..5f99bd3fe6 --- /dev/null +++ b/app/sidekiq/execute_communication_set_job.rb @@ -0,0 +1,613 @@ +require 'csv' + +class ExecuteCommunicationSetJob + include Sidekiq::Job + include Sidekiq::Status::Worker + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args[0], args[1], 'communication-set'] }, + on_conflict: :reject, + retry: 1 + + def perform(communication_set_id, target_rule_id = nil) + communication_set = CommunicationSet.find(communication_set_id) + rules = communication_set.communication_rules.to_a + target_rule_id = target_rule_id&.to_i + + if target_rule_id.present? && rules.none? { |rule| rule.id == target_rule_id } + raise ActiveRecord::RecordNotFound, "CommunicationRule #{target_rule_id} not found in set #{communication_set_id}" + end + + eligible_projects = communication_set.eligible_projects + remaining_projects = eligible_projects.dup + executed_rules = [] + action_results = [] + + rules_to_process = + if target_rule_id.present? + cutoff_index = rules.index { |rule| rule.id == target_rule_id } + rules.first(cutoff_index + 1) + else + rules + end + + at(0) + total(rules_to_process.length.nonzero? || 1) + + rules_to_process.each_with_index do |rule, index| + matched_projects = rule.matching_projects(remaining_projects) + + executed_rules << { + rule_id: rule.id, + rule_name: rule.name, + matched_project_ids: matched_projects.map(&:id) + } + + should_execute_actions = target_rule_id.present? ? rule.id == target_rule_id : true + + if should_execute_actions + rule_action_results = rule.communication_actions.flat_map do |action| + execute_action(action, matched_projects, communication_set.unit, rule) + end + + action_results.concat(rule_action_results) + if rule.send_log_to_convenors? + action_results.concat( + send_action_log_to_convenors( + matched_projects, + communication_set.unit, + rule, + rule_action_results + ) + ) + end + end + + remaining_projects -= matched_projects + at(index + 1) + end + + store( + result: { + communication_set_id: communication_set.id, + target_rule_id: target_rule_id, + executed_rule_ids: executed_rules.map { |item| item[:rule_id] }, + remaining_project_ids: remaining_projects.map(&:id), + rules: executed_rules, + actions: action_results + } + ) + rescue StandardError => e + logger.error("ExecuteCommunicationSetJob failed: #{e.class} #{e.message}") + raise e + end + + private + + def execute_action(action, projects, unit, rule) + case action.type + when 'ChangeTargetGradeAction' + execute_change_target_grade_action(action, projects) + when 'EmailStudentAction' + execute_email_student_action(action, projects, unit, rule) + when 'EmailStaffAction' + execute_email_staff_action(action, projects, unit, rule) + when 'TaskCommentAction' + execute_task_comment_action(action, projects, unit, rule) + else + [{ + action_id: action.id, + action_type: action.type, + status: 'skipped', + reason: 'unsupported action type' + }] + end + end + + def execute_change_target_grade_action(action, projects) + projects.map do |project| + previous_target_grade = project.target_grade + + project.update!(target_grade: action.target_grade) + + { + action_id: action.id, + action_type: action.type, + status: 'updated', + project_id: project.id, + username: project.user&.username, + previous_target_grade: previous_target_grade, + target_grade: action.target_grade + } + end + end + + def execute_email_student_action(action, projects, unit, rule) + projects.filter_map do |project| + recipient = project.user + sender = sender_for(unit) + + if recipient&.email.blank? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + reason: 'student email missing' + } + end + + if sender.blank? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + reason: 'sender email missing' + } + end + + subject = render_template(action.subject, project, unit, rule, projects.length) + body = render_template(action.body, project, unit, rule, projects.length) + + CommunicationsMailer.communication_email( + to: formatted_email(recipient), + from: sender, + subject: subject, + body: body, + recipient: recipient, + sender: sender_user_for(unit), + unit: unit, + rule: rule + ).deliver_now + + { + action_id: action.id, + action_type: action.type, + status: 'sent', + project_id: project.id, + username: recipient.username, + recipient_email: recipient.email + } + end + end + + def execute_email_staff_action(action, projects, unit, rule) + projects.flat_map do |project| + sender = sender_for(unit) + + if sender.blank? + next [{ + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + username: project.user&.username, + reason: 'sender email missing' + }] + end + + staff_recipients_for(project, unit, action).map do |recipient| + subject = render_template(action.subject, project, unit, rule, projects.length) + body = render_template(action.body, project, unit, rule, projects.length) + + CommunicationsMailer.communication_email( + to: formatted_email(recipient), + from: sender, + subject: subject, + body: body, + recipient: recipient, + sender: sender_user_for(unit), + unit: unit, + rule: rule + ).deliver_now + + { + action_id: action.id, + action_type: action.type, + status: 'sent', + project_id: project.id, + username: project.user&.username, + recipient_email: recipient.email, + recipient_username: recipient.username + } + end + end + end + + def execute_task_comment_action(action, projects, unit, rule) + comment_author = sender_user_for(unit) + + if comment_author.blank? + return [{ + action_id: action.id, + action_type: action.type, + status: 'skipped', + reason: 'comment author missing' + }] + end + + comment_text_template = action.body.to_s.strip + + projects.map do |project| + task_definition = action.task_definition || unit.task_definitions.find_by(id: action.task_definition_id) + if task_definition.blank? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + username: project.user&.username, + reason: 'task definition missing' + } + end + + task = project.task_for_task_definition(task_definition) + rendered_comment = render_template(comment_text_template, project, unit, rule, projects.length) + + if rendered_comment.blank? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + username: project.user&.username, + reason: 'comment text missing' + } + end + + comment = task.add_text_comment(comment_author, rendered_comment) + + if comment.nil? + next { + action_id: action.id, + action_type: action.type, + status: 'skipped', + project_id: project.id, + username: project.user&.username, + reason: 'duplicate comment' + } + end + + { + action_id: action.id, + action_type: action.type, + status: 'commented', + project_id: project.id, + username: project.user&.username, + task_definition_id: task_definition.id, + task_definition_name: task_definition.name, + comment_id: comment.id + } + end + end + + def send_action_log_to_convenors(projects, unit, rule, prior_action_results) + sender = sender_for(unit) + + if sender.blank? + return [{ + action_id: nil, + action_type: 'SendLogToConvenors', + status: 'skipped', + reason: 'sender email missing' + }] + end + + recipients = unit.convenors.includes(:user).map(&:user).select { |user| user&.email.present? }.uniq(&:id) + + if recipients.empty? + return [{ + action_id: nil, + action_type: 'SendLogToConvenors', + status: 'skipped', + reason: 'convenor email missing' + }] + end + + csv_content = build_action_log_csv(rule, projects, prior_action_results) + csv_filename = "communication-rule-#{rule.id}-action-log.csv" + body = action_log_email_body(rule, prior_action_results) + subject = action_log_email_subject(unit, rule) + + recipients.map do |recipient| + CommunicationsMailer.action_log_email( + to: formatted_email(recipient), + from: sender, + subject: subject, + body: body, + recipient: recipient, + sender: sender_user_for(unit), + unit: unit, + rule: rule, + csv_content: csv_content, + csv_filename: csv_filename, + affected_students_count: projects.length + ).deliver_now + + { + action_id: nil, + action_type: 'SendLogToConvenors', + status: 'sent', + recipient_email: recipient.email, + recipient_username: recipient.username, + attachment_filename: csv_filename, + affected_students_count: projects.length + } + end + end + + def staff_recipients_for(project, unit, action) + recipients = [] + + if action.email_tutors + recipients.concat( + project.tutorial_enrolments.filter_map do |tutorial_enrolment| + tutorial_enrolment.tutorial&.unit_role&.user + end + ) + end + + if action.email_convenors + recipients.concat(unit.convenors.includes(:user).map(&:user)) + end + + recipients.select { |recipient| recipient&.email.present? }.uniq(&:id) + end + + def render_template(template, project, unit, rule, affected_students_count, target_grade_override = nil, action_results = []) + return '' if template.blank? + + student = project&.user + target_grade_value = target_grade_override.nil? ? project&.target_grade : target_grade_override + + replacements = { + '{{student.first_name}}' => student&.first_name.to_s, + '{{student.last_name}}' => student&.last_name.to_s, + '{{student.preferred_name}}' => (student&.nickname.presence || student&.first_name).to_s, + '{{student.full_name}}' => [student&.first_name, student&.last_name].compact.join(' '), + '{{student.username}}' => student&.username.to_s, + '{{student.student_id}}' => student&.student_id.to_s, + '{{affected_students_count}}' => affected_students_count.to_s, + '{{unit.code}}' => unit.code.to_s, + '{{unit.name}}' => unit.name.to_s, + '{{rule.name}}' => rule.name.to_s, + '{{target_grade}}' => target_grade_name(target_grade_value, unit), + '{{conditions_summary}}' => conditions_summary(rule), + '{{actions_summary}}' => actions_summary(rule, action_results) + } + + replacements.reduce(template.dup) do |rendered, (token, value)| + rendered.gsub(token, value) + end + end + + def target_grade_name(value, unit = nil) + (unit&.grade_label(value) || GradeHelper.grade_for(value)).to_s + end + + def formatted_email(user) + return nil if user&.email.blank? + + %("#{user.name}" <#{user.email}>) + end + + def sender_for(unit) + formatted_email(sender_user_for(unit)) + end + + def sender_user_for(unit) + unit.main_convenor_user || unit.convenors.includes(:user).first&.user + end + + def conditions_summary(rule) + rule.communication_conditions.map do |condition| + "- #{human_condition_summary(condition)}" + end.join("\n") + end + + def actions_summary(rule, action_results) + return rule.communication_actions.map { |action| "- #{human_action_summary(action)}" }.join("\n") if action_results.blank? + + rule.communication_actions.map { |action| "- #{human_action_summary(action)}" }.join("\n") + end + + def build_action_log_csv(rule, projects, action_results) + action_order = rule.communication_actions.each_with_index.to_h { |action, index| [action.id, index] } + ordered_results = action_results.sort_by do |result| + project = projects.find { |item| item.id == result[:project_id] } + student = project&.user + + [ + student&.username.to_s, + action_order.fetch(result[:action_id], Float::INFINITY), + result[:recipient_email].to_s + ] + end + + CSV.generate(headers: true) do |csv| + csv << [ + 'student_username', + 'student_id', + 'student_name', + 'rule_name', + 'action_type', + 'status', + 'details', + # 'previous_target_grade', + # 'new_target_grade', + 'recipient_email', + 'executed_at' + ] + + ordered_results.each do |result| + project = projects.find { |item| item.id == result[:project_id] } + student = project&.user + details = if result[:status] == 'updated' + "Changed target grade from #{target_grade_name(result[:previous_target_grade], rule.unit)} to #{target_grade_name(result[:target_grade], rule.unit)}" + elsif result[:status] == 'commented' + task_definition = TaskDefinition.find_by(id: result[:task_definition_id]) + "Added comment to #{task_definition_label(task_definition)}" + elsif result[:recipient_email].present? + "Sent email to #{result[:recipient_email]}" + else + result[:reason].to_s + end + + csv << [ + student&.username, + student&.student_id, + student&.name, + rule.name, + result[:action_type], + result[:status], + details, + # target_grade_name(result[:previous_target_grade]), + # target_grade_name(result[:target_grade]), + result[:recipient_email], + Time.current.iso8601 + ] + end + end + end + + def action_log_email_subject(unit, rule) + "#{unit.code} #{rule.name} action log" + end + + def action_log_email_body(rule, action_results) + [ + action_log_conditions_intro(rule), + conditions_summary(rule), + 'The following actions have been applied to these students:', + actions_summary(rule, action_results) + ].join("\n") + end + + def action_log_conditions_intro(rule) + if rule.operator == 'or' + 'A scheduled rule has been run for students that match any of the following conditions:' + else + 'A scheduled rule has been run for students that match all of the following conditions:' + end + end + + def human_condition_summary(condition) + case condition.type + when 'TaskDefinitionStatusCondition' + predicate = condition.operator == 'not_equal_to' ? 'Not In' : 'In' + task = TaskDefinition.find_by(id: condition.task_definition_id) + task_label = if task + "Task #{task.abbreviation} #{task.name}" + else + "Task #{condition.task_definition_id}" + end + "Students that have #{task_label} #{predicate} [#{Array(condition.task_statuses).map { |status| status.to_s.titleize }.join(', ')}]" + when 'TargetGradeCondition' + "Students with a Target Grade #{operator_label(condition.operator)} #{target_grade_name(condition.target_grade, condition.communication.unit)}" + when 'TaskStatusCountCondition' + grade_label = target_grade_name(condition.task_target_grade, condition.communication.unit) + statuses = Array(condition.task_statuses).map { |status| status.to_s.titleize }.join(', ') + "Students that have #{operator_label(condition.operator)} #{condition.task_status_count} #{grade_label} tasks in [#{statuses}]" + when 'LoginStatusCondition' + relative_activity_summary(condition, 'signed in') + when 'UnitViewedStatusCondition' + relative_activity_summary(condition, 'viewed this unit') + when 'SpecConCondition' + "Students with Special Consideration Days #{operator_label(condition.operator)} #{condition.spec_con_days}" + when 'TutorialEnrolmentCondition' + tutorial = Tutorial.find_by(id: condition.tutorial_id) + tutorial_label = + if tutorial + [tutorial.abbreviation, tutorial.name].compact.join(' ') + else + "Tutorial #{condition.tutorial_id}" + end + "Students #{enrolment_label(condition.operator).downcase} #{tutorial_label}" + when 'TutorialStreamEnrolmentCondition' + tutorial_stream = TutorialStream.find_by(id: condition.tutorial_stream_id) + stream_label = + if tutorial_stream + [tutorial_stream.abbreviation, tutorial_stream.name].compact.join(' ') + else + "Tutorial Stream #{condition.tutorial_stream_id}" + end + "Students #{enrolment_label(condition.operator).downcase} #{stream_label}" + when 'CampusCondition' + campus = Campus.find_by(id: condition.campus_id) + campus_label = campus&.name || "Campus #{condition.campus_id}" + "Students #{enrolment_label(condition.operator).downcase} #{campus_label}" + else + "#{condition.type.to_s.underscore.humanize} #{condition.operator.to_s.humanize}" + end + end + + def relative_activity_summary(condition, activity) + duration = "#{condition.activity_days} #{'day'.pluralize(condition.activity_days)}" + + if condition.operator == 'more_than' + "Students who have not #{activity} for more than #{duration}" + else + "Students who #{activity} within the last #{duration}" + end + end + + def human_action_summary(action) + case action.type + when 'EmailStudentAction' + 'Send email' + when 'EmailStaffAction' + 'Send staff email' + when 'ChangeTargetGradeAction' + "Change Target Grade to #{target_grade_name(action.target_grade, action.communication_rule.unit)}" + when 'TaskCommentAction' + "Add comment to #{task_definition_label(action.task_definition)}" + else + human_action_type_name(action.type) + end + end + + def human_action_type_name(type) + case type + when 'EmailStudentAction' + 'Send email' + when 'EmailStaffAction' + 'Send staff email' + when 'ChangeTargetGradeAction' + 'Change target grade' + when 'TaskCommentAction' + 'Add task comment' + else + type.to_s.underscore.humanize + end + end + + def operator_label(operator) + case operator.to_s + when 'greater_than' + 'Greater Than' + when 'greater_than_or_equal_to' + 'Greater Than Or Equal To' + when 'less_than' + 'Less Than' + when 'less_than_or_equal_to' + 'Less Than Or Equal To' + when 'equal_to' + 'Equal To' + when 'not_equal_to' + 'Not Equal To' + else + operator.to_s.humanize + end + end + + def enrolment_label(operator) + operator.to_s == 'not_enrolled_in' ? 'Not Enrolled In' : 'Enrolled In' + end + + def task_definition_label(task_definition) + return 'Task' if task_definition.blank? + + "Task #{task_definition.abbreviation} #{task_definition.name}" + end +end diff --git a/app/sidekiq/execute_communication_set_schedule_job.rb b/app/sidekiq/execute_communication_set_schedule_job.rb new file mode 100644 index 0000000000..6960a96dd4 --- /dev/null +++ b/app/sidekiq/execute_communication_set_schedule_job.rb @@ -0,0 +1,24 @@ +class ExecuteCommunicationSetScheduleJob + include Sidekiq::Job + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first, 'communication-set-schedule'] }, + on_conflict: :reject, + retry: 1 + + def perform(schedule_id) + schedule = CommunicationSetSchedule.find(schedule_id) + return unless schedule.active? + return unless schedule.unit.active? + + now = Time.zone.now + return unless schedule.due?(now) + + ExecuteCommunicationSetJob.perform_async(schedule.communication_set_id) + schedule.update!( + last_enqueued_at: now, + last_run_at: now, + next_run_at: schedule.next_occurrence_after(now + 1.second) + ) + end +end diff --git a/app/sidekiq/import_batch_feedback_job.rb b/app/sidekiq/import_batch_feedback_job.rb new file mode 100644 index 0000000000..e8504d6eb2 --- /dev/null +++ b/app/sidekiq/import_batch_feedback_job.rb @@ -0,0 +1,39 @@ +class ImportBatchFeedbackJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { args.first(3) }, + on_conflict: :reject, + retry: false + + def perform(unit_id, assessor_id, task_definition_id, path_to_upload) + logger.info "Starting batch feedback import..." + + at(0, 'Preparing batch feedback import') + total(1) + + unit = Unit.find(unit_id) + task_definition = unit.task_definitions.find(task_definition_id) + assessor = User.find(assessor_id) + + file = File.open(path_to_upload, 'rb') + result = unit.upload_batch_feedback_csv( + assessor, + task_definition, + { 'tempfile' => file }, + progress_callback: lambda { |message: nil, total_rows: nil, rows_processed: nil| + total(total_rows) if total_rows + at(rows_processed, message) unless rows_processed.nil? + } + ) + + store(result: result.to_json) + + logger.info "Completed batch feedback import!" + ensure + file&.close if defined?(file) && file.present? + FileUtils.rm_f(path_to_upload) if path_to_upload.present? + end +end diff --git a/app/sidekiq/import_staff_notes_csv_job.rb b/app/sidekiq/import_staff_notes_csv_job.rb new file mode 100644 index 0000000000..b41379943d --- /dev/null +++ b/app/sidekiq/import_staff_notes_csv_job.rb @@ -0,0 +1,39 @@ +require 'csv' + +class ImportStaffNotesCsvJob + include Sidekiq::Job + include Sidekiq::Status::Worker + include LogHelper + include ApplicationHelper + include FileHelper + include MimeCheckHelpers + include CsvHelper + + sidekiq_options lock: :until_executed, + lock_args_method: ->(args) { [args.first] }, + on_conflict: :reject, + retry: false + + def perform(unit_id, author_id, path_to_csv) + logger.info "Starting staff note imports..." + + at(0) + + ensure_csv!(path_to_csv) + + unit = Unit.find(unit_id) + result = unit.import_staff_notes_from_csv(path_to_csv, author_id, progress_callback: lambda { |message: nil, total_rows: nil, rows_processed: nil| + total(total_rows) if total_rows + at(rows_processed, message) if rows_processed + }) + + store(result: result.to_json) + + FileUtils.rm(path_to_csv) + + logger.info "Completed staff note imports!" + rescue StandardError => e + logger.error e + raise e + end +end diff --git a/app/sidekiq/notify_discuss_timeout_job.rb b/app/sidekiq/notify_discuss_timeout_job.rb new file mode 100644 index 0000000000..3a1de38a04 --- /dev/null +++ b/app/sidekiq/notify_discuss_timeout_job.rb @@ -0,0 +1,12 @@ +class NotifyDiscussTimeoutJob + include Sidekiq::Job + + sidekiq_options lock: :until_executed, + lock_args_method: ->(_args) { ['notify-discuss-timeout'] }, + on_conflict: :reject, + retry: 1 + + def perform + Unit.notify_discuss_timeouts! + end +end diff --git a/app/sidekiq/poll_communication_set_schedules_job.rb b/app/sidekiq/poll_communication_set_schedules_job.rb new file mode 100644 index 0000000000..b5f0f803f2 --- /dev/null +++ b/app/sidekiq/poll_communication_set_schedules_job.rb @@ -0,0 +1,19 @@ +class PollCommunicationSetSchedulesJob + include Sidekiq::Job + + sidekiq_options lock: :until_executed, + lock_args_method: ->(_args) { ['poll-communication-set-schedules'] }, + on_conflict: :reject, + retry: 1 + + def perform + CommunicationSetSchedule + .includes(:communication_set) + .active + .with_active_unit + .due(Time.zone.now) + .find_each do |schedule| + ExecuteCommunicationSetScheduleJob.perform_async(schedule.id) + end + end +end diff --git a/app/sidekiq/refresh_moderation_feedback_timestamps_job.rb b/app/sidekiq/refresh_moderation_feedback_timestamps_job.rb index de38dc9a80..ad3d8abf77 100644 --- a/app/sidekiq/refresh_moderation_feedback_timestamps_job.rb +++ b/app/sidekiq/refresh_moderation_feedback_timestamps_job.rb @@ -6,6 +6,8 @@ class RefreshModerationFeedbackTimestampsJob def perform ModeratedTask .where(state: %i[open waiting_for_new_feedback]) + .joins(task: { project: :unit }) + .where(units: { active: true }) .includes(task: :comments) .find_each do |moderated_task| task = moderated_task.task diff --git a/app/sidekiq/send_discuss_timeout_email_job.rb b/app/sidekiq/send_discuss_timeout_email_job.rb new file mode 100644 index 0000000000..76349e7835 --- /dev/null +++ b/app/sidekiq/send_discuss_timeout_email_job.rb @@ -0,0 +1,24 @@ +class SendDiscussTimeoutEmailJob + include Sidekiq::Job + + sidekiq_options retry: 5 + + def perform(task_id, sender_id, notification_type, expiry_date = nil) + task = Task.find_by(id: task_id) + sender = User.find_by(id: sender_id) + return if task.blank? || sender.blank? + return unless task.unit.send_notifications + return unless task.project.student.receive_feedback_notifications + + mail = case notification_type + when 'approaching' + NotificationsMailer.discussion_deadline_approaching(task, sender, Date.iso8601(expiry_date)) + when 'missed' + NotificationsMailer.discussion_deadline_missed(task, sender) + else + raise ArgumentError, "Unknown discussion deadline notification type: #{notification_type}" + end + + mail.deliver_now + end +end diff --git a/app/views/communications_mailer/action_log_email.html.erb b/app/views/communications_mailer/action_log_email.html.erb new file mode 100644 index 0000000000..7df5658bf1 --- /dev/null +++ b/app/views/communications_mailer/action_log_email.html.erb @@ -0,0 +1,69 @@ + + + + + + +
+

<%= @doubtfire_product_name %> Notification

+ +

Hi <%= @recipient&.nickname.presence || @recipient&.first_name || @recipient&.name %>,

+ + <% @body_paragraphs.each do |paragraph| %> +

<%= paragraph %>

+ <% end %> + +
+ +

+ <%= @affected_students_count %> + students matched this rule and were affected by the configured actions. +

+ +

A full log of actions has been attached as a CSV.

+ +

+ Cheers,
+ The <%= @doubtfire_product_name %> Team +

+ +
+ + diff --git a/app/views/communications_mailer/action_log_email.text.erb b/app/views/communications_mailer/action_log_email.text.erb new file mode 100644 index 0000000000..f980a817f3 --- /dev/null +++ b/app/views/communications_mailer/action_log_email.text.erb @@ -0,0 +1,18 @@ +Hi <%= @recipient&.nickname.presence || @recipient&.first_name || @recipient&.name %>, + +<% @body_paragraphs.each do |paragraph| %> +<%= paragraph %> + +<% end %> +--- + +<%= @affected_students_count %> students matched this rule and were affected by the configured actions. + +A full log of actions has been attached as a CSV. + +Cheers, +The <%= @doubtfire_product_name %> Team + +--- + +Generated with <%= @doubtfire_product_name %> diff --git a/app/views/communications_mailer/communication_email.html.erb b/app/views/communications_mailer/communication_email.html.erb new file mode 100644 index 0000000000..7933f82475 --- /dev/null +++ b/app/views/communications_mailer/communication_email.html.erb @@ -0,0 +1,53 @@ + + + + + + +
+

<%= @doubtfire_product_name %> Notification

+ + <% @body_paragraphs.each do |paragraph| %> +

<%= paragraph %>

+ <% end %> + +
+ + diff --git a/app/views/communications_mailer/communication_email.text.erb b/app/views/communications_mailer/communication_email.text.erb new file mode 100644 index 0000000000..5ad89c15fb --- /dev/null +++ b/app/views/communications_mailer/communication_email.text.erb @@ -0,0 +1,12 @@ +<%# Hi <%= @recipient.nickname.presence || @recipient.first_name %> %> + +<% @body_paragraphs.each do |paragraph| %> +<%= paragraph %> + +<% end %> +<%# Cheers, +The <%= @doubtfire_product_name %> Team on behalf of <%= @sender.name %> %> + +--- + +Generated with <%= @doubtfire_product_name %> diff --git a/app/views/layouts/application.pdf.erbtex b/app/views/layouts/application.pdf.erbtex index b051661808..4c1c48e55f 100644 --- a/app/views/layouts/application.pdf.erbtex +++ b/app/views/layouts/application.pdf.erbtex @@ -27,6 +27,12 @@ } \usepackage[fencedCode,hashEnumerators,pipeTables,texMathDollars]{markdown} +\markdownSetup{rendererPrototypes={ + ulBeginTight={\begin{itemize}}, + ulEndTight={\end{itemize}}, + olBeginTight={\begin{enumerate}}, + olEndTight={\end{enumerate}} +}} \usepackage{luatextra} \defaultfontfeatures{Ligatures=TeX} diff --git a/app/views/layouts/discussion_deadline_mailer.html.erb b/app/views/layouts/discussion_deadline_mailer.html.erb new file mode 100644 index 0000000000..36b8b3f822 --- /dev/null +++ b/app/views/layouts/discussion_deadline_mailer.html.erb @@ -0,0 +1,59 @@ + + + + + + +
+ <%= yield %> + +

+ Cheers,
+ The <%= @doubtfire_product_name %> Team on behalf of <%= @sender.name %> +

+
+ + + diff --git a/app/views/layouts/discussion_deadline_mailer.text.erb b/app/views/layouts/discussion_deadline_mailer.text.erb new file mode 100644 index 0000000000..37f0bddbd7 --- /dev/null +++ b/app/views/layouts/discussion_deadline_mailer.text.erb @@ -0,0 +1 @@ +<%= yield %> diff --git a/app/views/layouts/jupynotex.py b/app/views/layouts/jupynotex.py index 525a424f48..db3e9ca18d 100644 --- a/app/views/layouts/jupynotex.py +++ b/app/views/layouts/jupynotex.py @@ -9,6 +9,7 @@ """ import base64 +import binascii import json import os import re @@ -17,6 +18,7 @@ import tempfile import textwrap import traceback +import urllib.parse # basic verbatim start/end @@ -27,6 +29,13 @@ MARKDOWN_BEGIN = [r"\begin{markdown}"] MARKDOWN_END = [r"\end{markdown}"+"\n"] +ATTACHMENT_MIMETYPES = ( + ('image/png', '.png'), + ('image/jpeg', '.jpg'), + ('image/gif', '.gif'), + ('image/bmp', '.bmp'), +) + # highlighers for different languages (block beginning and ending) HIGHLIGHTERS = { 'python': ([r'\begin{minted}[fontsize=\footnotesize,breaklines,breakanywhere,tabsize=4]{python}'], [r'\end{minted}']), @@ -60,6 +69,13 @@ def _validator_positive_int(value): return value +def _as_lines(value): + """Return notebook text stored as either a string or list of strings.""" + if isinstance(value, str): + return value.splitlines() + return value + + def _process_plain_text(lines, config_options=None): """Wrap a series of lines around a verbatim indication.""" if config_options is None: @@ -67,6 +83,7 @@ def _process_plain_text(lines, config_options=None): result = [] result.extend(VERBATIM_BEGIN) + lines = _as_lines(lines) for line in lines: line = line.strip() @@ -117,17 +134,18 @@ def process_plain_text(self, lines): def process_png(self, image_data): """Process a PNG: just save the received b64encoded data to a temp file.""" - _, fname = tempfile.mkstemp(suffix='.png') - with open(fname, 'wb') as fh: + descriptor, fname = tempfile.mkstemp(suffix='.png') + with os.fdopen(descriptor, 'wb') as fh: fh.write(base64.b64decode(image_data)) return fname def process_svg(self, image_data): """Process a SVG: save the data, transform to PDF, and then use that.""" - _, svg_fname = tempfile.mkstemp(suffix='.svg') - _, pdf_fname = tempfile.mkstemp(suffix='.pdf') + svg_descriptor, svg_fname = tempfile.mkstemp(suffix='.svg') + pdf_descriptor, pdf_fname = tempfile.mkstemp(suffix='.pdf') + os.close(pdf_descriptor) raw_svg = ''.join(image_data).encode('utf8') - with open(svg_fname, 'wb') as fh: + with os.fdopen(svg_descriptor, 'wb') as fh: fh.write(raw_svg) cmd = ['rsvg-convert', '--format=pdf', '--output={}'.format(pdf_fname), svg_fname] @@ -169,8 +187,8 @@ def __init__(self, path, config_options): with open(path, 'rt', encoding='utf8') as fh: nb_data = json.load(fh) - # get the languaje, to highlight - lang = nb_data['metadata']['language_info']['name'] + # get the language, when available, to highlight + lang = nb_data.get('metadata', {}).get('language_info', {}).get('name') self._highlight_delimiters = HIGHLIGHTERS.get(lang, HIGHLIGHTERS[None]) # get all cells @@ -184,9 +202,73 @@ def _validate_config(self, config): config[key] = new_value return config + def _save_markdown_attachment(self, mime_bundle): + """Save a supported attachment MIME bundle and return its temporary path.""" + for mimetype, suffix in ATTACHMENT_MIMETYPES: + if mimetype not in mime_bundle: + continue + + image_data = mime_bundle[mimetype] + if isinstance(image_data, list): + image_data = ''.join(image_data) + + descriptor, fname = tempfile.mkstemp(suffix=suffix) + try: + with os.fdopen(descriptor, 'wb') as fh: + fh.write(base64.b64decode(image_data)) + except Exception: + os.unlink(fname) + raise + return fname + + if 'image/svg+xml' in mime_bundle: + return ItemProcessor(self.cell_options, self.config_options).process_svg( + mime_bundle['image/svg+xml'] + ) + + return None + + def _process_markdown_attachments(self, content, source): + """Extract embedded notebook images and rewrite their attachment URLs.""" + source_text = '\n'.join(source) + + # Use the longest names first so one attachment name cannot partially + # replace another, for example image.png and my-image.png. + attachments = content.get('attachments') or {} + for name in sorted(attachments, key=len, reverse=True): + try: + attachment_path = self._save_markdown_attachment(attachments[name]) + except (OSError, TypeError, ValueError, binascii.Error): + attachment_path = None + + if not attachment_path: + continue + + # Notebook editors may leave spaces literal or URL-encode them. + references = { + f'attachment:{name}', + f'attachment:{urllib.parse.quote(name)}', + f'attachment:{urllib.parse.quote(name, safe="")}', + } + for reference in references: + source_text = source_text.replace(reference, attachment_path) + + # A missing, corrupt, or unsupported embedded image should not make the + # whole submission PDF fail. Keep a visible marker in its place. + source_text = re.sub( + r'!\[([^\]]*)\]\(attachment:([^)]+)\)', + lambda match: ( + f'**[Image attachment unavailable: ' + f'{match.group(1) or match.group(2)}]**' + ), + source_text, + ) + + return source_text.splitlines() + def _proc_src(self, content): """Process the source of a cell.""" - source = content['source'] + source = _as_lines(content['source']) result = [] if content['cell_type'] == 'code': begin, end = self._highlight_delimiters @@ -194,6 +276,7 @@ def _proc_src(self, content): result.extend(textwrap.fill(line[:1000] + ' [The rest of this line has been truncated by the system to improve readability.] ' * (len(line) > 1000), width=90, subsequent_indent=' ') for line in source) result.extend(end) elif content['cell_type'] == 'markdown': + source = self._process_markdown_attachments(content, source) result.extend(MARKDOWN_BEGIN) result.extend(line.replace('```markdown', '```md').strip() for line in source) result.extend(MARKDOWN_END) diff --git a/app/views/notifications_mailer/discussion_deadline_approaching.html.erb b/app/views/notifications_mailer/discussion_deadline_approaching.html.erb new file mode 100644 index 0000000000..80f1278ab6 --- /dev/null +++ b/app/views/notifications_mailer/discussion_deadline_approaching.html.erb @@ -0,0 +1,16 @@ +

<%= @unit.name %> - Discussion Deadline Approaching

+

<%= @task.task_definition.abbreviation %> - <%= @task.task_definition.name %>

+ +

Hi <%= @student.first_name %>,

+ +

+ You must discuss task <%= @task.task_definition.abbreviation %> with your tutor before + <%= @deadline %>. +

+ +

+ If it is not discussed by then, it will move to Fix and Resubmit, and you will + need to resubmit your work. +

+ +

View this task in <%= @doubtfire_product_name %>.

diff --git a/app/views/notifications_mailer/discussion_deadline_approaching.text.erb b/app/views/notifications_mailer/discussion_deadline_approaching.text.erb new file mode 100644 index 0000000000..37d86d5171 --- /dev/null +++ b/app/views/notifications_mailer/discussion_deadline_approaching.text.erb @@ -0,0 +1,14 @@ +Hi <%= @student.first_name %>, + +You must discuss task <%= @task.task_definition.abbreviation %> with your tutor before <%= @deadline %>. + +If it is not discussed by then, it will move to Fix and Resubmit, and you will need to resubmit your work. + +View this task in <%= @doubtfire_product_name %>: <%= @task_url %> + +Cheers, +The <%= @doubtfire_product_name %> Team on behalf of <%= @sender.name %> + +--- + +Visit <%= @unsubscribe_url %> to unsubscribe from these notifications. diff --git a/app/views/notifications_mailer/discussion_deadline_missed.html.erb b/app/views/notifications_mailer/discussion_deadline_missed.html.erb new file mode 100644 index 0000000000..eade95411a --- /dev/null +++ b/app/views/notifications_mailer/discussion_deadline_missed.html.erb @@ -0,0 +1,13 @@ +

<%= @unit.name %> - Discussion Deadline Missed

+

<%= @task.task_definition.abbreviation %> - <%= @task.task_definition.name %>

+ +

Hi <%= @student.first_name %>,

+ +

+ Task <%= @task.task_definition.abbreviation %> moved to + Fix and Resubmit because it was not discussed by the deadline. +

+ +

Review any feedback and resubmit it when you are ready.

+ +

View this task in <%= @doubtfire_product_name %>.

diff --git a/app/views/notifications_mailer/discussion_deadline_missed.text.erb b/app/views/notifications_mailer/discussion_deadline_missed.text.erb new file mode 100644 index 0000000000..bbc19c18a5 --- /dev/null +++ b/app/views/notifications_mailer/discussion_deadline_missed.text.erb @@ -0,0 +1,14 @@ +Hi <%= @student.first_name %>, + +<%= @task.task_definition.abbreviation %> - <%= @task.task_definition.name %> moved to Fix and Resubmit because it was not discussed by the deadline. + +Review any feedback and resubmit it when you are ready. + +View this task in <%= @doubtfire_product_name %>: <%= @task_url %> + +Cheers, +The <%= @doubtfire_product_name %> Team on behalf of <%= @sender.name %> + +--- + +Visit <%= @unsubscribe_url %> to unsubscribe from these notifications. diff --git a/app/views/portfolio/portfolio_pdf.pdf.erb b/app/views/portfolio/portfolio_pdf.pdf.erb index 64704249c1..6e47a8d719 100644 --- a/app/views/portfolio/portfolio_pdf.pdf.erb +++ b/app/views/portfolio/portfolio_pdf.pdf.erb @@ -96,7 +96,12 @@ No Tutor % DATE SECTION %---------------------------------------------------------------------------------------- -{\large \today}\\[3cm] % Date, change the \today to a set date if you want to be precise +<% if @formatted_submission_date.present? %> +{\large <%= lesc @formatted_submission_date %>}\\[0.2cm] +{\large \textcolor{gray}{<%= lesc @formatted_submission_time %>}}\\[3cm] +<% else %> +\vspace{3cm} +<% end %> %---------------------------------------------------------------------------------------- % LOGO SECTION @@ -125,7 +130,7 @@ No Tutor \textbf{Task} & \textbf{Status} & \textbf{Times Assessed} \\ \hline <% @task_defs.each do |td| %> - <%= lesc td.name %> & + <%= lesc "#{td.abbreviation} #{td.name}" %> & <% task = task_for_def(td) if task.nil? %> diff --git a/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb new file mode 100644 index 0000000000..95fcda7a4e --- /dev/null +++ b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.html.erb @@ -0,0 +1,72 @@ + + + + + + +
+

<%= @doubtfire_product_name %> Notification

+

Hi <%= @student.first_name %>,

+

+ Automated assessment did not pass for the tasks below. Please review the Overseer report in <%= @doubtfire_product_name %>, verify your output, and resubmit. +

+

+ Tasks to review: +

+

+

+ Cheers,
+ The <%= @doubtfire_product_name %> Team on behalf of <%= @tutor.name %> +

+
+ + + diff --git a/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb new file mode 100644 index 0000000000..37248debfa --- /dev/null +++ b/app/views/portfolio_evidence_mailer/overseer_assessment_failed.text.erb @@ -0,0 +1,17 @@ +Hi <%= @student.first_name %>, + +Automated assessment did not pass for the tasks below. Please review the Overseer report in <%= @doubtfire_product_name %>, verify your output, and resubmit. + +Tasks to review: +<% @tasks.each do |task| %> + * <%= task.task_definition.abbreviation %> - <%= task.task_definition.name %> +<% end %> + +Cheers, +The <%= @doubtfire_product_name %> Team on behalf of <%= @tutor.name %> + +--- + +Visit <%= @unsubscribe_url %> to unsubscribe from these notifications. + +Generated with <%= @doubtfire_product_name %> diff --git a/app/views/shared/_file.pdf.erb b/app/views/shared/_file.pdf.erb index 036b4b2635..d97d945e7a 100644 --- a/app/views/shared/_file.pdf.erb +++ b/app/views/shared/_file.pdf.erb @@ -26,3 +26,9 @@ <% end # end for %> <% end %> + +<% if %w[zip archive].include?(file_type) %> +\begin{tcolorbox}[colback=blue!5!white,colframe=blue!75!black] + This zip file was uploaded with the portfolio. Zip contents are not expanded into this PDF. +\end{tcolorbox} +<% end %> diff --git a/app/views/task/task_pdf.pdf.erb b/app/views/task/task_pdf.pdf.erb index 44e8a83861..e279bff132 100644 --- a/app/views/task/task_pdf.pdf.erb +++ b/app/views/task/task_pdf.pdf.erb @@ -136,6 +136,45 @@ No Tutor % Supervisor's Name \includepdf[pages={<%= page_idx %>-<%= page_idx %>},fitpaper]{<%= file[:path] %>} <% end # end for %> <% end # end if %> + <% if %w[zip archive].include?(file[:type]) %> +<% archive_tree = FileHelper.zip_file_tree(file[:path], File.basename(file[:path])) %> +\begin{tcolorbox}[enhanced,breakable,colback=orange!8!white,colframe=orange!80!black,boxrule=0.9pt,left=4mm,right=4mm,top=3mm,bottom=3mm] +\begin{minipage}{0.12\textwidth} +\centering +{\Huge \faFileArchiveO} +\end{minipage} +\hfill +\begin{minipage}{0.82\textwidth} +\textbf{Zip submission file}\\[0.35em] +This zip file was uploaded with the submission. Zip contents are not expanded into this PDF.\\[0.65em] +\href{<%= @submitted_files_url %>}{\faDownload\ Download submitted files} {\footnotesize (Must be already authenticated in <%= lesc @doubtfire_product_name %>)} +\end{minipage} + +\tcblower +\textbf{Zip contents}\\[0.35em] +<% if archive_tree[:error] %> +The zip file tree could not be read while generating this PDF. Download the submitted files to inspect the contents. +<% elsif archive_tree[:lines].empty? %> +No readable file entries were found in this zip. +<% else %> +{\footnotesize\ttfamily +\begin{tabular}{@{}p{0.96\textwidth}@{}} +<% archive_tree[:lines].each do |line| %> +<% + indent = line[/\A */].length + content = line.lstrip + indent_width = indent * 0.45 +%> +\hspace*{<%= indent_width %>em}<%= raw lesc(content).to_s.gsub('↳', '\ensuremath{\hookrightarrow}') %>\\ +<% end %> +\end{tabular} +} +<% if archive_tree[:truncated] %> +\\[0.35em]\emph{Showing the first <%= archive_tree[:lines].length %> of <%= archive_tree[:tree_lines] %> tree lines. Download the submitted files to inspect everything.} +<% end %> +<% end %> +\end{tcolorbox} + <% end %> <% end # list of documents %> <% diff --git a/config/application.rb b/config/application.rb index fe280a2899..2a2e907876 100644 --- a/config/application.rb +++ b/config/application.rb @@ -5,6 +5,7 @@ require 'csv' require 'yaml' require 'bunny-pub-sub/services_manager' +require_relative '../app/middleware/sentry_tunnel_middleware' # Precompile assets before deploying to production if defined?(Bundler) @@ -17,6 +18,8 @@ module Doubtfire # class Application < Rails::Application config.load_defaults 7.0 + config.credentials.content_path = Rails.root.join('config/credentials/credentials.yml.enc') + config.credentials.key_path = Rails.root.join('config/credentials/master.key') # Remove Action Mailbox and Active Storage routes - not used initializer(:remove_action_mailbox_and_activestorage_routes, after: :add_routing_paths) do |app| @@ -34,6 +37,8 @@ class Application < Rails::Application # are: database, ldap, aaf, or saml. It can be overridden using the DF_AUTH_METHOD # environment variable. config.auth_method = (ENV['DF_AUTH_METHOD'] || :database).to_sym + config.access_token_expiry = ENV.fetch('DF_ACCESS_TOKEN_EXPIRY_SECONDS', 2.hours.to_i).to_i.seconds + config.refresh_token_expiry = ENV.fetch('DF_REFRESH_TOKEN_EXPIRY_SECONDS', 1.week.to_i).to_i.seconds # ==> Student work directory # File server location for storing student's work. Defaults to `student_work` @@ -52,9 +57,26 @@ class Application < Rails::Application # Period for which to keep units config.unit_archive_after_period = ENV.fetch('DF_UNIT_ARCHIVE_PERIOD', 2).to_f * 1.year + # Minimum time to wait before notifying a student about an unread failed overseer assessment + config.overseer_student_notification_grace_period = ENV.fetch('OVERSEER_STUDENT_NOTIFICATION_GRACE_PERIOD_MINUTES', 30).to_i.minutes + # Limit number of pdf generators to run at once config.pdfgen_max_processes = ENV['DF_MAX_PDF_GEN_PROCESSES'] || 2 + # Each Word document conversion runs a short-lived, network-isolated + # Gotenberg container. The image includes the conversion entrypoint. + config.gotenberg_image = ENV.fetch('GOTENBERG_IMAGE', nil) + + # Absolute host path to tmp/gotenberg. Production uses this to mount only + # the current conversion's work directory into its one-shot container. + config.gotenberg_workdir_volume_mount = ENV.fetch('GOTENBERG_WORKDIR_VOLUME_MOUNT', nil) + + # Development fallback matching Overseer. This exposes the fallback + # container's entire tmp/gotenberg mount to each conversion container. + config.gotenberg_fallback_volume_container = ENV.fetch('GOTENBERG_FALLBACK_VOLUME_CONTAINER', nil) + config.word_document_build_path = ENV.fetch('WORD_DOCUMENT_BUILD_PATH', '/gotenberg/word_document_build.sh') + config.word_document_conversion_timeout_seconds = ENV.fetch('WORD_DOCUMENT_CONVERSION_TIMEOUT_SECONDS', 120) + # Date range for auditors to view config.auditor_unit_access_years = ENV.fetch('DF_AUDITOR_UNIT_ACCESS_YEARS', 2).to_f * 1.year @@ -64,38 +86,65 @@ def self.fetch_boolean_env(name) %w'true 1'.include?(ENV.fetch(name, 'false').downcase) end + def self.fetch_credential_or_env(*credential_path, env_key:, default: nil) + credential_value = + if credential_path.length == 1 && credentials.respond_to?(credential_path.first) + credentials.public_send(credential_path.first) + else + credentials.dig(*credential_path) + end + + credential_value.nil? ? ENV.fetch(env_key, default) : credential_value + end + # ==> Log to stdout config.log_to_stdout = Application.fetch_boolean_env('DF_LOG_TO_STDOUT') # Have rails report errors and log messages to the following email address where present config.email_errors_to = ENV.fetch('DF_EMAIL_ERRORS_TO', nil) - # ==> JPLAG report directory - # File server location for storing JPLAG reports. Defaults to `jplag/results` - # directory under root but is overridden using DF_JPLAG_REPORT_DIR environment - # variable. - config.jplag_report_dir = ENV['DF_JPLAG_REPORT_DIR'] || Rails.root.join('jplag/results').to_s + # Tunes the comparison sensitivity by adjusting the minimum token required to be + # counted as a matching section. A smaller value increases the sensitivity + # but might lead to more false-positives config.jplag_min_tokens = ENV.fetch('DF_JPLAG_MIN_TOKENS', -1) + + # Skips the cluster calculation config.jplag_skip_cluster_check = ENV['DF_JPLAG_SKIP_CLUSTER_CHECK'].present? && (ENV['DF_JPLAG_SKIP_CLUSTER_CHECK'].to_s.downcase == "true" || ENV['DF_JPLAG_SKIP_CLUSTER_CHECK'].to_i == 1) + # The maximum number of comparisons that will be shown in the generated report + # if set to -1 all comparisons will be shown + config.jplag_max_shown_comparisons = ENV.fetch('DF_JPLAG_MAX_SHOWN_COMPARISONS', 2500) + # ==> File size limits # Sets the global file size limit per upload requirement # Defaults to 10MB (10,000,000 bytes) config.max_file_size = ENV.fetch('DF_MAX_FILE_SIZE', 10_000_000) - # ==> Load credentials from env - credentials.secret_key_base = ENV.fetch('DF_SECRET_KEY_BASE', Rails.env.production? ? nil : '9e010ee2f52af762916406fd2ac488c5694a6cc784777136e657511f8bbc7a73f96d59c0a9a778a0d7cf6406f8ecbf77efe4701dfbd63d8248fc7cc7f32dea97') - credentials.secret_key_attr = ENV.fetch('DF_SECRET_KEY_ATTR', Rails.env.production? ? nil : 'e69fc5960ca0e8700844a3a25fe80373b41c0a265d342eba06950113f3766fd983bad9ec51bf36eb615d9711bfe1dd90b8e35f01841b323f604ffee857e32055') - credentials.secret_key_devise = ENV.fetch('DF_SECRET_KEY_DEVISE', Rails.env.production? ? nil : 'f4e23c4388dc600e503a09ad057b8271d8fcf4c2cd6723b44f33db638e49075fe96bc545eed9110ded0c5df505625d4e1c838b718349eecf1d39270d0829d5b9') - credentials.secret_key_aaf = ENV.fetch('DF_SECRET_KEY_AAF', Rails.env.production? ? nil : 'secretsecret12345') - credentials.secret_key_moss = ENV.fetch('DF_SECRET_KEY_MOSS', nil) + # Max files inside an uploaded zip. If denied for "too many files", + # remove generated folders (e.g. node_modules/build) or raise this limit. + config.zip_entry_limit = ENV.fetch('DF_ZIP_ENTRY_LIMIT', 1_000) + + # Max zip compression ratio. If denied for "compression ratio is too high", + # check for repetitive/generated data before raising this zip-bomb guard. + config.zip_compression_ratio_limit = ENV.fetch('DF_ZIP_COMPRESSION_RATIO_LIMIT', 100) + + # Max expanded zip size as a multiple of DF_MAX_FILE_SIZE. + # Eg. If max file size is 10MB, the uncompressed size can be a maximum of 100MB + config.zip_uncompressed_size_multiplier = ENV.fetch('DF_ZIP_UNCOMPRESSED_SIZE_MULTIPLIER', 10) + + # Prefer encrypted Rails credentials, while keeping env vars as a safe fallback. + credentials.secret_key_base = Application.fetch_credential_or_env(:secret_key_base, env_key: 'DF_SECRET_KEY_BASE', default: Rails.env.production? ? nil : '9e010ee2f52af762916406fd2ac488c5694a6cc784777136e657511f8bbc7a73f96d59c0a9a778a0d7cf6406f8ecbf77efe4701dfbd63d8248fc7cc7f32dea97') + credentials.secret_key_attr = Application.fetch_credential_or_env(:secret_key_attr, env_key: 'DF_SECRET_KEY_ATTR', default: Rails.env.production? ? nil : 'e69fc5960ca0e8700844a3a25fe80373b41c0a265d342eba06950113f3766fd983bad9ec51bf36eb615d9711bfe1dd90b8e35f01841b323f604ffee857e32055') + credentials.secret_key_devise = Application.fetch_credential_or_env(:secret_key_devise, env_key: 'DF_SECRET_KEY_DEVISE', default: Rails.env.production? ? nil : 'f4e23c4388dc600e503a09ad057b8271d8fcf4c2cd6723b44f33db638e49075fe96bc545eed9110ded0c5df505625d4e1c838b718349eecf1d39270d0829d5b9') + credentials.secret_key_aaf = Application.fetch_credential_or_env(:aaf, :secret_key, env_key: 'DF_SECRET_KEY_AAF', default: Rails.env.production? ? nil : 'secretsecret12345') + credentials.secret_key_moss = Application.fetch_credential_or_env(:moss, :secret_key, env_key: 'DF_SECRET_KEY_MOSS') # ==> LTI settings # If enabled, mounts the LTI routes and enables LTI authentication. config.lti_enabled = ENV.fetch('LTI_ENABLED', false).to_s.downcase == "true" # Shared secret between Ruby on Rails API and the LTI.js API # LTI.js will send signed JWT tokens using this secret - config.lti_api_secret = ENV.fetch('LTI_SHARED_API_SECRET', nil) + config.lti_api_secret = Application.fetch_credential_or_env(:lti, :shared_api_secret, env_key: 'LTI_SHARED_API_SECRET') # ==> Moderation settings config.moderation_score_factor = Float(ENV.fetch('MODERATION_SCORE_FACTOR', 1.0)) @@ -117,6 +166,7 @@ def self.fetch_boolean_env(name) config.institution[:plagiarism] = ENV['DF_INSTITUTION_PLAGIARISM'] if ENV['DF_INSTITUTION_PLAGIARISM'] # Institution host becomes localhost in development config.institution[:host] ||= 'http://localhost:4200' if Rails.env.development? + config.institution[:settings] = ENV['DF_INSTITUTION_SETTINGS_RB'] if ENV['DF_INSTITUTION_SETTINGS_RB'] config.institution[:ffmpeg] = ENV['DF_FFMPEG_PATH'] || 'ffmpeg' @@ -232,6 +282,7 @@ def self.fetch_boolean_env(name) config.autoload_paths << Rails.root.join('app') << Rails.root.join('app/models/comments') << + Rails.root.join('app/models/communication') << Rails.root.join('app/models/turn_it_in') << Rails.root.join('app/models/similarity') << Rails.root.join('app/models/d2l') @@ -239,15 +290,35 @@ def self.fetch_boolean_env(name) config.eager_load_paths << Rails.root.join('app') << Rails.root.join('app/models/comments') << + Rails.root.join('app/models/communication') << Rails.root.join('app/models/turn_it_in') << Rails.root.join('app/models/similarity') << Rails.root.join('app/models/d2l') # CORS config + # Configure a strict allowlist. Override per environment via: + # CORS_ALLOWED_ORIGINS="http://localhost:4200,https://frontend.example.edu" + default_cors_origins = [ + 'http://localhost:4200', + "https://#{config.institution[:host]}" + ].uniq + allowed_cors_origins = ENV.fetch('CORS_ALLOWED_ORIGINS', default_cors_origins.join(',')) + .split(',') + .map(&:strip) + .reject(&:empty?) + .uniq + + config.middleware.insert_before Rack::MethodOverride, SentryTunnelMiddleware + config.middleware.insert_before Warden::Manager, Rack::Cors do allow do - origins '*' - resource '*', headers: :any, methods: %i(get post put delete options) + origins do |source, _env| + allowed_cors_origins.include?(source) + end + + resource '*', + headers: %w[Content-Type Authorization Accept], + methods: %i[get post put delete options] end end @@ -269,14 +340,14 @@ def self.fetch_boolean_env(name) config.sm_instance = nil config.overseer_enabled = ENV['OVERSEER_ENABLED'].present? && ENV['OVERSEER_ENABLED'].to_s.downcase != "false" && ENV['OVERSEER_ENABLED'].to_i != 0 - if (config.overseer_enabled) - config.docker_config = { - DOCKER_REGISTRY_URL: ENV.fetch('DOCKER_REGISTRY_URL', nil), - DOCKER_PROXY_URL: ENV.fetch('DOCKER_PROXY_URL', nil), - DOCKER_TOKEN: ENV.fetch('DOCKER_TOKEN', nil), - DOCKER_USER: ENV.fetch('DOCKER_USER', nil) - } + config.docker_config = { + DOCKER_REGISTRY_URL: ENV.fetch('DOCKER_REGISTRY_URL', nil), + DOCKER_PROXY_URL: ENV.fetch('DOCKER_PROXY_URL', nil), + DOCKER_TOKEN: ENV.fetch('DOCKER_TOKEN', nil), + DOCKER_USER: ENV.fetch('DOCKER_USER', nil) + } + if (config.overseer_enabled) # Path to a physical directory on the host used for mounting overseer task work directories. # # Example (macOS development): diff --git a/config/database.yml b/config/database.yml index 06faaed683..81ebb08640 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,29 +1,33 @@ development: - adapter: <%= ENV['DF_DEV_DB_ADAPTER'] %> - database: <%= ENV['DF_DEV_DB_DATABASE'] %> - username: <%= ENV['DF_DEV_DB_USERNAME'] %> - password: <%= ENV['DF_DEV_DB_PASSWORD'] %> - host: <%= ENV['DF_DEV_DB_HOST'] %> + adapter: <%= Rails.application.credentials.dig(:database, :development, :adapter) || ENV['DF_DEV_DB_ADAPTER'] %> + database: <%= Rails.application.credentials.dig(:database, :development, :database) || ENV['DF_DEV_DB_DATABASE'] %> + username: <%= Rails.application.credentials.dig(:database, :development, :username) || ENV['DF_DEV_DB_USERNAME'] %> + password: <%= Rails.application.credentials.dig(:database, :development, :password) || ENV['DF_DEV_DB_PASSWORD'] %> + host: <%= Rails.application.credentials.dig(:database, :development, :host) || ENV['DF_DEV_DB_HOST'] %> min_messages: warning + encoding: utf8mb4 + collation: utf8mb4_general_ci test: - adapter: <%= ENV['DF_TEST_DB_ADAPTER'] %> - database: <%= ENV['DF_TEST_DB_DATABASE'] %> - username: <%= ENV['DF_TEST_DB_USERNAME'] %> - password: <%= ENV['DF_TEST_DB_PASSWORD'] %> - host: <%= ENV['DF_TEST_DB_HOST'] %> + adapter: <%= Rails.application.credentials.dig(:database, :test, :adapter) || ENV['DF_TEST_DB_ADAPTER'] %> + database: <%= Rails.application.credentials.dig(:database, :test, :database) || ENV['DF_TEST_DB_DATABASE'] %> + username: <%= Rails.application.credentials.dig(:database, :test, :username) || ENV['DF_TEST_DB_USERNAME'] %> + password: <%= Rails.application.credentials.dig(:database, :test, :password) || ENV['DF_TEST_DB_PASSWORD'] %> + host: <%= Rails.application.credentials.dig(:database, :test, :host) || ENV['DF_TEST_DB_HOST'] %> min_messages: warning - + encoding: utf8mb4 + collation: utf8mb4_general_ci + staging: - adapter: <%= ENV['DF_STAGING_DB_ADAPTER'] %> - host: <%= ENV['DF_STAGING_DB_HOST'] %> - database: <%= ENV['DF_STAGING_DB_DATABASE'] %> - username: <%= ENV['DF_STAGING_DB_USERNAME'] %> - password: <%= ENV['DF_STAGING_DB_PASSWORD'] %> + adapter: <%= Rails.application.credentials.dig(:database, :staging, :adapter) || ENV['DF_STAGING_DB_ADAPTER'] %> + host: <%= Rails.application.credentials.dig(:database, :staging, :host) || ENV['DF_STAGING_DB_HOST'] %> + database: <%= Rails.application.credentials.dig(:database, :staging, :database) || ENV['DF_STAGING_DB_DATABASE'] %> + username: <%= Rails.application.credentials.dig(:database, :staging, :username) || ENV['DF_STAGING_DB_USERNAME'] %> + password: <%= Rails.application.credentials.dig(:database, :staging, :password) || ENV['DF_STAGING_DB_PASSWORD'] %> production: - adapter: <%= ENV['DF_PRODUCTION_DB_ADAPTER'] %> - host: <%= ENV['DF_PRODUCTION_DB_HOST'] %> - database: <%= ENV['DF_PRODUCTION_DB_DATABASE'] %> - username: <%= ENV['DF_PRODUCTION_DB_USERNAME'] %> - password: <%= ENV['DF_PRODUCTION_DB_PASSWORD'] %> + adapter: <%= Rails.application.credentials.dig(:database, :production, :adapter) || ENV['DF_PRODUCTION_DB_ADAPTER'] %> + host: <%= Rails.application.credentials.dig(:database, :production, :host) || ENV['DF_PRODUCTION_DB_HOST'] %> + database: <%= Rails.application.credentials.dig(:database, :production, :database) || ENV['DF_PRODUCTION_DB_DATABASE'] %> + username: <%= Rails.application.credentials.dig(:database, :production, :username) || ENV['DF_PRODUCTION_DB_USERNAME'] %> + password: <%= Rails.application.credentials.dig(:database, :production, :password) || ENV['DF_PRODUCTION_DB_PASSWORD'] %> diff --git a/config/environments/development.rb b/config/environments/development.rb index 7dd6f8112f..05d01df74c 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -106,7 +106,7 @@ # pdfgen log verbosity config.pdfgen_quiet = false - config.active_record.encryption.key_derivation_salt = ENV['DF_ENCRYPTION_KEY_DERIVATION_SALT'] || 'U9jurHMfZbMpzlbDTMe5OSAhUJYHla9Z' - config.active_record.encryption.deterministic_key = ENV['DF_ENCRYPTION_DETERMINISTIC_KEY'] || 'zYtzYUlLFaWdvdUO5eIINRT6ZKDddcgx' - config.active_record.encryption.primary_key = ENV['DF_ENCRYPTION_PRIMARY_KEY'] || '92zoF7RJaQ01JEExOgHbP9bRWldNQUz5' + config.active_record.encryption.key_derivation_salt = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :key_derivation_salt, env_key: 'DF_ENCRYPTION_KEY_DERIVATION_SALT', default: 'U9jurHMfZbMpzlbDTMe5OSAhUJYHla9Z') + config.active_record.encryption.deterministic_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :deterministic_key, env_key: 'DF_ENCRYPTION_DETERMINISTIC_KEY', default: 'zYtzYUlLFaWdvdUO5eIINRT6ZKDddcgx') + config.active_record.encryption.primary_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :primary_key, env_key: 'DF_ENCRYPTION_PRIMARY_KEY', default: '92zoF7RJaQ01JEExOgHbP9bRWldNQUz5') end diff --git a/config/environments/production.rb b/config/environments/production.rb index 37b429654a..b673b52f68 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -50,7 +50,7 @@ config.action_mailer.smtp_settings[:authentication] = nil if %w[no_auth none].include?(config.action_mailer.smtp_settings[:authentication]) end - config.active_record.encryption.key_derivation_salt = ENV.fetch('DF_ENCRYPTION_KEY_DERIVATION_SALT', nil) - config.active_record.encryption.deterministic_key = ENV.fetch('DF_ENCRYPTION_DETERMINISTIC_KEY', nil) - config.active_record.encryption.primary_key = ENV.fetch('DF_ENCRYPTION_PRIMARY_KEY', nil) + config.active_record.encryption.key_derivation_salt = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :key_derivation_salt, env_key: 'DF_ENCRYPTION_KEY_DERIVATION_SALT') + config.active_record.encryption.deterministic_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :deterministic_key, env_key: 'DF_ENCRYPTION_DETERMINISTIC_KEY') + config.active_record.encryption.primary_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :primary_key, env_key: 'DF_ENCRYPTION_PRIMARY_KEY') end diff --git a/config/environments/test.rb b/config/environments/test.rb index 24fcb3d4cc..fd2b7ce8f9 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -37,9 +37,9 @@ # Logging level (:debug, :info, :warn, :error, :fatal) config.log_level = :warn - config.active_record.encryption.key_derivation_salt = ENV['DF_ENCRYPTION_KEY_DERIVATION_SALT'] || 'U9jurHMfZbMpzlbDTMe5OSAhUJYHla9Z' - config.active_record.encryption.deterministic_key = ENV['DF_ENCRYPTION_KEY_DERIVATION_SALT'] || 'zYtzYUlLFaWdvdUO5eIINRT6ZKDddcgx' - config.active_record.encryption.primary_key = ENV['DF_ENCRYPTION_KEY_DERIVATION_SALT'] || '92zoF7RJaQ01JEExOgHbP9bRWldNQUz5' + config.active_record.encryption.key_derivation_salt = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :key_derivation_salt, env_key: 'DF_ENCRYPTION_KEY_DERIVATION_SALT', default: 'U9jurHMfZbMpzlbDTMe5OSAhUJYHla9Z') + config.active_record.encryption.deterministic_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :deterministic_key, env_key: 'DF_ENCRYPTION_DETERMINISTIC_KEY', default: 'zYtzYUlLFaWdvdUO5eIINRT6ZKDddcgx') + config.active_record.encryption.primary_key = Doubtfire::Application.fetch_credential_or_env(:active_record_encryption, :primary_key, env_key: 'DF_ENCRYPTION_PRIMARY_KEY', default: '92zoF7RJaQ01JEExOgHbP9bRWldNQUz5') # Set turn it in environment ENV.store('TCA_SIGNING_KEY', 'test') diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb new file mode 100644 index 0000000000..f09d28752b --- /dev/null +++ b/config/initializers/sentry.rb @@ -0,0 +1,101 @@ +module OnTrackSentryRedaction + FILTERED = "[Filtered]".freeze + PII_KEY_PATTERN = /(^|_)(email|first_name|last_name|login_id|name|student_id|username|user_id)\z/i + RAILS_LATEX_TASK_WORK_DIR = %r{(tmp/rails-latex/task-\d{8}-\d{4}-)[^/\s]+?(-[^/\s-]+-\d+-\d+(?:-retry)?)(?=/|\s|$)} + RAILS_LATEX_PORTFOLIO_WORK_DIR = %r{(tmp/rails-latex/portfolio-\d{8}-\d{4}-)[^/\s]+?(-\d+-\d+(?:-retry)?)(?=/|\s|$)} + SCORM_PATH = %r{(/scorm/[^/?#]+/)[^/?#]+(/)[^/?#]+}i + + module_function + + def scrub_event(event) + scrub_request(event.request) if event.respond_to?(:request) + + event.message = scrub_string(event.message) if event.respond_to?(:message) && event.respond_to?(:message=) + event.user.replace(scrub_hash(event.user)) if event.respond_to?(:user) && event.user.respond_to?(:replace) + event.extra.replace(scrub_hash(event.extra)) if event.respond_to?(:extra) && event.extra.respond_to?(:replace) + + scrub_exception(event.exception) if event.respond_to?(:exception) + scrub_breadcrumbs(event.breadcrumbs) if event.respond_to?(:breadcrumbs) + + event + end + + def scrub_request(request) + return unless request + + request.url = scrub_string(request.url) if request.respond_to?(:url) && request.respond_to?(:url=) + request.headers&.each_key do |key| + request.headers[key] = FILTERED if key.casecmp("Username").zero? + end + end + + def scrub_exception(exception) + exception_values = exception&.values + exception_values&.each do |value| + value.value = scrub_string(value.value) if value.respond_to?(:value) && value.respond_to?(:value=) + + next unless value.respond_to?(:stacktrace) + + value.stacktrace&.frames&.each do |frame| + %i[abs_path context_line filename function module pre_context post_context vars].each do |attribute| + next unless frame.respond_to?(attribute) && frame.respond_to?(:"#{attribute}=") + + frame.public_send(:"#{attribute}=", scrub_value(frame.public_send(attribute))) + end + end + end + end + + def scrub_breadcrumbs(breadcrumbs) + breadcrumbs&.each do |breadcrumb| + breadcrumb.message = scrub_string(breadcrumb.message) if breadcrumb.respond_to?(:message) && breadcrumb.respond_to?(:message=) + breadcrumb.data = scrub_hash(breadcrumb.data) if breadcrumb.respond_to?(:data) && breadcrumb.respond_to?(:data=) + end + end + + def scrub_value(value) + case value + when Hash + scrub_hash(value) + when Array + value.map { |item| scrub_value(item) } + when String + scrub_string(value) + else + value + end + end + + def scrub_hash(hash) + hash.each_with_object({}) do |(key, value), result| + key = key.to_s + result[key] = key.match?(PII_KEY_PATTERN) ? FILTERED : scrub_value(value) + end + end + + def scrub_string(value) + return value unless value.is_a?(String) + + value + .gsub(RAILS_LATEX_TASK_WORK_DIR, "\\1[username]\\2") + .gsub(RAILS_LATEX_PORTFOLIO_WORK_DIR, "\\1[username]\\2") + .gsub(SCORM_PATH, "\\1[username]\\2[Filtered]") + end +end + +if ENV["SENTRY_DSN"].present? + Sentry.init do |config| + config.dsn = ENV.fetch("SENTRY_DSN", nil) + # Rails log breadcrumbs are excluded because app logs can include user details. + config.breadcrumbs_logger = [:http_logger] + config.environment = ENV.fetch("SENTRY_ENVIRONMENT", Rails.env) + config.release = ENV["SENTRY_RELEASE"] if ENV["SENTRY_RELEASE"].present? + # Add data like request headers and IP for users, if applicable; + # see https://docs.sentry.io/platforms/ruby/data-management/data-collected/ for more info + config.send_default_pii = false + + config.before_send = lambda do |event, _hint| + OnTrackSentryRedaction.scrub_event(event) + end + end +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index dc00e55825..942c043b5d 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -14,6 +14,11 @@ Sidekiq::Status.configure_server_middleware(config, expiration: 30.minutes.to_i) Sidekiq::Status.configure_client_middleware(config, expiration: 30.minutes.to_i) + + config.on(:startup) do + schedule_file = Rails.root.join('config/schedule.yml') + Sidekiq::Cron::Job.load_from_hash!(YAML.load_file(schedule_file)) if File.exist?(schedule_file) + end end Sidekiq.configure_client do |config| diff --git a/config/initializers/uri_credential_leak_mitigation.rb b/config/initializers/uri_credential_leak_mitigation.rb new file mode 100644 index 0000000000..1096c5545a --- /dev/null +++ b/config/initializers/uri_credential_leak_mitigation.rb @@ -0,0 +1,21 @@ +require 'uri' + +# Mitigates credential leakage when combining a credential-bearing base URI +# with a relative URI via URI#+ by stripping userinfo from the merged result. +module UriCredentialLeakMitigation + def +(other) + combined = super + + return combined unless other.respond_to?(:absolute?) && !other.absolute? + return combined unless combined.respond_to?(:user=) || combined.respond_to?(:password=) + + sanitized = combined.dup + sanitized.user = nil if sanitized.respond_to?(:user=) + sanitized.password = nil if sanitized.respond_to?(:password=) + sanitized + rescue URI::InvalidComponentError + combined + end +end + +URI::Generic.prepend(UriCredentialLeakMitigation) diff --git a/config/schedule.yml b/config/schedule.yml index b0bd370f02..adb2a1f282 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -16,6 +16,18 @@ refresh_moderation_feedback_timestamps: cron: "every 60 minutes" class: "RefreshModerationFeedbackTimestampsJob" +aggregate_task_completion_stats: + cron: "every day at 11:55pm" + class: "AggregateTaskCompletionStatsJob" + +poll_communication_set_schedules: + cron: "every 5 minutes" + class: "PollCommunicationSetSchedulesJob" + +notify_discuss_timeout: + cron: "every day at 8am" + class: "NotifyDiscussTimeoutJob" + # archive_old_units: # cron: "every 6 months" # class: "ArchiveOldUnitsJob" diff --git a/db/migrate/20260327041457_add_enforce_feedback_before_discussed_in_class.rb b/db/migrate/20260327041457_add_enforce_feedback_before_discussed_in_class.rb new file mode 100644 index 0000000000..ba60168e01 --- /dev/null +++ b/db/migrate/20260327041457_add_enforce_feedback_before_discussed_in_class.rb @@ -0,0 +1,5 @@ +class AddEnforceFeedbackBeforeDiscussedInClass < ActiveRecord::Migration[8.0] + def change + add_column :units, :enforce_feedback_before_discussed_in_class, :boolean, null: false, default: false + end +end diff --git a/db/migrate/20260604031804_add_student_notified_at_to_overseer_assessments.rb b/db/migrate/20260604031804_add_student_notified_at_to_overseer_assessments.rb new file mode 100644 index 0000000000..a0b9be1695 --- /dev/null +++ b/db/migrate/20260604031804_add_student_notified_at_to_overseer_assessments.rb @@ -0,0 +1,7 @@ +class AddStudentNotifiedAtToOverseerAssessments < ActiveRecord::Migration[8.0] + def change + add_column :overseer_assessments, :student_notified_at, :datetime + add_index :overseer_assessments, [:status, :student_notified_at, :updated_at], + name: 'index_overseer_assessments_on_status_notified_updated' + end +end diff --git a/db/migrate/20260604070032_add_communications_feat.rb b/db/migrate/20260604070032_add_communications_feat.rb new file mode 100644 index 0000000000..b28e520cc4 --- /dev/null +++ b/db/migrate/20260604070032_add_communications_feat.rb @@ -0,0 +1,123 @@ +class AddCommunicationsFeat < ActiveRecord::Migration[8.0] + def change + create_table :communication_sets do |t| + t.references :unit, null: false + t.string :name + t.boolean :active, null: false, default: true + t.timestamps + end + + create_table :communication_rules do |t| + t.references :communication_set, null: false + + t.integer :position, null: false, default: 0 + t.string :name + # AND | OR + t.string :operator # AND | OR + + t.boolean :send_log_to_convenors, null: false, default: false + + t.boolean :active, null: false, default: true + + t.timestamps + end + + create_table :communication_conditions do |t| + t.string :type, null: false + t.references :communication, null: false + + # TargetGradeCondition + t.integer :target_grade + # t.string :target_grade_comparison # gt|gte|lt|lte|notequal|equal + + # TaskDefinitionStatusCondition + t.references :task_definition + t.json :task_statuses + # t.string :task_status_comparison # equal|notequal + + # LoginStatusCondition + t.datetime :last_sign_in_at + # t.string :last_sign_in_comparison # before|after + + # TutorialEnrolmentCondition + t.references :tutorial + # t.string :tutorial_comparison # enrolled|notenrolled + + # TutorialStreamEnrolmentCondition + t.references :tutorial_stream + # t.string :tutorial_stream_comparison # enrolled|notenrolled + + # CampusCondition + t.references :campus + # t.string :campus_comparison # enrolled|notenrolled + + # TaskStatusCountCondition + t.integer :task_status_count + t.integer :task_target_grade + + # SpecConDaysCondition + t.integer :spec_con_days + + t.string :operator, null: false + + t.timestamps + end + + create_table :communication_actions do |t| + t.string :type, null: false + t.references :communication_rule, null: false + + # EmailStudentAction / EmailStaffAction + t.string :subject + t.text :body + + # EmailStaffAction + t.boolean :email_tutors, null: false, default: false + t.boolean :email_convenors, null: false, default: false + + # ChangeTargetGradeAction + t.integer :target_grade + + # TaskCommentAction + t.references :task_definition, null: true + + t.timestamps + end + + create_table :communication_set_schedules do |t| + t.references :communication_set, null: false + t.string :name + t.boolean :active, null: false, default: true + + # Anchor the schedule to the unit's teaching calendar. The actual start + # datetime is resolved through Unit#date_for_week_and_day. + t.integer :anchor_week, null: false + t.string :anchor_day, null: false + t.integer :hour, null: false, default: 8 + t.integer :minute, null: false, default: 0 + t.string :timezone, null: false, default: "UTC" + + # Canonical recurrence settings to hydrate IceCube rules from. + t.string :recurrence, null: false, default: "none" + t.integer :interval, null: false, default: 1 + + # Optional limits for recurring schedules. + t.integer :repeat_count + t.datetime :until_at + + # Serialized payload to rebuild an IceCube schedule without guessing. + t.json :ice_cube_schedule + + # Derived state for the worker that enqueues due communication runs. + t.datetime :next_run_at + t.datetime :last_run_at + t.datetime :last_enqueued_at + + t.timestamps + end + + add_index :communication_actions, :type + add_index :communication_conditions, :type + add_index :communication_set_schedules, [:active, :next_run_at] + end +end diff --git a/db/migrate/20260610065531_create_overflow_task_claim_logs.rb b/db/migrate/20260610065531_create_overflow_task_claim_logs.rb new file mode 100644 index 0000000000..d02ea8762a --- /dev/null +++ b/db/migrate/20260610065531_create_overflow_task_claim_logs.rb @@ -0,0 +1,17 @@ +class CreateOverflowTaskClaimLogs < ActiveRecord::Migration[8.0] + def change + create_table :overflow_task_claim_logs do |t| + t.references :unit, null: false + t.references :task, null: false + t.references :claimed_by_unit_role, null: false + t.references :claimed_by_user, null: false + t.references :original_tutor_user + t.references :student_user, null: false + t.integer :days_awaiting_feedback, null: false + t.datetime :claimed_at, null: false + t.timestamps + end + + add_index :overflow_task_claim_logs, [:unit_id, :claimed_at] + end +end diff --git a/db/migrate/20260611053643_create_submission_histories.rb b/db/migrate/20260611053643_create_submission_histories.rb new file mode 100644 index 0000000000..6661f139c1 --- /dev/null +++ b/db/migrate/20260611053643_create_submission_histories.rb @@ -0,0 +1,243 @@ +# rubocop:disable Rails/SkipsModelValidations +require 'zip' + +class CreateSubmissionHistories < ActiveRecord::Migration[8.0] + # Migration-local models provide direct access to the tables as they exist at + # this migration point, without using future application callbacks or validations. + class MigrationTaskDefinition < ApplicationRecord + self.table_name = 'task_definitions' + end + + class MigrationOverseerAssessment < ApplicationRecord + self.table_name = 'overseer_assessments' + end + + class MigrationSubmissionHistory < ApplicationRecord + self.table_name = 'submission_histories' + end + + class MigrationTask < ApplicationRecord + self.table_name = 'tasks' + belongs_to :project, class_name: 'CreateSubmissionHistories::MigrationProject' + end + + class MigrationProject < ApplicationRecord + self.table_name = 'projects' + belongs_to :unit, class_name: 'CreateSubmissionHistories::MigrationUnit' + belongs_to :user, class_name: 'CreateSubmissionHistories::MigrationUser' + + def student + user + end + end + + class MigrationUnit < ApplicationRecord + self.table_name = 'units' + end + + class MigrationUser < ApplicationRecord + self.table_name = 'users' + end + + # `up` applies the migration when moving the database to this version. + def up + # Store one completed submission-history archive per task and timestamp. + create_table :submission_histories do |t| + t.references :task, null: false + t.string :submission_timestamp, null: false + + t.timestamps + end + + add_index :submission_histories, + [:task_id, :submission_timestamp], + unique: true, + name: 'index_submission_histories_on_task_and_timestamp' + + # Enable submission history for existing Overseer task definitions. Preserve + # an explicit value if this key has already been added to a requirement. + MigrationTaskDefinition.reset_column_information + MigrationTaskDefinition.find_each do |task_definition| + requirements = JSON.parse(task_definition.upload_requirements.presence || '[]') + next unless requirements.is_a?(Array) + + requirements.each do |requirement| + next unless requirement.is_a?(Hash) + next if requirement.key?('submission_history') + + requirement['submission_history'] = task_definition.assessment_enabled? + end + + task_definition.update_columns(upload_requirements: requirements.to_json) + rescue JSON::ParserError + next + end + + # Add the association as nullable first because existing Overseer assessments + # do not have a submission-history row yet. + add_reference :overseer_assessments, :submission_history, index: false + + # Refresh Active Record's cached columns after creating the table and column. + MigrationOverseerAssessment.reset_column_information + MigrationSubmissionHistory.reset_column_information + + # Preserve existing Overseer archives by creating a history with the same + # task and timestamp, then linking the assessment to that history. + MigrationOverseerAssessment.find_each do |assessment| + history = MigrationSubmissionHistory.find_or_create_by!( + task_id: assessment.task_id, + submission_timestamp: assessment.submission_timestamp + ) + assessment.update_columns(submission_history_id: history.id) + end + + # Replace each task's timestamp directories with one history.zip. Every + # directory and filename is preserved inside the archive. + MigrationSubmissionHistory.distinct.pluck(:task_id).each do |task_id| + task = MigrationTask.find_by(id: task_id) + archive_existing_history_directories(task) if task + end + + # Once all existing rows are linked, make the association required and + # ensure a submission history cannot belong to multiple Overseer assessments. + change_column_null :overseer_assessments, :submission_history_id, false + add_index :overseer_assessments, :submission_history_id, unique: true + end + + # `down` reverses `up` when rolling the database back from this version. + def down + # Restore timestamp directories before removing the records that identify + # which task archives need to be unpacked. + MigrationSubmissionHistory.distinct.pluck(:task_id).each do |task_id| + task = MigrationTask.find_by(id: task_id) + restore_history_directories(task) if task + end + + # Remove the dependent association before removing its referenced table. + remove_reference :overseer_assessments, :submission_history + + # Restore upload requirements to their shape before this migration. + MigrationTaskDefinition.find_each do |task_definition| + requirements = JSON.parse(task_definition.upload_requirements.presence || '[]') + next unless requirements.is_a?(Array) + + requirements.each do |requirement| + requirement.delete('submission_history') if requirement.is_a?(Hash) + end + + task_definition.update_columns(upload_requirements: requirements.to_json) + rescue JSON::ParserError + next + end + + # Submission-history rows are no longer needed after the association and + # upload-requirement configuration have been removed. + drop_table :submission_histories + end + + private + + def archive_existing_history_directories(task) + task_path = FileHelper.task_submission_identifier_path(:done, task) + return unless Dir.exist?(task_path) + + directories = Dir.children(task_path).filter_map do |name| + path = File.join(task_path, name) + [name, path] if File.directory?(path) + end + return if directories.empty? + + archive_path = File.join(task_path, 'history.zip') + temporary_path = "#{archive_path}.tmp-#{SecureRandom.hex(8)}" + + Zip::File.open(temporary_path, create: true) do |archive| + copy_zip_entries(archive_path, archive) if File.exist?(archive_path) + directories.each { |name, path| add_directory_to_zip(archive, name, path) } + end + + FileUtils.mv(temporary_path, archive_path) + directories.each { |directory| FileUtils.rm_rf(directory.last) } + ensure + FileUtils.rm_f(temporary_path) if temporary_path + end + + def restore_history_directories(task) + task_path = FileHelper.task_submission_identifier_path(:done, task) + archive_path = File.join(task_path, 'history.zip') + return unless File.exist?(archive_path) + + submission_entries = Hash.new { |entries, timestamp| entries[timestamp] = [] } + Zip::File.open(archive_path) do |archive| + archive.each do |entry| + next if entry.name_is_directory? + + timestamp, relative_path = entry.name.split('/', 2) + next unless relative_path + + if relative_path.start_with?("#{task.id}/") + submission_entries[timestamp] << [relative_path, entry.get_input_stream.read] + else + destination = File.join(task_path, timestamp, relative_path) + FileUtils.mkdir_p(File.dirname(destination)) + entry.extract(destination) { true } + end + end + end + + submission_entries.each do |timestamp, entries| + timestamp_path = File.join(task_path, timestamp) + FileUtils.mkdir_p(timestamp_path) + Zip::File.open(File.join(timestamp_path, 'submission.zip'), create: true) do |submission_zip| + entries.each do |relative_path, contents| + submission_zip.get_output_stream(relative_path) { |output| output.write(contents) } + end + end + end + + FileUtils.rm_f(archive_path) + end + + def add_directory_to_zip(archive, root_name, path) + archive.mkdir("#{root_name}/") unless archive.find_entry("#{root_name}/") + + Dir.glob(File.join(path, '**', '*'), File::FNM_DOTMATCH).sort.each do |source| + next if ['.', '..'].include?(File.basename(source)) + + relative_path = File.join(root_name, source.delete_prefix("#{path}/")) + if File.directory?(source) + archive.mkdir("#{relative_path}/") unless archive.find_entry("#{relative_path}/") + elsif File.basename(source) == 'submission.zip' + add_submission_zip_to_history(archive, root_name, source) + else + archive.add(relative_path, source) + end + end + end + + def add_submission_zip_to_history(archive, root_name, submission_path) + Zip::File.open(submission_path) do |submission_zip| + submission_zip.each do |entry| + next if entry.name_is_directory? + + archive.get_output_stream(File.join(root_name, entry.name)) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + end + end + + def copy_zip_entries(source_path, destination) + Zip::File.open(source_path) do |source| + source.each do |entry| + if entry.name_is_directory? + destination.mkdir(entry.name) unless destination.find_entry(entry.name) + else + destination.get_output_stream(entry.name) do |output| + entry.get_input_stream { |input| IO.copy_stream(input, output) } + end + end + end + end + end +end +# rubocop:enable Rails/SkipsModelValidations diff --git a/db/migrate/20260618001447_create_engagements.rb b/db/migrate/20260618001447_create_engagements.rb new file mode 100644 index 0000000000..c66c8a759f --- /dev/null +++ b/db/migrate/20260618001447_create_engagements.rb @@ -0,0 +1,27 @@ +class CreateEngagements < ActiveRecord::Migration[8.0] + def change + create_table :engagements do |t| + t.references :project, null: false, index: true + t.references :user, null: false, index: true + t.string :engagement_type, null: false + t.text :note, null: false + t.datetime :occurred_at, null: false + t.text :evidence_url + t.string :content_type + t.string :attachment_extension + + t.timestamps + end + + create_table :engagement_comments do |t| + t.references :engagement, null: false, index: true + t.references :user, null: false, index: true + t.references :reply_to, null: true, index: true + t.text :comment, null: false + + t.timestamps + end + + add_index :engagements, [:project_id, :occurred_at] + end +end diff --git a/db/migrate/20260618033139_add_grade_values_to_units.rb b/db/migrate/20260618033139_add_grade_values_to_units.rb new file mode 100644 index 0000000000..1cc8ca1b4c --- /dev/null +++ b/db/migrate/20260618033139_add_grade_values_to_units.rb @@ -0,0 +1,17 @@ +class AddGradeValuesToUnits < ActiveRecord::Migration[8.0] + DEFAULT_GRADE_VALUES = [0, 1, 2, 3].to_json + + def up + add_column :units, :grade_values, :json + execute "UPDATE units SET grade_values = #{connection.quote(DEFAULT_GRADE_VALUES)}" + + change_column_null :task_definition_grade_due_dates, :target_due_date, true + add_column :task_definition_grade_due_dates, :start_date, :datetime, null: true unless column_exists?(:task_definition_grade_due_dates, :start_date) + end + + def down + remove_column :task_definition_grade_due_dates, :start_date if column_exists?(:task_definition_grade_due_dates, :start_date) + change_column_null :task_definition_grade_due_dates, :target_due_date, false + remove_column :units, :grade_values + end +end diff --git a/db/migrate/20260625055323_create_task_completion_snapshots.rb b/db/migrate/20260625055323_create_task_completion_snapshots.rb new file mode 100644 index 0000000000..7b66ae68d4 --- /dev/null +++ b/db/migrate/20260625055323_create_task_completion_snapshots.rb @@ -0,0 +1,12 @@ +class CreateTaskCompletionSnapshots < ActiveRecord::Migration[8.0] + def change + create_table :task_completion_snapshots do |t| + t.references :unit, null: false + t.string :snapshot_timestamp, null: false + + t.timestamps + end + + add_index :task_completion_snapshots, [:unit_id, :snapshot_timestamp], unique: true + end +end diff --git a/db/migrate/20260709014859_add_rediscuss_status.rb b/db/migrate/20260709014859_add_rediscuss_status.rb new file mode 100644 index 0000000000..509144a88c --- /dev/null +++ b/db/migrate/20260709014859_add_rediscuss_status.rb @@ -0,0 +1,23 @@ +class AddRediscussStatus < ActiveRecord::Migration[8.0] + DESCRIPTION = "You attempted to discuss this task, but it was not adequate. " \ + "Brush up your knowledge and return for another discussion " \ + "to get the task signed off.".freeze + + def up + status = TaskStatus.find_by(id: 15) || TaskStatus.find_by(name: "Rediscuss") + + if status + status.update!(name: "Rediscuss", description: DESCRIPTION) + else + TaskStatus.create!(id: 15, name: "Rediscuss", description: DESCRIPTION) + end + + Rails.cache.delete("task_statuses/15") + end + + def down + status = TaskStatus.find_by(id: 15, name: "Rediscuss") + status&.destroy! + Rails.cache.delete("task_statuses/15") + end +end diff --git a/db/migrate/20260713232535_add_unit_content_management.rb b/db/migrate/20260713232535_add_unit_content_management.rb new file mode 100644 index 0000000000..076a54b8d6 --- /dev/null +++ b/db/migrate/20260713232535_add_unit_content_management.rb @@ -0,0 +1,31 @@ +class AddUnitContentManagement < ActiveRecord::Migration[8.0] + def change + create_table :unit_content_sites do |t| + t.references :unit, null: false + t.string :name, null: false + t.string :original_filename, null: false + t.string :archive_path, null: false + t.string :root_dir, null: false, default: '/' + t.boolean :is_main, null: false, default: false + + t.timestamps + end + + add_index :unit_content_sites, [:unit_id, :name], unique: true + + create_table :unit_content_links do |t| + t.references :unit, null: false + t.references :unit_content_site, null: false + t.string :context_type, null: false + t.string :context_key, null: false + t.string :route, null: false, default: '/' + + t.timestamps + end + + add_index :unit_content_links, + [:unit_id, :context_type, :context_key], + unique: true, + name: 'index_unit_content_links_on_context' + end +end diff --git a/db/migrate/20260722015317_add_activity_timestamps.rb b/db/migrate/20260722015317_add_activity_timestamps.rb new file mode 100644 index 0000000000..fe781cc5f8 --- /dev/null +++ b/db/migrate/20260722015317_add_activity_timestamps.rb @@ -0,0 +1,6 @@ +class AddActivityTimestamps < ActiveRecord::Migration[8.0] + def change + add_column :users, :last_access_at, :datetime + add_column :projects, :last_viewed_at, :datetime + end +end diff --git a/db/migrate/20260722030000_add_activity_days_to_communication_conditions.rb b/db/migrate/20260722030000_add_activity_days_to_communication_conditions.rb new file mode 100644 index 0000000000..14adb011b5 --- /dev/null +++ b/db/migrate/20260722030000_add_activity_days_to_communication_conditions.rb @@ -0,0 +1,5 @@ +class AddActivityDaysToCommunicationConditions < ActiveRecord::Migration[8.0] + def change + add_column :communication_conditions, :activity_days, :integer + end +end diff --git a/db/migrate/20260722065239_add_discuss_timeout.rb b/db/migrate/20260722065239_add_discuss_timeout.rb new file mode 100644 index 0000000000..413de251e5 --- /dev/null +++ b/db/migrate/20260722065239_add_discuss_timeout.rb @@ -0,0 +1,14 @@ +class AddDiscussTimeout < ActiveRecord::Migration[8.0] + def change + add_column :units, :discuss_timeout_enabled, :boolean, null: false, default: false + add_column :units, :discuss_timeout_warning_days, :integer, null: false, default: 7 + add_column :units, :discuss_timeout_expire_days, :integer, null: false, default: 14 + add_column :tasks, :moved_to_discuss_at, :datetime + add_column :tasks, :notified_discuss_warning_at, :datetime + add_column :tasks, :notified_discuss_expiry_at, :datetime + + add_index :tasks, [:task_status_id, :moved_to_discuss_at] + add_index :tasks, [:task_status_id, :notified_discuss_warning_at] + add_index :tasks, [:task_status_id, :notified_discuss_expiry_at] + end +end diff --git a/db/migrate/20260724015355_add_submitted_portfolio_to_communication_conditions.rb b/db/migrate/20260724015355_add_submitted_portfolio_to_communication_conditions.rb new file mode 100644 index 0000000000..7deb42a5c5 --- /dev/null +++ b/db/migrate/20260724015355_add_submitted_portfolio_to_communication_conditions.rb @@ -0,0 +1,5 @@ +class AddSubmittedPortfolioToCommunicationConditions < ActiveRecord::Migration[8.0] + def change + add_column :communication_conditions, :submitted_portfolio, :boolean + end +end diff --git a/db/migrate/20260728033837_create_engagement_projects.rb b/db/migrate/20260728033837_create_engagement_projects.rb new file mode 100644 index 0000000000..1a32df4b5f --- /dev/null +++ b/db/migrate/20260728033837_create_engagement_projects.rb @@ -0,0 +1,11 @@ +class CreateEngagementProjects < ActiveRecord::Migration[8.0] + def change + create_table :engagement_projects do |t| + t.references :engagement, null: false, index: true + t.references :project, null: false, index: true + t.timestamps + end + + add_index :engagement_projects, [:engagement_id, :project_id], unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 73fc5ebd58..2a0252e4ce 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2026_03_22_230239) do +ActiveRecord::Schema[8.0].define(version: 2026_07_28_033837) do create_table "activity_types", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name", null: false t.string "abbreviation", null: false @@ -73,6 +73,94 @@ t.index ["user_id"], name: "index_comments_read_receipts_on_user_id" end + create_table "communication_actions", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.string "type", null: false + t.bigint "communication_rule_id", null: false + t.string "subject" + t.text "body" + t.boolean "email_tutors", default: false, null: false + t.boolean "email_convenors", default: false, null: false + t.integer "target_grade" + t.bigint "task_definition_id" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["communication_rule_id"], name: "index_communication_actions_on_communication_rule_id" + t.index ["task_definition_id"], name: "index_communication_actions_on_task_definition_id" + t.index ["type"], name: "index_communication_actions_on_type" + end + + create_table "communication_conditions", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.string "type", null: false + t.bigint "communication_id", null: false + t.integer "target_grade" + t.bigint "task_definition_id" + t.text "task_statuses", size: :long, collation: "utf8mb4_bin" + t.datetime "last_sign_in_at" + t.bigint "tutorial_id" + t.bigint "tutorial_stream_id" + t.bigint "campus_id" + t.integer "task_status_count" + t.integer "task_target_grade" + t.integer "spec_con_days" + t.string "operator", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.integer "activity_days" + t.boolean "submitted_portfolio" + t.index ["campus_id"], name: "index_communication_conditions_on_campus_id" + t.index ["communication_id"], name: "index_communication_conditions_on_communication_id" + t.index ["task_definition_id"], name: "index_communication_conditions_on_task_definition_id" + t.index ["tutorial_id"], name: "index_communication_conditions_on_tutorial_id" + t.index ["tutorial_stream_id"], name: "index_communication_conditions_on_tutorial_stream_id" + t.index ["type"], name: "index_communication_conditions_on_type" + t.check_constraint "json_valid(`task_statuses`)", name: "task_statuses" + end + + create_table "communication_rules", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "communication_set_id", null: false + t.integer "position", default: 0, null: false + t.string "name" + t.string "operator" + t.boolean "send_log_to_convenors", default: false, null: false + t.boolean "active", default: true, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["communication_set_id"], name: "index_communication_rules_on_communication_set_id" + end + + create_table "communication_set_schedules", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "communication_set_id", null: false + t.string "name" + t.boolean "active", default: true, null: false + t.integer "anchor_week", null: false + t.string "anchor_day", null: false + t.integer "hour", default: 8, null: false + t.integer "minute", default: 0, null: false + t.string "timezone", default: "UTC", null: false + t.string "recurrence", default: "none", null: false + t.integer "interval", default: 1, null: false + t.integer "repeat_count" + t.datetime "until_at" + t.text "ice_cube_schedule", size: :long, collation: "utf8mb4_bin" + t.datetime "next_run_at" + t.datetime "last_run_at" + t.datetime "last_enqueued_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active", "next_run_at"], name: "index_communication_set_schedules_on_active_and_next_run_at" + t.index ["communication_set_id"], name: "index_communication_set_schedules_on_communication_set_id" + t.check_constraint "json_valid(`ice_cube_schedule`)", name: "ice_cube_schedule" + end + + create_table "communication_sets", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.string "name" + t.boolean "active", default: true, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["unit_id"], name: "index_communication_sets_on_unit_id" + end + create_table "d2l_assessment_mappings", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "unit_id", null: false t.string "org_unit_id" @@ -99,6 +187,44 @@ t.index ["task_definition_id"], name: "index_discussion_prompts_on_task_definition_id" end + create_table "engagement_comments", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "engagement_id", null: false + t.bigint "user_id", null: false + t.bigint "reply_to_id" + t.text "comment", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["engagement_id"], name: "index_engagement_comments_on_engagement_id" + t.index ["reply_to_id"], name: "index_engagement_comments_on_reply_to_id" + t.index ["user_id"], name: "index_engagement_comments_on_user_id" + end + + create_table "engagement_projects", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "engagement_id", null: false + t.bigint "project_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["engagement_id", "project_id"], name: "index_engagement_projects_on_engagement_id_and_project_id", unique: true + t.index ["engagement_id"], name: "index_engagement_projects_on_engagement_id" + t.index ["project_id"], name: "index_engagement_projects_on_project_id" + end + + create_table "engagements", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "project_id", null: false + t.bigint "user_id", null: false + t.string "engagement_type", null: false + t.text "note", null: false + t.datetime "occurred_at", null: false + t.text "evidence_url" + t.string "content_type" + t.string "attachment_extension" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["project_id", "occurred_at"], name: "index_engagements_on_project_id_and_occurred_at" + t.index ["project_id"], name: "index_engagements_on_project_id" + t.index ["user_id"], name: "index_engagements_on_user_id" + end + create_table "feedback_chips", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "type" t.text "chip_text" @@ -226,6 +352,26 @@ t.index ["task_id"], name: "index_moderated_tasks_on_task_id" end + create_table "overflow_task_claim_logs", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.bigint "task_id", null: false + t.bigint "claimed_by_unit_role_id", null: false + t.bigint "claimed_by_user_id", null: false + t.bigint "original_tutor_user_id" + t.bigint "student_user_id", null: false + t.integer "days_awaiting_feedback", null: false + t.datetime "claimed_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["claimed_by_unit_role_id"], name: "index_overflow_task_claim_logs_on_claimed_by_unit_role_id" + t.index ["claimed_by_user_id"], name: "index_overflow_task_claim_logs_on_claimed_by_user_id" + t.index ["original_tutor_user_id"], name: "index_overflow_task_claim_logs_on_original_tutor_user_id" + t.index ["student_user_id"], name: "index_overflow_task_claim_logs_on_student_user_id" + t.index ["task_id"], name: "index_overflow_task_claim_logs_on_task_id" + t.index ["unit_id", "claimed_at"], name: "index_overflow_task_claim_logs_on_unit_id_and_claimed_at" + t.index ["unit_id"], name: "index_overflow_task_claim_logs_on_unit_id" + end + create_table "overflow_task_claims", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "task_id", null: false t.bigint "claimed_by_unit_role_id", null: false @@ -242,6 +388,10 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "total_steps" + t.datetime "student_notified_at" + t.bigint "submission_history_id", null: false + t.index ["status", "student_notified_at", "updated_at"], name: "index_overseer_assessments_on_status_notified_updated" + t.index ["submission_history_id"], name: "index_overseer_assessments_on_submission_history_id", unique: true t.index ["task_id", "submission_timestamp"], name: "index_overseer_assessments_on_task_id_and_submission_timestamp", unique: true t.index ["task_id"], name: "index_overseer_assessments_on_task_id" end @@ -329,6 +479,7 @@ t.integer "spec_con_days", default: 0, null: false t.bigint "assessor_id" t.datetime "portfolio_submission_date" + t.datetime "last_viewed_at" t.index ["assessor_id"], name: "index_projects_on_assessor_id" t.index ["campus_id"], name: "index_projects_on_campus_id" t.index ["enrolled"], name: "index_projects_on_enrolled" @@ -373,6 +524,15 @@ t.index ["user_id"], name: "index_staff_notes_on_user_id" end + create_table "submission_histories", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "task_id", null: false + t.string "submission_timestamp", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["task_id", "submission_timestamp"], name: "index_submission_histories_on_task_and_timestamp", unique: true + t.index ["task_id"], name: "index_submission_histories_on_task_id" + end + create_table "task_comments", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "task_id", null: false t.bigint "user_id", null: false @@ -405,6 +565,15 @@ t.index ["user_id"], name: "index_task_comments_on_user_id" end + create_table "task_completion_snapshots", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.string "snapshot_timestamp", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["unit_id", "snapshot_timestamp"], name: "idx_on_unit_id_snapshot_timestamp_e923c3ae10", unique: true + t.index ["unit_id"], name: "index_task_completion_snapshots_on_unit_id" + end + create_table "task_definition_grade_due_dates", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "task_definition_id", null: false t.integer "target_grade", null: false @@ -546,10 +715,16 @@ t.datetime "target_start_date" t.datetime "target_due_date" t.datetime "last_tutor_feedback_at" + t.datetime "moved_to_discuss_at" + t.datetime "notified_discuss_warning_at" + t.datetime "notified_discuss_expiry_at" t.index ["group_submission_id"], name: "index_tasks_on_group_submission_id" t.index ["project_id", "task_definition_id"], name: "tasks_uniq_proj_task_def", unique: true t.index ["project_id"], name: "index_tasks_on_project_id" t.index ["task_definition_id"], name: "index_tasks_on_task_definition_id" + t.index ["task_status_id", "moved_to_discuss_at"], name: "index_tasks_on_task_status_id_and_moved_to_discuss_at" + t.index ["task_status_id", "notified_discuss_expiry_at"], name: "index_tasks_on_task_status_id_and_notified_discuss_expiry_at" + t.index ["task_status_id", "notified_discuss_warning_at"], name: "index_tasks_on_task_status_id_and_notified_discuss_warning_at" t.index ["task_status_id"], name: "index_tasks_on_task_status_id" end @@ -696,6 +871,32 @@ t.index ["unit_role_id"], name: "index_tutorials_on_unit_role_id" end + create_table "unit_content_links", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.bigint "unit_content_site_id", null: false + t.string "context_type", null: false + t.string "context_key", null: false + t.string "route", default: "/", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["unit_content_site_id"], name: "index_unit_content_links_on_unit_content_site_id" + t.index ["unit_id", "context_type", "context_key"], name: "index_unit_content_links_on_context", unique: true + t.index ["unit_id"], name: "index_unit_content_links_on_unit_id" + end + + create_table "unit_content_sites", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.bigint "unit_id", null: false + t.string "name", null: false + t.string "original_filename", null: false + t.string "archive_path", null: false + t.string "root_dir", default: "/", null: false + t.boolean "is_main", default: false, null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["unit_id", "name"], name: "index_unit_content_sites_on_unit_id_and_name", unique: true + t.index ["unit_id"], name: "index_unit_content_sites_on_unit_id" + end + create_table "unit_roles", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "user_id" t.bigint "tutorial_id" @@ -743,10 +944,16 @@ t.boolean "mark_late_submissions_as_assess_in_portfolio", default: false, null: false t.integer "feedback_warning_threshold_days", default: 5 t.integer "feedback_overflow_threshold_days", default: 7 + t.boolean "enforce_feedback_before_discussed_in_class", default: false, null: false + t.text "grade_values", size: :long, collation: "utf8mb4_bin" + t.boolean "discuss_timeout_enabled", default: false, null: false + t.integer "discuss_timeout_warning_days", default: 7, null: false + t.integer "discuss_timeout_expire_days", default: 14, null: false t.index ["draft_task_definition_id"], name: "index_units_on_draft_task_definition_id" t.index ["main_convenor_id"], name: "index_units_on_main_convenor_id" t.index ["overseer_image_id"], name: "index_units_on_overseer_image_id" t.index ["teaching_period_id"], name: "index_units_on_teaching_period_id" + t.check_constraint "json_valid(`grade_values`)", name: "grade_values" end create_table "user_oauth_states", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| @@ -797,6 +1004,7 @@ t.string "tii_eula_version" t.datetime "tii_eula_date" t.boolean "tii_eula_version_confirmed", default: false, null: false + t.datetime "last_access_at" t.index ["email"], name: "index_users_on_email", unique: true t.index ["login_id"], name: "index_users_on_login_id", unique: true t.index ["role_id"], name: "index_users_on_role_id" diff --git a/gotenberg.Dockerfile b/gotenberg.Dockerfile new file mode 100644 index 0000000000..6a23edcbbc --- /dev/null +++ b/gotenberg.Dockerfile @@ -0,0 +1,12 @@ +FROM gotenberg/gotenberg:8-libreoffice + +USER root + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY lib/shell/word_document_build.sh /gotenberg/word_document_build.sh +RUN chmod 755 /gotenberg/word_document_build.sh + +USER gotenberg diff --git a/lib/helpers/database_populator.rb b/lib/helpers/database_populator.rb index 57911a98bc..b415a30ec2 100644 --- a/lib/helpers/database_populator.rb +++ b/lib/helpers/database_populator.rb @@ -64,6 +64,7 @@ def initialize(scale = :small) generate_global_learning_outcomes() generate_campuses generate_activity_types + generate_overseer_images end def generate_teaching_periods @@ -166,6 +167,18 @@ def generate_activity_types ActivityType.create! data end + def generate_overseer_images + echo_line "--> Generating overseer images" + + overseer_image = OverseerImage.create!( + name: 'Bash', + tag: 'bash:latest' + ) + + echo_line "---> Pulling overseer image #{overseer_image.tag}" + overseer_image.pull_from_docker + end + # # Generate some users. Pass in an optional filter(s) for: # Role.admin, Role.convenor, Role.tutor, Role.student diff --git a/lib/shell/latex_build.sh b/lib/shell/latex_build.sh index c853bdb20f..a729f11414 100644 --- a/lib/shell/latex_build.sh +++ b/lib/shell/latex_build.sh @@ -15,7 +15,12 @@ cd work lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex RESULT=$? if [ $RESULT -eq 0 ]; then - echo "Running lualatex a second time to remove temporary last page..." + echo "Running lualatex a second time to remove temporary last page and update references..." + lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex + RESULT=$? +fi +if [ $RESULT -eq 0 ]; then + echo "Running lualatex a third time to stabilise page references..." lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex RESULT=$? fi diff --git a/lib/shell/send_overseer_notifications.sh b/lib/shell/send_overseer_notifications.sh new file mode 100755 index 0000000000..e5be6b305e --- /dev/null +++ b/lib/shell/send_overseer_notifications.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +#Get path to script +APP_PATH=`echo $0 | awk '{split($0,patharr,"/"); idx=1; while(patharr[idx+1] != "") { if (patharr[idx] != "/") {printf("%s/", patharr[idx]); idx++ }} }'` +APP_PATH=`cd "$APP_PATH"; pwd` + +ROOT_PATH=`cd "$APP_PATH"/../..; pwd` + +cd "$ROOT_PATH" + +DF_LOG_TO_STDOUT=true rails overseer_notifications:send_failed_assessment_notifications diff --git a/lib/shell/word_document_build.sh b/lib/shell/word_document_build.sh new file mode 100755 index 0000000000..42bcb917e1 --- /dev/null +++ b/lib/shell/word_document_build.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +set -eu + +WORK_ID=${1:-} + +if [ -z "$WORK_ID" ]; then + echo "Usage: word_document_build.sh " >&2 + exit 2 +fi + +case "$WORK_ID" in + *[!A-Za-z0-9_-]*) + echo "Invalid Gotenberg work id" >&2 + exit 2 + ;; +esac + +WORK_DIR="/workdir/gotenberg/$WORK_ID" +INPUT_FILE="$WORK_DIR/input.docx" +OUTPUT_FILE="$WORK_DIR/output.pdf" +TEMP_OUTPUT="$WORK_DIR/output.pdf.tmp" +GOTENBERG_URL="http://localhost:3000" +GOTENBERG_PID= + +if [ ! -f "$INPUT_FILE" ]; then + echo "Word document input was not found" >&2 + exit 1 +fi + +cleanup() { + exit_status=$? + trap - EXIT INT TERM + + if [ -n "$GOTENBERG_PID" ] && kill -0 "$GOTENBERG_PID" 2>/dev/null; then + kill "$GOTENBERG_PID" 2>/dev/null || true + wait "$GOTENBERG_PID" 2>/dev/null || true + fi + + rm -f "$TEMP_OUTPUT" + exit "$exit_status" +} + +start_gotenberg() { + # Docker replaces the image's normal command with this script, so start the + # bundled API before making the local conversion request. + gotenberg --gotenberg-graceful-shutdown-duration=0s & + GOTENBERG_PID=$! + + curl \ + --fail \ + --silent \ + --show-error \ + --retry 30 \ + --retry-connrefused \ + --retry-delay 1 \ + --connect-timeout 1 \ + --max-time 30 \ + "$GOTENBERG_URL/health" \ + >/dev/null +} + +convert_document() { + curl \ + --fail-with-body \ + --silent \ + --show-error \ + --connect-timeout 5 \ + --max-time "${WORD_DOCUMENT_CONVERSION_TIMEOUT_SECONDS:-120}" \ + --request POST \ + --form "files=@$INPUT_FILE" \ + --output "$TEMP_OUTPUT" \ + "$GOTENBERG_URL/forms/libreoffice/convert" + + if [ ! -s "$TEMP_OUTPUT" ]; then + echo "Gotenberg did not produce a PDF" >&2 + exit 1 + fi + + mv "$TEMP_OUTPUT" "$OUTPUT_FILE" +} + +trap cleanup EXIT INT TERM + +start_gotenberg +convert_document diff --git a/lib/tasks/generate_pdfs.rake b/lib/tasks/generate_pdfs.rake index f558bd2a1f..5db26d07e7 100644 --- a/lib/tasks/generate_pdfs.rake +++ b/lib/tasks/generate_pdfs.rake @@ -140,6 +140,7 @@ namespace :submission do success = false begin # Notify system admin + Sentry.capture_exception(e) if defined?(Sentry) mail = ErrorLogMailer.error_message("Failed portfolio job: #{project.log_details}", "Failed to create portfolio for project #{project.log_details}", e) mail.deliver if mail.present? rescue StandardError => e diff --git a/lib/tasks/init.rake b/lib/tasks/init.rake index 2023e509a3..4137a32533 100644 --- a/lib/tasks/init.rake +++ b/lib/tasks/init.rake @@ -1,4 +1,39 @@ +require 'io/console' + namespace :db do + def prompt_for_admin_password + password = ENV.fetch('DF_INITIAL_ADMIN_PASSWORD', nil) + if password.present? + unless Devise.password_length.cover?(password.length) + raise "DF_INITIAL_ADMIN_PASSWORD must be #{Devise.password_length} characters long" + end + + return password + end + + console = IO.console + if console.nil? + raise 'An interactive terminal is required to set the initial admin password. ' \ + 'Set DF_INITIAL_ADMIN_PASSWORD for non-interactive setup.' + end + + loop do + password = console.getpass('Enter password for the initial admin account: ') + confirmation = console.getpass('Confirm password: ') + + raise 'Password entry was cancelled' if password.nil? || confirmation.nil? + + unless Devise.password_length.cover?(password.length) + puts "Password must be #{Devise.password_length} characters long." + next + end + + return password if password == confirmation + + puts 'Passwords do not match. Try again.' + end + end + # # Generate roles # @@ -42,7 +77,8 @@ namespace :db do Fail: "You did not successfully demonstrate the required learning in this task.", "Time Exceeded": "You did not submit or complete the task before the appropriate deadline.", "Assess in Portfolio": "This task will not be signed off as complete by your tutor, and will be marked directly in your portfolio.", - "Attention Required": "This task needs to be discussed with your tutor so that you can get back on track." + "Attention Required": "This task needs to be discussed with your tutor so that you can get back on track.", + Rediscuss: "You attempted to discuss this task, but it was not adequate. Brush up your knowledge and return for another discussion to get the task signed off." } statuses.each do |name, desc| print "." @@ -73,8 +109,9 @@ namespace :db do profile[:login_id] ||= username if AuthenticationHelpers.db_auth? - profile[:password] = 'password' - profile[:password_confirmation] = 'password' + password = Rails.env.production? ? prompt_for_admin_password : 'password' + profile[:password] = password + profile[:password_confirmation] = password end user = User.create!(profile) diff --git a/lib/tasks/maintenance.rake b/lib/tasks/maintenance.rake index f6ba8ebfcb..d7053b6e2b 100644 --- a/lib/tasks/maintenance.rake +++ b/lib/tasks/maintenance.rake @@ -1,6 +1,181 @@ require_all 'lib/helpers' +require 'sidekiq/api' +# rubocop:disable Metrics/BlockLength namespace :maintenance do + def sidekiq_job_present_in_workers_or_default_queue?(&matcher) + Sidekiq::Workers.new.each do |_process_id, _thread_id, work| + payload = work['payload'].is_a?(String) ? JSON.parse(work['payload']) : work['payload'] + + return true if matcher.call(payload['class'], payload['args']) + end + + # TODO: We may need to iterate through each queue when we implement parallel sidekiq jobs + Sidekiq::Queue.new("default").each do |job| + return true if matcher.call(job.klass, job.args) + end + + false + end + + def accept_submission_job_matches_task?(job_class, job_args, task_id) + job_class == 'AcceptSubmissionJob' && job_args.first.to_i == task_id + end + + def accept_submission_job_present?(task_id) + sidekiq_job_present_in_workers_or_default_queue? do |job_class, job_args| + accept_submission_job_matches_task?(job_class, job_args, task_id) + end + end + + def accept_overseer_job_matches_assessment?(job_class, job_args, overseer_assessment_id) + job_class == 'AcceptOverseerJob' && job_args.last.to_i == overseer_assessment_id + end + + def accept_overseer_job_present?(overseer_assessment_id) + sidekiq_job_present_in_workers_or_default_queue? do |job_class, job_args| + accept_overseer_job_matches_assessment?(job_class, job_args, overseer_assessment_id) + end + end + + def create_submission_history_job_present?(task_id) + sidekiq_job_present_in_workers_or_default_queue? do |job_class, job_args| + job_class == 'CreateSubmissionHistoryJob' && job_args.first.to_i == task_id + end + end + + def notify_failed_submission(task, message) + if task.project.student.receive_task_notifications + begin + PortfolioEvidenceMailer.task_pdf_failed(task.project, [task]).deliver_now + rescue StandardError => e + Rails.logger.error "Failed to send task pdf failed email for project #{task.project.id}!\n#{e.message}" + end + end + + if defined?(Sentry) + Sentry.capture_message( + "Cleared abandoned in-process submission for task #{task.id}", + level: :info, + extra: { + task_id: task.id, + task_definition: task.task_definition.abbreviation, + username: task.project.user.username, + detail: message + } + ) + end + + begin + exception = StandardError.new(message) + mail = ErrorLogMailer.error_message('Accept Submission Cleanup', message, exception) + mail.deliver_now if mail.present? + rescue StandardError => e + Rails.logger.error "Failed to send error log to admin for task #{task.id}!\n#{e.message}" + end + end + + def mark_task_for_resubmission(task, _message) + tutor = task.project.tutor_for(task.task_definition) + + task.trigger_transition(trigger: 'fix', by_user: tutor) + task.add_text_comment(tutor, "**Automated Comment**: Something went wrong with compiling your submission. Please resubmit the task.") + rescue StandardError => e + Rails.logger.error "Failed to move task #{task.id} to fix and add automated comment!\n#{e.message}" + end + + def mark_task_for_overseer_resubmission(task) + tutor = task.project.tutor_for(task.task_definition) + + task.trigger_transition(trigger: 'fix', by_user: tutor) + task.add_text_comment(tutor, "**Automated Comment**: Something went wrong while running the automated tests for this submission. Please resubmit the task.") + rescue StandardError => e + Rails.logger.error "Failed to move task #{task.id} to fix and add Overseer automated comment!\n#{e.message}" + end + + def clear_abandoned_submissions! + in_process_path = FileHelper.student_work_dir(:in_process) + return unless Dir.exist?(in_process_path) + + abandoned_submission_timeout = 10.minutes + stale_before = abandoned_submission_timeout.ago + + Dir.foreach(in_process_path) do |entry| + next unless entry.match?(/^\d+$/) + + task_path = File.join(in_process_path, entry) + next unless File.directory?(task_path) + next unless File.mtime(task_path) < stale_before + + task = Task.includes(project: [:user, :unit]).find_by(id: entry.to_i) + next if task.nil? + + if accept_submission_job_present?(task.id) + Rails.logger.info "Skipping abandoned submission cleanup for task #{task.id} because AcceptSubmissionJob is still running or queued" + next + end + + message = "Abandoned in-process submission detected for task #{task.log_details}. The stale in-process folder was older than #{abandoned_submission_timeout / 1.minute} minutes with no running or queued AcceptSubmissionJob, has now been cleared, and the task requires resubmission." + Rails.logger.error message + + mark_task_for_resubmission(task, message) + task.clear_in_process + notify_failed_submission(task, message) + end + end + + def clear_abandoned_overseer_assessments! + abandoned_assessment_timeout = 10.minutes + stale_before = abandoned_assessment_timeout.ago + + OverseerAssessment + .pre_queued + .includes(task: [project: :user]) + .where('created_at < ?', stale_before) + .find_each do |assessment| + if accept_overseer_job_present?(assessment.id) + Rails.logger.info "Skipping abandoned OverseerAssessment cleanup for assessment #{assessment.id} because AcceptOverseerJob is still running or queued" + next + end + + task = assessment.task + message = "Abandoned OverseerAssessment detected for task #{task.log_details}. Assessment #{assessment.id} remained pre_queued for more than #{abandoned_assessment_timeout / 1.minute} minutes with no running or queued AcceptOverseerJob and has been marked failed." + Rails.logger.error message + + assessment.update!(status: :failed) + mark_task_for_overseer_resubmission(task) + + if defined?(Sentry) + Sentry.capture_message( + "Marked stale OverseerAssessment failed for task #{task.id}", + level: :warning, + extra: { + task_id: task.id, + overseer_assessment_id: assessment.id, + task_definition: task.task_definition.abbreviation, + username: task.project.user.username, + detail: message + } + ) + end + end + end + + def clear_abandoned_submission_history_markers! + stale_before = 10.minutes.ago + marker_pattern = File.join(FileHelper.root_submission_history_dir, '**', 'pending', '*', 'submission-history') + + Dir.glob(marker_pattern).each do |marker_path| + next unless File.mtime(marker_path) < stale_before + + task_id = File.basename(File.dirname(marker_path)).to_i + next if create_submission_history_job_present?(task_id) + + Rails.logger.error "Clearing abandoned submission history marker for task #{task_id}" + FileUtils.rm_f(marker_path) + end + end + desc 'Cleanup temporary files' task cleanup: [:environment] do path = FileHelper.tmp_file_dir @@ -31,6 +206,24 @@ namespace :maintenance do .find_each(&:destroy!) AuthToken.destroy_old_tokens + clear_abandoned_submissions! + clear_abandoned_submission_history_markers! + clear_abandoned_overseer_assessments! + end + + desc 'Clear abandoned in-process submission folders and notify affected users' + task clear_abandoned_submissions: [:environment] do + clear_abandoned_submissions! + end + + desc 'Clear abandoned pre-queued Overseer assessments and request resubmission' + task clear_abandoned_overseer_assessments: [:environment] do + clear_abandoned_overseer_assessments! + end + + desc 'Clear stale submission history markers with no queued or running job' + task clear_abandoned_submission_history_markers: [:environment] do + clear_abandoned_submission_history_markers! end desc 'Remove PDFs from old submissions and archive units' @@ -74,3 +267,4 @@ namespace :maintenance do `find #{FileHelper.root_portfolio_dir} -name "*pdf.old" -exec rm {} \;` end end +# rubocop:enable Metrics/BlockLength diff --git a/lib/tasks/notifications.rake b/lib/tasks/notifications.rake new file mode 100644 index 0000000000..577b0f3a48 --- /dev/null +++ b/lib/tasks/notifications.rake @@ -0,0 +1,7 @@ +namespace :notifications do + desc 'Warn students about stale Discuss tasks and expire overdue Discuss tasks' + task notify_discuss_timeout: :environment do + count = Unit.notify_discuss_timeouts! + Rails.logger.info "Discuss timeout notification pass created #{count} comment(s)." + end +end diff --git a/lib/tasks/overseer_notifications.rake b/lib/tasks/overseer_notifications.rake new file mode 100644 index 0000000000..2b69ee3f9e --- /dev/null +++ b/lib/tasks/overseer_notifications.rake @@ -0,0 +1,28 @@ +namespace :overseer_notifications do + def notify_failed_overseer_assessments! + assessments = OverseerAssessment + .awaiting_student_failure_notification + .includes(task: [{ project: :unit }, :task_definition]) + + assessments.group_by(&:project).each do |project, project_assessments| + tasks = project_assessments.map(&:task).uniq + + begin + mail = PortfolioEvidenceMailer.overseer_assessment_failed(project, tasks) + next if mail.blank? + + mail.deliver_now + project_assessments.each do |assessment| + assessment.update!(student_notified_at: Time.current) + end + rescue StandardError => e + Rails.logger.error "Failed to send overseer assessment email for project #{project.id}!\n#{e.message}" + end + end + end + + desc 'Send overdue overseer assessment failure notifications to students' + task send_failed_assessment_notifications: [:environment] do + notify_failed_overseer_assessments! + end +end diff --git a/test/api/auth_test.rb b/test/api/auth_test.rb index cc3f737601..00aaa66c6f 100644 --- a/test/api/auth_test.rb +++ b/test/api/auth_test.rb @@ -2,6 +2,7 @@ class AuthTest < ActiveSupport::TestCase include Rack::Test::Methods + include ActiveSupport::Testing::TimeHelpers include TestHelpers::AuthHelper include TestHelpers::JsonHelper @@ -62,6 +63,19 @@ def test_auth_post assert_match(/username=#{User.first.username};/, last_response.cookies['username'].to_s, 'Expect username to be set') end + def test_auth_records_sign_in_and_access_time + user = User.find_by!(username: 'aadmin') + sign_in_time = Time.zone.parse('2026-07-21 10:00:00 UTC') + + travel_to sign_in_time do + post_json '/api/auth.json', username: user.username, password: 'password' + end + + assert_equal 201, last_response.status + assert_equal sign_in_time, user.reload.last_sign_in_at + assert_equal sign_in_time, user.last_access_at + end + def test_auth_no_remember data_to_post = { username: 'aadmin', @@ -231,6 +245,22 @@ def test_refresh_token assert_equal last_response_body['auth_token'], new_new_token.authentication_token end + def test_refresh_token_updates_access_but_not_sign_in_time + user = FactoryBot.create(:user, last_sign_in_at: 1.day.ago, last_access_at: 1.hour.ago) + original_sign_in_time = user.last_sign_in_at + token = user.generate_authentication_token!(token_type: :refresh_token) + access_time = Time.zone.parse('2026-07-21 11:00:00 UTC') + + set_cookie "username=#{user.username}" + set_cookie "refresh_token=#{token.authentication_token}" + + travel_to(access_time) { post '/api/auth/access-token' } + + assert_equal 201, last_response.status + assert_equal original_sign_in_time, user.reload.last_sign_in_at + assert_equal access_time, user.last_access_at + end + def test_token_signout_works_with_multiple user = FactoryBot.create(:user) # Create 2 auth tokens diff --git a/test/api/comments/comment_test.rb b/test/api/comments/comment_test.rb index 1e1fc558a1..961b2a5e0d 100644 --- a/test/api/comments/comment_test.rb +++ b/test/api/comments/comment_test.rb @@ -2,6 +2,7 @@ class CommentTest < ActiveSupport::TestCase include Rack::Test::Methods + include ActiveSupport::Testing::TimeHelpers include TestHelpers::AuthHelper include TestHelpers::JsonHelper include TestHelpers::TestFileHelper @@ -221,6 +222,100 @@ def test_student_reply_to_themselve_in_different_task_same_project assert_equal 404, last_response.status end + def test_student_can_edit_own_comment_within_10_minutes + project = Project.first + user = project.student + task_definition = project.unit.task_definitions.first + + add_auth_header_for(user: user) + post_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments", comment: 'Original comment' + assert_equal 201, last_response.status + + comment_id = last_response_body['id'] + + travel_to 9.minutes.from_now do + put_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments/#{comment_id}", comment: 'Edited comment' + assert_equal 200, last_response.status, last_response.body + end + + assert_equal 'Edited comment', TaskComment.find(comment_id).read_attribute(:comment) + assert_equal 'Edited comment', last_response_body['comment'] + end + + def test_student_cannot_edit_own_comment_after_10_minutes + project = Project.first + user = project.student + task_definition = project.unit.task_definitions.first + + add_auth_header_for(user: user) + post_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments", comment: 'Original comment' + assert_equal 201, last_response.status + + comment_id = last_response_body['id'] + + travel_to 11.minutes.from_now do + put_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments/#{comment_id}", comment: 'Too late' + assert_equal 403, last_response.status, last_response.body + end + + assert_equal 'Original comment', TaskComment.find(comment_id).read_attribute(:comment) + end + + def test_student_cannot_edit_other_users_comment + project = Project.first + task_definition = project.unit.task_definitions.first + tutor = project.tutor_for(task_definition) + user = project.student + task = project.task_for_task_definition(task_definition) + comment = task.add_text_comment(tutor, 'Tutor comment') + + add_auth_header_for(user: user) + put_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments/#{comment.id}", comment: 'Edited by student' + + assert_equal 403, last_response.status, last_response.body + assert_equal 'Tutor comment', comment.reload.read_attribute(:comment) + end + + def test_special_task_comments_cannot_be_deleted + project = FactoryBot.create(:project) + task_definition = project.unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + tutor = project.tutor_for(task_definition) + student = project.student + + submission_history = FactoryBot.create(:submission_history, task: task) + overseer_assessment = FactoryBot.create( + :overseer_assessment, + task: task, + submission_history: submission_history, + submission_timestamp: submission_history.submission_timestamp, + status: :failed + ) + + protected_comments = [ + task.add_status_comment(student, TaskStatus.ready_for_feedback), + task.add_discussed_comment(tutor), + task.add_feedback_review_request_comment(student), + AssessmentComment.create!( + task: task, + user: tutor, + recipient: student, + comment: 'Automated tests failed', + commentable: overseer_assessment + ) + ] + + protected_comments.each do |comment| + add_auth_header_for(user: comment.user) + delete_json "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/comments/#{comment.id}" + assert_equal 403, last_response.status, "Expected #{comment.class.name} delete to be rejected" + end + + deleted_comment_types = protected_comments.reject { |comment| TaskComment.exists?(comment.id) }.map(&:content_type) + + assert_empty deleted_comment_types, "Expected protected comment types to remain after delete attempt: #{deleted_comment_types.join(', ')}" + end + def test_student_reply_to_other_student_in_same_group unit = FactoryBot.create :unit diff --git a/test/api/communication_rules_api_test.rb b/test/api/communication_rules_api_test.rb new file mode 100644 index 0000000000..07b9a680ed --- /dev/null +++ b/test/api/communication_rules_api_test.rb @@ -0,0 +1,86 @@ +require 'test_helper' + +class CommunicationRulesApiTest < ActiveSupport::TestCase + include Rack::Test::Methods + include TestHelpers::AuthHelper + include TestHelpers::JsonHelper + + def app + Rails.application + end + + def test_create_and_update_relative_activity_condition + admin = FactoryBot.create(:user, :admin) + unit = FactoryBot.create(:unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0) + communication_set = unit.communication_sets.create!(name: 'Activity', active: true) + rule = communication_set.communication_rules.create!(name: 'Inactive students', operator: 'and', position: 0) + add_auth_header_for(user: admin) + + post_json "/api/units/#{unit.id}/communication_rules/#{rule.id}/conditions", + communication_condition: { + type: 'LoginStatusCondition', + operator: 'more_than', + activity_days: 7 + } + + assert_equal 201, last_response.status + condition_response = last_response_body + assert_equal 'LoginStatusCondition', condition_response['type'] + assert_equal 'more_than', condition_response['operator'] + assert_equal 7, condition_response['activity_days'] + + put_json "/api/units/#{unit.id}/communication_rules/#{rule.id}/conditions/#{condition_response['id']}", + communication_condition: { + type: 'UnitViewedStatusCondition', + operator: 'within_last', + activity_days: 3 + } + + assert_equal 200, last_response.status + condition_response = last_response_body + assert_equal 'UnitViewedStatusCondition', condition_response['type'] + assert_equal 'within_last', condition_response['operator'] + assert_equal 3, condition_response['activity_days'] + + viewed_at = 1.day.ago.change(usec: 0) + project = FactoryBot.create(:project, unit: unit, last_viewed_at: viewed_at) + project.user.update!(last_sign_in_at: 2.days.ago.change(usec: 0)) + + get "/api/units/#{unit.id}/communication_sets/#{communication_set.id}" + + assert_equal 200, last_response.status + student = last_response_body['previews'].first['allocations'].first['students'].first + assert_equal project.user.username, student['username'] + assert_equal false, student['has_portfolio'] + assert_equal viewed_at.iso8601(3), Time.zone.parse(student['last_viewed_at']).iso8601(3) + assert_equal project.user.last_sign_in_at.iso8601(3), Time.zone.parse(student['last_sign_in_at']).iso8601(3) + end + + def test_create_and_update_portfolio_submitted_condition + admin = FactoryBot.create(:user, :admin) + unit = FactoryBot.create(:unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0) + communication_set = unit.communication_sets.create!(name: 'Portfolio', active: true) + rule = communication_set.communication_rules.create!(name: 'Portfolio submitted', operator: 'and', position: 0) + add_auth_header_for(user: admin) + + post_json "/api/units/#{unit.id}/communication_rules/#{rule.id}/conditions", + communication_condition: { + type: 'PortfolioSubmittedCondition', + operator: 'equal_to', + submitted_portfolio: true + } + + assert_equal 201, last_response.status + condition_response = last_response_body + assert_equal true, condition_response['submitted_portfolio'] + + put_json "/api/units/#{unit.id}/communication_rules/#{rule.id}/conditions/#{condition_response['id']}", + communication_condition: { + submitted_portfolio: false + } + + assert_equal 200, last_response.status + assert_equal false, last_response_body['submitted_portfolio'] + assert_equal false, rule.communication_conditions.find(condition_response['id']).submitted_portfolio + end +end diff --git a/test/api/csv_test.rb b/test/api/csv_test.rb index 39845dcd38..42a9f8ebdd 100644 --- a/test/api/csv_test.rb +++ b/test/api/csv_test.rb @@ -181,7 +181,7 @@ def test_csv_upload_normalises_bad_upload_requirement_file_keys # Ensure our test csv has unordered or duplicate file keys # This scenario can happen when moving upload requirements around from different tasks expected_upload_requirements_by_task = { - '1.1P' => '[{"key":"file1","name":"HelloWorld.pas","type":"code"},{"key":"file1","name":"Screenshot","type":"image"}]', + '1.1P' => '[{"key":"file1","name":"HelloWorld.cpp","type":"code"},{"key":"file1","name":"Screenshot","type":"image"}]', '1.2P' => '[{"key":"file0","name":"PictureDrawing.pas","type":"code"},{"key":"file5","name":"Screenshot","type":"image"}]', '1.3P' => '[{"key":"file0","name":"PictureDrawing.pas","type":"code"},{"key":"file3","name":"Screenshot","type":"image"},{"key":"file2","name":"Screenshot","type":"image"}]' } @@ -989,6 +989,60 @@ def test_download_csv_stats_tutor_assessed end end + # GET /api/csv/units/{id}/overflow_task_claims + def test_download_csv_overflow_task_claims + Sidekiq::Testing.inline! do + unit = create(:unit, student_count: 1, task_count: 1, stream_count: 0, tutorials: 1) + convenor = unit.main_convenor.user + tutor_user = create(:user, :tutor, first_name: 'ATutor', last_name: 'Tutor') + tutor_role = unit.employ_staff(tutor_user, Role.tutor) + tutorial = unit.tutorials.first + tutorial.update!(unit_role: tutor_role) + + project = unit.active_projects.first + student = project.student + student.update!(username: 'student-one', student_id: 's1234567') + + task_definition = unit.task_definitions.first + task_definition.update!(abbreviation: 'T1') + + task = project.task_for_task_definition(task_definition) + claimed_at = Time.zone.parse('2026-04-01 10:30:00 UTC') + create( + :overflow_task_claim_log, + unit_id: unit.id, + task_id: task.id, + claimed_by_unit_role_id: tutor_role.id, + claimed_by_user_id: tutor_user.id, + original_tutor_user_id: tutor_user.id, + student_user_id: student.id, + days_awaiting_feedback: 12, + claimed_at: claimed_at + ) + + add_auth_header_for(user: convenor) + + get "/api/csv/units/#{unit.id}/overflow_task_claims" + + assert_equal 200, last_response.status + assert_equal unit.overflow_task_claims_csv, last_response_body['result'] + + rows = CSV.parse(last_response_body['result'], headers: true) + assert_equal 1, rows.length + assert_equal 'ATutor Tutor', rows[0]['Tutor who claimed'] + assert_equal tutor_role.id.to_s, rows[0]['Claiming Unit Role ID'] + assert_equal 'ATutor Tutor', rows[0]['Original Tutor'] + assert_equal 'student-one', rows[0]['Student Username'] + assert_equal 's1234567', rows[0]['Student ID'] + assert_equal task.id.to_s, rows[0]['Task ID'] + assert_equal 'T1', rows[0]['Task Definition'] + assert_equal '12', rows[0]['Days Awaiting Feedback'] + assert_equal claimed_at.to_s, rows[0]['Timestamp'] + assert_equal claimed_at, Time.zone.parse(rows[0]['Timestamp']) + Sidekiq::Testing.fake! + end + end + #47: Testing for unit ID error with empty user ID #GET /api/csv/units/{id}/tutor_assessments def test_download_csv_stats_tutor_assessed_with_empty_unit_id diff --git a/test/api/engagements_api_test.rb b/test/api/engagements_api_test.rb new file mode 100644 index 0000000000..b5353c314b --- /dev/null +++ b/test/api/engagements_api_test.rb @@ -0,0 +1,424 @@ +require 'test_helper' + +class EngagementsApiTest < ActiveSupport::TestCase + include Rack::Test::Methods + include ActiveSupport::Testing::TimeHelpers + include TestHelpers::AuthHelper + include TestHelpers::JsonHelper + include TestHelpers::TestFileHelper + + def app + Rails.application + end + + def setup + @unit = FactoryBot.create(:unit, with_students: false) + @student = FactoryBot.create(:user, :student) + @project = @unit.enrol_student(@student, nil) + @tutor = FactoryBot.create(:user, :tutor) + @unit.employ_staff(@tutor, Role.tutor) + @convenor = @unit.main_convenor_user + end + + def engagement_params(overrides = {}) + { + engagement_type: 'attendance', + note: 'Attended the weekly discussion.', + occurred_at: Time.zone.now.iso8601 + }.merge(overrides) + end + + def create_engagement(user: @tutor, overrides: {}) + add_auth_header_for(user: user) + post_json "/api/projects/#{@project.id}/engagements", engagement_params(overrides) + assert_equal 201, last_response.status, last_response.body + Engagement.find(last_response_body['id']) + end + + def test_tutor_can_create_and_student_can_read_engagements + later = create_engagement(overrides: { engagement_type: 'forum', occurred_at: 1.day.from_now.iso8601 }) + earlier = create_engagement(overrides: { engagement_type: 'negative', occurred_at: 1.day.ago.iso8601 }) + + add_auth_header_for(user: @student) + get "/api/projects/#{@project.id}/engagements" + + assert_equal 200, last_response.status + assert_equal [earlier.id, later.id], (last_response_body.map { |engagement| engagement['id'] }) + assert_equal 'negative', last_response_body.first['engagement_type'] + assert_equal @tutor.id, last_response_body.first.dig('user', 'id') + end + + def test_tutor_can_create_one_engagement_for_multiple_students + other_student = FactoryBot.create(:user, :student) + other_project = @unit.enrol_student(other_student, nil) + + engagement = create_engagement( + overrides: { project_ids: [@project.id, other_project.id] } + ) + + assert_equal [engagement.id], @project.engagements.pluck(:id) + assert_equal [engagement.id], other_project.shared_engagements.pluck(:id) + assert_equal [other_project.id], engagement.additional_projects.pluck(:id) + + add_auth_header_for(user: other_student) + get "/api/projects/#{other_project.id}/engagements/#{engagement.id}" + assert_equal [@student.id, other_student.id].sort, + last_response_body['students'].pluck('id').sort + end + + def test_records_automatic_class_discussion_with_tutorial_context + tutorial = @unit.tutorials.first + tutorial.campus.update!(timezone: 'UTC') + @project.enrol_in(tutorial) + tutorial.update!(meeting_day: 'Monday', meeting_time: '10:00') + add_auth_header_for(user: @tutor) + task_definition = @unit.task_definitions.first + + assert_difference 'Engagement.count', 1 do + travel_to(Time.zone.parse('2026-07-20 10:30:00 UTC')) do + post_json( + "/api/projects/#{@project.id}/engagements/class_discussion", + { + task_status_updates: [ + { + task_definition_id: task_definition.id, + from_status: 'ready_for_feedback', + to_status: 'complete' + } + ] + } + ) + end + end + + assert_equal 201, last_response.status + engagement = @project.engagements.last + assert_equal 'Discussion', engagement.engagement_type + assert_equal( + "Updated task statuses during tutorial. #{task_definition.abbreviation}: Ready for Feedback → Complete.", + engagement.note + ) + assert_equal @tutor, engagement.user + end + + def test_records_class_discussion_without_status_updates_outside_tutorial + add_auth_header_for(user: @tutor) + + assert_difference 'Engagement.count', 1 do + post_json( + "/api/projects/#{@project.id}/engagements/class_discussion", + { task_status_updates: [] } + ) + end + + assert_equal 201, last_response.status + assert_equal( + 'Class discussion outside of tutorial; no task statuses were updated.', + @project.engagements.last.note + ) + end + + def test_debounces_automatic_class_discussion_engagements + add_auth_header_for(user: @tutor) + path = "/api/projects/#{@project.id}/engagements/class_discussion" + task_definitions = @unit.task_definitions.first(2) + + assert_difference 'Engagement.count', 1 do + post_json( + path, + { + task_status_updates: [ + { + task_definition_id: task_definitions.first.id, + from_status: 'ready_for_feedback', + to_status: 'complete' + } + ] + } + ) + post_json( + path, + { + task_status_updates: [ + { + task_definition_id: task_definitions.second.id, + from_status: 'discuss', + to_status: 'rediscuss' + } + ] + } + ) + end + + assert_equal false, last_response_body['recorded'] + note = @project.engagements.last.note + assert_includes note, "#{task_definitions.first.abbreviation}: Ready for Feedback → Complete." + assert_includes note, "#{task_definitions.second.abbreviation}: Discuss → Rediscuss." + end + + def test_student_cannot_record_automatic_class_discussion + add_auth_header_for(user: @student) + + assert_no_difference 'Engagement.count' do + post_json( + "/api/projects/#{@project.id}/engagements/class_discussion", + { task_status_updates: [] } + ) + end + + assert_equal 403, last_response.status + end + + def test_student_and_unrelated_tutor_cannot_create_engagements + add_auth_header_for(user: @student) + post_json "/api/projects/#{@project.id}/engagements", engagement_params + assert_equal 403, last_response.status + + unrelated_tutor = FactoryBot.create(:user, :tutor) + add_auth_header_for(user: unrelated_tutor) + post_json "/api/projects/#{@project.id}/engagements", engagement_params + assert_equal 403, last_response.status + end + + def test_only_author_with_current_teaching_access_can_edit + engagement = create_engagement + other_tutor = FactoryBot.create(:user, :tutor) + @unit.employ_staff(other_tutor, Role.tutor) + + add_auth_header_for(user: other_tutor) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}", + { note: 'Changed by another tutor.' } + ) + assert_equal 403, last_response.status + + add_auth_header_for(user: @tutor) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}", + { note: 'Corrected note.', occurred_at: 2.days.ago.iso8601 } + ) + assert_equal 200, last_response.status + assert_equal 'Corrected note.', engagement.reload.note + + @unit.unit_role_for(@tutor).destroy! + assert Engagement.exists?(engagement.id) + + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}", + { note: 'Changed after role removal.' } + ) + assert_equal 403, last_response.status + assert_equal 'Corrected note.', engagement.reload.note + end + + def test_only_convenor_can_delete + engagement = create_engagement + + add_auth_header_for(user: @tutor) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}" + assert_equal 403, last_response.status + + add_auth_header_for(user: @convenor) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}" + assert_equal 200, last_response.status + assert_nil Engagement.find_by(id: engagement.id) + end + + def test_student_and_teaching_staff_can_comment + engagement = create_engagement + + add_auth_header_for(user: @student) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'I have added the supporting context.' } + ) + assert_equal 201, last_response.status + assert_equal @student.id, last_response_body.dig('user', 'id') + + add_auth_header_for(user: @tutor) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'Thanks, this clarifies the evidence.' } + ) + assert_equal 201, last_response.status + + unrelated_student = FactoryBot.create(:user, :student) + add_auth_header_for(user: unrelated_student) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'Unrelated comment.' } + ) + assert_equal 403, last_response.status + + add_auth_header_for(user: @student) + get "/api/projects/#{@project.id}/engagements/#{engagement.id}" + assert_equal 200, last_response.status + assert_equal 2, last_response_body['comments'].length + assert_equal( + ['I have added the supporting context.', 'Thanks, this clarifies the evidence.'], + last_response_body['comments'].map { |comment| comment['comment'] } + ) + end + + def test_comment_can_reply_to_comment_in_same_engagement + engagement = create_engagement + + add_auth_header_for(user: @student) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'Original comment.' } + ) + assert_equal 201, last_response.status + original_comment_id = last_response_body['id'] + + add_auth_header_for(user: @tutor) + post_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments", + { comment: 'Reply comment.', reply_to_id: original_comment_id } + ) + assert_equal 201, last_response.status + assert_equal original_comment_id, last_response_body['reply_to_id'] + + get "/api/projects/#{@project.id}/engagements/#{engagement.id}" + assert_equal original_comment_id, last_response_body['comments'].last['reply_to_id'] + end + + def test_comment_cannot_reply_to_comment_in_another_engagement + engagement = create_engagement + other_engagement = create_engagement(overrides: { note: 'Another engagement.' }) + original_comment = engagement.engagement_comments.create!( + user: @student, + comment: 'Comment on the first engagement.' + ) + + add_auth_header_for(user: @tutor) + post_json( + "/api/projects/#{@project.id}/engagements/#{other_engagement.id}/comments", + { comment: 'Invalid reply.', reply_to_id: original_comment.id } + ) + assert_equal 404, last_response.status + end + + def test_comment_author_can_edit_within_ten_minutes + engagement = create_engagement + comment = engagement.engagement_comments.create!(user: @student, comment: 'Original comment.') + + add_auth_header_for(user: @student) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{comment.id}", + { comment: 'Updated comment.' } + ) + + assert_equal 200, last_response.status + assert_equal 'Updated comment.', comment.reload.comment + end + + def test_comment_cannot_be_edited_after_ten_minutes_or_by_another_user + engagement = create_engagement + comment = engagement.engagement_comments.create!(user: @student, comment: 'Original comment.') + + add_auth_header_for(user: @tutor) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{comment.id}", + { comment: 'Tutor edit.' } + ) + assert_equal 403, last_response.status + + comment.update_column(:created_at, 11.minutes.ago) + add_auth_header_for(user: @student) + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{comment.id}", + { comment: 'Late edit.' } + ) + assert_equal 403, last_response.status + assert_equal 'Original comment.', comment.reload.comment + end + + def test_comment_author_and_convenor_can_delete + engagement = create_engagement + student_comment = engagement.engagement_comments.create!( + user: @student, + comment: 'Student comment.' + ) + + add_auth_header_for(user: @tutor) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{student_comment.id}" + assert_equal 403, last_response.status + + add_auth_header_for(user: @student) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{student_comment.id}" + assert_equal 200, last_response.status + assert_not EngagementComment.exists?(student_comment.id) + + tutor_comment = engagement.engagement_comments.create!(user: @tutor, comment: 'Tutor comment.') + add_auth_header_for(user: @convenor) + delete "/api/projects/#{@project.id}/engagements/#{engagement.id}/comments/#{tutor_comment.id}" + assert_equal 200, last_response.status + assert_not EngagementComment.exists?(tutor_comment.id) + end + + def test_rejects_file_and_url_together + add_auth_header_for(user: @tutor) + data = engagement_params( + evidence_url: 'https://example.com/evidence', + attachment: upload_file('test_files/submissions/boo.png', 'image/png') + ) + + post "/api/projects/#{@project.id}/engagements", data + + assert_equal 400, last_response.status + assert_equal 0, @project.engagements.count + end + + def test_image_attachment_can_be_retrieved_and_replaced_with_url + add_auth_header_for(user: @tutor) + data = engagement_params( + attachment: upload_file('test_files/submissions/Deakin_Logo.jpeg', 'image/jpeg') + ) + + post "/api/projects/#{@project.id}/engagements", data + assert_equal 201, last_response.status, last_response.body + + engagement = Engagement.find(last_response_body['id']) + assert engagement.attachment? + assert File.exist?(engagement.attachment_path) + + add_auth_header_for(user: @student) + get "/api/projects/#{@project.id}/engagements/#{engagement.id}/attachment" + assert_equal 200, last_response.status + + add_auth_header_for(user: @tutor) + old_attachment_path = engagement.attachment_path + put_json( + "/api/projects/#{@project.id}/engagements/#{engagement.id}", + { evidence_url: 'https://example.com/replacement' } + ) + assert_equal 200, last_response.status + + engagement.reload + assert_not engagement.attachment? + assert_not File.exist?(old_attachment_path) + assert_equal 'https://example.com/replacement', engagement.evidence_url + end + + def test_pdf_attachment_is_accepted_and_removed_with_project + add_auth_header_for(user: @tutor) + data = engagement_params( + attachment: upload_file('test_files/submissions/00_question.pdf', 'application/pdf') + ) + + post "/api/projects/#{@project.id}/engagements", data + assert_equal 201, last_response.status, last_response.body + + engagement = Engagement.find(last_response_body['id']) + assert_equal 'pdf', engagement.content_type + attachment_path = engagement.attachment_path + assert File.exist?(attachment_path) + + @project.tutorial_enrolments.destroy_all + @project.destroy! + + assert_nil Engagement.find_by(id: engagement.id) + assert_not File.exist?(attachment_path) + end +end diff --git a/test/api/feedback/learning_outcome_api_test.rb b/test/api/feedback/learning_outcome_api_test.rb index 0dcf445623..14431c27c5 100644 --- a/test/api/feedback/learning_outcome_api_test.rb +++ b/test/api/feedback/learning_outcome_api_test.rb @@ -404,6 +404,17 @@ def test_unit_rollover assert_equal chip.learning_outcome_id, new_td_lo.id, 'Each feedback chip should have the correct learning outcome id' end + copied_group_chip = new_td_lo.feedback_group_chips.find_by(chip_text: group_chip_td_lo.chip_text) + copied_template_chip = new_td_lo.feedback_template_chips.find_by(chip_text: template_chip_td_lo.chip_text) + + assert_not_nil copied_group_chip, 'The task learning outcome feedback group should be copied' + assert_not_nil copied_template_chip, 'The task learning outcome feedback template should be copied' + assert_equal copied_group_chip, copied_template_chip.parent_chip + assert_equal template_chip_td_lo.description, copied_template_chip.description + assert_equal template_chip_td_lo.task_status, copied_template_chip.task_status + assert_equal template_chip_td_lo.summary_text, copied_template_chip.summary_text + assert_equal template_chip_td_lo.comment_text, copied_template_chip.comment_text + new_link = LearningOutcomeLink.find_by(source_id: new_lo1.id, target_id: global_learning_outcome.id) assert_not_nil new_link, 'New learning outcome 1 should be linked to the global learning outcome' diff --git a/test/api/marking_sessions_api_test.rb b/test/api/marking_sessions_api_test.rb index 7a0615f7db..f5079fd36d 100644 --- a/test/api/marking_sessions_api_test.rb +++ b/test/api/marking_sessions_api_test.rb @@ -223,6 +223,8 @@ def test_assesment_activities assert_equal project.id, last_activity.project.id assert_equal "delete-comment", last_activity.action + project.task_for_task_definition(td).add_text_comment(tutor, 'Manual tutor feedback') + # Test asessment put "/api/projects/#{project.id}/task_def_id/#{td.id}", { trigger: 'complete' } assert_equal 200, last_response.status @@ -364,7 +366,7 @@ def test_marking_sessions_tutorial_split # unit = Unit.first project = unit.projects.first - task = project.tasks.first + task = project.task_for_task_definition(unit.task_definitions.first) while current_time <= end_time travel_to current_time do diff --git a/test/api/overseer_image_api_test.rb b/test/api/overseer_image_api_test.rb index 65025a8803..99e45cd4d8 100644 --- a/test/api/overseer_image_api_test.rb +++ b/test/api/overseer_image_api_test.rb @@ -26,7 +26,7 @@ def test_get_all_overseer_images assert_equal 200, last_response.status assert_equal expected_data.count, last_response_body.count, last_response_body - response_keys = %w[name tag pulled_image_text pulled_image_status last_pulled_date] + response_keys = %w[name tag pulled_image_text pulled_image_status] last_response_body.each do |data| expected_data = OverseerImage.find(data['id']) diff --git a/test/api/projects_api_test.rb b/test/api/projects_api_test.rb index 30407838e6..b6a41ccc75 100644 --- a/test/api/projects_api_test.rb +++ b/test/api/projects_api_test.rb @@ -4,6 +4,7 @@ class ProjectsApiTest < ActiveSupport::TestCase include Rack::Test::Methods + include ActiveSupport::Testing::TimeHelpers include TestHelpers::AuthHelper include TestHelpers::JsonHelper include TestHelpers::TestFileHelper @@ -86,6 +87,103 @@ def test_get_project_response_is_correct assert_json_matches_model project, last_response_body, key_test end + def test_get_project_records_when_student_viewed_it + user = FactoryBot.create(:user, :student, enrol_in: 1) + project = user.projects.first + viewed_at = Time.zone.parse('2026-07-21 12:00:00 UTC') + add_auth_header_for(user: user) + + travel_to(viewed_at) { get "/api/projects/#{project.id}" } + + assert_equal 200, last_response.status + assert_equal viewed_at, project.reload.last_viewed_at + end + + def test_get_project_does_not_record_staff_view_as_student_view + project = FactoryBot.create(:project) + admin = FactoryBot.create(:user, :admin) + add_auth_header_for(user: admin) + + get "/api/projects/#{project.id}" + + assert_equal 200, last_response.status + assert_nil project.reload.last_viewed_at + end + + def test_get_project_can_record_attendance_during_enrolled_tutorial + project, tutor, tutorial = attendance_test_data + attended_at = Time.zone.parse('2026-07-20 10:30:00 UTC') + tutorial.update!(meeting_day: 'Monday', meeting_time: '10:00') + add_auth_header_for(user: tutor) + + assert_difference 'Engagement.count', 1 do + travel_to(attended_at) { get "/api/projects/#{project.id}?record_attendance=true" } + end + + assert_equal 200, last_response.status + attendance = project.engagements.last + assert_equal tutor, attendance.user + assert_equal 'Attendance', attendance.engagement_type + assert_equal 'Attended tutorial and QR scanned.', attendance.note + assert_equal attended_at, attendance.occurred_at + end + + def test_get_project_does_not_record_attendance_outside_enrolled_tutorial + project, tutor, tutorial = attendance_test_data + tutorial.update!(meeting_day: 'Monday', meeting_time: '10:00') + add_auth_header_for(user: tutor) + + assert_no_difference 'Engagement.count' do + travel_to(Time.zone.parse('2026-07-20 12:00:00 UTC')) do + get "/api/projects/#{project.id}?record_attendance=true" + end + end + + assert_equal 200, last_response.status + end + + def test_get_project_debounces_attendance_for_fifteen_minutes + project, tutor, tutorial = attendance_test_data + tutorial.update!(meeting_day: 'Monday', meeting_time: '10:00') + add_auth_header_for(user: tutor) + + assert_difference 'Engagement.count', 1 do + travel_to(Time.zone.parse('2026-07-20 10:30:00 UTC')) do + get "/api/projects/#{project.id}?record_attendance=true" + end + end + + assert_no_difference 'Engagement.count' do + travel_to(Time.zone.parse('2026-07-20 10:44:00 UTC')) do + get "/api/projects/#{project.id}?record_attendance=true" + end + end + + assert_difference 'Engagement.count', 1 do + travel_to(Time.zone.parse('2026-07-20 10:46:00 UTC')) do + get "/api/projects/#{project.id}?record_attendance=true" + end + end + end + + def test_get_project_rejects_attendance_trigger_without_unit_teaching_access + project, = attendance_test_data + add_auth_header_for(user: project.student) + + assert_no_difference 'Engagement.count' do + get "/api/projects/#{project.id}?record_attendance=true" + end + + assert_equal 403, last_response.status + + add_auth_header_for(user: FactoryBot.create(:user, :admin)) + assert_no_difference 'Engagement.count' do + get "/api/projects/#{project.id}?record_attendance=true" + end + + assert_equal 403, last_response.status + end + def test_projects_works_with_inactive_units user = FactoryBot.create(:user, :student, enrol_in: 2) Unit.last.update(active: false) @@ -192,4 +290,19 @@ def test_download_portfolio ensure FileUtils.rm_f(project.portfolio_path) end + + private + + def attendance_test_data + unit = FactoryBot.create(:unit, with_students: false, task_count: 0) + tutorial = unit.tutorials.first + tutorial.campus.update!(timezone: 'UTC') + student = FactoryBot.create(:user, :student) + project = unit.enrol_student(student, tutorial.campus) + project.enrol_in(tutorial) + tutor = FactoryBot.create(:user, :tutor) + unit.employ_staff(tutor, Role.tutor) + + [project, tutor, tutorial] + end end diff --git a/test/api/staff_notes_api_test.rb b/test/api/staff_notes_api_test.rb index 300d30eb77..09e4a69f14 100644 --- a/test/api/staff_notes_api_test.rb +++ b/test/api/staff_notes_api_test.rb @@ -93,6 +93,83 @@ def test_tutor_can_create_staff_notes assert_equal response_staff_note.id, second_staff_note.reply_to_id end + def test_only_convenors_and_admins_can_upload_staff_notes_csv + unit = FactoryBot.create(:unit, code: 'COS10001', with_students: false) + admin_unit = FactoryBot.create(:unit, code: 'COS10002', with_students: false) + student = FactoryBot.create(:user, :student, login_id: 'staff-note-student') + tutor = FactoryBot.create(:user, :tutor) + convenor = FactoryBot.create(:user, :convenor) + admin = FactoryBot.create(:user, :admin) + unit.enrol_student(student, nil) + admin_unit.enrol_student(student, nil) + unit.employ_staff(tutor, Role.tutor) + unit.employ_staff(convenor, Role.convenor) + + csv_file = Tempfile.new(['staff-notes-permissions', '.csv']) + csv_file.write("username,comment\n#{student.username},Permission test note\n") + csv_file.rewind + + Sidekiq::Testing.inline! do + [[convenor, unit], [admin, admin_unit]].each do |user, upload_unit| + add_auth_header_for(user: user) + post "/api/csv/units/#{upload_unit.id}/staff_notes", file: Rack::Test::UploadedFile.new(csv_file.path, 'text/csv') + assert_equal 201, last_response.status, "#{user.role.name}: #{last_response.body}" + end + + [tutor, student].each do |user| + add_auth_header_for(user: user) + post "/api/csv/units/#{unit.id}/staff_notes", file: Rack::Test::UploadedFile.new(csv_file.path, 'text/csv') + assert_equal 403, last_response.status, user.role.name + end + ensure + Sidekiq::Testing.fake! + end + ensure + csv_file&.close + csv_file&.unlink + end + + def test_staff_notes_csv_imports_by_username_or_login_id + unit = FactoryBot.create(:unit, code: 'COS10001', with_students: false) + convenor = FactoryBot.create(:user, :convenor) + first_student = FactoryBot.create(:user, :student, login_id: 'staff-note-login-1') + second_student = FactoryBot.create(:user, :student, login_id: 'staff-note-login-2') + first_project = unit.enrol_student(first_student, nil) + second_project = unit.enrol_student(second_student, nil) + unit.employ_staff(convenor, Role.convenor) + + csv_file = Tempfile.new(['staff-notes-import', '.csv']) + csv_file.write(CSV.generate do |csv| + csv << %w[username login_id comment] + csv << [first_student.username, nil, 'Username note'] + csv << [nil, second_student.login_id, 'Login ID note'] + csv << [first_student.username, first_student.login_id, 'Duplicate note'] + csv << [first_student.username, first_student.login_id, 'Duplicate note'] + csv << [first_student.username, second_student.login_id, 'Mismatched identifiers note'] + end) + csv_file.rewind + + Sidekiq::Testing.inline! do + add_auth_header_for(user: convenor) + post "/api/csv/units/#{unit.id}/staff_notes", file: Rack::Test::UploadedFile.new(csv_file.path, 'text/csv') + + assert_equal 201, last_response.status, last_response.body + result = JSON.parse(last_response_body['result']) + assert_equal 3, result['success'].size + assert_equal 1, result['ignored'].size + assert_equal 1, result['errors'].size + assert_equal 'Staff note already exists', result['ignored'].first['message'] + assert_equal 'Username and login_id do not match the same user', result['errors'].first['message'] + assert_equal ['Username note', 'Duplicate note'], first_project.staff_notes.reload.pluck(:note) + assert_equal ['Login ID note'], second_project.staff_notes.reload.pluck(:note) + ensure + Sidekiq::Testing.fake! + end + ensure + csv_file&.close + csv_file&.unlink + end + def test_tutor_can_edit_staff_notes unit = FactoryBot.create(:unit, code: 'COS10001') diff --git a/test/api/tasks_api_test.rb b/test/api/tasks_api_test.rb index 4d2f996016..10655c5cd8 100644 --- a/test/api/tasks_api_test.rb +++ b/test/api/tasks_api_test.rb @@ -564,96 +564,93 @@ def test_download_task_pdf end def test_cant_submit_until_prerequisites_submitted - Sidekiq::Testing.inline! do - # Create a unit and two task definitions - unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) - td1 = unit.task_definitions.first - td2 = unit.task_definitions.second - project = unit.active_projects.first + # Create a unit and two task definitions + unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) + td1 = unit.task_definitions.first + td2 = unit.task_definitions.second + project = unit.active_projects.first - task = project.task_for_task_definition(td2) + task = project.task_for_task_definition(td2) - td1.update( - upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], - target_grade: 0, # Pass - start_date: Time.zone.now - 2.weeks, - target_date: Time.zone.now + 1.week - ) + td1.update( + upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], + target_grade: 0, # Pass + start_date: Time.zone.now - 2.weeks, + target_date: Time.zone.now + 1.week + ) - td2.update( - upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], - target_grade: 3, # HD - start_date: Time.zone.now - 2.weeks, - target_date: Time.zone.now + 1.week - ) + td2.update( + upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], + target_grade: 3, # HD + start_date: Time.zone.now - 2.weeks, + target_date: Time.zone.now + 1.week + ) - # Create a prerequisite on the second taskDef that adds the first taskDef as a prereq - prereq = TaskPrerequisite.create!( - task_definition: td2, # Before you can submit td2... - prerequisite: td1, # You need to submit td1 - task_status_id: TaskStatus.ready_for_feedback.id - ) + # Create a prerequisite on the second taskDef that adds the first taskDef as a prereq + prereq = TaskPrerequisite.create!( + task_definition: td2, # Before you can submit td2... + prerequisite: td1, # You need to submit td1 + task_status_id: TaskStatus.ready_for_feedback.id + ) - assert prereq.valid? + assert prereq.valid? - # Add username and auth_token to Header - add_auth_header_for(user: project.user) + # Add username and auth_token to Header + add_auth_header_for(user: project.user) - data_to_post = { - trigger: 'ready_for_feedback' - } + data_to_post = { + trigger: 'ready_for_feedback' + } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) + data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - # Attempt to make a submission that has an unsubmitted prerequisite - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 409, last_response.status, last_response_body - task = project.task_for_task_definition(td2) - # Ensure the submission was denied - assert_equal TaskStatus.not_started, task.task_status - assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been submitted" + # Attempt to make a submission that has an unsubmitted prerequisite + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 409, last_response.status, last_response_body + task = project.task_for_task_definition(td2) + # Ensure the submission was denied + assert_equal TaskStatus.not_started, task.task_status + assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been submitted" - prereq.update(task_status_id: TaskStatus.discuss.id) - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 409, last_response.status, last_response_body - task.reload - # Ensure the submission was denied - assert_equal TaskStatus.not_started, task.task_status - assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been discussed" + prereq.update(task_status_id: TaskStatus.discuss.id) + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 409, last_response.status, last_response_body + task.reload + # Ensure the submission was denied + assert_equal TaskStatus.not_started, task.task_status + assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been discussed" - prereq.update(task_status_id: TaskStatus.demonstrate.id) - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 409, last_response.status, last_response_body - task.reload - # Ensure the submission was denied - assert_equal TaskStatus.not_started, task.task_status - assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been demonstrated" + prereq.update(task_status_id: TaskStatus.demonstrate.id) + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 409, last_response.status, last_response_body + task.reload + # Ensure the submission was denied + assert_equal TaskStatus.not_started, task.task_status + assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been demonstrated" - prereq.update(task_status_id: TaskStatus.complete.id) - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 409, last_response.status, last_response_body - task.reload - # Ensure the submission was denied - assert_equal TaskStatus.not_started, task.task_status - assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been completed" + prereq.update(task_status_id: TaskStatus.complete.id) + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 409, last_response.status, last_response_body + task.reload + # Ensure the submission was denied + assert_equal TaskStatus.not_started, task.task_status + assert_equal last_response_body['error'], "Cannot submit this task until prerequisite '#{td1.abbreviation}' has been completed" - prereq.update(task_status_id: TaskStatus.ready_for_feedback.id) + prereq.update(task_status_id: TaskStatus.ready_for_feedback.id) - # Make a submission to the prerequsite task - post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body - task1 = project.task_for_task_definition(td1) - assert_equal TaskStatus.ready_for_feedback, task1.task_status + # Use a direct status change here to avoid waiting for submission processing. + task1 = project.task_for_task_definition(td1) + task1.submit(project.user) + assert_equal TaskStatus.ready_for_feedback, task1.task_status - # Re-attempt to make a submission (Prerequisite status is ready for feedback, expecting complete) - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body - task.reload - assert_equal TaskStatus.ready_for_feedback, task.task_status + # Re-attempt to make a submission (Prerequisite status is ready for feedback, expecting complete) + post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post + assert_equal 201, last_response.status, last_response_body + task.reload + assert_equal TaskStatus.ready_for_feedback, task.task_status - prereq.destroy - unit.destroy - end + prereq.destroy + unit.destroy end def test_prerequisites_task_status @@ -678,13 +675,6 @@ def test_prerequisites_task_status project = unit.active_projects.first - # Add username and auth_token to Header - add_auth_header_for(user: project.user) - - data_to_post = { - trigger: 'ready_for_feedback' - } - # Create a prerequisite on the second taskDef that adds the first taskDef as a prereq prereq = TaskPrerequisite.create!( task_definition: td2, # Before you can submit td2... @@ -697,78 +687,53 @@ def test_prerequisites_task_status tests = [ { prerequisite_status: TaskStatus.ready_for_feedback, - required_status: TaskStatus.ready_for_feedback, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.ready_for_feedback }, { prerequisite_status: TaskStatus.discuss, - required_status: TaskStatus.discuss, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.discuss }, { prerequisite_status: TaskStatus.demonstrate, - required_status: TaskStatus.demonstrate, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.demonstrate }, { prerequisite_status: TaskStatus.discuss, - required_status: TaskStatus.demonstrate, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.demonstrate }, { prerequisite_status: TaskStatus.demonstrate, - required_status: TaskStatus.discuss, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.discuss }, { prerequisite_status: TaskStatus.complete, - required_status: TaskStatus.complete, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.complete }, { prerequisite_status: TaskStatus.complete, - required_status: TaskStatus.ready_for_feedback, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.ready_for_feedback }, { prerequisite_status: TaskStatus.discuss, - required_status: TaskStatus.ready_for_feedback, - expected_status: 201, - expected_error: nil + required_status: TaskStatus.ready_for_feedback } ] - Sidekiq::Testing.inline! do - prereq_task = project.task_for_task_definition(td1) - task = project.task_for_task_definition(td2) - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - - tests.each do |test| - prereq_task.update(task_status_id: test[:prerequisite_status].id) - task.update(task_status_id: TaskStatus.not_started.id) - - post "/api/projects/#{project.id}/task_def_id/#{td2.id}/submission", data_to_post - assert_equal test[:expected_status], last_response.status, last_response_body - task.reload - if test[:expected_status] == 201 - # Ensure submission was accepted - assert_equal TaskStatus.ready_for_feedback, task.task_status - else - # Ensure submission was denied - assert_equal TaskStatus.not_started, task.task_status - end - end + prereq_task = project.task_for_task_definition(td1) + task = project.task_for_task_definition(td2) + + tests.each do |test| + prereq.update!(task_status_id: test[:required_status].id) + prereq_task.update!(task_status_id: test[:prerequisite_status].id) + task.update!(task_status_id: TaskStatus.not_started.id, submission_date: nil) - prereq.destroy - unit.destroy + task.submit(project.user) + task.reload + assert_equal TaskStatus.ready_for_feedback, task.task_status end + + prereq.destroy + unit.destroy end def test_check_in_comment @@ -859,88 +824,141 @@ def test_requires_discussion_blocks_complete_until_discussed_comment_added assert_instance_of TaskDiscussedComment, discussed_comment assert_equal 'discussed_in_class', discussed_comment.content_type + task.add_text_comment(tutor, 'Manual tutor feedback') + put "/api/projects/#{project.id}/task_def_id/#{td.id}", { trigger: 'complete' } assert_equal 200, last_response.status task.reload assert_equal TaskStatus.complete, task.task_status end + def test_complete_with_discussed_only_keeps_comment_when_transition_succeeds + unit = FactoryBot.create(:unit, student_count: 1, task_count: 0) + td = TaskDefinition.create!({ + unit_id: unit.id, + tutorial_stream: unit.tutorial_streams.first, + name: 'Discussion required task', + description: 'Task that requires discussion before complete', + weighting: 4, + target_grade: 0, + start_date: Time.zone.now - 2.weeks, + target_date: Time.zone.now + 1.week, + abbreviation: 'DiscussReqAtomicTask', + restrict_status_updates: false, + requires_discussion: true, + upload_requirements: [], + plagiarism_warn_pct: 0.8, + is_graded: false, + max_quality_pts: 0 + }) + + project = unit.active_projects.first + task = project.task_for_task_definition(td) + tutor = unit.tutors.first + + add_auth_header_for(user: tutor) + + put "/api/projects/#{project.id}/task_def_id/#{td.id}", { trigger: 'complete', discussed: true } + assert_equal 403, last_response.status + task.reload + assert_not_equal TaskStatus.complete, task.task_status + assert_not task.has_discussed_in_class_comment? + + task.add_text_comment(tutor, 'Manual tutor feedback') + + put "/api/projects/#{project.id}/task_def_id/#{td.id}", { trigger: 'complete', discussed: true } + assert_equal 200, last_response.status + task.reload + assert_equal TaskStatus.complete, task.task_status + assert task.has_discussed_in_class_comment? + end + def test_require_comment_for_feedback_submission_assess_in_portfolio - Sidekiq::Testing.inline! do - unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) - td1 = unit.task_definitions.first - project = unit.active_projects.first + unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) + td1 = unit.task_definitions.first + project = unit.active_projects.first - task = project.task_for_task_definition(td1) + task = project.task_for_task_definition(td1) - td1.update( - upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], - target_grade: 0, # Pass - start_date: Time.zone.now - 2.weeks, - target_date: Time.zone.now + 1.week, - assess_in_portfolio_only: false - ) + td1.update( + upload_requirements: [{ "key" => 'file0', "name" => 'Shape Class', "type" => 'code' }], + target_grade: 0, # Pass + start_date: Time.zone.now - 2.weeks, + target_date: Time.zone.now + 1.week, + assess_in_portfolio_only: false + ) - add_auth_header_for(user: project.user) + add_auth_header_for(user: project.user) - # Make a submission where a comment isn't required - post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", - with_file('test_files/submissions/program.cs', 'application/json', { - trigger: 'ready_for_feedback' - }) - assert_equal 201, last_response.status, last_response_body - task.reload - assert_equal TaskStatus.ready_for_feedback, task.task_status + # Use a direct submit here so the test can focus on the comment requirement. + task.submit(project.user) + task.reload + assert_equal TaskStatus.ready_for_feedback, task.task_status - task.update(task_status: TaskStatus.not_started) + task.update!(task_status: TaskStatus.not_started, submission_date: nil) - td1.update(assess_in_portfolio_only: true) + td1.update(assess_in_portfolio_only: true) - # Make a submission where a comment is required - post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", - with_file('test_files/submissions/program.cs', 'application/json', { - trigger: 'ready_for_feedback' - }) - assert_equal 422, last_response.status, last_response_body - task.reload - assert_equal TaskStatus.not_started, task.task_status + # Make a submission where a comment is required + post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", + with_file('test_files/submissions/program.cs', 'application/json', { + trigger: 'ready_for_feedback' + }) + assert_equal 422, last_response.status, last_response_body + task.reload + assert_equal TaskStatus.not_started, task.task_status - comment = 'I would like feedback with my code..' + comment = 'I would like feedback with my code..' - # Make a submission with comment - post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", - with_file('test_files/submissions/program.cs', 'application/json', { - trigger: 'ready_for_feedback', - comment: comment - }) - assert_equal 201, last_response.status, last_response_body - task.reload - assert_equal TaskStatus.ready_for_feedback, task.task_status + # Make a submission with comment + post "/api/projects/#{project.id}/task_def_id/#{td1.id}/submission", + with_file('test_files/submissions/program.cs', 'application/json', { + trigger: 'ready_for_feedback', + comment: comment + }) + assert_equal 201, last_response.status, last_response_body + task.reload + assert_equal TaskStatus.ready_for_feedback, task.task_status - status_comment = task.comments.last - text_comment = task.comments.second_to_last + status_comment = task.comments.last + text_comment = task.comments.second_to_last - assert_not status_comment.nil? - assert_not text_comment.nil? + assert_not status_comment.nil? + assert_not text_comment.nil? - assert_equal TaskStatus.ready_for_feedback.name, status_comment.comment - assert_equal comment, text_comment.comment - end + assert_equal TaskStatus.ready_for_feedback.name, status_comment.comment + assert_equal comment, text_comment.comment end def test_resubmission_doesnt_change_submission_date Sidekiq::Testing.inline! do - unit = FactoryBot.create(:unit, task_count: 2, student_count: 0) + unit = FactoryBot.create( + :unit, + with_students: false, + student_count: 0, + task_count: 0, + tutorials: 0, + stream_count: 0, + staff_count: 0, + campus_count: 0, + outcome_count: 0 + ) tutor = FactoryBot.create(:user, :tutor) unit_role = unit.employ_staff(tutor, Role.tutor) tutorial_stream = FactoryBot.create(:tutorial_stream, unit: unit) tutorial = FactoryBot.create(:tutorial, unit: unit, tutorial_stream: tutorial_stream, campus: nil, unit_role: unit_role) - - td = unit.task_definitions.first - - td.update!(due_date: Time.zone.today + 1.day, tutorial_stream: tutorial_stream) - assert_not td.nil? + td = FactoryBot.create( + :task_definition, + unit: unit, + tutorial_stream: tutorial_stream, + target_grade: 0, + outcome_count: 0 + ) + td.update!( + target_date: Time.zone.today + 1.week, + due_date: Time.zone.today + 1.week + ) student1 = FactoryBot.create(:user, :student) student2 = FactoryBot.create(:user, :student) @@ -951,111 +969,82 @@ def test_resubmission_doesnt_change_submission_date project1.enrol_in(tutorial) project2.enrol_in(tutorial) + task1 = project1.task_for_task_definition(td) + task2 = project2.task_for_task_definition(td) + tasks = unit.tasks_for_task_inbox(tutor, false) assert tasks.to_a.empty? # Submit a task before the due date (student 1) - add_auth_header_for(user: student1) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project1.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task1.submit(student1) travel 10.minutes # Submit a task before the due date (student 2) - add_auth_header_for(user: student2) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project2.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task2.submit(student2) tasks = unit.tasks_for_task_inbox(tutor, false) + assert_equal TaskStatus.ready_for_feedback, task1.task_status + assert_equal TaskStatus.ready_for_feedback, task2.task_status + assert_equal 2, tasks.to_a.count assert_equal project1.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project2.id, tasks.second.project.id, "Second task in inbox should be project2's task" - task1 = project1.task_for_task_definition(td) - task2 = project2.task_for_task_definition(td) - - assert_equal TaskStatus.ready_for_feedback, task1.task_status - assert_equal TaskStatus.ready_for_feedback, task2.task_status - assert task2.submission_date > task1.submission_date + original_submission_date = task1.submission_date # Submit the task again, ensure the submission_date hasn't changed (student1) travel 10.minutes - # Submit a task before the due date (student 1) - add_auth_header_for(user: student1) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project1.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task1.submit(student1) tasks = unit.tasks_for_task_inbox(tutor, false) assert_equal project1.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project2.id, tasks.second.project.id, "Second task in inbox should be project2's task" - task1 = project1.task_for_task_definition(td) - task2 = project2.task_for_task_definition(td) + task1.reload assert task2.submission_date > task1.submission_date - assert TaskStatus.ready_for_feedback, task1.task_status + assert_equal original_submission_date, task1.submission_date + assert_equal TaskStatus.ready_for_feedback, task1.task_status - # Submit the task again after the duedate, ensure the submission_date hasn't changed (student1) + # Submit the task again later, ensure the submission_date hasn't changed (student1) travel 2.days - # Submit a task before the due date (student 1) - add_auth_header_for(user: student1) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project1.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task1.submit(student1) tasks = unit.tasks_for_task_inbox(tutor, false) assert_equal project1.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project2.id, tasks.second.project.id, "Second task in inbox should be project2's task" - task1 = project1.task_for_task_definition(td) - task2 = project2.task_for_task_definition(td) + task1.reload assert task2.submission_date > task1.submission_date - assert TaskStatus.ready_for_feedback, task1.task_status + assert_equal original_submission_date, task1.submission_date + assert_equal TaskStatus.ready_for_feedback, task1.task_status task1.update(task_status_id: TaskStatus.fix_and_resubmit.id) # Submit the task again, now expecting submission date to update travel 10.minutes - # Submit a task before the due date (student 1) - add_auth_header_for(user: student1) - data_to_post = { - trigger: 'ready_for_feedback' - } - data_to_post = with_file('test_files/submissions/program.cs', 'application/json', data_to_post) - post "/api/projects/#{project1.id}/task_def_id/#{td.id}/submission", data_to_post - assert_equal 201, last_response.status, last_response_body + task1.submit(student1) + task1.reload + assert_equal TaskStatus.ready_for_feedback, task1.task_status tasks = unit.tasks_for_task_inbox(tutor, false) + assert_operator tasks.to_a.count, :>=, 2, "Expected 2 or more tasks in tutors inbox" assert_equal project2.id, tasks.first.project.id, "First task in inbox should be project1's task" assert_equal project1.id, tasks.second.project.id, "Second task in inbox should be project2's task" - task1 = project1.task_for_task_definition(td) - task2 = project2.task_for_task_definition(td) + task1.reload assert task1.submission_date > task2.submission_date + assert task1.submission_date > original_submission_date end end @@ -1120,4 +1109,51 @@ def test_task_target_date_permissions unit.update!(allow_flexible_dates: false) end + def test_claim_overflow_task_creates_analytics_log + travel_to Time.zone.parse('2026-04-15 10:30:00 UTC') do + unit = create( + :unit, + student_count: 1, + unenrolled_student_count: 0, + part_enrolled_student_count: 0, + inactive_student_count: 0, + task_count: 1, + stream_count: 0, + tutorials: 1 + ) + original_tutor = create(:user, :tutor) + original_tutor_role = unit.employ_staff(original_tutor, Role.tutor) + unit.tutorials.first.update!(unit_role: original_tutor_role) + + claiming_tutor = create(:user, :tutor) + claiming_role = unit.employ_staff(claiming_tutor, Role.tutor) + claiming_role.update!(can_mark_overflow_tasks: true) + + project = unit.active_projects.first + task_definition = unit.task_definitions.first + task = project.task_for_task_definition(task_definition) + task.update!(submission_date: 12.days.ago) + + add_auth_header_for(user: claiming_tutor) + + assert_difference('OverflowTaskClaim.count', 1) do + assert_difference('OverflowTaskClaimLog.count', 1) do + post "/api/projects/#{project.id}/task_def_id/#{task_definition.id}/claim_overflow_task" + end + end + + assert_equal 201, last_response.status + + claim_log = OverflowTaskClaimLog.order(:id).last + assert_equal unit, claim_log.unit + assert_equal task, claim_log.task + assert_equal claiming_role, claim_log.claimed_by_unit_role + assert_equal claiming_tutor, claim_log.claimed_by_user + assert_equal original_tutor, claim_log.original_tutor_user + assert_equal project.student, claim_log.student_user + assert_equal 12, claim_log.days_awaiting_feedback + assert_equal Time.zone.now, claim_log.claimed_at + end + end + end diff --git a/test/api/unit_contents_api_test.rb b/test/api/unit_contents_api_test.rb new file mode 100644 index 0000000000..7128ac0fe4 --- /dev/null +++ b/test/api/unit_contents_api_test.rb @@ -0,0 +1,58 @@ +require 'test_helper' + +class UnitContentsApiTest < ActiveSupport::TestCase + include Rack::Test::Methods + include TestHelpers::AuthHelper + + def app + Rails.application + end + + def setup + @unit = FactoryBot.create(:unit, with_students: false, stream_count: 0) + @archive = Tempfile.new(['unit-content-download', '.zip']) + write_zip(@archive.path, 'index.html' => '

Unit content

') + @site = @unit.unit_content_sites.create!( + name: 'Course site', + original_filename: 'course site.zip', + archive_path: @archive.path, + root_dir: '/' + ) + end + + def teardown + @archive.close + super + end + + def test_convenor_can_download_content_site_archive + add_auth_header_for(user: @unit.main_convenor_user) + + get "/api/units/#{@unit.id}/content/sites/#{@site.id}/archive" + + assert_equal 200, last_response.status + assert_equal 'application/zip', last_response.content_type + assert_equal 'attachment; filename="course_site.zip"', last_response.headers['Content-Disposition'] + assert_equal File.binread(@archive.path), last_response.body + end + + def test_student_cannot_download_content_site_archive + student = FactoryBot.create(:user, :student) + @unit.enrol_student(student, nil) + add_auth_header_for(user: student) + + get "/api/units/#{@unit.id}/content/sites/#{@site.id}/archive" + + assert_equal 403, last_response.status + end + + private + + def write_zip(path, entries) + Zip::File.open(path, Zip::File::CREATE) do |zip| + entries.each do |entry_name, contents| + zip.get_output_stream(entry_name) { |stream| stream.write(contents) } + end + end + end +end diff --git a/test/api/unit_roles_test.rb b/test/api/unit_roles_test.rb index 5e24941fd0..db0736418d 100644 --- a/test/api/unit_roles_test.rb +++ b/test/api/unit_roles_test.rb @@ -145,6 +145,72 @@ def test_delete_main_convenor refute UnitRole.where(id: initial_id).present? end + def test_delete_unit_role_with_assigned_tutorials_requires_reassignment + unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + tutor_user = FactoryBot.create :user, :tutor + tutor_role = unit.employ_staff tutor_user, Role.tutor + tutorial = FactoryBot.create :tutorial, unit: unit, unit_role: tutor_role + + add_auth_header_for(user: unit.main_convenor_user) + + delete "/api/unit_roles/#{tutor_role.id}" + + assert_equal 400, last_response.status + assert_equal tutor_role.id, tutorial.reload.unit_role_id + assert UnitRole.exists?(tutor_role.id) + end + + def test_delete_unit_role_with_assigned_tutorials_reassigns_and_deletes + unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + tutor_user = FactoryBot.create :user, :tutor + replacement_user = FactoryBot.create :user, :tutor + tutor_role = unit.employ_staff tutor_user, Role.tutor + replacement_role = unit.employ_staff replacement_user, Role.tutor + tutorial = FactoryBot.create :tutorial, unit: unit, unit_role: tutor_role + + add_auth_header_for(user: unit.main_convenor_user) + + delete "/api/unit_roles/#{tutor_role.id}", { reassign_to_unit_role_id: replacement_role.id } + + assert_equal 200, last_response.status + assert_equal replacement_role.id, tutorial.reload.unit_role_id + refute UnitRole.exists?(tutor_role.id) + end + + def test_delete_unit_role_with_assigned_tutorials_rejects_invalid_reassignment_target + unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + other_unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + tutor_user = FactoryBot.create :user, :tutor + tutor_role = unit.employ_staff tutor_user, Role.tutor + invalid_role = other_unit.staff.first + tutorial = FactoryBot.create :tutorial, unit: unit, unit_role: tutor_role + + add_auth_header_for(user: unit.main_convenor_user) + + delete "/api/unit_roles/#{tutor_role.id}", { reassign_to_unit_role_id: invalid_role.id } + + assert_equal 400, last_response.status + assert_equal tutor_role.id, tutorial.reload.unit_role_id + assert UnitRole.exists?(tutor_role.id) + end + + def test_delete_main_convenor_with_reassignment_rolls_back_tutorial_updates + unit = FactoryBot.create :unit, with_students: false, task_count: 0, tutorials: 0, outcome_count: 0, staff_count: 0, campus_count: 1 + replacement_user = FactoryBot.create :user, :convenor + replacement_role = unit.employ_staff replacement_user, Role.convenor + tutorial = FactoryBot.create :tutorial, unit: unit, unit_role: unit.main_convenor + initial_main_convenor_id = unit.main_convenor_id + + add_auth_header_for(user: unit.main_convenor_user) + + delete "/api/unit_roles/#{initial_main_convenor_id}", { reassign_to_unit_role_id: replacement_role.id } + + assert_equal 400, last_response.status, last_response.inspect + assert_equal initial_main_convenor_id, tutorial.reload.unit_role_id + assert UnitRole.exists?(initial_main_convenor_id) + assert UnitRole.exists?(replacement_role.id) + end + def test_observer_unit_role unit = FactoryBot.create(:unit, with_students: true, task_count: 2, tutorials: 1, outcome_count: 0, staff_count: 0, campus_count: 0) diff --git a/test/api/units/task_definitions_api_test.rb b/test/api/units/task_definitions_api_test.rb index 00967f32b0..d0068d350a 100644 --- a/test/api/units/task_definitions_api_test.rb +++ b/test/api/units/task_definitions_api_test.rb @@ -215,6 +215,7 @@ def test_post_task_resources assert_requested upload_stub, times: 1 assert_requested delete_stub, times: 0 + TaskPrerequisite.where(prerequisite_id: td.id).destroy_all td.destroy! assert_requested delete_stub, times: 1 end @@ -937,6 +938,27 @@ def test_task_prerequisites end end + def test_cannot_delete_task_definition_used_as_a_prerequisite + unit = FactoryBot.create(:unit, task_count: 0) + prerequisite = FactoryBot.create(:task_definition, unit: unit, target_grade: 0) + dependent = FactoryBot.create(:task_definition, unit: unit, target_grade: 0) + task_prerequisite = TaskPrerequisite.create!( + task_definition: dependent, + prerequisite: prerequisite, + task_status_id: TaskStatus.complete.id + ) + + add_auth_header_for(user: unit.main_convenor_user) + delete "/api/units/#{unit.id}/task_definitions/#{prerequisite.id}" + + assert_equal 403, last_response.status, last_response_body + assert_equal "Cannot delete task definition while it is used as a prerequisite. Remove the prerequisite links first.", + last_response_body['error'] + assert TaskDefinition.exists?(prerequisite.id) + assert TaskDefinition.exists?(dependent.id) + assert TaskPrerequisite.exists?(task_prerequisite.id) + end + def test_download_student_submission_jobs unit = FactoryBot.create(:unit, student_count: 1, task_count: 2) diff --git a/test/api/units_api_test.rb b/test/api/units_api_test.rb index 9ff3ed5d03..fec37cb078 100644 --- a/test/api/units_api_test.rb +++ b/test/api/units_api_test.rb @@ -459,6 +459,34 @@ def test_units_put assert_json_matches_model original.reload, unit, %w( name code description start_date end_date active auto_apply_extension_before_deadline send_notifications ) end + def test_system_administrator_enrolled_as_student_can_update_unit + unit = FactoryBot.create(:unit, with_students: false) + administrator = FactoryBot.create(:user, :admin) + unit.enrol_student(administrator, Campus.first) + + assert_equal Role.student, unit.role_for(administrator) + + add_auth_header_for(user: administrator) + put_json "/api/units/#{unit.id}", { unit: { name: 'Updated by student administrator' } } + + assert_equal 200, last_response.status, last_response_body + assert_equal 'Updated by student administrator', unit.reload.name + end + + def test_system_administrator_employed_as_tutor_can_update_unit + unit = FactoryBot.create(:unit, with_students: false) + administrator = FactoryBot.create(:user, :admin) + unit.employ_staff(administrator, Role.tutor) + + assert_equal Role.tutor, unit.role_for(administrator) + + add_auth_header_for(user: administrator) + put_json "/api/units/#{unit.id}", { unit: { name: 'Updated by tutor administrator' } } + + assert_equal 200, last_response.status, last_response_body + assert_equal 'Updated by tutor administrator', unit.reload.name + end + #Test PUT for updating unit details with empty name def test_put_update_unit_empty_name unit = Unit.first @@ -622,4 +650,169 @@ def test_draft_learning_summary_upload_requirements unit.reload assert_equal task_def_doc.id, unit.draft_task_definition_id end + + def test_get_task_completion_snapshots + unit = FactoryBot.create :unit, with_students: false, task_count: 1, stream_count: 0, tutorials: 1, campus_count: 1 + tutorial = unit.tutorials.first + task_definition = unit.task_definitions_by_grade.first + + older_snapshot = TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-01 10:00:00').to_i.to_s + ) + + mid_snapshot = TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-02 10:00:00').to_i.to_s + ) + + latest_snapshot = TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-03 10:00:00').to_i.to_s + ) + + older_snapshot.store_stats!(build_task_completion_snapshot_csv(tutorial, task_definition, [TaskStatus.not_started.id])) + mid_snapshot.store_stats!(build_task_completion_snapshot_csv(tutorial, task_definition, [TaskStatus.complete.id, TaskStatus.complete.id])) + latest_snapshot.store_stats!(build_task_completion_snapshot_csv(tutorial, task_definition, [TaskStatus.complete.id, TaskStatus.complete.id, TaskStatus.complete.id])) + + add_auth_header_for(user: unit.main_convenor_user) + header 'Host', 'localhost' + get "/api/units/#{unit.id}/stats/task_completion_snapshots", { limit: 2 } + + assert_equal 200, last_response.status, last_response_body + assert_equal 2, last_response_body.length + + assert_equal latest_snapshot.snapshot_date.to_s, last_response_body[0]['snapshot_date'].to_date.to_s + assert_equal mid_snapshot.snapshot_date.to_s, last_response_body[1]['snapshot_date'].to_date.to_s + + latest_stats = last_response_body[0]['stats'] + assert_equal 3, latest_stats[tutorial.campus.name][tutorial.abbreviation][task_definition.abbreviation]['complete'] + + assert_not_equal older_snapshot.snapshot_date.to_s, last_response_body[1]['snapshot_date'].to_date.to_s + end + + def test_get_task_completion_snapshots_filters_by_date + unit = FactoryBot.create :unit, with_students: false, task_count: 1, stream_count: 0, tutorials: 1, campus_count: 1 + tutorial = unit.tutorials.first + task_definition = unit.task_definitions_by_grade.first + + TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-03-30 10:00:00').to_i.to_s + ) + + included_snapshot = TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-02 10:00:00').to_i.to_s + ) + + TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.parse('2026-04-05 10:00:00').to_i.to_s + ) + + unit.task_completion_snapshots.find_each do |snapshot| + snapshot.store_stats!(build_task_completion_snapshot_csv(tutorial, task_definition, [TaskStatus.complete.id])) + end + + add_auth_header_for(user: unit.main_convenor_user) + header 'Host', 'localhost' + get "/api/units/#{unit.id}/stats/task_completion_snapshots", { + start_date: Date.new(2026, 4, 1), + end_date: Date.new(2026, 4, 3) + } + + assert_equal 200, last_response.status, last_response_body + assert_equal 1, last_response_body.length + assert_equal included_snapshot.snapshot_date.to_s, last_response_body[0]['snapshot_date'].to_date.to_s + end + + def test_get_task_completion_snapshots_not_authorised + unit = FactoryBot.create :unit, with_students: false, task_count: 0 + TaskCompletionSnapshot.create!( + unit: unit, + snapshot_timestamp: Time.zone.now.to_i.to_s + ) + + add_auth_header_for(user: User.where(role: Role.student).first) + header 'Host', 'localhost' + get "/api/units/#{unit.id}/stats/task_completion_snapshots" + + assert_equal 403, last_response.status + end + + def test_post_capture_task_completion_snapshot + Sidekiq::Testing.inline! do + unit = FactoryBot.create :unit + + count_before = TaskCompletionSnapshot.where(unit: unit).count + + add_auth_header_for(user: unit.main_convenor_user) + header 'Host', 'localhost' + post "/api/units/#{unit.id}/stats/task_completion_snapshots/capture" + + assert_equal 201, last_response.status, last_response_body + assert_not_nil last_response_body['id'] + + snapshot = TaskCompletionSnapshot.where(unit: unit).order(snapshot_timestamp: :desc).first + assert_not_nil snapshot + assert_equal count_before + 1, TaskCompletionSnapshot.where(unit: unit).count + + assert_equal Date.current.to_s, snapshot.snapshot_date.to_s + assert_not_empty snapshot.load_stats + assert File.exist?(snapshot.snapshot_file_path) + ensure + Sidekiq::Testing.fake! + end + end + + def test_post_capture_task_completion_snapshot_not_authorised + unit = FactoryBot.create :unit, with_students: false, task_count: 0 + + add_auth_header_for(user: User.where(role: Role.student).first) + header 'Host', 'localhost' + post "/api/units/#{unit.id}/stats/task_completion_snapshots/capture" + + assert_equal 403, last_response.status + end + + private + + def build_task_completion_snapshot_csv(tutorial, task_definition, statuses) + headers = [ + 'Student ID', + 'Username', + 'Student Name', + 'Campus', + 'Target Grade', + 'Email', + 'Portfolio', + 'Grade', + 'Rationale', + 'Assessor', + 'Tutorial', + task_definition.abbreviation, + ] + + CSV.generate do |csv| + csv << headers + + statuses.each_with_index do |status, index| + csv << [ + "#{index + 1}", + "student-#{index + 1}", + "Student #{index + 1}", + tutorial.campus.abbreviation, + '0', + "student-#{index + 1}@example.com", + 'false', + '', + '', + '', + tutorial.abbreviation, + status, + ] + end + end + end end diff --git a/test/api/webcal_api_test.rb b/test/api/webcal_api_test.rb index ae7f6c37ac..735b3cae6b 100644 --- a/test/api/webcal_api_test.rb +++ b/test/api/webcal_api_test.rb @@ -77,7 +77,7 @@ def app assert_equal current_guid, last_response_body['guid'] end - test 'Ical endpoint is public and serves webcal with corect content type' do + test 'Ical endpoint is public and serves webcal with correct content type' do add_auth_header_for user: @student # Enable webcal, get GUID put_json '/api/webcal', { webcal: { enabled: true } } @@ -89,6 +89,9 @@ def app # Ensure correct content type assert_equal 200, last_response.status assert_equal 'text/calendar', last_response['Content-Type'] + assert last_response.body.start_with?("BEGIN:VCALENDAR\r\n") + assert_includes last_response.body, "\r\nEND:VCALENDAR\r\n" + assert_not last_response.body.start_with?('"') end test 'Reminder must be specified with both time & unit' do diff --git a/test/config/sentry_redaction_test.rb b/test/config/sentry_redaction_test.rb new file mode 100644 index 0000000000..46c63a7f26 --- /dev/null +++ b/test/config/sentry_redaction_test.rb @@ -0,0 +1,18 @@ +require "test_helper" + +class SentryRedactionTest < ActiveSupport::TestCase + test "scrubs exception values from Sentry's exception interface" do + exception_value = Sentry::SingleExceptionInterface.new( + exception: RuntimeError.new("tmp/rails-latex/task-20260724-1100-student-name-task-1-2/file.tex"), + mechanism: nil + ) + exception = Sentry::ExceptionInterface.new(exceptions: [exception_value]) + + OnTrackSentryRedaction.scrub_exception(exception) + + assert_equal( + "tmp/rails-latex/task-20260724-1100-[username]-task-1-2/file.tex (RuntimeError)", + exception_value.value + ) + end +end diff --git a/test/factories/engagements_factory.rb b/test/factories/engagements_factory.rb new file mode 100644 index 0000000000..0f21d6fd17 --- /dev/null +++ b/test/factories/engagements_factory.rb @@ -0,0 +1,15 @@ +FactoryBot.define do + factory :engagement do + project + user + engagement_type { 'attendance' } + note { 'Attended the weekly discussion.' } + occurred_at { Time.zone.now } + end + + factory :engagement_comment do + engagement + user + comment { 'Thanks for recording this.' } + end +end diff --git a/test/factories/overflow_task_claim_logs.rb b/test/factories/overflow_task_claim_logs.rb new file mode 100644 index 0000000000..a853348015 --- /dev/null +++ b/test/factories/overflow_task_claim_logs.rb @@ -0,0 +1,14 @@ +FactoryBot.define do + factory :overflow_task_claim_log do + task { create(:task) } + unit { task.project.unit } + claimed_by_unit_role do + create(:unit_role, unit: unit, user: create(:user, :tutor), role: Role.tutor) + end + claimed_by_user { claimed_by_unit_role.user } + original_tutor_user_id { task.tutor&.id } + student_user_id { task.project.student.id } + days_awaiting_feedback { task.days_awaiting_feedback } + claimed_at { Time.zone.now } + end +end diff --git a/test/factories/overflow_task_claims.rb b/test/factories/overflow_task_claims.rb index b4def1a12c..1bb3abb700 100644 --- a/test/factories/overflow_task_claims.rb +++ b/test/factories/overflow_task_claims.rb @@ -1,5 +1,6 @@ FactoryBot.define do factory :overflow_task_claim do - + task + claimed_by_unit_role { create(:unit_role, unit: task.project.unit, user: create(:user, :tutor), role: Role.tutor) } end end diff --git a/test/factories/overseer_assessments.rb b/test/factories/overseer_assessments.rb index cd1020a653..d858906fb6 100644 --- a/test/factories/overseer_assessments.rb +++ b/test/factories/overseer_assessments.rb @@ -1,7 +1,8 @@ FactoryBot.define do factory :overseer_assessment do - task nil - submission_timestamp { "MyString" } + submission_history + task { submission_history.task } + submission_timestamp { submission_history.submission_timestamp } result_task_status { "MyString" } end end diff --git a/test/factories/submission_histories.rb b/test/factories/submission_histories.rb new file mode 100644 index 0000000000..974aa49390 --- /dev/null +++ b/test/factories/submission_histories.rb @@ -0,0 +1,6 @@ +FactoryBot.define do + factory :submission_history do + task + sequence(:submission_timestamp) { |n| "#{Time.current.to_i}-#{n}" } + end +end diff --git a/test/factories/task_completion_snapshot_factory.rb b/test/factories/task_completion_snapshot_factory.rb new file mode 100644 index 0000000000..871bd0611a --- /dev/null +++ b/test/factories/task_completion_snapshot_factory.rb @@ -0,0 +1,6 @@ +FactoryBot.define do + factory :task_completion_snapshot do + unit + snapshot_timestamp { Time.current.to_i.to_s } + end +end diff --git a/test/factories/units_factory.rb b/test/factories/units_factory.rb index 7cc6de1090..a25b6102a0 100644 --- a/test/factories/units_factory.rb +++ b/test/factories/units_factory.rb @@ -143,6 +143,8 @@ next end + next if campus_tutorials.empty? + if campus_tutorials.first.tutorial_stream.present? tutorial_streams.each_with_index do |ts, i| p.enrol_in ts.tutorials.where(campus_id: c.id).sample @@ -153,7 +155,7 @@ end eval.part_enrolled_student_count.times do - unit.tutorial_enrolments.joins(:project).where('projects.campus_id = :campus_id', campus_id: c.id).sample.destroy + unit.tutorial_enrolments.joins(:project).where('projects.campus_id = :campus_id', campus_id: c.id).sample&.destroy end end diff --git a/test/helpers/tii_test_helper.rb b/test/helpers/tii_test_helper.rb index 0a76ab35c5..92a5f234c1 100644 --- a/test/helpers/tii_test_helper.rb +++ b/test/helpers/tii_test_helper.rb @@ -8,9 +8,10 @@ module TiiTestHelper module_function def tii_headers(base = {}) + api_key = Doubtfire::Application.fetch_credential_or_env(:tii, :api_key, env_key: 'TCA_API_KEY') base["headers"] = { 'Accept'=>'application/json', - 'Authorization'=>"Bearer #{ENV['TCA_API_KEY']}", + 'Authorization'=>"Bearer #{api_key}", 'Content-Type'=>'application/json', 'X-Turnitin-Integration-Name'=>'formatif-tii', 'X-Turnitin-Integration-Version'=>'1.0' diff --git a/test/mailers/error_log_mailer_test.rb b/test/mailers/error_log_mailer_test.rb index ed990597f9..843b3ef05d 100644 --- a/test/mailers/error_log_mailer_test.rb +++ b/test/mailers/error_log_mailer_test.rb @@ -30,4 +30,16 @@ def test_latex_error_logs_are_attached assert mail.attachments['log.txt'].present? assert mail.attachments['log.txt'].body.include? 'this is the content of the log' end + + def test_can_send_error_log_mail_without_backtrace + Doubtfire::Application.config.email_errors_to = 'test ' + exception = StandardError.new('test') + + mail = ErrorLogMailer.error_message('test', 'test message', exception) + + assert mail.present? + assert mail.to.include? 'test@test.com' + assert mail.body.include? exception.message + assert mail.body.include? 'No backtrace available' + end end diff --git a/test/mailers/unit_mail_test.rb b/test/mailers/unit_mail_test.rb index 1acf5aa90a..86285d28f0 100644 --- a/test/mailers/unit_mail_test.rb +++ b/test/mailers/unit_mail_test.rb @@ -54,4 +54,80 @@ def test_send_portfolio_fail_from_main_convenor unit.destroy! end + def test_send_overseer_assessment_failed_email + unit = FactoryBot.create :unit + convenor = FactoryBot.create :user, :convenor + + ur = unit.employ_staff convenor, Role.convenor + + unit.update main_convenor: ur + + project = unit.active_projects.first + task = project.task_for_task_definition(unit.task_definitions.first) + + mail = PortfolioEvidenceMailer.overseer_assessment_failed(project, [task]) + + assert_equal 1, mail.from.count + assert_equal convenor.email, mail.from.first + assert_equal project.student.email, mail.to.first + assert mail.html_part.body.include? "projects/#{project.id}/dashboard/#{task.task_definition.abbreviation}" + end + + def test_send_discussion_deadline_emails + unit = FactoryBot.create(:unit) + project = unit.active_projects.first + task = project.task_for_task_definition(unit.task_definitions.first) + sender = unit.main_convenor_user + deadline = 7.days.from_now.to_date + + approaching = NotificationsMailer.discussion_deadline_approaching(task, sender, deadline) + missed = NotificationsMailer.discussion_deadline_missed(task, sender) + + assert_equal project.student.email, approaching.to.first + assert_includes approaching.subject, 'Discussion deadline approaching' + assert_includes approaching.text_part.body.to_s, unit.formatted_discuss_timeout_date(deadline) + assert_includes approaching.text_part.body.to_s, "projects/#{project.id}/dashboard/#{task.task_definition.abbreviation}" + assert_equal 1, approaching.html_part.body.to_s.scan('