diff --git a/.ci_cd/README.md b/.ci_cd/README.md index 106b78fe80..adc30fee2d 100644 --- a/.ci_cd/README.md +++ b/.ci_cd/README.md @@ -1,7 +1,7 @@ # CI/CD workflow files -The files in this directory are used by the [CI/CD workflow](../.github/workflows/ci_cd.yml) which provides a standard -github workflow for CI/CD tasks in this repo and custom projects/forks on a linux host via SSH. +The files in this directory are used by the [CI/CD workflows](../.github/workflows) of this repo and custom +projects/forks, which deploy to a linux host via SSH (see [deployment.yml](../.github/workflows/deployment.yml)). # `ci_cd.json` File @@ -115,7 +115,6 @@ Standard variables: - `OR_HOSTNAME` - FQDN for the host (default: env/secrets.OR_HOSTNAME) - `SSH_USER` - SSH username (default: root) - `SSH_KEY` - SSH private key (this is written to `ssh.key` file and loaded by `deploy.sh` for SSH/SCP commands) -- `SSH_PASSWORD` - SSH password (alternative to private key authentication) - `SSH_PORT` - SSH port (default: 22) - `AWS_KEY` - AWS access key ID (should be prefixed with `_TEMP_` - `AWS_SECRET` - AWS access key secret diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 036427a5c2..0990cf4d04 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -15,16 +15,14 @@ # 'deployment/build.gradle' must exist with an installDist task that prepares the deployment image in the # 'deployment/build' directory; if this condition is not met the deployment will fail. # -# Secrets, inputs and environment variables are combined for consumption by the '.ci_cd/deploy.sh' bash script, they are -# combined in the following priority: +# Secrets, inputs and environment variables are combined for consumption by the '.ci_cd/deploy.sh' bash script, later +# entries override earlier ones: # +# - secrets and inputs (inputs from manual trigger using workflow_dispatch override secrets with the same name) # - '.ci_cd/env/.env' # - '.ci_cd/env/{ENVIRONMENT}.env' -# - secrets -# - inputs (from manual trigger using workflow_dispatch) # -# The above variables are output to 'temp/env' file for each requested deployment. If a secret called 'SSH_PASSWORD' is -# found that is output to 'ssh.env' file (so it is not copied to the host), if a secret called 'SSH_KEY' is +# The above variables are output to 'temp/env' file for each requested deployment. If a secret called 'SSH_KEY' is # found that is output to 'ssh.key' file (so it is not copied to the host). Sensitive credentials should always be stored # in github secrets so they are encrypted, do not store within repo files (even private repos as these are not encrypted). # @@ -33,7 +31,8 @@ # MANUAL TRIGGER # # This workflow can be triggered by push, release, manually or schedule (from any repo other than openremote/core) -# if triggered manually then the following inputs are available and these override any set in env files and/or secrets: +# if triggered manually then the following inputs are available, they override secrets with the same name but not +# values from the '.ci_cd/env' files: # # Inputs: # @@ -47,7 +46,6 @@ # - COMMIT - Which branch/SHA should be checked out for the deployment (defaults to trigger commit) # - OR_HOSTNAME - FQDN of host to deploy to (e.g. demo.openremote.app) # - SSH_USER - Set/override the SSH user to use for SSH/SCP commands -# - SSH_PASSWORD - Set/override the SSH password to use for SSH/SCP commands (SSH key should be preferred) # - SSH_PORT - Set/override the SSH port to use for SSH/SCP commands # - OR_ADMIN_PASSWORD - The admin password to set for clean installs # --------------------------------------------------------------------------------------------------------------------- diff --git a/.github/workflows/custom-deploy.yml b/.github/workflows/custom-deploy.yml index ee73b59667..5d515de555 100644 --- a/.github/workflows/custom-deploy.yml +++ b/.github/workflows/custom-deploy.yml @@ -1,4 +1,26 @@ -# Manual deployments for custom projects, must be called from a workflow_dispatch workflow +# --------------------------------------------------------------------------------------------------------------------- +# This environment workflow runs for manual deployments of custom projects +# (https://github.com/openremote/custom-project), it handles: +# +# - Building and deploying the selected commit (see deployment.yml for how deployments work) +# +# MANUAL TRIGGER +# +# The calling workflow declares the following inputs on its manual trigger and passes them as INPUTS, they override +# secrets with the same name but not values from the '.ci_cd/env' files: +# +# - ENVIRONMENT - Which environment to deploy, leave empty to only use '.ci_cd/env/.env' and secrets without +# an environment prefix +# - CLEAN_INSTALL - Should the .ci_cd/host_init/clean.sh script be run during deployment (warning this will delete +# assets, rules, etc.) +# - COMMIT - Which branch/SHA should be checked out for the deployment (defaults to trigger commit) +# - OR_HOSTNAME - FQDN of host to deploy to (e.g. demo.openremote.app) +# - OR_ADMIN_PASSWORD - The admin password to set for clean installs +# +# The calling workflow must be on the default branch before it can be triggered from the GitHub UI, and must not use +# the same concurrency group as this workflow, GitHub cancels the run as a deadlock. +# --------------------------------------------------------------------------------------------------------------------- + name: Custom deploy on: diff --git a/.github/workflows/custom-review.yml b/.github/workflows/custom-review.yml index f5754c3289..425c631302 100644 --- a/.github/workflows/custom-review.yml +++ b/.github/workflows/custom-review.yml @@ -1,3 +1,16 @@ +# --------------------------------------------------------------------------------------------------------------------- +# This environment workflow runs for pull requests of custom projects +# (https://github.com/openremote/custom-project), it handles: +# +# - Linting +# - Building the project including the deployment, nothing is published +# - Running unit and integration tests +# +# Jobs are skipped based on which files changed (see the detect-changes job). +# +# The calling workflow must not use the same concurrency group as this workflow, GitHub cancels the run as a deadlock. +# --------------------------------------------------------------------------------------------------------------------- + name: Custom review on: diff --git a/.github/workflows/custom-staging.yml b/.github/workflows/custom-staging.yml index f733108f9a..f415dfb49e 100644 --- a/.github/workflows/custom-staging.yml +++ b/.github/workflows/custom-staging.yml @@ -1,3 +1,17 @@ +# --------------------------------------------------------------------------------------------------------------------- +# This environment workflow runs for pushed commits of custom projects +# (https://github.com/openremote/custom-project), it handles: +# +# - Linting, building and testing, as in custom-review.yml +# - Deploying when '.ci_cd/ci_cd.json' has a 'deploy' entry for the pushed branch (see deployment.yml) +# +# Without such an entry the deployment job is skipped, use custom-deploy.yml for manual deployments. An entry without +# an environment, e.g. { "push": { "main": { "deploy": {} } } }, only uses '.ci_cd/env/.env' and secrets without an +# environment prefix. +# +# The calling workflow must not use the same concurrency group as this workflow, GitHub cancels the run as a deadlock. +# --------------------------------------------------------------------------------------------------------------------- + name: Custom staging on: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9600ae61dc..5a51a78195 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,7 +1,28 @@ -# This workflow only exists to allow manual deployments e.g. to test servers. +# --------------------------------------------------------------------------------------------------------------------- +# This environment workflow runs for manual deployments of this repo, it handles: # -# It leverages existing reusable workflows similar to the staging.yml environment workflow but unlike the staging.yml workflow -# it only includes the required jobs for deployments: install-dist, build-manager-image and deploy. +# - Building and deploying the selected commit (see deployment.yml for how deployments work) +# +# MANUAL TRIGGER +# +# The following inputs are available, they override secrets with the same name but not values from the '.ci_cd/env' +# files: +# +# - ENVIRONMENT - Which environment to deploy (equivalent to deploy/environment in '.ci_cd/ci_cd.json') +# - MANAGER_TAG - Which manager docker tag to deploy (equivalent to deploy/managerTag in '.ci_cd/ci_cd.json'), +# leave empty to build a manager image from the checked out commit +# - CLEAN_INSTALL - Should the .ci_cd/host_init/clean.sh script be run during deployment (warning this will delete +# assets, rules, etc.) +# - COMMIT - Which branch/SHA should be checked out for the deployment (defaults to trigger commit) +# - OR_HOSTNAME - FQDN of host to deploy to (e.g. demo.openremote.app) +# - OR_ADMIN_PASSWORD - The admin password to set for clean installs +# +# When triggered on a tag the 'workflow_dispatch' deploy entry in '.ci_cd/ci_cd.json' is used instead of ENVIRONMENT +# and MANAGER_TAG. +# +# Custom projects use custom-deploy.yml instead. +# --------------------------------------------------------------------------------------------------------------------- + name: Deploy on: diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index ac9c3e59d7..4555084346 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -1,4 +1,42 @@ +# --------------------------------------------------------------------------------------------------------------------- +# This job workflow does the deployments for custom-staging.yml, deploy.yml and custom-deploy.yml. +# +# The repo must contain a 'deployment/Dockerfile' and a 'deployment/build.gradle' with an installDist task that prepares +# the deployment image in the 'deployment/build' directory; if this condition is not met the deployment will fail. +# Deployments are only done for repos owned by the openremote organisation. +# +# Each deployment consists of an environment and an openremote/manager docker tag, taken from: +# +# - The ENVIRONMENT input (and MANAGER_TAG in this repo) when manually triggered on a branch +# - Otherwise the 'deploy' entries in '.ci_cd/ci_cd.json' for the triggering event (see .ci_cd/README.md) +# +# In this repo a missing manager tag builds the manager image from the checked out commit. Custom projects always deploy +# the manager image matching the openremote version in 'gradle/libs.versions.toml' and use the '.ci_cd' scripts of that +# core release; the project's own '.ci_cd/deploy.sh', '.ci_cd/host_init' and '.ci_cd/aws' files take precedence. +# +# The environment selects '.ci_cd/env/{ENVIRONMENT}.env', the '_{ENVIRONMENT}_' prefixed secrets (in uppercase) and the +# docker compose file (see .ci_cd/README.md). +# +# Secrets, inputs and environment variables are combined for consumption by the '.ci_cd/deploy.sh' bash script and +# written to the 'temp/env' file, which is copied to the host. Later entries override earlier ones: +# +# - secrets and inputs (inputs override secrets with the same name) +# - '.ci_cd/env/.env' +# - '.ci_cd/env/{ENVIRONMENT}.env' +# +# Secrets prefixed with '_TEMP_' are written to the 'temp.env' file instead, which overrides 'temp/env' and is not +# copied to the host; a secret called 'SSH_KEY' is written to the 'ssh.key' file, which is not copied to the host +# either. Other secrets starting with '_' and secrets containing '.' are ignored. Sensitive credentials should always be +# stored in github secrets so they are encrypted, do not store within repo files (even private repos as these are not +# encrypted). +# +# When the '_TEMP_GFE_PASSWORD' secret is set it is used to decrypt the encrypted files of the deployment build. +# +# See https://github.com/openremote/core/tree/main/.ci_cd for details of standard variables and handling. +# --------------------------------------------------------------------------------------------------------------------- + name: Deployment + on: workflow_call: inputs: diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index ace8382fde..89a361274f 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -1,3 +1,12 @@ +# --------------------------------------------------------------------------------------------------------------------- +# This environment workflow runs for pull requests and merge queue checks of this repo, it handles: +# +# - Linting, building and testing +# - Checking that the packages can be published +# +# Custom projects use custom-review.yml instead. +# --------------------------------------------------------------------------------------------------------------------- + name: Review on: diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index e89711e3a6..5d9be3d5bf 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1,3 +1,17 @@ +# --------------------------------------------------------------------------------------------------------------------- +# This environment workflow runs for commits pushed to main and releases of this repo, it handles: +# +# - Linting, building and testing +# - Publishing the manager image, its tags are configured in '.ci_cd/ci_cd.json' +# - Publishing the packages +# - Updating dependent repositories to the new version +# +# Releases are created by release.yml, which pushes the release tag and then manually triggers this workflow on that +# tag, so the 'release' section of '.ci_cd/ci_cd.json' applies (see .ci_cd/README.md for more details). +# +# This workflow does not deploy, use deploy.yml for manual deployments. Custom projects use custom-staging.yml instead. +# --------------------------------------------------------------------------------------------------------------------- + name: Staging on: diff --git a/AGENTS.md b/AGENTS.md index a7764e789d..af9f0a1abf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,3 +47,7 @@ Access control belongs in the resource implementation, not in the service. Exten ## UI Frontend conventions live in `ui/AGENTS.md`. Read it before changing anything under `ui/`. + +## CI/CD pipeline + +Pipeline concepts and conventions live in the [CI/CD pipeline guide](https://docs.openremote.io/docs/developer-guide/ci-cd-pipeline). Read it before changing anything under `.github/` or `.ci_cd/`; the deployment scripts and their variables are covered by `.ci_cd/README.md`.