diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 45f5a9d..dac5dc4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "name": "LocalStack DooD setup", + "name": "Floci DooD setup", "dockerComposeFile": [ "../docker/networks.yml", "../docker/infrastructure.yml", @@ -13,14 +13,6 @@ "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { "upgradePackages": true }, - "ghcr.io/localstack/devcontainer-feature/localstack-cli:latest": { - "version": "latest", - "awslocal": true, // if true, add in features manually: ghcr.io/devcontainers/features/aws-cli - "cdklocal": false, // if true, add in features manually: ghcr.io/devcontainers-contrib/features/aws-cdk - "pulumilocal": false, // if true, add in features manually: ghcr.io/devcontainers-contrib/features/pulumi - "samlocal": false, // if true, add in features manually: ghcr.io/customink/codespaces-features/sam-cli - "tflocal": false // if true, add in features manually: ghcr.io/devcontainers-contrib/features/terraform-asdf - }, "ghcr.io/devcontainers/features/aws-cli:1": {} }, "customizations": { diff --git a/.devcontainer/devcontainer.yml b/.devcontainer/devcontainer.yml index 4461fab..6e67afd 100644 --- a/.devcontainer/devcontainer.yml +++ b/.devcontainer/devcontainer.yml @@ -15,7 +15,7 @@ services: env_file: - .env dns: - # Set the DNS server to be the LocalStack container + # Set the DNS server to be the Floci container - 192.168.0.20 networks: - ls diff --git a/.vscode/launch.json b/.vscode/launch.json index 1ac16f5..63df850 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,17 +1,6 @@ { "version": "0.2.0", "configurations": [ - { - // # The address of the host running the Lambda function to be debugged. - "address": "${env:DEBUG_HOST_ADDRESS}", - "localRoot": "${workspaceFolder}", - "name": "Attach to Remote Node.js", - "port": 9229, - "remoteRoot": "/var/task/", - "request": "attach", - "type": "node", - "preLaunchTask": "Wait Remote Debugger Server" - }, { "type": "node", "request": "launch", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 101dfbc..6a33c54 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,10 +1,4 @@ { "version": "2.0.0", - "tasks": [ - { - "label": "Wait Remote Debugger Server", - "type": "shell", - "command": "while [[ -z $(docker ps | grep :9229) ]]; do sleep 1; done; sleep 1;" - } - ] + "tasks": [] } diff --git a/docker/.env b/docker/.env index 3ce413f..1d686d6 100644 --- a/docker/.env +++ b/docker/.env @@ -1,47 +1,14 @@ -# Activate LocalStack Pro: https://docs.localstack.cloud/getting-started/auth-token/ -LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN:-} # required for Pro, not processed via template due to security reasons -LOCALSTACK_API_KEY=${LOCALSTACK_API_KEY:-} -# LocalStack configuration: https://docs.localstack.cloud/references/configuration/ -ACTIVATE_PRO=false -DEBUG=false -LS_LOG=info -PERSISTENCE=false -AWS_ENDPOINT_URL=http://localhost.localstack.cloud:4566 -LOCALSTACK_HOST=localhost.localstack.cloud:4566 -AUTO_LOAD_POD= -ENFORCE_IAM=false -AWS_REGION=eu-west-2 +FLOCI_SERVICES_LAMBDA_HOT_RELOAD_ENABLED=true +# FLOCI_STORAGE_MODE=memory +AWS_ENDPOINT_URL=http://localhost:4566 +AWS_ACCESS_KEY_ID=test +AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION=eu-west-2 -# LocalStack community edition is sufficient -IMAGE_NAME=localstack/localstack:latest +AWS_REGION=eu-west-2 +AWS_PAGER= +# FLOCI_HOSTNAME=floci +FLOCI_DEFAULT_REGION=eu-west-2 LAMBDA_TIMEOUT=9999999 -# Comment the line below and create a new containerised development environment to disable debugging of Lambda functions. -# IMPORTANT - If cloning the remote repository into a container volume, the change must be pushed to a pull request -# branch from which a new containerised development environment is created. If a new containerised development environment -# is not created, running multiple Lambda functions without remote debug limitations will not be possible. -LAMBDA_DOCKER_FLAGS="-e NODE_OPTIONS=--inspect-brk=0.0.0.0:9229 -p 9229:9229" -# Let Docker select a matching image based on the host operating system and architecture. -LAMBDA_IGNORE_ARCHITECTURE=1 - -# The address of the host running the Lambda function to be debugged. -# -# IMPORTANT -# When using a development container based local development environment, if the address does not match -# the address of the container running the Lambda function to be debugged, the debugger will NOT attach. -# The default value is for use in a development container based local development environment running -# four containers on the host machine BEFORE the container running the Lambda function to be debugged. -# - Dev container -# - LocalStack container -# - Postgres container -# - PgAdmin4 container -# -# When not using a development container the Lambda function to be debugged will run on the host machine -# (IP address 127.0.0.1). When not using a development container and using Visual Studio Code for debugging, -# this environment variable MUST be set to 127.0.0.1 and available to the session used to launch Visual Studio -# Code WITHOUT modifying this file. -# IMPORTANT - A change to this value MUST NOT be pushed to the remote repository as it will prevent -# debugging when cloning the remote repository into a container volume. -DEBUG_HOST_ADDRESS=192.168.0.5 CPX_DB_HOST=capxmldb NODEJS_VERSION=24 PGADMIN_DEFAULT_EMAIL=ubuntu@localhost.localdomain diff --git a/docker/infrastructure.yml b/docker/infrastructure.yml index ff88ba3..be7d1b4 100644 --- a/docker/infrastructure.yml +++ b/docker/infrastructure.yml @@ -1,15 +1,14 @@ services: - localstack: - container_name: "localstack-main" - image: localstack/localstack:4.14.0 + floci: + container_name: "floci-main" + image: floci/floci:2.0.1 ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4566:4566" # Floci AWS Gateway - "127.0.0.1:4510-4559:4510-4559" # external services port range env_file: - .env volumes: - "${DOCKER_SOCK:-/run/user/1000/docker.sock}:/var/run/docker.sock" - - "./.volume:/var/lib/localstack" networks: ls: # Set the container IP address in the 192.168.0.0/24 subnet diff --git a/docker/networks.yml b/docker/networks.yml index b5d92a0..2d2e19d 100644 --- a/docker/networks.yml +++ b/docker/networks.yml @@ -3,8 +3,6 @@ networks: ipam: config: # Specify the subnet range for IP address allocation - # This differs from the default subnet in LocalStack networking documentation - # (see https://blog.localstack.cloud/2024-03-04-making-connecting-to-localstack-easier/) - # to avoid conflicts with Oracle VirtualBox networking. + # This differs from common default emulator subnets to avoid conflicts with Oracle VirtualBox networking. # If using a different virtualization platform, the subnet range might need to be adjusted. - subnet: 192.168.0.0/24 diff --git a/docker/scripts/load-dummy-data.sh b/docker/scripts/load-dummy-data.sh index da2b4d8..92b5059 100755 --- a/docker/scripts/load-dummy-data.sh +++ b/docker/scripts/load-dummy-data.sh @@ -4,11 +4,13 @@ set -e +. docker/scripts/set-local-aws-environment.sh + # Constants BASE_GUID="4eb3b7350ab7aa443650fc9351f02940E" BASE_AREA="TESTAREA" DATA_FILE="test/lib/functions/data/nws-alert.xml" -LAMBDA_URL=http://$(awslocal apigateway get-rest-apis | jq -r ".items[0].id").execute-api.localhost.localstack.cloud:4566/local/message +LAMBDA_URL="$AWS_ENDPOINT_URL/restapis/$(aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway get-rest-apis --query "items[?name=='CPX API Gateway'].id | [0]" --output text)/local/_user_request_/message" # Calculate tomorrow's date TOMORROW=$(date -u -d "+1 day" +"%Y-%m-%dT%H:%M:%S+00:00") diff --git a/docker/scripts/register-api-gateway.sh b/docker/scripts/register-api-gateway.sh index d8d6c33..2696dce 100755 --- a/docker/scripts/register-api-gateway.sh +++ b/docker/scripts/register-api-gateway.sh @@ -3,12 +3,15 @@ # See https://containers.dev/implementors/json_reference/. set -e +. docker/scripts/set-local-aws-environment.sh + main() { - # Reference - https://docs.localstack.cloud/user-guide/aws/apigateway/ + # Reference - https://docs.aws.amazon.com/cli/latest/reference/apigateway/ echo "Creating API Gateway" + echo $AWS_ENDPOINT_URL - cap_xml_rest_api_id=$(awslocal apigateway create-rest-api --name "FWS API Gateway" | jq -r '.id') - cap_xml_rest_api_root_resource_id=$(awslocal apigateway get-resources --rest-api-id $cap_xml_rest_api_id | jq -r '.items[0].id') + cap_xml_rest_api_id=$(aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway create-rest-api --name "CPX API Gateway" | jq -r '.id') + cap_xml_rest_api_root_resource_id=$(aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway get-resources --rest-api-id "$cap_xml_rest_api_id" | jq -r '.items[] | select(.path == "/") | .id') lambda_functions_dir="lib/functions" find "$lambda_functions_dir" -type f -name "*.js" | while read -r lambda_function; do @@ -37,9 +40,13 @@ main() { done - awslocal apigateway create-deployment \ - --rest-api-id $cap_xml_rest_api_id \ - --stage-name local + deployment_id=$(aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway create-deployment \ + --rest-api-id $cap_xml_rest_api_id | jq -r '.id') + + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway create-stage \ + --rest-api-id $cap_xml_rest_api_id \ + --stage-name local \ + --deployment-id $deployment_id echo "Created API Gateway deployment" return 0 @@ -94,11 +101,24 @@ register_api_gateway_support_for_process_message() { } create_resource() { - cap_xml_rest_api_root_resource_id=$1 + cap_xml_rest_api_parent_resource_id=$1 cap_xml_rest_api_path_part=$2 - echo $(awslocal apigateway create-resource \ + + # A resource with the same parent and path part may already have been created + # by another lambda function (e.g. GET /message/{id} and POST /message share /message). + existing_resource_id=$(aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway get-resources \ + --rest-api-id $cap_xml_rest_api_id \ + | jq -r --arg parent "$cap_xml_rest_api_parent_resource_id" --arg path "$cap_xml_rest_api_path_part" \ + '.items[] | select(.parentId == $parent and .pathPart == $path) | .id') + + if [ -n "$existing_resource_id" ]; then + echo $existing_resource_id + return 0 + fi + + echo $(aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway create-resource \ --rest-api-id $cap_xml_rest_api_id \ - --parent-id $cap_xml_rest_api_root_resource_id \ + --parent-id $cap_xml_rest_api_parent_resource_id \ --path-part $cap_xml_rest_api_path_part | jq -r '.id') return 0 } @@ -106,7 +126,7 @@ create_resource() { put_method_and_integration() { resource_id=$1 - awslocal apigateway put-method \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-method \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -131,7 +151,7 @@ put_integration() { case $lambda_function_name in getMessage|getMessage_v2) - awslocal apigateway put-integration \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-integration \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -145,7 +165,7 @@ put_integration() { put_responses_for_get_message ;; getMessagesAtom|getMessagesAtom_v2) - awslocal apigateway put-integration \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-integration \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -158,7 +178,7 @@ put_integration() { put_responses_for_get_messages_atom ;; processMessage) - awslocal apigateway put-integration \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-integration \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -188,7 +208,7 @@ put_method_response_for_http_200_status_code() { # by a function. This results in some duplication. case $lambda_function_name in getMessage|getMessagesAtom) - awslocal apigateway put-method-response \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-method-response \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -196,7 +216,7 @@ put_method_response_for_http_200_status_code() { --response-models '{"application/xml": "Empty"}' ;; processMessage) - awslocal apigateway put-method-response \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-method-response \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -217,7 +237,7 @@ put_responses_for_get_message() { put_responses_for_http_200_get - awslocal apigateway put-integration-response \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-integration-response \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -237,7 +257,7 @@ put_responses_for_get_messages_atom() { } put_responses_for_process_message() { - awslocal apigateway put-integration-response \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-integration-response \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -252,7 +272,7 @@ put_responses_for_http_200_get() { put_method_response_for_http_200_status_code - awslocal apigateway put-integration-response \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-integration-response \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ @@ -264,7 +284,7 @@ put_responses_for_http_200_get() { put_integration_response_for_http_500() { - awslocal apigateway put-integration-response \ + aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway put-integration-response \ --rest-api-id $cap_xml_rest_api_id \ --resource-id $resource_id \ --http-method $http_method \ diff --git a/docker/scripts/register-lambda-functions.sh b/docker/scripts/register-lambda-functions.sh index 3e40b79..019d842 100755 --- a/docker/scripts/register-lambda-functions.sh +++ b/docker/scripts/register-lambda-functions.sh @@ -3,8 +3,10 @@ # See https://containers.dev/implementors/json_reference/. set -e +. docker/scripts/set-local-aws-environment.sh + lambda_functions_dir="lib/functions" -deployed_cpx_agw_url=http://$(awslocal apigateway get-rest-apis | jq -r ".items[0].id").execute-api.localhost.localstack.cloud:4566/local +deployed_cpx_agw_url="$AWS_ENDPOINT_URL/restapis/$(aws --endpoint-url "$AWS_ENDPOINT_URL" apigateway get-rest-apis --query "items[?name=='CPX API Gateway'].id | [0]" --output text)/local/_user_request_" # Prepare a comma separated list of custom environment variables required by # each Lambda function. @@ -24,6 +26,9 @@ node_tls_reject_unauthorized=$(echo NODE_TLS_REJECT_UNAUTHORIZED=$NODE_TLS_REJEC set -- $cpx_db_username $cpx_db_password $cpx_db_name $cpx_db_host $cpx_agw_url $cpx_redis_host $cpx_redis_port $cpx_redis_tls $cpx_meteoalarm_api_url $cpx_meteoalarm_api_username $cpx_meteoalarm_api_password $node_tls_reject_unauthorized $cpx_meteoalarm_disable custom_environment_variables=$(printf '%s,' "$@" | sed 's/,*$//g') +# Create the hot-reload bucket so Floci can serve Lambda code from the local filesystem. +aws --endpoint-url "$AWS_ENDPOINT_URL" s3 mb s3://hot-reload 2>/dev/null || true + # Iterate over each file in lambda_functions_dir find "$lambda_functions_dir" -type f -name "*.js" | while read -r lambda_function; do if [ -f "$lambda_function" ]; then @@ -43,9 +48,9 @@ find "$lambda_functions_dir" -type f -name "*.js" | while read -r lambda_functio ;; esac - echo Registering $function_name with LocalStack + echo Registering $function_name with Floci - awslocal lambda create-function \ + aws --endpoint-url "$AWS_ENDPOINT_URL" lambda create-function \ --function-name "$function_name" \ --code S3Bucket="hot-reload",S3Key="$(pwd)/" \ --runtime nodejs${NODEJS_VERSION}.x \ @@ -59,12 +64,12 @@ find "$lambda_functions_dir" -type f -name "*.js" | while read -r lambda_functio fi done -echo "All Lambda functions have been registered with LocalStack." +echo "All Lambda functions have been registered with Floci." -awslocal lambda create-function-url-config --function-name archiveMessages --auth-type NONE +aws --endpoint-url "$AWS_ENDPOINT_URL" lambda create-function-url-config --function-name archiveMessages --auth-type NONE echo "Created function URL config for archiveMessages function" -echo Function URL for archiveMessages is $(awslocal lambda get-function-url-config --function-name archiveMessages | jq -r .FunctionUrl) -echo API Gateway root URL is http://$(awslocal apigateway get-rest-apis | jq -r ".items[0].id").execute-api.localhost.localstack.cloud:4566/local +echo Function URL for archiveMessages is $(aws --endpoint-url "$AWS_ENDPOINT_URL" lambda get-function-url-config --function-name archiveMessages | jq -r .FunctionUrl) +echo API Gateway base URL is $deployed_cpx_agw_url diff --git a/docker/scripts/set-local-aws-environment.sh b/docker/scripts/set-local-aws-environment.sh new file mode 100644 index 0000000..75e7db5 --- /dev/null +++ b/docker/scripts/set-local-aws-environment.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Sourced (not executed) by scripts that call the AWS CLI, so that every call is +# guaranteed to target the local Floci emulator rather than a real AWS account. + +AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://localhost:4566}" + +# Fail fast rather than provision resources in a real AWS account. +case "$AWS_ENDPOINT_URL" in + http://localhost:* | http://127.0.0.1:* | http://floci:* | http://floci-main:*) ;; + *) + echo "Refusing to run: AWS_ENDPOINT_URL is '$AWS_ENDPOINT_URL', which is not a local Floci endpoint." >&2 + echo "Unset it, or set it to the local Floci endpoint (http://localhost:4566), before retrying." >&2 + exit 1 + ;; +esac + +# Dummy values so the CLI can never fall back to a real profile or region. +AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-test}" +AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test}" +AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-2}" +AWS_PROFILE="" + +export AWS_ENDPOINT_URL AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION +unset AWS_PROFILE diff --git a/docker/scripts/setup-aws-cli-command-completion.sh b/docker/scripts/setup-aws-cli-command-completion.sh index 21fb95e..d92b8c8 100755 --- a/docker/scripts/setup-aws-cli-command-completion.sh +++ b/docker/scripts/setup-aws-cli-command-completion.sh @@ -3,4 +3,3 @@ set -e echo "" >> ~/.bashrc echo complete -C '/usr/local/bin/aws_completer' aws >> ~/.bashrc -echo complete -C '/usr/local/bin/aws_completer' awslocal >> ~/.bashrc diff --git a/docs/local-development/common/running-and-debugging-lambda-functions.md b/docs/local-development/common/running-and-debugging-lambda-functions.md index 53a6e4f..18d9290 100644 --- a/docs/local-development/common/running-and-debugging-lambda-functions.md +++ b/docs/local-development/common/running-and-debugging-lambda-functions.md @@ -1,50 +1,32 @@ -# Running And Debugging AWS Lambda Functions +# Running AWS Lambda Functions -## Default Configuration +## Step Debugging Is Not Currently Supported -[The Docker environment variable file](../../../docker/.env) is configured to debug a -LocalStack hosted AWS Lambda function by default through the following environment variable: +**WARNING** - Step debugging of AWS Lambda functions in the local development environment is **not currently +supported under Floci**. It was previously possible under LocalStack and was lost when the local development +environment moved to Floci. -```sh -LAMBDA_DOCKER_FLAGS=-e NODE_OPTIONS=--inspect-brk=0.0.0.0:9229 -p 9229:9229 -``` - -This environment variable allows the Visual Studio Code debugger to attach to the standard Node.js debug port (9229) -in a LocalStack Docker container used to run an AWS Lambda function. - -## Disabling Debug Functionality - -AWS Lambda function debugging can be disabled by: - -* Commenting out the **LAMBDA_DOCKER_FLAGS** environment variable in [the development container environment variable file](../../../docker/.env). -* Replacing ([Teardown](../dev-container/additional-dev-container-considerations.md#teardown) and recreate) the existing containerised development environment with a new containerised development environment using - the revised configuration - * **IMPORTANT** - If cloning the remote repository into a container volume, the configuration change must be pushed to a branch from which the new containerised development environment **must** be created. - * **IMPORTANT** - If a new containerised dev environment is not created, running multiple Lambda functions without - remote debug limitations will **not** be possible. - -## Re-enabling Debug Functionality +LocalStack supported a **LAMBDA_DOCKER_FLAGS** environment variable, which this project used to pass +`-e NODE_OPTIONS=--inspect-brk=0.0.0.0:9229 -p 9229:9229` to each Lambda container so that the Visual Studio Code +debugger could attach to the standard Node.js debug port. Floci provides no equivalent - it has no mechanism for +injecting arbitrary Docker flags or environment variables into the containers it spawns for Lambda functions, so +neither the inspector nor the published debug port can be enabled. -Uncomment the **LAMBDA_DOCKER_FLAGS** environment variable in [the Docker environment variable file](../../../docker/.env) and replace the containerised development environment as described above. +Floci also starts a **fresh container for every invocation** (see [Container Lifecycle](#container-lifecycle) below), +so even if the inspector could be enabled, the debugger would need to reattach on every single request. -## Preparing To Debug An AWS Lambda Function +In the meantime: -### Visual Studio Code - -Run the Visual Studio Code **Attach to Remote Node.js (cap-xml)** debug configuration **before** AWS Lambda function -invocation. This waits for the standard Node.js debug port to be made available by a Docker container running an -AWS Lambda function before attempting to attach the debugger. Please consult the [LocalStack Lambda debugging documentation](https://hashnode.localstack.cloud/debugging-nodejs-lambda-functions-locally-using-localstack) for further details. - -### Other Software - -Please consult appropriate documentation. +* Unit tests can still be debugged with the Visual Studio Code **debug unit tests** configuration. +* Lambda function behaviour can be inspected at runtime using container logs, for example + `docker logs <>`, and by adding logging to the function under investigation. ## Invoking AWS Lambda Functions -From within the development container, use the [LocalStack AWS Command Line interface](https://docs.localstack.cloud/user-guide/integrations/aws-cli/) to retrieve the identifier of the deployed REST API from the API Gateway. For example, the command below can be used when an initial attempt to create a containerised development environment succeeds (resulting in the creation of one REST API instance) +From within the development container, use the AWS CLI (configured with the **AWS_ENDPOINT_URL** environment variable pointing at Floci) to retrieve the identifier of the deployed REST API from the API Gateway. For example, the command below can be used when an initial attempt to create a containerised development environment succeeds (resulting in the creation of one REST API instance) ```sh -awslocal apigateway get-rest-apis | jq -r '.items[0].id' +aws apigateway get-rest-apis | jq -r '.items[0].id' ``` IMPORTANT @@ -52,9 +34,7 @@ IMPORTANT In the following examples of invoking Lambda functions through API Gateway endpoints: * Angled bracket based placeholders such as **<<REST-API-ID>>** placeholder **must** be replaced. -* For convenience, LocalStack is configured not to require an API key when calling API Gateway endpoints. -* When the debugger attaches, it breaks before running the AWS Lambda function. As such, debugging **must** be resumed to reach - configured breakpoints. +* For convenience, Floci is configured not to require an API key when calling API Gateway endpoints. ### Loading dummy data @@ -69,7 +49,7 @@ npm run load-dummy-data Use the REST API identifier to call the **/messages.atom** endpoint linked to the **getMessagesAtom** AWS Lambda function. For example, the following curl command can be used: ```sh -curl "http://<>.execute-api.localhost.localstack.cloud:4566/local/messages.atom" +curl "http://localhost:4566/restapis/<>/local/_user_request_/messages.atom" ``` ### Making A HTTP GET Request To The /message Endpoint @@ -77,14 +57,14 @@ curl "http://<>.execute-api.localhost.localstack.cloud:4566/local/m Use the REST API identifier to call the **/message/<<MESSAGE-ID>>** endpoint linked to the **getMessage** AWS Lambda function. For example, the following curl command can be used: ```sh -curl "http://<>.execute-api.localhost.localstack.cloud:4566/local/message/<>" +curl "http://localhost:4566/restapis/<>/local/_user_request_/message/<>" ``` ### Making A HTTP POST To The /message Endpoint AWS API Gateway request templates are used to ensure XML message content is embedded within AWS Lambda JSON event objects. Real AWS API Gateway software appears capable of embedding raw XML within AWS Lambda JSON event objects without further configuration. -At the time of writing, LocalStack API Gateway software appears to require raw XML message content to be embedded as a string +The configured Floci API Gateway integration requires raw XML message content to be embedded as a string within a JSON document and slightly different request templates to be able to provide the AWS Lambda function with an event object reflecting that received from real AWS API Gateway software. For example, real AWS API Gateway software appears capable of handling raw XML such as the following: @@ -94,7 +74,7 @@ such as the following: ``` -LocalStack API Gateway software appears to require a JSON structure such as the following for Lambda functions to receive an event object +The configured Floci API Gateway integration requires a JSON structure such as the following for Lambda functions to receive an event object consistent with that received from real AWS API Gateway software. ```sh @@ -106,12 +86,12 @@ consistent with that received from real AWS API Gateway software. The following command can be used as a guide to making a HTTP POST request to the **/message** endpoint using **curl**: ```sh -curl -H "Content-Type: text/xml" -d "@<>" "http://<>.execute-api.localhost.localstack.cloud:4566/local/message" +curl -H "Content-Type: text/xml" -d "@<>" "http://localhost:4566/restapis/<>/local/_user_request_/message" ``` IMPORTANT -* The request header **Content-Type: text/xml** or **Content-Type: text/html** **must** be used to ensure correct processing by the LocalStack API Gateway request template. +* The request header **Content-Type: text/xml** or **Content-Type: text/html** **must** be used to ensure correct processing by the Floci API Gateway request template. ### Invoking The archiveMessages AWS Lambda Function @@ -119,33 +99,42 @@ This function is not invoked through an API Gateway endpoint and needs to be inv An AWS Lambda Function URL is created and output during local environment provisioning. The URL can be retrieved using the following command: ```sh -awslocal lambda get-function-url-config --function-name archiveMessages | jq -r .FunctionUrl +aws lambda get-function-url-config --function-name archiveMessages | jq -r .FunctionUrl ``` The AWS Lambda Function URL can be used with a suitable HTTP client such as curl. For example, the following command can be used in a Linux environment: ```sh -curl $(awslocal lambda get-function-url-config --function-name archiveMessages | jq -r .FunctionUrl) +curl $(aws lambda get-function-url-config --function-name archiveMessages | jq -r .FunctionUrl) ``` ## Making Code Changes -Code changes can be made without having to redeploy Lambda functions to LocalStack. Please consult [LocalStack Lambda debugging documentation](https://hashnode.localstack.cloud/debugging-nodejs-lambda-functions-locally-using-localstack) for more details. +Lambda functions are registered with Floci in bind-mount hot-reload mode, so code changes are picked up without +having to redeploy. The repository working directory is mounted into each Lambda container at **/var/task**, meaning +every invocation runs the files exactly as they currently exist on disk. -## Debug Limitations +## Container Lifecycle -LocalStack uses a different Docker container for each available AWS Lambda function (i.e. while the same container can -process multiple requests invoking the same AWS Lambda function, a new container will be created to invoke a different -AWS Lambda function). +Hot-reload containers are **always ephemeral**. Floci starts a fresh container for **every invocation** and removes it +once the invocation completes, which is what guarantees the next invocation sees the current state of the directory. -At a particular point in time, the standard Node.js debug port can only be used by one LocalStack Docker container -used to run an AWS Lambda function. This restriction means that a particular AWS Lambda function cannot be debugged -if a container for debugging a different AWS Lambda function is running. As such, while Docker containers used to run -Lambda functions are ephemeral, manual container removal or timeout (causing automatic removal) is needed when invoking different AWS Lambda functions in quick succession. The timeout for a Lambda function is specified by the LAMBDA_TIMEOUT -environment variable within [the Docker environment variable file](../../../docker/.env). +This means: + +* There is no warm container reuse, so every request pays a cold start penalty. End to end functional tests are + correspondingly slower than they would be against warm containers. +* This is inherent to hot-reload mode rather than a misconfiguration. Setting **FLOCI_SERVICES_LAMBDA_EPHEMERAL** to + false does **not** change it, because hot-reload overrides that setting. Warm container reuse would require giving up + hot-reload and deploying function code as a ZIP instead. + +The invocation timeout for a Lambda function is specified by the LAMBDA_TIMEOUT environment variable within +[the Docker environment variable file](../../../docker/.env), which the registration script passes to Floci as the +function **--timeout**. ## Known Issues -* Calls to the **getMessagesAtom** and **getMessage** LocalStack API Gateway endpoints return a Content-Type HTTP response header of **text/plain** rather than **application/xml** returned by calls to AWS API Gateway endpoints. This affects markup formatting in browsers. +**NOTE** - The issues below were originally observed under LocalStack and have not yet been re-verified against Floci; treat them as unconfirmed until retested. + +* Calls to the **getMessagesAtom** and **getMessage** API Gateway endpoints return a Content-Type HTTP response header of **text/plain** rather than **application/xml** returned by calls to AWS API Gateway endpoints. This affects markup formatting in browsers. * When running/debugging in a development container created by cloning the repository into a container volume, calls to **getMessagesAtom** return XML content containing incorrect URLs that use the REST API ID configured on the host machine rather than the REST API ID configured in the development container. * To workaround this issue, ensure that the url attribute within config/config.json on the host machine matches that configured in the development container. diff --git a/docs/local-development/common/troubleshooting.md b/docs/local-development/common/troubleshooting.md index da95d55..c9d1837 100644 --- a/docs/local-development/common/troubleshooting.md +++ b/docs/local-development/common/troubleshooting.md @@ -1,10 +1,10 @@ # Troubleshooting -## LocalStack Container Fails To Start +## Floci Container Fails To Start -* This could be caused by an existing application using the subnet configured for LocalStack to act as a DNS server (192.168.0.0/24). +* This could be caused by an existing application using the subnet configured for Floci to act as a DNS server (192.168.0.0/24). * The configured subnet avoids conflict with Oracle VirtualBox networking. -* If the configured subnet conflicts with another application that cannot be stopped, try changing the networking configuration in the [development container Docker Compose file](../.devcontainer/devcontainer.yml) to use a different subnet (such as 10.0.2.0/24 as used in [LocalStack network connectivity documentation](https://blog.localstack.cloud/2024-03-04-making-connecting-to-localstack-easier/)), [teardown](../common/teardown.md) existing development container based resources and create a new development container. +* If the configured subnet conflicts with another application that cannot be stopped, try changing the networking configuration in the [development container Docker Compose file](../.devcontainer/devcontainer.yml) to use a different subnet (such as 10.0.2.0/24), [teardown](../common/teardown.md) existing development container based resources and create a new development container. **IMPORTANT** - If cloning the remote repository into a container volume, the configuration change must be pushed to a branch from which the new containerised development environment **must** be created. @@ -15,33 +15,10 @@ * [Rootless Docker based configuration with development containers](../dev-container/rootless-docker-configuration.md) * [Rootless Docker based configuration without development containers](../manual-configuration/rootless-docker-configuration.md) -### Lambda Function Cannot Bind To Standard Node.js Debug Port +### Lambda Function Step Debugging -* Remove any existing Lambda function container or other process using the port. - -### Lambda Function Timeout - -* If using Visual Studio Code run the **Attach to Remote Node.js (cap-xml)** debug configuration **before** Lambda function -invocation. - -### Visual Studio Code Debugger Does Not Attach To Lambda Function Container - -#### Dev Container Based Local Development Environment - -* Check that the IP address of the Lambda function container matches that configured in the DEBUG_HOST_ADDRESS - environment variable within [the Docker environment variable file](../../../docker/.env). - * This check could be difficult to perform if Lambda function containers only exist for a short amount of time. - * In a standard debugging scenario, a Lambda function container should have the IP address **192.168.0.5** based - on these four containers running before the Lambda function container is created: - * Dev container - * LocalStack container - * Postgres container - * PgAdmin4 container - -#### Non-Dev Container Based Local Development Environment - -* Ensure that the DEBUG_HOST_ADDRESS environment variable on the host running the Lambda function container is set to 127.0.0.1 - **before** launching Visual Studio Code. +* Step debugging of Lambda functions is **not currently supported under Floci** - see + [Running AWS Lambda Functions](./running-and-debugging-lambda-functions.md#step-debugging-is-not-currently-supported). ### Node.js Module Import Errors @@ -59,4 +36,4 @@ Ensure that the [local cap-xml repository location](../dev-container/local-repos ### Unsuccessful Resolution Of Problems -If problems persist, prerequisites and associated configuration should be reviewed followed by a [teardown](../common/teardown.md) and rebuild of the containerised development environment. If LocalStack resources for other projects are present (for example [fws-api](https://github.com/DEFRA/fws-api)), try removing these resources before rebuilding to eliminate potential conflicts. +If problems persist, prerequisites and associated configuration should be reviewed followed by a [teardown](../common/teardown.md) and rebuild of the containerised development environment. If Floci resources for other projects are present (for example [fws-api](https://github.com/DEFRA/fws-api)), try removing these resources before rebuilding to eliminate potential conflicts. diff --git a/docs/local-development/dev-container/dev-container-creation.md b/docs/local-development/dev-container/dev-container-creation.md index 18d6c83..dda39a7 100644 --- a/docs/local-development/dev-container/dev-container-creation.md +++ b/docs/local-development/dev-container/dev-container-creation.md @@ -20,7 +20,7 @@ After a local directory has been selected or a repository URL has been entered i * Issue the command **docker ps** * Four containers associated with CAP XML should be running on the **Docker host** (due to use of Docker outside of Docker Compose): * Development container - * LocalStack container + * Floci container * Postgres container * PgAdmin4 container diff --git a/docs/local-development/dev-container/local-repository-creation.md b/docs/local-development/dev-container/local-repository-creation.md index d048f19..f980f80 100644 --- a/docs/local-development/dev-container/local-repository-creation.md +++ b/docs/local-development/dev-container/local-repository-creation.md @@ -22,7 +22,7 @@ A development container can be created from either a local repository on the dev * A local repository provides easier access to the code outside the container using a bind mount. * When using a remote repository URL the code used by the container is located in a Docker volume. * While [Microsoft recommend this approach for performance reasons](https://code.visualstudio.com/remote/advancedcontainers/improve-performance), seemingly - incompatible constraints when using this approach with [Docker outside Of Docker Compose](https://github.com/devcontainers/templates/tree/main/src/docker-outside-of-docker-compose), [LocalStack integration](https://docs.localstack.cloud/user-guide/integrations/devcontainers/#docker-outside-of-docker) and either macOS or Windows has led to this option only seeming viable with native Linux at the time of writing. + incompatible constraints when using this approach with [Docker outside Of Docker Compose](https://github.com/devcontainers/templates/tree/main/src/docker-outside-of-docker-compose) and either macOS or Windows has led to this option only seeming viable with native Linux at the time of writing. Regardless of how a development container is created, repository contents are held in a workspace folder within the development container. diff --git a/docs/local-development/dev-container/rootless-docker-configuration.md b/docs/local-development/dev-container/rootless-docker-configuration.md index 41164a8..00357e5 100644 --- a/docs/local-development/dev-container/rootless-docker-configuration.md +++ b/docs/local-development/dev-container/rootless-docker-configuration.md @@ -26,7 +26,7 @@ The repository can be located anywhere on the Linux host accessible to the user * Rootful Docker socket backup. * Replacement of rootful Docker socket with a symbolic link to the rootless Docker socket. - * This is required because the Docker socket used by LocalStack and [Docker outside Of Docker Compose](https://github.com/devcontainers/templates/tree/main/src/docker-outside-of-docker-compose) development containers does not appear to be configurable at the time of writing. + * This is required because the Docker socket used by Floci and [Docker outside Of Docker Compose](https://github.com/devcontainers/templates/tree/main/src/docker-outside-of-docker-compose) development containers does not appear to be configurable at the time of writing. * Grant of rootless Docker socket read write access to the host SUBGID for the development container **vscode** user. To facilitate development container creation from the local cap-xml repository, ownership of the local cap-xml repository is **transferred** to the host SUBUID and SUBGID for the development container **vscode** user. diff --git a/docs/local-development/dev-container/setup-and-teardown.md b/docs/local-development/dev-container/setup-and-teardown.md index aa5e894..7deea01 100644 --- a/docs/local-development/dev-container/setup-and-teardown.md +++ b/docs/local-development/dev-container/setup-and-teardown.md @@ -8,15 +8,15 @@ A [development container](https://code.visualstudio.com/docs/remote/containers) * CAP XML database creation and reference data population using a containerised Postgres instance. * Use of a [containerised Postgres Graphical User Interface](https://www.pgadmin.org/download/pgadmin-4-container/) for performing database operations. -* [LocalStack](https://www.localstack.cloud/) AWS API Gateway and AWS Lambda - provisioning to facilitate local running and debugging of CAP XML calls without round tripping +* [Floci](https://floci.io/) AWS API Gateway and AWS Lambda + provisioning to facilitate local running of CAP XML calls without round tripping to AWS infrastructure. - * The [LocalStack AWS Command Line interface](https://docs.localstack.cloud/user-guide/integrations/aws-cli/) is + * The AWS CLI (configured with the **AWS_ENDPOINT_URL** environment variable pointing at Floci) is used during provisioning and can also be used at runtime. * Runtime environment variable configuration. * Configuration generation providing connectivity to local development environment resources. -Development containers are based on [Docker outside Of Docker Compose](https://github.com/devcontainers/templates/tree/main/src/docker-outside-of-docker-compose) with [LocalStack integration](https://docs.localstack.cloud/user-guide/integrations/devcontainers/#docker-outside-of-docker). +Development containers are based on [Docker outside Of Docker Compose](https://github.com/devcontainers/templates/tree/main/src/docker-outside-of-docker-compose). Development containers run as an unprivileged **vscode** user with passwordless sudo access. @@ -33,7 +33,7 @@ If an existing local development environment has been configured manually **it i * [Rootless Docker Configuration](./rootless-docker-configuration.md) * **Only** applicable when using native Linux or systemd enabled WSL 2 with native Docker. * [Dev Container Creation](./dev-container-creation.md) -* [Running And Debugging Lambda Functions](../common/running-and-debugging-lambda-functions.md) +* [Running AWS Lambda Functions](../common/running-and-debugging-lambda-functions.md) * [Troubleshooting](../common/troubleshooting.md) * [Teardown](../common/teardown.md) * [Additional Development Container Considerations](./additional-dev-container-considerations.md) diff --git a/docs/local-development/manual-configuration/bootstrapping.md b/docs/local-development/manual-configuration/bootstrapping.md index 0bd6f68..ef05b52 100644 --- a/docs/local-development/manual-configuration/bootstrapping.md +++ b/docs/local-development/manual-configuration/bootstrapping.md @@ -7,15 +7,6 @@ | LOCAL_CAP_XML_DIR | The **absolute** path to the root of a local cap-xml repository. | | DOCKER_SOCK | (Optional) absolute path to docker.sock for rootless installation, defaults to /run/user/1000/docker.sock -### Mandatory Environment Variables For Visual Studio Code Debugging - -* If using Visual Studio Code for debugging, the environment variable DEBUG_HOST_ADDRESS **must** be set to 127.0.0.1 - **before** launching Visual Studio Code. For example, the following command can be used on native Linux: - - ```sh - export DEBUG_HOST_ADDRESS=127.0.0.1 && code - ``` - ## Run Bootstrap Script * Run the bootstrap-debug npm script from the repository root. @@ -30,12 +21,12 @@ * Docker named volume creation. * Docker custom network creation. * Container creation: - * LocalStack. + * Floci. * Postgres database. * Pgadmin4. * Liquibase. * Changesets are run to create the containerised Postgres database structure. - * AWS Lambda function registration with LocalStack. - * AWS API Gateway registration with LocalStack. + * AWS Lambda function registration with Floci. + * AWS API Gateway registration with Floci. * Configuration generation providing connectivity to local development environment resources. * Liquibase container removal. diff --git a/docs/local-development/manual-configuration/prerequisites.md b/docs/local-development/manual-configuration/prerequisites.md index 8a73740..6a68d6d 100644 --- a/docs/local-development/manual-configuration/prerequisites.md +++ b/docs/local-development/manual-configuration/prerequisites.md @@ -4,11 +4,6 @@ - [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) - [Configuring command completion is recommended](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-completion.html). -- [LocalStack AWS CLI](https://github.com/localstack/awscli-local) - - [Configuring command completion is recommended](https://github.com/localstack/awscli-local?tab=readme-ov-file#completion). - -**IMPORTANT** - [LocalStack initialisation hooks](https://docs.localstack.cloud/aws/capabilities/config/initialization-hooks/) are **not** used due -to [LocalStack Docker containers providing no support for the AWS CLI v2 currently](https://github.com/localstack/localstack/issues/11948). ## Additional Prerequisites diff --git a/docs/local-development/manual-configuration/rootless-docker-configuration.md b/docs/local-development/manual-configuration/rootless-docker-configuration.md index 9058b5c..c2caa18 100644 --- a/docs/local-development/manual-configuration/rootless-docker-configuration.md +++ b/docs/local-development/manual-configuration/rootless-docker-configuration.md @@ -19,7 +19,7 @@ The location **/opt/workspaces/** (resulting in a local repository root of **/ * Rootful Docker socket backup. * Replacement of rootful Docker socket with a symbolic link to the rootless Docker socket. - * This is required because the Docker socket used by LocalStack does not appear to be configurable at the time of writing. + * This is required because the Docker socket used by Floci does not appear to be configurable at the time of writing. **IMPORTANT** - If [setup-for-rootless-docker-without-dev-container.sh](../../../docker/scripts/setup-for-rootless-docker-without-dev-container.sh) is run using the sudo command, the user session from which the sudo command is run **must** ensure that mandatory environment variables are available to the script. This is because user session environment variables are not available to a sudo command by default. For example, if the environment variables are available to the user session running the sudo command, they can be preserved for availability to [setup-for-rootless-docker-without-dev-container.sh](../../../docker/scripts/setup-for-rootless-docker-without-dev-container.sh) using the following sudo command: diff --git a/docs/local-development/manual-configuration/setup-and-teardown.md b/docs/local-development/manual-configuration/setup-and-teardown.md index f123cc7..99a31d3 100644 --- a/docs/local-development/manual-configuration/setup-and-teardown.md +++ b/docs/local-development/manual-configuration/setup-and-teardown.md @@ -6,6 +6,6 @@ * [Rootless Docker Configuration](./rootless-docker-configuration.md). * **Only** applicable when using native Linux or systemd enabled WSL 2 with native Docker. * [Bootstrapping](./bootstrapping.md). -* [Running And Debugging Lambda Functions](../common/running-and-debugging-lambda-functions.md). +* [Running AWS Lambda Functions](../common/running-and-debugging-lambda-functions.md). * [Troubleshooting](../common/troubleshooting.md). * [Teardown](../common/teardown.md) diff --git a/package-lock.json b/package-lock.json index e761539..3cd8e5c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "cap-xml", - "version": "4.0.1", + "version": "4.1.0", "license": "OGL", "dependencies": { "@aws-sdk/client-sns": "3.1022.0", diff --git a/package.json b/package.json index df9fa89..9b1dd08 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "unit-test": "lab test -v -c -r console -o stdout -r lcov -o coverage/lcov.info -m 5000", "test": "npm run lint && npm run unit-test", "validate-config": "node ./config/index.js", - "bootstrap-debug": "docker/scripts/non-dev-container-bootstrap.sh", + "bootstrap": "docker/scripts/non-dev-container-bootstrap.sh", "load-dummy-data": "docker/scripts/load-dummy-data.sh", "teardown": "docker/scripts/teardown.sh" },