From ffe679eb9874f6c02d91172583a978fa2bcc477f Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 09:59:41 +0200
Subject: [PATCH 01/15] Handle JSON null in numeric All datapoint queries
---
.../query/AssetDatapointAllQuery.java | 3 +-
.../assets/AssetDatapointNullQueryTest.groovy | 93 +++++++++++++++++++
2 files changed, 95 insertions(+), 1 deletion(-)
create mode 100644 test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
index 26943b55b8..03c6188903 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
@@ -50,7 +50,8 @@ public String getSQLQuery(String tableName, Class attributeType) throws Illeg
String conditionSuffix =
" where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? order by timestamp desc";
if (isNumber) {
- return "select timestamp as X, cast(value as numeric) as Y from "
+ // JSON null must become SQL NULL before it can be cast to a number.
+ return "select timestamp as X, cast(nullif(value, 'null'::jsonb) as numeric) as Y from "
+ tableName
+ conditionSuffix;
} else if (isBoolean) {
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
new file mode 100644
index 0000000000..07ad616960
--- /dev/null
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2026, OpenRemote Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+package org.openremote.test.assets
+
+import org.openremote.manager.asset.AssetStorageService
+import org.openremote.manager.datapoint.AssetDatapointService
+import org.openremote.manager.datapoint.AssetPredictedDatapointService
+import org.openremote.manager.setup.SetupService
+import org.openremote.model.asset.impl.ThingAsset
+import org.openremote.model.attribute.Attribute
+import org.openremote.model.datapoint.ValueDatapoint
+import org.openremote.model.datapoint.query.AssetDatapointAllQuery
+import org.openremote.model.value.ValueType
+import org.openremote.setup.integration.KeycloakTestSetup
+import org.openremote.test.ManagerContainerTrait
+import spock.lang.Specification
+
+import java.time.temporal.ChronoUnit
+
+class AssetDatapointNullQueryTest extends Specification implements ManagerContainerTrait {
+
+ AssetStorageService assetStorageService
+ AssetDatapointService historicalService
+ AssetPredictedDatapointService predictedService
+ ThingAsset asset
+ String realm
+ long start
+
+ def setup() {
+ def container = startContainer(defaultConfig(), defaultServices())
+ stopPseudoClock()
+ assetStorageService = container.getService(AssetStorageService.class)
+ historicalService = container.getService(AssetDatapointService.class)
+ predictedService = container.getService(AssetPredictedDatapointService.class)
+ realm = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class).realmBuilding.name
+ start = getInstantTimeOf(container).truncatedTo(ChronoUnit.SECONDS).toEpochMilli()
+ }
+
+ def cleanup() {
+ if (asset?.id) {
+ assert assetStorageService.delete([asset.id])
+ }
+ }
+
+ def "numeric All query preserves #description for #storage #valueType.name datapoints"() {
+ given: "numeric datapoints containing explicitly stored JSON nulls"
+ asset = assetStorageService.merge(new ThingAsset("Null query test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def timestamps = (0..
+ new ValueDatapoint<>(timestamps[index], value)
+ }
+ // The query must include both boundary timestamps and exclude their neighbours.
+ datapoints.add(new ValueDatapoint<>(timestamps.first() - 60_000L, 111G))
+ datapoints.add(new ValueDatapoint<>(timestamps.last() + 60_000L, 222G))
+ service.upsertValues(asset.id, "value", datapoints)
+
+ when: "the inclusive range is queried without aggregation"
+ def result = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointAllQuery(timestamps.first(), timestamps.last()))
+
+ then: "every timestamp is returned newest first, retaining unavailable values"
+ result.collect { it.timestamp } == timestamps.reverse()
+ result.collect { it.value } == values.reverse()
+
+ where:
+ [storage, valueType, description, values] << ["historical", "predicted"].collectMany { storage ->
+ [ValueType.NUMBER, ValueType.BIG_NUMBER].collectMany { type ->
+ [
+ [storage, type, "mixed values", [-5G, 0G, null, 90.125G]],
+ [storage, type, "an entirely null range", [null, null, null]]
+ ]
+ }
+ }
+ }
+}
From 3b1352222d916b7f329d1aab8abbd1b43d1f8e8e Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 10:36:56 +0200
Subject: [PATCH 02/15] Handle JSON null in nearest datapoint queries
---
.../query/AssetDatapointNearestQuery.java | 6 ++-
.../assets/AssetDatapointNullQueryTest.groovy | 38 +++++++++++++++++++
2 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.java
index 4842794682..41042f919c 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.java
@@ -31,7 +31,8 @@
description =
"Returns at most one numeric datapoint: the closest value at or before the requested "
+ "time. Use `fromTimestamp` as a Unix-seconds timestamp at the HTTP API boundary. `toTimestamp`, "
- + "`fromTime`, and `toTime` are ignored. The equality boundary is inclusive.")
+ + "`fromTime`, and `toTime` are ignored. The equality boundary is inclusive. "
+ + "If the nearest datapoint has a null value, its timestamp and null value are returned.")
public class AssetDatapointNearestQuery extends AssetDatapointQuery {
public AssetDatapointNearestQuery() {}
@@ -43,7 +44,8 @@ public AssetDatapointNearestQuery(long timestamp) {
@Override
public String getSQLQuery(String tableName, Class attributeType) throws IllegalStateException {
- return "SELECT timestamp as X, cast(value as numeric) as Y "
+ // Preserve an unavailable nearest value instead of rejecting its JSON null.
+ return "SELECT timestamp as X, cast(nullif(value, 'null'::jsonb) as numeric) as Y "
+ " FROM "
+ tableName
+ " "
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 07ad616960..466e85d0df 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -26,6 +26,7 @@ import org.openremote.model.asset.impl.ThingAsset
import org.openremote.model.attribute.Attribute
import org.openremote.model.datapoint.ValueDatapoint
import org.openremote.model.datapoint.query.AssetDatapointAllQuery
+import org.openremote.model.datapoint.query.AssetDatapointNearestQuery
import org.openremote.model.value.ValueType
import org.openremote.setup.integration.KeycloakTestSetup
import org.openremote.test.ManagerContainerTrait
@@ -43,6 +44,8 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
long start
def setup() {
+ // Match the pseudo clock's UTC zone before JDBC connections capture the JVM time zone.
+ TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
def container = startContainer(defaultConfig(), defaultServices())
stopPseudoClock()
assetStorageService = container.getService(AssetStorageService.class)
@@ -90,4 +93,39 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
}
}
}
+
+ def "numeric Nearest query preserves unavailable values for #storage #valueType.name datapoints"() {
+ given: "an unavailable datapoint between older and newer available prices"
+ asset = assetStorageService.merge(new ThingAsset("Nearest null query test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))
+ def service = storage == "historical" ? historicalService : predictedService
+ service.upsertValues(asset.id, "value", [
+ new ValueDatapoint<>(start, -5G),
+ new ValueDatapoint<>(start + 60_000L, null),
+ new ValueDatapoint<>(start + 120_000L, 0G),
+ new ValueDatapoint<>(start + 180_000L, 90.125G)
+ ])
+ def queryAt = { long offset ->
+ service.queryDatapoints(asset.id, "value", new AssetDatapointNearestQuery(start + offset)).collect {
+ [it.timestamp, it.value]
+ }
+ }
+
+ expect: "an exact match retains the null value and its timestamp"
+ queryAt(60_000L) == [[start + 60_000L, null]]
+
+ and: "a query between observations returns the null row, not an older or newer price"
+ queryAt(90_000L) == [[start + 60_000L, null]]
+
+ and: "a query before the first observation returns no datapoint"
+ queryAt(-1_000L).isEmpty()
+
+ and: "available observations remain selectable at and after their timestamps"
+ queryAt(0) == [[start, -5G]]
+ queryAt(120_000L) == [[start + 120_000L, 0G]]
+ queryAt(240_000L) == [[start + 180_000L, 90.125G]]
+
+ where:
+ [storage, valueType] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER]].combinations()
+ }
}
From 446f09d594581225be5b0eb58dd7e1f208488553 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 10:46:10 +0200
Subject: [PATCH 03/15] Handle JSON null in standard numeric datapoint
aggregates
---
.../query/AssetDatapointIntervalQuery.java | 9 ++--
.../assets/AssetDatapointNullQueryTest.groovy | 50 +++++++++++++++++++
2 files changed, 55 insertions(+), 4 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index de10cb4ee2..2a4fb05732 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -57,7 +57,8 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
+ "value; `DIFFERENCE` calculates the change between the last values of consecutive buckets; "
+ "`COUNT` counts datapoints; `SUM` totals numeric values (or true values for booleans); `MODE` "
+ "selects the most frequent value; and `MEDIAN` selects the middle value. `DIFFERENCE` is numeric "
- + "only.",
+ + "only. Numeric MIN, AVG, MAX, SUM and MEDIAN ignore null values and return null "
+ + "for buckets with no available values.",
requiredMode = Schema.RequiredMode.REQUIRED)
public Formula formula;
@@ -137,7 +138,7 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + "SUM(cast(value as numeric)) AS total_sum "
+ + "SUM(cast(nullif(value, 'null'::jsonb) as numeric)) AS total_sum "
+ "FROM "
+ tableName
+ " "
@@ -170,7 +171,7 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + "PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY cast(value as numeric)) as median_value "
+ + "PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY cast(nullif(value, 'null'::jsonb) as numeric)) as median_value "
+ "FROM "
+ tableName
+ " "
@@ -181,7 +182,7 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
+ function
+ "(cast(? as interval), timestamp) AS x, "
+ this.formula.toString().toLowerCase()
- + "(cast(value as numeric)) FROM "
+ + "(cast(nullif(value, 'null'::jsonb) as numeric)) FROM "
+ tableName
+ " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? GROUP BY x ORDER by x ASC";
}
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 466e85d0df..d53e53d57f 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -26,12 +26,15 @@ import org.openremote.model.asset.impl.ThingAsset
import org.openremote.model.attribute.Attribute
import org.openremote.model.datapoint.ValueDatapoint
import org.openremote.model.datapoint.query.AssetDatapointAllQuery
+import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery
+import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery.Formula
import org.openremote.model.datapoint.query.AssetDatapointNearestQuery
import org.openremote.model.value.ValueType
import org.openremote.setup.integration.KeycloakTestSetup
import org.openremote.test.ManagerContainerTrait
import spock.lang.Specification
+import java.time.Instant
import java.time.temporal.ChronoUnit
class AssetDatapointNullQueryTest extends Specification implements ManagerContainerTrait {
@@ -128,4 +131,51 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
where:
[storage, valueType] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER]].combinations()
}
+
+ def "numeric #formula handles unavailable values for #storage #valueType.name with gapFill=#gapFill"() {
+ given: "mixed, entirely null, empty, and zero-valued minute buckets"
+ asset = assetStorageService.merge(new ThingAsset("Interval null query test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()
+ service.upsertValues(asset.id, "value", [
+ [-1_000L, 999G],
+ [0L, -6G],
+ [10_000L, 0G],
+ [20_000L, null],
+ [30_000L, 6G],
+ [40_000L, 12G],
+ [70_000L, null],
+ [100_000L, null],
+ [190_000L, 0G],
+ [240_000L, 999G]
+ ].collect { new ValueDatapoint<>(bucketStart + it[0], it[1]) })
+
+ when: "the four-minute range is aggregated"
+ def result = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 239_999L,
+ "1 minute", formula, gapFill))
+
+ then: "null-only buckets remain present and empty buckets follow the gap-fill setting"
+ result.collect {
+ it.timestamp
+ } == (gapFill ? [0L, 60_000L, 120_000L, 180_000L] : [0L, 60_000L, 180_000L])
+ .collect { bucketStart + it }
+ result.collect { it.value } == (gapFill
+ ? [mixedValue, nullBucketValue, null, zeroBucketValue]
+ : [mixedValue, nullBucketValue, zeroBucketValue])
+
+ where:
+ [storage, valueType, gapFill, formula, mixedValue, nullBucketValue, zeroBucketValue] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER], [false, true]].combinations().collectMany { storage, type, gapFill ->
+ [
+ [Formula.MIN, -6G, null, 0G],
+ [Formula.MAX, 12G, null, 0G],
+ [Formula.AVG, 3G, null, 0G],
+ [Formula.SUM, 12G, null, 0G],
+ [Formula.MEDIAN, 3G, null, 0G],
+ // COUNT retains row-count semantics, including stored nulls.
+ [Formula.COUNT, 5G, 2G, 1G]
+ ].collect { [storage, type, gapFill, *it] }
+ }
+ }
}
From 923b46d1e1a1f220550cae3d85a205efa6d58288 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 10:53:39 +0200
Subject: [PATCH 04/15] Fix null handling in boolean datapoint All queries
---
.../query/AssetDatapointAllQuery.java | 6 ++--
.../assets/AssetDatapointNullQueryTest.groovy | 30 +++++++++++++++++++
2 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
index 03c6188903..e034286592 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
@@ -29,7 +29,8 @@
description =
"Returns every stored datapoint in the inclusive range, ordered from newest to oldest. "
+ "Use this when the complete series is needed without aggregation or downsampling. Supports numeric, "
- + "boolean, text, and JSON-compatible complex attribute values.")
+ + "boolean, text, and JSON-compatible complex attribute values. "
+ + "Boolean values are returned as 1 for true, 0 for false, and null for unavailable values.")
public class AssetDatapointAllQuery extends AssetDatapointQuery {
public AssetDatapointAllQuery() {}
@@ -55,7 +56,8 @@ public String getSQLQuery(String tableName, Class attributeType) throws Illeg
+ tableName
+ conditionSuffix;
} else if (isBoolean) {
- return "select timestamp as X, (case when cast(cast(value as text) as boolean) is true then 1 else 0 end) as Y from "
+ // Preserve unavailable values instead of treating SQL NULL as false.
+ return "select timestamp as X, (case cast(cast(nullif(value, 'null'::jsonb) as text) as boolean) when true then 1 when false then 0 else null end) as Y from "
+ tableName
+ conditionSuffix;
} else {
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index d53e53d57f..418472ca38 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -97,6 +97,36 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
}
}
+ def "boolean All query preserves #description for #storage datapoints"() {
+ given: "boolean datapoints containing explicitly stored JSON nulls"
+ asset = assetStorageService.merge(new ThingAsset("Boolean null query test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", ValueType.BOOLEAN)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def timestamps = (0..
+ new ValueDatapoint<>(timestamps[index], value)
+ }
+ datapoints.add(new ValueDatapoint<>(timestamps.first() - 60_000L, true))
+ datapoints.add(new ValueDatapoint<>(timestamps.last() + 60_000L, false))
+ service.upsertValues(asset.id, "value", datapoints)
+
+ when: "the inclusive range is queried without aggregation"
+ def result = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointAllQuery(timestamps.first(), timestamps.last()))
+
+ then: "true, false, and unavailable remain distinct at their original timestamps"
+ result.collect { it.timestamp } == timestamps.reverse()
+ result.collect { it.value } == expectedValues.reverse()
+
+ where:
+ [storage, description, values, expectedValues] << ["historical", "predicted"].collectMany { storage ->
+ [
+ [storage, "mixed values", [true, false, null], [1D, 0D, null]],
+ [storage, "an entirely null range", [null, null, null], [null, null, null]]
+ ]
+ }
+ }
+
def "numeric Nearest query preserves unavailable values for #storage #valueType.name datapoints"() {
given: "an unavailable datapoint between older and newer available prices"
asset = assetStorageService.merge(new ThingAsset("Nearest null query test")
From fedca78a63cfabb294fa3b12d9d89449a27ed425 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 11:03:12 +0200
Subject: [PATCH 05/15] Fix null handling in boolean interval aggregates
---
.../query/AssetDatapointIntervalQuery.java | 18 +++++--
.../assets/AssetDatapointNullQueryTest.groovy | 50 +++++++++++++++++++
2 files changed, 63 insertions(+), 5 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index 2a4fb05732..75f2057139 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -57,7 +57,7 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
+ "value; `DIFFERENCE` calculates the change between the last values of consecutive buckets; "
+ "`COUNT` counts datapoints; `SUM` totals numeric values (or true values for booleans); `MODE` "
+ "selects the most frequent value; and `MEDIAN` selects the middle value. `DIFFERENCE` is numeric "
- + "only. Numeric MIN, AVG, MAX, SUM and MEDIAN ignore null values and return null "
+ + "only. For numeric and boolean attributes, MIN, AVG, MAX, SUM and MEDIAN ignore null values and return null "
+ "for buckets with no available values.",
requiredMode = Schema.RequiredMode.REQUIRED)
public Formula formula;
@@ -189,6 +189,9 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
}
protected String getSQLQueryForBooleans(String tableName, String function) {
+ // Keep unavailable values null so aggregates ignore them instead of counting them as false.
+ String booleanAsNumber =
+ "CASE cast(nullif(value, 'null'::jsonb) as text)::boolean WHEN true THEN 1 WHEN false THEN 0 ELSE NULL END";
switch (this.formula) {
case DIFFERENCE:
throw new IllegalStateException(
@@ -207,7 +210,9 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + "SUM(CASE WHEN cast(value as text)::boolean THEN 1 ELSE 0 END) AS true_count_sum "
+ + "SUM("
+ + booleanAsNumber
+ + ") AS true_count_sum "
+ "FROM "
+ tableName
+ " "
@@ -240,8 +245,9 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + "CASE WHEN PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY (CASE WHEN cast(value as text)::boolean THEN 1 ELSE 0 END)) >= 0.5 "
- + "THEN true ELSE false END AS median_value "
+ + "(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ("
+ + booleanAsNumber
+ + ")) >= 0.5)::integer AS median_value "
+ "FROM "
+ tableName
+ " "
@@ -252,7 +258,9 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
+ function
+ "(cast(? as interval), timestamp) AS x, "
+ this.formula.toString().toLowerCase()
- + "(case when cast(cast(value as text) as boolean) is true then 1 else 0 end) FROM "
+ + "("
+ + booleanAsNumber
+ + ") FROM "
+ tableName
+ " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? GROUP BY x ORDER by x ASC";
}
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 418472ca38..7b70b404bf 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -208,4 +208,54 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
].collect { [storage, type, gapFill, *it] }
}
}
+
+ def "boolean #formula handles unavailable values for #storage with gapFill=#gapFill"() {
+ given: "mixed, entirely null, empty, false, and true minute buckets"
+ asset = assetStorageService.merge(new ThingAsset("Boolean interval null query test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", ValueType.BOOLEAN)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()
+ service.upsertValues(asset.id, "value", [
+ [-1_000L, true],
+ [0L, true],
+ [10_000L, false],
+ [20_000L, null],
+ [30_000L, null],
+ [40_000L, null],
+ [70_000L, null],
+ [100_000L, null],
+ [190_000L, false],
+ [200_000L, null],
+ [250_000L, true],
+ [260_000L, null],
+ [300_000L, false]
+ ].collect { new ValueDatapoint<>(bucketStart + it[0], it[1]) })
+
+ when: "the five-minute range is aggregated"
+ def result = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 299_999L,
+ "1 minute", formula, gapFill))
+
+ then: "unavailable values are distinct from false and empty buckets follow the gap-fill setting"
+ result.collect { it.timestamp } == (gapFill
+ ? [0L, 60_000L, 120_000L, 180_000L, 240_000L]
+ : [0L, 60_000L, 180_000L, 240_000L]).collect { bucketStart + it }
+ result.collect { it.value } == (gapFill
+ ? [mixedValue, nullBucketValue, null, falseBucketValue, trueBucketValue]
+ : [mixedValue, nullBucketValue, falseBucketValue, trueBucketValue])
+
+ where:
+ [storage, gapFill, formula, mixedValue, nullBucketValue, falseBucketValue, trueBucketValue] << [["historical", "predicted"], [false, true]].combinations().collectMany { storage, gapFill ->
+ [
+ [Formula.MIN, 0D, null, 0D, 1D],
+ [Formula.MAX, 1D, null, 0D, 1D],
+ [Formula.AVG, 0.5D, null, 0D, 1D],
+ [Formula.SUM, 1D, null, 0D, 1D],
+ // MEDIAN retains the existing true result when available values are tied.
+ [Formula.MEDIAN, 1D, null, 0D, 1D],
+ // COUNT retains row-count semantics, including stored nulls.
+ [Formula.COUNT, 5D, 2D, 2D, 2D]
+ ].collect { [storage, gapFill, *it] }
+ }
+ }
}
From 3133168d68783cf291a621f4958a0b55bf616168 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 11:50:02 +0200
Subject: [PATCH 06/15] Fix null handling in numeric and boolean MODE queries
---
.../query/AssetDatapointIntervalQuery.java | 12 ++---
.../assets/AssetDatapointNullQueryTest.groovy | 52 +++++++++++++++++++
2 files changed, 58 insertions(+), 6 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index 75f2057139..41ebb1ec5b 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -57,7 +57,7 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
+ "value; `DIFFERENCE` calculates the change between the last values of consecutive buckets; "
+ "`COUNT` counts datapoints; `SUM` totals numeric values (or true values for booleans); `MODE` "
+ "selects the most frequent value; and `MEDIAN` selects the middle value. `DIFFERENCE` is numeric "
- + "only. For numeric and boolean attributes, MIN, AVG, MAX, SUM and MEDIAN ignore null values and return null "
+ + "only. For numeric and boolean attributes, MIN, AVG, MAX, SUM, MODE and MEDIAN ignore null values and return null "
+ "for buckets with no available values.",
requiredMode = Schema.RequiredMode.REQUIRED)
public Formula formula;
@@ -149,7 +149,7 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
+ " SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + " cast(value as numeric) AS num_value "
+ + " cast(nullif(value, 'null'::jsonb) as numeric) AS num_value "
+ " FROM "
+ tableName
+ " "
@@ -159,7 +159,7 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
+ " SELECT num_value FROM ( "
+ " SELECT num_value, COUNT(*) AS freq "
+ " FROM bucketed b2 "
- + " WHERE b2.x = d.x "
+ + " WHERE b2.x = d.x AND num_value IS NOT NULL "
+ " GROUP BY num_value "
+ " ORDER BY freq DESC, num_value "
+ " LIMIT 1 "
@@ -223,17 +223,17 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
+ " SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + " cast(value as text)::boolean AS bool_value "
+ + " cast(nullif(value, 'null'::jsonb) as text)::boolean AS bool_value "
+ " FROM "
+ tableName
+ " "
+ " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ ") "
+ "SELECT d.x, ( "
- + " SELECT bool_value FROM ( "
+ + " SELECT bool_value::integer FROM ( "
+ " SELECT bool_value, COUNT(*) AS freq "
+ " FROM bucketed b2 "
- + " WHERE b2.x = d.x "
+ + " WHERE b2.x = d.x AND bool_value IS NOT NULL "
+ " GROUP BY bool_value "
+ " ORDER BY freq DESC, bool_value DESC "
+ " LIMIT 1 "
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 7b70b404bf..0999c24e89 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -258,4 +258,56 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
].collect { [storage, gapFill, *it] }
}
}
+
+ def "MODE ignores unavailable values for #storage #valueType.name with gapFill=#gapFill"() {
+ given: "nulls outnumber available values, followed by null-only, empty, tied, zero, and high-value buckets"
+ asset = assetStorageService.merge(new ThingAsset("Mode null query test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()
+ def buckets = [
+ [low, low, high, null, null, null],
+ [null, null],
+ [],
+ [low, high, null],
+ [zero, null],
+ [high, high, low, null]
+ ]
+ def datapoints = buckets.withIndex().collectMany { values, bucket ->
+ values.withIndex().collect { value, index ->
+ new ValueDatapoint<>(bucketStart + bucket * 60_000L + index * 1_000L, value)
+ }
+ }
+ datapoints.add(new ValueDatapoint<>(bucketStart - 1_000L, high))
+ datapoints.add(new ValueDatapoint<>(bucketStart + 360_000L, low))
+ service.upsertValues(asset.id, "value", datapoints)
+
+ when: "the six-minute range is queried for its most frequent available values"
+ def result = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 359_999L,
+ "1 minute", Formula.MODE, gapFill))
+
+ then: "nulls cannot win and existing numeric and boolean tie-breaking is preserved"
+ // MODE currently omits empty buckets even with gapFill=true; fixing that is a separate step.
+ result.collect { it.timestamp } == [0L, 60_000L, 180_000L, 240_000L, 300_000L]
+ .collect { bucketStart + it }
+ result.collect { it.value } == [lowResult, null, tieResult, 0D, highResult]
+
+ when: "only the entirely unavailable bucket is queried"
+ def unavailable = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart + 60_000L, bucketStart + 119_999L,
+ "1 minute", Formula.MODE, gapFill))
+
+ then: "the bucket and its null result remain present"
+ unavailable.collect { [it.timestamp, it.value] } == [[bucketStart + 60_000L, null]]
+
+ where:
+ [storage, gapFill, valueType, low, high, zero, lowResult, highResult, tieResult] << [["historical", "predicted"], [false, true]].combinations().collectMany { storage, gapFill ->
+ [
+ [ValueType.NUMBER, -5G, 7G, 0G, -5D, 7D, -5D],
+ [ValueType.BIG_NUMBER, -5G, 7G, 0G, -5D, 7D, -5D],
+ [ValueType.BOOLEAN, false, true, false, 0D, 1D, 1D]
+ ].collect { [storage, gapFill, *it] }
+ }
+ }
}
From 70412b4d1c833498c1777098c12ca583d17754c2 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 12:16:57 +0200
Subject: [PATCH 07/15] Fix: honor gap filling in MODE datapoint queries
Replace correlated frequency queries with grouped MODE aggregates.
Preserve null handling and tie-breaking, and cover empty ranges and leading/trailing gaps with regression tests.
BREAKING CHANGE: MODE with gapFill=true now includes empty buckets as null-valued results across the inclusive query range.
Results may contain more rows and exceed the configured datapoint query limit.
Use gapFill=false to retain populated-bucket-only results.
---
.../query/AssetDatapointIntervalQuery.java | 48 +++++--------------
.../assets/AssetDatapointNullQueryTest.groovy | 34 ++++++++++---
.../assets/AssetDatapointQueryTest.groovy | 8 +++-
3 files changed, 45 insertions(+), 45 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index 41ebb1ec5b..24aa7c9314 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -58,7 +58,7 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
+ "`COUNT` counts datapoints; `SUM` totals numeric values (or true values for booleans); `MODE` "
+ "selects the most frequent value; and `MEDIAN` selects the middle value. `DIFFERENCE` is numeric "
+ "only. For numeric and boolean attributes, MIN, AVG, MAX, SUM, MODE and MEDIAN ignore null values and return null "
- + "for buckets with no available values.",
+ + "for buckets with no available values. MODE ties select the smaller numeric value or true for booleans.",
requiredMode = Schema.RequiredMode.REQUIRED)
public Formula formula;
@@ -145,28 +145,15 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
+ "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ "GROUP BY x ORDER by x ASC";
case MODE:
- return "WITH bucketed AS ( "
- + " SELECT "
+ return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + " cast(nullif(value, 'null'::jsonb) as numeric) AS num_value "
- + " FROM "
+ + "MODE() WITHIN GROUP (ORDER BY cast(nullif(value, 'null'::jsonb) as numeric) ASC) AS mode_value "
+ + "FROM "
+ tableName
+ " "
- + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
- + ") "
- + "SELECT d.x, ( "
- + " SELECT num_value FROM ( "
- + " SELECT num_value, COUNT(*) AS freq "
- + " FROM bucketed b2 "
- + " WHERE b2.x = d.x AND num_value IS NOT NULL "
- + " GROUP BY num_value "
- + " ORDER BY freq DESC, num_value "
- + " LIMIT 1 "
- + " ) AS mode_sub "
- + ") AS mode_value "
- + "FROM (SELECT DISTINCT x FROM bucketed) d "
- + "ORDER BY d.x ASC";
+ + "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ + "GROUP BY x ORDER by x ASC";
case MEDIAN:
return "SELECT "
+ function
@@ -219,28 +206,15 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
+ "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ "GROUP BY x ORDER by x ASC";
case MODE:
- return "WITH bucketed AS ( "
- + " SELECT "
+ return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + " cast(nullif(value, 'null'::jsonb) as text)::boolean AS bool_value "
- + " FROM "
+ + "MODE() WITHIN GROUP (ORDER BY cast(nullif(value, 'null'::jsonb) as text)::boolean DESC)::integer AS mode_value "
+ + "FROM "
+ tableName
+ " "
- + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
- + ") "
- + "SELECT d.x, ( "
- + " SELECT bool_value::integer FROM ( "
- + " SELECT bool_value, COUNT(*) AS freq "
- + " FROM bucketed b2 "
- + " WHERE b2.x = d.x AND bool_value IS NOT NULL "
- + " GROUP BY bool_value "
- + " ORDER BY freq DESC, bool_value DESC "
- + " LIMIT 1 "
- + " ) AS mode_sub "
- + ") AS mode_value "
- + "FROM (SELECT DISTINCT x FROM bucketed) d "
- + "ORDER BY d.x ASC";
+ + "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ + "GROUP BY x ORDER by x ASC";
case MEDIAN:
return "SELECT "
+ function
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 0999c24e89..06b9c36c03 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -278,8 +278,8 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
new ValueDatapoint<>(bucketStart + bucket * 60_000L + index * 1_000L, value)
}
}
- datapoints.add(new ValueDatapoint<>(bucketStart - 1_000L, high))
- datapoints.add(new ValueDatapoint<>(bucketStart + 360_000L, low))
+ datapoints.add(new ValueDatapoint<>(bucketStart - 61_000L, high))
+ datapoints.add(new ValueDatapoint<>(bucketStart + 420_000L, low))
service.upsertValues(asset.id, "value", datapoints)
when: "the six-minute range is queried for its most frequent available values"
@@ -288,10 +288,32 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
"1 minute", Formula.MODE, gapFill))
then: "nulls cannot win and existing numeric and boolean tie-breaking is preserved"
- // MODE currently omits empty buckets even with gapFill=true; fixing that is a separate step.
- result.collect { it.timestamp } == [0L, 60_000L, 180_000L, 240_000L, 300_000L]
- .collect { bucketStart + it }
- result.collect { it.value } == [lowResult, null, tieResult, 0D, highResult]
+ result.collect { it.timestamp } == (gapFill
+ ? [0L, 60_000L, 120_000L, 180_000L, 240_000L, 300_000L]
+ : [0L, 60_000L, 180_000L, 240_000L, 300_000L]).collect { bucketStart + it }
+ result.collect { it.value } == (gapFill
+ ? [lowResult, null, null, tieResult, 0D, highResult]
+ : [lowResult, null, tieResult, 0D, highResult])
+
+ when: "the range includes empty leading and trailing buckets"
+ def extended = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart - 60_000L, bucketStart + 419_999L,
+ "1 minute", Formula.MODE, gapFill))
+
+ then: "gap filling covers the requested range rather than just the observed timestamps"
+ extended.collect { [it.timestamp, it.value] } == (gapFill
+ ? [[bucketStart - 60_000L, null]] + result.collect {
+ [it.timestamp, it.value]
+ } + [[bucketStart + 360_000L, null]]
+ : result.collect { [it.timestamp, it.value] })
+
+ when: "only the empty bucket is queried"
+ def empty = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart + 120_000L, bucketStart + 179_999L,
+ "1 minute", Formula.MODE, gapFill))
+
+ then: "an entirely empty range also follows the gap-fill setting"
+ empty.collect { [it.timestamp, it.value] } == (gapFill ? [[bucketStart + 120_000L, null]] : [])
when: "only the entirely unavailable bucket is queried"
def unavailable = service.queryDatapoints(asset.id, "value",
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
index 79cd81f6ec..5f43f910a5 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
@@ -444,19 +444,21 @@ class AssetDatapointQueryTest extends Specification implements ManagerContainerT
)
then: "MODE datapoints should be correct"
- assert modeDatapoints1.size() == 6
+ assert modeDatapoints1.size() == 7 // Include the empty bucket at the inclusive end timestamp.
assert modeDatapoints1[0].timestamp == time30.toEpochMilli()
assert modeDatapoints1[1].timestamp == time30.plus(5, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[2].timestamp == time30.plus(10, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[3].timestamp == time30.plus(15, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[4].timestamp == time30.plus(20, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[5].timestamp == time30.plus(25, ChronoUnit.MINUTES).toEpochMilli()
+ assert modeDatapoints1[6].timestamp == timeNow.toEpochMilli()
assert modeDatapoints1[0].value == 10
assert modeDatapoints1[1].value == 15
assert modeDatapoints1[2].value == 90
assert modeDatapoints1[3].value == 20
assert modeDatapoints1[4].value == 25
assert modeDatapoints1[5].value == 30
+ assert modeDatapoints1[6].value == null
when: "requesting interval datapoints using MEDIAN"
def medianDatapoints1 = assetDatapointService.queryDatapoints(
@@ -489,9 +491,11 @@ class AssetDatapointQueryTest extends Specification implements ManagerContainerT
)
then: "MODE datapoints should return 90, because it is most common within the interval"
- assert modeDatapoints2.size() == 1
+ assert modeDatapoints2.size() == 2 // Include the empty bucket at the inclusive end timestamp.
assert modeDatapoints2[0].timestamp == time60.toEpochMilli()
assert modeDatapoints2[0].value == 90
+ assert modeDatapoints2[1].timestamp == timeNow.toEpochMilli()
+ assert modeDatapoints2[1].value == null
}
def "All query should return the correct data when the maximum is respected"() {
From 69b56f682dd6d8e3c3c4224986837d622ec7e9f5 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 12:27:35 +0200
Subject: [PATCH 08/15] Fix: preserve unavailable values in DIFFERENCE queries
Normalize JSON nulls before casting and remove the fallback that replaced unavailable differences with absolute readings.
Document carry-forward semantics and add regression coverage.
BREAKING CHANGE: With gap filling enabled, the first reading after a leading empty bucket now yields null instead of its absolute value as the difference.
---
.../query/AssetDatapointIntervalQuery.java | 10 ++-
.../assets/AssetDatapointNullQueryTest.groovy | 63 +++++++++++++++++++
2 files changed, 70 insertions(+), 3 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index 24aa7c9314..f8a6a5a55b 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -58,7 +58,11 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
+ "`COUNT` counts datapoints; `SUM` totals numeric values (or true values for booleans); `MODE` "
+ "selects the most frequent value; and `MEDIAN` selects the middle value. `DIFFERENCE` is numeric "
+ "only. For numeric and boolean attributes, MIN, AVG, MAX, SUM, MODE and MEDIAN ignore null values and return null "
- + "for buckets with no available values. MODE ties select the smaller numeric value or true for booleans.",
+ + "for buckets with no available values. MODE ties select the smaller numeric value or true for booleans. "
+ + "DIFFERENCE uses the last stored value per bucket. With gap filling enabled, empty buckets carry "
+ + "the previous bucket value, while explicit null values interrupt carry-forward. A difference is "
+ + "null if either endpoint is unavailable. The first returned bucket has a difference of zero "
+ + "if its value is available; observations outside the query range are not used.",
requiredMode = Schema.RequiredMode.REQUIRED)
public Formula formula;
@@ -118,12 +122,12 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
return "WITH interval_data AS ("
+ "SELECT "
+ function
- + "(cast(? as interval), timestamp) AS x, public.locf(public.last(value::DOUBLE PRECISION, timestamp)) AS numeric_value "
+ + "(cast(? as interval), timestamp) AS x, public.locf(public.last(nullif(value, 'null'::jsonb)::DOUBLE PRECISION, timestamp)) AS numeric_value "
+ "FROM "
+ tableName
+ " "
+ "WHERE ENTITY_ID = ? AND ATTRIBUTE_NAME = ? AND TIMESTAMP >= ? AND TIMESTAMP <= ? GROUP BY x )"
- + "SELECT x, COALESCE(numeric_value - LAG(numeric_value, 1, numeric_value) OVER (ORDER BY x), numeric_value) AS delta FROM interval_data ORDER BY x ASC";
+ + "SELECT x, numeric_value - LAG(numeric_value, 1, numeric_value) OVER (ORDER BY x) AS delta FROM interval_data ORDER BY x ASC";
case COUNT:
return "SELECT "
+ function
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 06b9c36c03..4a9131051a 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -332,4 +332,67 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
].collect { [storage, gapFill, *it] }
}
}
+
+ def "DIFFERENCE preserves unavailable values for #storage #valueType.name with gapFill=#gapFill"() {
+ given: "last values, explicit unavailability, recovery, and empty minute buckets"
+ asset = assetStorageService.merge(new ThingAsset("Difference null query test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()
+ service.upsertValues(asset.id, "value", [
+ [-61_000L, 999G],
+ [0L, 75G],
+ [10_000L, 80G],
+ [120_000L, 90G],
+ [130_000L, null],
+ [240_000L, null],
+ [250_000L, 100G],
+ [300_000L, 110G],
+ [360_000L, 0G],
+ [420_000L, -5G],
+ [540_000L, -10G],
+ [660_000L, 999G]
+ ].collect { new ValueDatapoint<>(bucketStart + it[0], it[1]) })
+
+ when: "the eleven-minute range is queried for changes between bucket values"
+ def result = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 659_999L,
+ "1 minute", Formula.DIFFERENCE, gapFill))
+
+ then: "explicit nulls interrupt differences while empty buckets carry forward when enabled"
+ result.collect { it.timestamp } == (gapFill
+ ? (0..10).toList() : [0, 2, 4, 5, 6, 7, 9]).collect { bucketStart + it * 60_000L }
+ result.collect { it.value } == (gapFill
+ ? [0D, 0D, null, null, null, 10D, -110D, -5D, 0D, -5D, 0D]
+ : [0D, null, null, 10D, -110D, -5D, -5D])
+
+ when: "the query starts with an empty bucket before the first available value"
+ def leadingGap = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart - 60_000L, bucketStart + 119_999L,
+ "1 minute", Formula.DIFFERENCE, gapFill))
+
+ then: "an unavailable predecessor does not become an assumed zero baseline"
+ leadingGap.collect { [it.timestamp, it.value] } == (gapFill
+ ? [[bucketStart - 60_000L, null], [bucketStart, null], [bucketStart + 60_000L, 0D]]
+ : [[bucketStart, 0D]])
+
+ when: "only an empty bucket is requested"
+ def empty = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart + 60_000L, bucketStart + 119_999L,
+ "1 minute", Formula.DIFFERENCE, gapFill))
+
+ then: "carry-forward does not read observations outside the query range"
+ empty.collect { [it.timestamp, it.value] } == (gapFill ? [[bucketStart + 60_000L, null]] : [])
+
+ when: "only explicitly unavailable observations are requested"
+ def unavailable = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart + 130_000L, bucketStart + 179_999L,
+ "1 minute", Formula.DIFFERENCE, gapFill))
+
+ then: "the initial difference stays unavailable"
+ unavailable.collect { [it.timestamp, it.value] } == [[bucketStart + 120_000L, null]]
+
+ where:
+ [storage, valueType, gapFill] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER], [false, true]].combinations()
+ }
}
From e38cb9f49f800a98da285548d90fea5fb51bc888 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 14:08:09 +0200
Subject: [PATCH 09/15] Fix null handling in numeric and boolean LTTB queries
---
.../query/AssetDatapointLTTBQuery.java | 10 +--
.../assets/AssetDatapointNullQueryTest.groovy | 72 +++++++++++++++++++
2 files changed, 78 insertions(+), 4 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java
index 265a1767a2..58b0fb759d 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java
@@ -30,13 +30,15 @@
"Downsamples numeric or boolean datapoints from the inclusive range with the "
+ "Largest-Triangle-Three-Buckets (LTTB) algorithm while preserving the series' visual shape. Use this "
+ "to reduce a large series for charts. Results are chronological; boolean values are processed as "
- + "1 for true and 0 for false. Other attribute value types are rejected.")
+ + "1 for true and 0 for false. Null-valued observations are omitted before downsampling, so their "
+ + "timestamps are not returned as gap markers. A range with no available values returns no points. "
+ + "Other attribute value types are rejected.")
public final class AssetDatapointLTTBQuery extends AssetDatapointQuery {
@Schema(
description =
"Requested maximum number of representative datapoints after downsampling. Fewer "
- + "points are returned when the range contains fewer datapoints.",
+ + "points are returned when the range contains fewer available datapoints.",
example = "500",
requiredMode = Schema.RequiredMode.REQUIRED)
public int amountOfPoints;
@@ -60,11 +62,11 @@ public String getSQLQuery(String tableName, Class attributeType) throws Illeg
boolean isNumber = Number.class.isAssignableFrom(attributeType);
boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);
if (isNumber) {
- return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), cast(value as double precision), ?) from "
+ return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), cast(nullif(value, 'null'::jsonb) as double precision), ?) from "
+ tableName
+ " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ?))";
} else if (isBoolean) {
- return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), (case when cast(cast(value as text) as boolean) is true then 1 else 0 end), ?) from "
+ return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), (case cast(cast(nullif(value, 'null'::jsonb) as text) as boolean) when true then 1 when false then 0 else null end), ?) from "
+ tableName
+ " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ?))";
} else {
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 4a9131051a..4a4ecb83b7 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -28,6 +28,7 @@ import org.openremote.model.datapoint.ValueDatapoint
import org.openremote.model.datapoint.query.AssetDatapointAllQuery
import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery
import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery.Formula
+import org.openremote.model.datapoint.query.AssetDatapointLTTBQuery
import org.openremote.model.datapoint.query.AssetDatapointNearestQuery
import org.openremote.model.value.ValueType
import org.openremote.setup.integration.KeycloakTestSetup
@@ -395,4 +396,75 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
where:
[storage, valueType, gapFill] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER], [false, true]].combinations()
}
+
+ def "LTTB omits unavailable values in #description for #storage #valueType.name datapoints"() {
+ given: "datapoints with unavailable observations and out-of-range neighbours"
+ asset = assetStorageService.merge(new ThingAsset("LTTB null query test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def timestamps = (0..
+ new ValueDatapoint<>(timestamps[index], value)
+ }
+ def outsideValue = valueType == ValueType.BOOLEAN ? true : 999G
+ datapoints.add(new ValueDatapoint<>(timestamps.first() - 60_000L, outsideValue))
+ datapoints.add(new ValueDatapoint<>(timestamps.last() + 60_000L, outsideValue))
+ service.upsertValues(asset.id, "value", datapoints)
+
+ when: "the requested resolution exceeds the available observations"
+ def result = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointLTTBQuery(timestamps.first(), timestamps.last(), 10))
+
+ then: "only available observations are returned, chronologically and at their original timestamps"
+ result.collect { [it.timestamp, it.value] } == expected.collect { index, value ->
+ [timestamps[index], value]
+ }
+
+ where:
+ [storage, valueType, description, values, expected] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER, ValueType.BOOLEAN]].combinations().collectMany { storage, type ->
+ def isBoolean = type == ValueType.BOOLEAN
+ [
+ [
+ storage,
+ type,
+ "a mixed range",
+ isBoolean ? [null, true, null, false, true, null, false, null] : [null, -5G, null, 0G, 90G, null, 40G, null],
+ isBoolean ? [[1, 1D], [3, 0D], [4, 1D], [6, 0D]] : [[1, -5D], [3, 0D], [4, 90D], [6, 40D]]
+ ],
+ [storage, type, "an entirely null range", [null, null, null], []],
+ [storage, type, "a single available observation", [null, isBoolean ? false : 0G, null], [[1, 0D]]]
+ ]
+ }
+ }
+
+ def "LTTB downsampling ignores unavailable observations for #storage #valueType.name datapoints"() {
+ given: "the same available series stored with and without null observations"
+ asset = assetStorageService.merge(new ThingAsset("LTTB downsampling null test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType), new Attribute<>("available", valueType)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def values = valueType == ValueType.BOOLEAN
+ ? [null, false, true, null, false, true, null, true, false, null, true, false, null]
+ : [null, 0G, 12G, null, -4G, 50G, null, 3G, 0G, null, 9G, 2G, null]
+ def datapoints = values.withIndex().collect { value, index ->
+ new ValueDatapoint<>(start + index * 60_000L, value)
+ }
+ service.upsertValues(asset.id, "value", datapoints)
+ service.upsertValues(asset.id, "available", datapoints.findAll { it.value != null })
+
+ when: "both series are reduced from eight available points to four"
+ def query = new AssetDatapointLTTBQuery(start, start + 720_000L, 4)
+ def result = service.queryDatapoints(asset.id, "value", query)
+ def reference = service.queryDatapoints(asset.id, "available", query)
+
+ then: "null observations do not affect point selection or the first and last available points"
+ result.size() == 4
+ result.collect { [it.timestamp, it.value] } == reference.collect { [it.timestamp, it.value] }
+ result.first().timestamp == start + 60_000L
+ result.last().timestamp == start + 660_000L
+ result.every { it.value != null }
+ result.collect { it.timestamp } == result.collect { it.timestamp }.sort()
+
+ where:
+ [storage, valueType] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER, ValueType.BOOLEAN]].combinations()
+ }
}
From 16758ed38f380cd79378affd65f18d7de8a27427 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 9 Oct 2026 14:15:43 +0200
Subject: [PATCH 10/15] Clarify some doc elements
---
.../model/datapoint/query/AssetDatapointAllQuery.java | 1 +
.../model/datapoint/query/AssetDatapointIntervalQuery.java | 7 +++++--
.../model/datapoint/query/AssetDatapointQuery.java | 6 ++++--
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
index e034286592..bcf6b76581 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java
@@ -30,6 +30,7 @@
"Returns every stored datapoint in the inclusive range, ordered from newest to oldest. "
+ "Use this when the complete series is needed without aggregation or downsampling. Supports numeric, "
+ "boolean, text, and JSON-compatible complex attribute values. "
+ + "Unavailable values retain their stored timestamps and are returned as null. "
+ "Boolean values are returned as 1 for true, 0 for false, and null for unavailable values.")
public class AssetDatapointAllQuery extends AssetDatapointQuery {
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index f8a6a5a55b..9f7eb8244b 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -55,10 +55,13 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
description =
"Aggregation applied per bucket: `MIN`, `AVG`, and `MAX` calculate the corresponding "
+ "value; `DIFFERENCE` calculates the change between the last values of consecutive buckets; "
- + "`COUNT` counts datapoints; `SUM` totals numeric values (or true values for booleans); `MODE` "
+ + "`COUNT` counts stored datapoints, including null-valued rows; `SUM` totals numeric values "
+ + "(or true values for booleans); `MODE` "
+ "selects the most frequent value; and `MEDIAN` selects the middle value. `DIFFERENCE` is numeric "
+ "only. For numeric and boolean attributes, MIN, AVG, MAX, SUM, MODE and MEDIAN ignore null values and return null "
- + "for buckets with no available values. MODE ties select the smaller numeric value or true for booleans. "
+ + "for buckets with no available values. Empty gap-filled buckets return null for COUNT. "
+ + "MODE ties select the smaller numeric value or true for booleans. Boolean MEDIAN returns 1 "
+ + "when the median of available 1/0 values is at least 0.5, otherwise 0; it returns null when unavailable. "
+ "DIFFERENCE uses the last stored value per bucket. With gap filling enabled, empty buckets carry "
+ "the previous bucket value, while explicit null values interrupt carry-forward. A difference is "
+ "null if either endpoint is unavailable. The first returned bucket has a difference of zero "
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java
index 1d22cf9d69..3869fdef35 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java
@@ -40,9 +40,11 @@
defaultImpl = AssetDatapointAllQuery.class)
@Schema(
description =
- "Polymorphic historical-datapoint query. Select `all` to return every value, "
+ "Polymorphic historical or predicted datapoint query. Select `all` to return every value, "
+ "`interval` to aggregate values into time buckets, `lttb` to downsample a numeric "
- + "or boolean series, or `nearest` to retrieve the last value at or before a time.",
+ + "or boolean series, or `nearest` to retrieve the last numeric value at or before a time. "
+ + "Returned datapoints use x for the timestamp and y for the value. When a query retains an "
+ + "unavailable datapoint, its JSON representation contains x but omits y.",
discriminatorProperty = "type",
discriminatorMapping = {
@DiscriminatorMapping(value = "all", schema = AssetDatapointAllQuery.class),
From b3503652f449fb6ee15338b82d6372628f07fd11 Mon Sep 17 00:00:00 2001
From: Eric Bariaux
Date: Fri, 23 Oct 2026 15:49:07 +0200
Subject: [PATCH 11/15] Fix pseudo-clock changing the JVM default time zone
---
.../container/timer/TimerService.java | 5 +--
.../container/timer/PseudoClockTest.groovy | 45 +++++++++++++++++++
.../assets/AssetDatapointNullQueryTest.groovy | 2 -
.../ResidenceNotifyAlarmTriggerTest.groovy | 14 +++---
4 files changed, 51 insertions(+), 15 deletions(-)
diff --git a/container/src/main/java/org/openremote/container/timer/TimerService.java b/container/src/main/java/org/openremote/container/timer/TimerService.java
index f84840baab..561b3e8476 100644
--- a/container/src/main/java/org/openremote/container/timer/TimerService.java
+++ b/container/src/main/java/org/openremote/container/timer/TimerService.java
@@ -22,7 +22,6 @@
import java.time.*;
import java.time.temporal.Temporal;
-import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Logger;
@@ -83,13 +82,11 @@ public void reset() {
PSEUDO {
protected AtomicLong offset = new AtomicLong();
protected volatile Long stopTime;
- // Explicitly define a UTC Clock
+ // Use UTC internally without changing the JVM's default time zone.
protected java.time.Clock utcClock = java.time.Clock.systemUTC();
@Override
public void init() {
- // Force JVM timezone to UTC for legacy date conversions to avoid DST issues in tests
- TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
long current = getCurrentTimeMillis();
LOG.info("Initialized pseudo clock to: " + current + " / " + Instant.ofEpochMilli(current));
}
diff --git a/container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovy b/container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovy
index 1bc52a8054..0d60e599a6 100644
--- a/container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovy
+++ b/container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovy
@@ -23,17 +23,62 @@ import spock.lang.Specification
import java.time.LocalDate
import java.time.LocalTime
import java.time.ZoneId
+import java.util.concurrent.TimeUnit
class PseudoClockTest extends Specification {
TimerService.Clock clock
+ TimeZone originalTimeZone
def setup() {
+ originalTimeZone = TimeZone.getDefault()
clock = TimerService.Clock.PSEUDO
+ clock.reset()
clock.init()
clock.stop()
}
+ def cleanup() {
+ try {
+ clock.reset()
+ } finally {
+ TimeZone.setDefault(originalTimeZone)
+ }
+ }
+
+ def "initializing the pseudo clock preserves the JVM time zone #zone"() {
+ given:
+ def timeZone = TimeZone.getTimeZone(zone)
+ TimeZone.setDefault(timeZone)
+
+ when:
+ clock.init()
+
+ then:
+ TimeZone.getDefault() == timeZone
+
+ where:
+ zone << ["Europe/Brussels", "America/New_York", "Asia/Kathmandu"]
+ }
+
+ def "setTime respects the #hours hour day on #date in Europe/Brussels"() {
+ given:
+ def zone = ZoneId.of("Europe/Brussels")
+ def startDate = LocalDate.parse(date)
+ def start = clock.setTime(startDate, LocalTime.MIDNIGHT, zone)
+
+ when:
+ def end = clock.setTime(startDate.plusDays(1), LocalTime.MIDNIGHT, zone)
+
+ then:
+ end - start == TimeUnit.HOURS.toMillis(hours)
+
+ where:
+ date | hours
+ "2026-03-29" | 23
+ "2026-10-25" | 25
+ }
+
def "setTime sets #date at midnight in #zone to #expected milliseconds"() {
expect:
clock.setTime(LocalDate.parse(date), LocalTime.MIDNIGHT, ZoneId.of(zone)) == expected
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 4a4ecb83b7..5013034ad4 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -48,8 +48,6 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
long start
def setup() {
- // Match the pseudo clock's UTC zone before JDBC connections capture the JVM time zone.
- TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
def container = startContainer(defaultConfig(), defaultServices())
stopPseudoClock()
assetStorageService = container.getService(AssetStorageService.class)
diff --git a/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy b/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy
index e3cd2b459c..9d3e156c6e 100644
--- a/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy
@@ -54,9 +54,6 @@ import spock.lang.Specification
import spock.util.concurrent.PollingConditions
import java.time.Instant
-import java.time.LocalDateTime
-import java.time.ZoneId
-import java.time.temporal.ChronoField
import java.time.temporal.ChronoUnit
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.TimeUnit
@@ -70,13 +67,12 @@ import static org.openremote.model.util.ValueUtil.parse
class ResidenceNotifyAlarmTriggerTest extends Specification implements ManagerContainerTrait {
def getOccurrenceFromTo(long baseTime, int occurrence) {
- def tomorrowMidnight = Instant.ofEpochMilli(baseTime).truncatedTo(ChronoUnit.DAYS).plus(1, ChronoUnit.DAYS).atZone(ZoneId.systemDefault())
- def from = LocalDateTime.of(tomorrowMidnight.get(ChronoField.YEAR), tomorrowMidnight.get(ChronoField.MONTH_OF_YEAR), tomorrowMidnight.get(ChronoField.DAY_OF_MONTH), 6, 0, 0)
- from = from.plus(occurrence, ChronoUnit.WEEKS)
- def to = from.plus(2, ChronoUnit.HOURS)
+ // CalendarEvent evaluates recurrences in UTC, so expected windows must also stay in UTC across DST.
+ def from = Instant.ofEpochMilli(baseTime).truncatedTo(ChronoUnit.DAYS)
+ .plus(1 + occurrence * 7L, ChronoUnit.DAYS).plus(6, ChronoUnit.HOURS)
return [
- from.atZone(ZoneId.systemDefault()).toDate(),
- to.atZone(ZoneId.systemDefault()).toDate()
+ Date.from(from),
+ Date.from(from.plus(2, ChronoUnit.HOURS))
]
}
From 5a8735fd46bce063cffa28ec1a679f19561e712f Mon Sep 17 00:00:00 2001
From: Eric Bariaux
Date: Fri, 23 Oct 2026 16:07:39 +0200
Subject: [PATCH 12/15] Separate MODE gap filling from the JSON-null fixes
Restore MODE's existing populated-bucket-only behavior for both gapFill
settings, retaining JSON-null handling and numeric/boolean tie-breaking.
Restore the corresponding expectations and document the MODE limitation.
The gap-fill behavior change and its regression coverage will be carried
by bugfix/mode-query-gap-fill as a separate follow-up.
---
.../query/AssetDatapointIntervalQuery.java | 49 ++++++++++++++-----
.../assets/AssetDatapointNullQueryTest.groovy | 34 +++----------
.../assets/AssetDatapointQueryTest.groovy | 8 +--
3 files changed, 46 insertions(+), 45 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index 9f7eb8244b..d5864a20ae 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -47,7 +47,8 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
description =
"When true, returns empty buckets across the requested range using time-bucket gap "
+ "filling; their aggregate is generally null. When false, only buckets containing datapoints are "
- + "returned. Defaults to false.",
+ + "returned. MODE currently returns only buckets containing datapoints regardless of this flag. "
+ + "Defaults to false.",
example = "true")
public boolean gapFill;
@@ -152,15 +153,28 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
+ "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ "GROUP BY x ORDER by x ASC";
case MODE:
- return "SELECT "
+ return "WITH bucketed AS ( "
+ + " SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + "MODE() WITHIN GROUP (ORDER BY cast(nullif(value, 'null'::jsonb) as numeric) ASC) AS mode_value "
- + "FROM "
+ + " cast(nullif(value, 'null'::jsonb) as numeric) AS num_value "
+ + " FROM "
+ tableName
+ " "
- + "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
- + "GROUP BY x ORDER by x ASC";
+ + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ + ") "
+ + "SELECT d.x, ( "
+ + " SELECT num_value FROM ( "
+ + " SELECT num_value, COUNT(*) AS freq "
+ + " FROM bucketed b2 "
+ + " WHERE b2.x = d.x AND num_value IS NOT NULL "
+ + " GROUP BY num_value "
+ + " ORDER BY freq DESC, num_value "
+ + " LIMIT 1 "
+ + " ) AS mode_sub "
+ + ") AS mode_value "
+ + "FROM (SELECT DISTINCT x FROM bucketed) d "
+ + "ORDER BY d.x ASC";
case MEDIAN:
return "SELECT "
+ function
@@ -213,15 +227,28 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
+ "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ "GROUP BY x ORDER by x ASC";
case MODE:
- return "SELECT "
+ return "WITH bucketed AS ( "
+ + " SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + "MODE() WITHIN GROUP (ORDER BY cast(nullif(value, 'null'::jsonb) as text)::boolean DESC)::integer AS mode_value "
- + "FROM "
+ + " cast(nullif(value, 'null'::jsonb) as text)::boolean AS bool_value "
+ + " FROM "
+ tableName
+ " "
- + "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
- + "GROUP BY x ORDER by x ASC";
+ + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ + ") "
+ + "SELECT d.x, ( "
+ + " SELECT bool_value::integer FROM ( "
+ + " SELECT bool_value, COUNT(*) AS freq "
+ + " FROM bucketed b2 "
+ + " WHERE b2.x = d.x AND bool_value IS NOT NULL "
+ + " GROUP BY bool_value "
+ + " ORDER BY freq DESC, bool_value DESC "
+ + " LIMIT 1 "
+ + " ) AS mode_sub "
+ + ") AS mode_value "
+ + "FROM (SELECT DISTINCT x FROM bucketed) d "
+ + "ORDER BY d.x ASC";
case MEDIAN:
return "SELECT "
+ function
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index 5013034ad4..dfeacc9d84 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -277,8 +277,8 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
new ValueDatapoint<>(bucketStart + bucket * 60_000L + index * 1_000L, value)
}
}
- datapoints.add(new ValueDatapoint<>(bucketStart - 61_000L, high))
- datapoints.add(new ValueDatapoint<>(bucketStart + 420_000L, low))
+ datapoints.add(new ValueDatapoint<>(bucketStart - 1_000L, high))
+ datapoints.add(new ValueDatapoint<>(bucketStart + 360_000L, low))
service.upsertValues(asset.id, "value", datapoints)
when: "the six-minute range is queried for its most frequent available values"
@@ -287,32 +287,10 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
"1 minute", Formula.MODE, gapFill))
then: "nulls cannot win and existing numeric and boolean tie-breaking is preserved"
- result.collect { it.timestamp } == (gapFill
- ? [0L, 60_000L, 120_000L, 180_000L, 240_000L, 300_000L]
- : [0L, 60_000L, 180_000L, 240_000L, 300_000L]).collect { bucketStart + it }
- result.collect { it.value } == (gapFill
- ? [lowResult, null, null, tieResult, 0D, highResult]
- : [lowResult, null, tieResult, 0D, highResult])
-
- when: "the range includes empty leading and trailing buckets"
- def extended = service.queryDatapoints(asset.id, "value",
- new AssetDatapointIntervalQuery(bucketStart - 60_000L, bucketStart + 419_999L,
- "1 minute", Formula.MODE, gapFill))
-
- then: "gap filling covers the requested range rather than just the observed timestamps"
- extended.collect { [it.timestamp, it.value] } == (gapFill
- ? [[bucketStart - 60_000L, null]] + result.collect {
- [it.timestamp, it.value]
- } + [[bucketStart + 360_000L, null]]
- : result.collect { [it.timestamp, it.value] })
-
- when: "only the empty bucket is queried"
- def empty = service.queryDatapoints(asset.id, "value",
- new AssetDatapointIntervalQuery(bucketStart + 120_000L, bucketStart + 179_999L,
- "1 minute", Formula.MODE, gapFill))
-
- then: "an entirely empty range also follows the gap-fill setting"
- empty.collect { [it.timestamp, it.value] } == (gapFill ? [[bucketStart + 120_000L, null]] : [])
+ // Preserve existing MODE behavior: empty buckets are omitted for either gapFill setting.
+ result.collect { it.timestamp } == [0L, 60_000L, 180_000L, 240_000L, 300_000L]
+ .collect { bucketStart + it }
+ result.collect { it.value } == [lowResult, null, tieResult, 0D, highResult]
when: "only the entirely unavailable bucket is queried"
def unavailable = service.queryDatapoints(asset.id, "value",
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
index 5f43f910a5..79cd81f6ec 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
@@ -444,21 +444,19 @@ class AssetDatapointQueryTest extends Specification implements ManagerContainerT
)
then: "MODE datapoints should be correct"
- assert modeDatapoints1.size() == 7 // Include the empty bucket at the inclusive end timestamp.
+ assert modeDatapoints1.size() == 6
assert modeDatapoints1[0].timestamp == time30.toEpochMilli()
assert modeDatapoints1[1].timestamp == time30.plus(5, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[2].timestamp == time30.plus(10, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[3].timestamp == time30.plus(15, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[4].timestamp == time30.plus(20, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[5].timestamp == time30.plus(25, ChronoUnit.MINUTES).toEpochMilli()
- assert modeDatapoints1[6].timestamp == timeNow.toEpochMilli()
assert modeDatapoints1[0].value == 10
assert modeDatapoints1[1].value == 15
assert modeDatapoints1[2].value == 90
assert modeDatapoints1[3].value == 20
assert modeDatapoints1[4].value == 25
assert modeDatapoints1[5].value == 30
- assert modeDatapoints1[6].value == null
when: "requesting interval datapoints using MEDIAN"
def medianDatapoints1 = assetDatapointService.queryDatapoints(
@@ -491,11 +489,9 @@ class AssetDatapointQueryTest extends Specification implements ManagerContainerT
)
then: "MODE datapoints should return 90, because it is most common within the interval"
- assert modeDatapoints2.size() == 2 // Include the empty bucket at the inclusive end timestamp.
+ assert modeDatapoints2.size() == 1
assert modeDatapoints2[0].timestamp == time60.toEpochMilli()
assert modeDatapoints2[0].value == 90
- assert modeDatapoints2[1].timestamp == timeNow.toEpochMilli()
- assert modeDatapoints2[1].value == null
}
def "All query should return the correct data when the maximum is respected"() {
From 05dc7c4159af4ff12671c12655f0566503747b44 Mon Sep 17 00:00:00 2001
From: Eric Bariaux
Date: Fri, 23 Oct 2026 16:16:22 +0200
Subject: [PATCH 13/15] Fix MODE queries to include empty buckets when gapFill
is enabled
Use grouped MODE aggregates so time_bucket_gapfill can produce empty
buckets across the inclusive requested range. Preserve JSON-null handling,
numeric and boolean tie-breaking, and populated-only results when disabled.
Migrate the internal, leading/trailing, entirely empty range and inclusive
end-boundary assertions from the JSON-null PR. Add coverage for sparse
queries exceeding the configured result limit after gap filling.
Release note / compatibility:
MODE with gapFill=true now returns null-valued empty buckets. Existing
callers may receive more results and exceed OR_DATA_POINTS_QUERY_LIMIT
(100,000 by default). Use gapFill=false to preserve populated-bucket-only
results, or reduce the range or increase the interval to request fewer
buckets. This applies to historical and predicted numeric/boolean data.
---
.../query/AssetDatapointIntervalQuery.java | 49 +++----------
.../assets/AssetDatapointNullQueryTest.groovy | 72 +++++++++++++++++--
.../assets/AssetDatapointQueryTest.groovy | 8 ++-
3 files changed, 83 insertions(+), 46 deletions(-)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index d5864a20ae..0a2fa2b4f4 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -47,8 +47,7 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
description =
"When true, returns empty buckets across the requested range using time-bucket gap "
+ "filling; their aggregate is generally null. When false, only buckets containing datapoints are "
- + "returned. MODE currently returns only buckets containing datapoints regardless of this flag. "
- + "Defaults to false.",
+ + "returned. Empty buckets count toward the datapoint query limit. Defaults to false.",
example = "true")
public boolean gapFill;
@@ -153,28 +152,15 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
+ "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ "GROUP BY x ORDER by x ASC";
case MODE:
- return "WITH bucketed AS ( "
- + " SELECT "
+ return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + " cast(nullif(value, 'null'::jsonb) as numeric) AS num_value "
- + " FROM "
+ + "MODE() WITHIN GROUP (ORDER BY cast(nullif(value, 'null'::jsonb) as numeric) ASC) AS mode_value "
+ + "FROM "
+ tableName
+ " "
- + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
- + ") "
- + "SELECT d.x, ( "
- + " SELECT num_value FROM ( "
- + " SELECT num_value, COUNT(*) AS freq "
- + " FROM bucketed b2 "
- + " WHERE b2.x = d.x AND num_value IS NOT NULL "
- + " GROUP BY num_value "
- + " ORDER BY freq DESC, num_value "
- + " LIMIT 1 "
- + " ) AS mode_sub "
- + ") AS mode_value "
- + "FROM (SELECT DISTINCT x FROM bucketed) d "
- + "ORDER BY d.x ASC";
+ + "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ + "GROUP BY x ORDER by x ASC";
case MEDIAN:
return "SELECT "
+ function
@@ -227,28 +213,15 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
+ "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ "GROUP BY x ORDER by x ASC";
case MODE:
- return "WITH bucketed AS ( "
- + " SELECT "
+ return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
- + " cast(nullif(value, 'null'::jsonb) as text)::boolean AS bool_value "
- + " FROM "
+ + "MODE() WITHIN GROUP (ORDER BY cast(nullif(value, 'null'::jsonb) as text)::boolean DESC)::integer AS mode_value "
+ + "FROM "
+ tableName
+ " "
- + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
- + ") "
- + "SELECT d.x, ( "
- + " SELECT bool_value::integer FROM ( "
- + " SELECT bool_value, COUNT(*) AS freq "
- + " FROM bucketed b2 "
- + " WHERE b2.x = d.x AND bool_value IS NOT NULL "
- + " GROUP BY bool_value "
- + " ORDER BY freq DESC, bool_value DESC "
- + " LIMIT 1 "
- + " ) AS mode_sub "
- + ") AS mode_value "
- + "FROM (SELECT DISTINCT x FROM bucketed) d "
- + "ORDER BY d.x ASC";
+ + "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ + "GROUP BY x ORDER by x ASC";
case MEDIAN:
return "SELECT "
+ function
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
index dfeacc9d84..164e4f86ee 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy
@@ -24,6 +24,7 @@ import org.openremote.manager.datapoint.AssetPredictedDatapointService
import org.openremote.manager.setup.SetupService
import org.openremote.model.asset.impl.ThingAsset
import org.openremote.model.attribute.Attribute
+import org.openremote.model.datapoint.DatapointQueryTooLargeException
import org.openremote.model.datapoint.ValueDatapoint
import org.openremote.model.datapoint.query.AssetDatapointAllQuery
import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery
@@ -277,8 +278,8 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
new ValueDatapoint<>(bucketStart + bucket * 60_000L + index * 1_000L, value)
}
}
- datapoints.add(new ValueDatapoint<>(bucketStart - 1_000L, high))
- datapoints.add(new ValueDatapoint<>(bucketStart + 360_000L, low))
+ datapoints.add(new ValueDatapoint<>(bucketStart - 61_000L, high))
+ datapoints.add(new ValueDatapoint<>(bucketStart + 420_000L, low))
service.upsertValues(asset.id, "value", datapoints)
when: "the six-minute range is queried for its most frequent available values"
@@ -287,10 +288,32 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
"1 minute", Formula.MODE, gapFill))
then: "nulls cannot win and existing numeric and boolean tie-breaking is preserved"
- // Preserve existing MODE behavior: empty buckets are omitted for either gapFill setting.
- result.collect { it.timestamp } == [0L, 60_000L, 180_000L, 240_000L, 300_000L]
- .collect { bucketStart + it }
- result.collect { it.value } == [lowResult, null, tieResult, 0D, highResult]
+ result.collect { it.timestamp } == (gapFill
+ ? [0L, 60_000L, 120_000L, 180_000L, 240_000L, 300_000L]
+ : [0L, 60_000L, 180_000L, 240_000L, 300_000L]).collect { bucketStart + it }
+ result.collect { it.value } == (gapFill
+ ? [lowResult, null, null, tieResult, 0D, highResult]
+ : [lowResult, null, tieResult, 0D, highResult])
+
+ when: "the range includes empty leading and trailing buckets"
+ def extended = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart - 60_000L, bucketStart + 419_999L,
+ "1 minute", Formula.MODE, gapFill))
+
+ then: "gap filling covers the requested range rather than just the observed timestamps"
+ extended.collect { [it.timestamp, it.value] } == (gapFill
+ ? [[bucketStart - 60_000L, null]] + result.collect {
+ [it.timestamp, it.value]
+ } + [[bucketStart + 360_000L, null]]
+ : result.collect { [it.timestamp, it.value] })
+
+ when: "only the empty bucket is queried"
+ def empty = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart + 120_000L, bucketStart + 179_999L,
+ "1 minute", Formula.MODE, gapFill))
+
+ then: "an entirely empty range also follows the gap-fill setting"
+ empty.collect { [it.timestamp, it.value] } == (gapFill ? [[bucketStart + 120_000L, null]] : [])
when: "only the entirely unavailable bucket is queried"
def unavailable = service.queryDatapoints(asset.id, "value",
@@ -310,6 +333,43 @@ class AssetDatapointNullQueryTest extends Specification implements ManagerContai
}
}
+ def "MODE counts gap-filled buckets toward the query limit for #storage #valueType.name"() {
+ given: "a sparse range with more buckets than the query limit"
+ asset = assetStorageService.merge(new ThingAsset("Mode query limit test")
+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))
+ def service = storage == "historical" ? historicalService : predictedService
+ def originalLimit = service.maxAmountOfQueryPoints
+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()
+ def value = valueType == ValueType.BOOLEAN ? true : 1G
+ service.upsertValues(asset.id, "value", [
+ new ValueDatapoint<>(bucketStart, value),
+ new ValueDatapoint<>(bucketStart + 180_000L, value)
+ ])
+ service.maxAmountOfQueryPoints = 2
+
+ when: "only populated buckets are requested"
+ def result = service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 239_999L,
+ "1 minute", Formula.MODE, false))
+
+ then: "the two populated buckets fit within the limit"
+ result.collect { it.timestamp } == [bucketStart, bucketStart + 180_000L]
+
+ when: "gap filling increases the result to four buckets"
+ service.queryDatapoints(asset.id, "value",
+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 239_999L,
+ "1 minute", Formula.MODE, true))
+
+ then: "the generated empty buckets also count toward the limit"
+ thrown(DatapointQueryTooLargeException)
+
+ cleanup:
+ service.maxAmountOfQueryPoints = originalLimit
+
+ where:
+ [storage, valueType] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER, ValueType.BOOLEAN]].combinations()
+ }
+
def "DIFFERENCE preserves unavailable values for #storage #valueType.name with gapFill=#gapFill"() {
given: "last values, explicit unavailability, recovery, and empty minute buckets"
asset = assetStorageService.merge(new ThingAsset("Difference null query test")
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
index 79cd81f6ec..5f43f910a5 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy
@@ -444,19 +444,21 @@ class AssetDatapointQueryTest extends Specification implements ManagerContainerT
)
then: "MODE datapoints should be correct"
- assert modeDatapoints1.size() == 6
+ assert modeDatapoints1.size() == 7 // Include the empty bucket at the inclusive end timestamp.
assert modeDatapoints1[0].timestamp == time30.toEpochMilli()
assert modeDatapoints1[1].timestamp == time30.plus(5, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[2].timestamp == time30.plus(10, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[3].timestamp == time30.plus(15, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[4].timestamp == time30.plus(20, ChronoUnit.MINUTES).toEpochMilli()
assert modeDatapoints1[5].timestamp == time30.plus(25, ChronoUnit.MINUTES).toEpochMilli()
+ assert modeDatapoints1[6].timestamp == timeNow.toEpochMilli()
assert modeDatapoints1[0].value == 10
assert modeDatapoints1[1].value == 15
assert modeDatapoints1[2].value == 90
assert modeDatapoints1[3].value == 20
assert modeDatapoints1[4].value == 25
assert modeDatapoints1[5].value == 30
+ assert modeDatapoints1[6].value == null
when: "requesting interval datapoints using MEDIAN"
def medianDatapoints1 = assetDatapointService.queryDatapoints(
@@ -489,9 +491,11 @@ class AssetDatapointQueryTest extends Specification implements ManagerContainerT
)
then: "MODE datapoints should return 90, because it is most common within the interval"
- assert modeDatapoints2.size() == 1
+ assert modeDatapoints2.size() == 2 // Include the empty bucket at the inclusive end timestamp.
assert modeDatapoints2[0].timestamp == time60.toEpochMilli()
assert modeDatapoints2[0].value == 90
+ assert modeDatapoints2[1].timestamp == timeNow.toEpochMilli()
+ assert modeDatapoints2[1].value == null
}
def "All query should return the correct data when the maximum is respected"() {
From 707bd7946278a31fb7424ad3818f0c1a3f089ac1 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Fri, 23 Oct 2026 17:05:19 +0200
Subject: [PATCH 14/15] Document PostgreSQL MODE tie-breaking dependency
---
.../model/datapoint/query/AssetDatapointIntervalQuery.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
index 0a2fa2b4f4..0843c285ee 100644
--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java
@@ -152,6 +152,7 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
+ "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ "GROUP BY x ORDER by x ASC";
case MODE:
+ // Relies on PostgreSQL MODE retaining the first tied value in sort order (ASC: smaller).
return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
@@ -213,6 +214,7 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
+ "WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
+ "GROUP BY x ORDER by x ASC";
case MODE:
+ // Relies on PostgreSQL MODE retaining the first tied value in sort order (DESC: true).
return "SELECT "
+ function
+ "(cast(? as interval), timestamp) AS x, "
From 9c56c422f968743207ae8fc4434eab760f1c8e82 Mon Sep 17 00:00:00 2001
From: Eric Bariaux <375613+ebariaux@users.noreply.github.com>
Date: Mon, 26 Oct 2026 13:25:17 +0100
Subject: [PATCH 15/15] Linting
---
.../rules/residence/ResidenceNotifyAlarmTriggerTest.groovy | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy b/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy
index 9d3e156c6e..e79561aeb8 100644
--- a/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy
@@ -70,10 +70,7 @@ class ResidenceNotifyAlarmTriggerTest extends Specification implements ManagerCo
// CalendarEvent evaluates recurrences in UTC, so expected windows must also stay in UTC across DST.
def from = Instant.ofEpochMilli(baseTime).truncatedTo(ChronoUnit.DAYS)
.plus(1 + occurrence * 7L, ChronoUnit.DAYS).plus(6, ChronoUnit.HOURS)
- return [
- Date.from(from),
- Date.from(from.plus(2, ChronoUnit.HOURS))
- ]
+ return [Date.from(from), Date.from(from.plus(2, ChronoUnit.HOURS))]
}
def "Trigger notification when presence is detected and alarm enabled"() {