diff --git a/pom.xml b/pom.xml
index 87840b0..b6e9d81 100644
--- a/pom.xml
+++ b/pom.xml
@@ -8,7 +8,7 @@
jflat
- 1.1.01-SNAPSHOT
+ 1.2.00-SNAPSHOT
JFlat Utility
JFlat Utility for Java
diff --git a/src/main/java/org/metricshub/jflat/JFlat.java b/src/main/java/org/metricshub/jflat/JFlat.java
index f2da0a9..7457be1 100644
--- a/src/main/java/org/metricshub/jflat/JFlat.java
+++ b/src/main/java/org/metricshub/jflat/JFlat.java
@@ -25,8 +25,11 @@
import java.io.StringReader;
import java.text.ParseException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
@@ -40,10 +43,18 @@
import javax.json.JsonStructure;
import javax.json.JsonValue;
import javax.json.stream.JsonLocation;
+import javax.json.stream.JsonParser;
+import javax.json.stream.JsonParser.Event;
import javax.json.stream.JsonParsingException;
/**
* Provides tools to convert a JSON-formated content to a flat structure, exported as a String.
+ *
+ * In event parsing mode (see {@link #JFlat(String, boolean)}), {@link #toCSV(String, String[], String)} reads the
+ * document again and flattens the value of the first element of the entry key one array element at a time,
+ * instead of keeping a map of the whole document: the memory used is bounded by the size of one element. The
+ * result is the same as in the default mode; the documents that the event parsing cannot process exactly (see
+ * {@link #toCSV(String, String[], String)}) are processed as in the default mode.
* @author Bertrand Martin
*
*/
@@ -55,20 +66,57 @@ public class JFlat {
private Reader inputReader;
private boolean parsed = false;
+ // Event parsing mode: the source is kept to be read again, and the map of the whole document is only built when
+ // needed (getFlatTree(), or a document that the event parsing cannot process exactly)
+ private boolean eventParsing;
+ private String source;
+ private boolean removeNodes;
+ private boolean fullMap;
+
/**
* Create a new JFlat instance
*
* @param pJsonReader A Reader object (can be StringReader, FileReader, etc.)
*/
public JFlat(Reader pJsonReader) {
+ this(pJsonReader, false);
+ }
+
+ /**
+ * Create a new JFlat instance
+ *
+ * @param pJsonReader A Reader object (can be StringReader, FileReader, etc.), read entirely by parse() in
+ * event parsing mode
+ * @param pEventParsing Whether toCSV() reads the document by events, one element at a time, instead of keeping a
+ * map of all its nodes
+ */
+ public JFlat(Reader pJsonReader, boolean pEventParsing) {
inputReader = pJsonReader;
+ eventParsing = pEventParsing;
}
/**
* @param pJsonSource JSON source to be parsed
*/
public JFlat(String pJsonSource) {
- this(pJsonSource == null ? new StringReader("") : new StringReader(pJsonSource));
+ this(pJsonSource, false);
+ }
+
+ /**
+ * @param pJsonSource JSON source to be parsed
+ * @param pEventParsing Whether toCSV() reads the document by events, one element at a time, instead of keeping a
+ * map of all its nodes
+ */
+ public JFlat(String pJsonSource, boolean pEventParsing) {
+ this(pJsonSource == null ? new StringReader("") : new StringReader(pJsonSource), pEventParsing);
+ source = pJsonSource == null ? "" : pJsonSource;
+ }
+
+ /**
+ * Container of the map of one unit of a document read by events
+ */
+ private JFlat() {
+ this((Reader) null, false);
}
/**
@@ -87,7 +135,8 @@ public void parse() throws ParseException, IOException, IllegalStateException {
/**
* Parse the JSON document
*
- * This call is mandatory before doing any other operation.
+ * This call is mandatory before doing any other operation. In event parsing mode, the document is only checked:
+ * nothing is kept but the source.
*
* @param removeNodes Whether to remove "artificial" nodes without values ({object} and {array})
*
@@ -96,6 +145,21 @@ public void parse() throws ParseException, IOException, IllegalStateException {
* @throws IllegalStateException when... actually never in a single-thread context
*/
public void parse(boolean removeNodes) throws ParseException, IOException, IllegalStateException {
+ if (eventParsing) {
+ this.removeNodes = removeNodes;
+ if (source == null) {
+ source = readAll(inputReader);
+ }
+ if (isEventParsable(source)) {
+ parsed = true;
+ return;
+ }
+
+ // Not a document the event parsing mode can read: parse it as usual, which throws the same exceptions
+ inputReader = new StringReader(source);
+ fullMap = true;
+ }
+
// Read the JSON source
JsonReader reader = Json.createReader(inputReader);
JsonStructure root;
@@ -132,6 +196,74 @@ public void parse(boolean removeNodes) throws ParseException, IOException, Illeg
parsed = true;
}
+ /**
+ * Read the whole content of a reader, and close it (even when it cannot be read)
+ *
+ * @param reader The reader
+ * @return The content
+ * @throws IOException when the reader cannot be read
+ */
+ private static String readAll(Reader reader) throws IOException {
+ try (Reader input = reader) {
+ StringBuilder content = new StringBuilder();
+ char[] buffer = new char[8192];
+ int length;
+ while ((length = input.read(buffer)) != -1) {
+ content.append(buffer, 0, length);
+ }
+ return content.toString();
+ }
+ }
+
+ /**
+ * Check, with the event parser, that the document is an object or an array that the default mode can read
+ * (what follows the root value is ignored, as by the default mode)
+ *
+ * @param json The JSON document
+ * @return true when the document can be read by events
+ */
+ private static boolean isEventParsable(String json) {
+ JsonParser parser = Json.createParser(new StringReader(json));
+ try {
+ Event event = parser.next();
+ if (event != Event.START_OBJECT && event != Event.START_ARRAY) {
+ return false;
+ }
+ int depth = 1;
+ while (depth > 0) {
+ event = parser.next();
+ if (event == Event.START_OBJECT || event == Event.START_ARRAY) {
+ depth++;
+ } else if (event == Event.END_OBJECT || event == Event.END_ARRAY) {
+ depth--;
+ } else if (event == Event.VALUE_NUMBER) {
+ // The default mode converts every number, which fails on an out-of-range exponent
+ parser.getValue();
+ }
+ }
+ return true;
+ } catch (RuntimeException e) {
+ return false;
+ } finally {
+ parser.close();
+ }
+ }
+
+ /**
+ * Build the map of the whole document of an event parsing instance
+ */
+ private void buildFullMap() {
+ eventParsing = false;
+ inputReader = new StringReader(source);
+ try {
+ parse(removeNodes);
+ } catch (ParseException | IOException e) {
+ // The document was checked by parse()
+ throw new IllegalStateException("JSON document cannot be parsed again", e);
+ }
+ fullMap = true;
+ }
+
/**
* Navigate the JSON tree and populate the hash map that will contain pairs of keys/value in the form below:
* obj1.propA = value
@@ -234,6 +366,11 @@ public StringBuilder getFlatTree(String valueSeparator, String replaceEndOfLines
throw new IllegalStateException("JSON document has not been parsed");
}
+ // The flat tree is the map of the whole document
+ if (eventParsing && !fullMap) {
+ buildFullMap();
+ }
+
// Use a StringBuilder to hold the result
StringBuilder result = new StringBuilder();
@@ -279,6 +416,16 @@ public StringBuilder getFlatTree() {
/**
* Translates (flattens) a JSON structure into a CSV string
+ *
+ * In event parsing mode, the document is read again and the value of the first element of the entry key
+ * (items in /items/status/conditions) is flattened one array element at a time (or
+ * as a single unit when it is not a non-empty array), skipping the nodes that neither the entry key nor the
+ * properties can reach. The document is processed as in the default mode when the event parsing cannot give the
+ * exact result:
+ * root array, entry key /, first element of the entry key that is a wildcard or contains an index,
+ * first element of the entry key present more than once in the root object (whatever the case), root key that
+ * contains / or [, property that refers to a node above the unit
+ * (../kind from /items), or a duplicate key in an object of a unit.
*
* @param csvEntryKey The key in the JSON data that will be shown as a new entry in the resulting CSV (i.e. a new line)
* @param csvProperties Array of strings specifying the properties of the entry key to be added to the CSV as new fields
@@ -324,6 +471,15 @@ public StringBuilder toCSV(String csvEntryKey, String[] csvProperties, String se
separator = ";";
}
+ // Event parsing mode: process the document one unit at a time, unless the event parsing cannot give the exact result
+ if (eventParsing && !fullMap) {
+ try {
+ return eventParsingToCSV(csvEntryKey, csvProperties, separator);
+ } catch (DefaultModeRequiredException e) {
+ buildFullMap();
+ }
+ }
+
// Initialize the StringBuilder to hold the result
StringBuilder csvResult = new StringBuilder();
@@ -377,7 +533,35 @@ public StringBuilder toCSV(String csvEntryKey, String[] csvProperties, String se
}
// Now, process each element, as described above
- for (String pathElement : pathElementArray) {
+ entries = expandEntries(entries, pathElementArray, 0);
+
+ // And now, build the CSV
+ try {
+ appendRows(entries, csvProperties, separator, null, csvResult);
+ } catch (DefaultModeRequiredException e) {
+ // Only raised for a unit of a document read by events
+ throw new IllegalStateException(e);
+ }
+
+ // Return
+ return csvResult;
+ }
+
+ /**
+ * Expand the entries with the elements of the entry key path
+ *
+ * @param entries The entries at the start
+ * @param pathElementArray The elements of the entry key path
+ * @param start The index of the first element to process
+ * @return The expanded entries
+ */
+ private ArrayList expandEntries(ArrayList entries, String[] pathElementArray, int start) {
+ int arrayLength;
+
+ // Now, process each element, as described above
+ for (int e = start; e < pathElementArray.length; e++) {
+ String pathElement = pathElementArray[e];
+
// Empty pathElement? Skip.
if (pathElement == null || pathElement.isEmpty()) {
continue;
@@ -523,6 +707,26 @@ public StringBuilder toCSV(String csvEntryKey, String[] csvProperties, String se
entries = newEntries;
}
+ return entries;
+ }
+
+ /**
+ * Append the CSV rows of the entries
+ *
+ * @param entries The entries
+ * @param csvProperties The properties of each entry
+ * @param separator The separator between fields
+ * @param unitPath The path of the unit read by events that holds the entries, null for the map of the whole document
+ * @param csvResult Where the rows are appended
+ * @throws DefaultModeRequiredException when a property refers to a node above the unit
+ */
+ private void appendRows(
+ List entries,
+ String[] csvProperties,
+ String separator,
+ String unitPath,
+ StringBuilder csvResult
+ ) throws DefaultModeRequiredException {
// And now, build the CSV
for (String entry : entries) {
// Check that the entry actually exists (in case, the user has put an invalid entryKey)
@@ -554,6 +758,10 @@ public StringBuilder toCSV(String csvEntryKey, String[] csvProperties, String se
while (path.contains("/../")) {
int pos2 = path.indexOf("/../");
int pos1 = path.lastIndexOf("/", pos2 - 1);
+ // A unit read by events only holds its own nodes
+ if (unitPath != null && pos1 < unitPath.length()) {
+ throw new DefaultModeRequiredException();
+ }
path = path.substring(0, pos1) + path.substring(pos2 + 3);
}
@@ -570,8 +778,346 @@ public StringBuilder toCSV(String csvEntryKey, String[] csvProperties, String se
// End of line, new record!
csvResult.append("\n");
}
+ }
+
+ /**
+ * Raised when a document must be processed as in the default mode
+ */
+ private static final class DefaultModeRequiredException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ DefaultModeRequiredException() {
+ super(null, null, false, false);
+ }
+ }
+
+ /**
+ * Translates the source into a CSV string in event parsing mode
+ *
+ * @param csvEntryKey The entry key (as specified)
+ * @param csvProperties The cleaned properties
+ * @param separator The separator between fields
+ * @return The CSV string
+ * @throws DefaultModeRequiredException when the event parsing cannot give the exact result
+ */
+ private StringBuilder eventParsingToCSV(String csvEntryKey, String[] csvProperties, String separator)
+ throws DefaultModeRequiredException {
+ // Same entry key as the default mode
+ String entryKey = csvEntryKey.isEmpty() ? "/" : csvEntryKey;
+ if (!entryKey.startsWith("/")) {
+ entryKey = "/" + entryKey;
+ }
+ String[] pathElementArray = entryKey.split("/");
+
+ // The first element selects the value of the root object that is read by events
+ int first = 0;
+ while (first < pathElementArray.length && pathElementArray[first].isEmpty()) {
+ first++;
+ }
+ if (first == pathElementArray.length) {
+ throw new DefaultModeRequiredException();
+ }
+ String firstElement = pathElementArray[first];
+ if ("*".equals(firstElement) || firstElement.indexOf('[') >= 0 || firstElement.indexOf(']') >= 0) {
+ throw new DefaultModeRequiredException();
+ }
+ if ("\\*".equals(firstElement)) {
+ firstElement = "*";
+ }
+
+ // What the units must keep, null to keep everything
+ PathNode needed = neededPaths(pathElementArray, first + 1, csvProperties);
+
+ StringBuilder csvResult = new StringBuilder();
+ JsonParser parser = Json.createParser(new StringReader(source));
+ try {
+ if (parser.next() != Event.START_OBJECT) {
+ throw new DefaultModeRequiredException();
+ }
+
+ boolean found = false;
+ Event event;
+ while ((event = parser.next()) != Event.END_OBJECT) {
+ String key = parser.getString();
+ if (key.indexOf('/') >= 0 || key.indexOf('[') >= 0) {
+ throw new DefaultModeRequiredException();
+ }
+ event = parser.next();
+ if (!key.equalsIgnoreCase(firstElement)) {
+ skipValue(parser, event);
+ continue;
+ }
+
+ // The default mode merges the keys that differ only by case and keeps the last duplicate
+ if (found) {
+ throw new DefaultModeRequiredException();
+ }
+ found = true;
+
+ String documentPath = "/" + key;
+ String entryPath = "/" + firstElement;
+ if (event == Event.START_ARRAY) {
+ event = parser.next();
+ if (event == Event.END_ARRAY) {
+ // An empty array is its own unit, as in the default mode
+ JFlat unit = new JFlat();
+ if (!removeNodes) {
+ unit.map.put(documentPath, "{array}");
+ }
+ unit.arrayPaths.add(documentPath);
+ unit.arrayLengths.add(0);
+ unit.appendUnitRows(entryPath, entryPath, pathElementArray, first, csvProperties, separator, csvResult);
+ continue;
+ }
+
+ // One unit per element of the array
+ int index = 0;
+ while (event != Event.END_ARRAY) {
+ JFlat unit = new JFlat();
+ unit.removeNodes = removeNodes;
+ unit.flatten(parser, event, documentPath + "[" + index + "]", needed);
+ // The parent array, for the wildcard that follows an array expansion
+ unit.arrayPaths.add(documentPath);
+ unit.arrayLengths.add(0);
+ String unitEntry = entryPath + "[" + index + "]";
+ unit.appendUnitRows(unitEntry, unitEntry, pathElementArray, first, csvProperties, separator, csvResult);
+ index++;
+ event = parser.next();
+ }
+ } else {
+ // Any other value is a single unit
+ JFlat unit = new JFlat();
+ unit.removeNodes = removeNodes;
+ unit.flatten(parser, event, documentPath, needed);
+ unit.appendUnitRows(entryPath, entryPath, pathElementArray, first, csvProperties, separator, csvResult);
+ }
+ }
+ } finally {
+ parser.close();
+ }
- // Return
return csvResult;
}
+
+ /**
+ * Append the rows of a unit
+ *
+ * @param unitEntry The entry of the unit, as built from the entry key
+ * @param unitPath The path of the unit
+ * @param pathElementArray The elements of the entry key path
+ * @param first The index of the element that selected the unit
+ * @param csvProperties The cleaned properties
+ * @param separator The separator between fields
+ * @param csvResult Where the rows are appended
+ * @throws DefaultModeRequiredException when a property refers to a node above the unit
+ */
+ private void appendUnitRows(
+ String unitEntry,
+ String unitPath,
+ String[] pathElementArray,
+ int first,
+ String[] csvProperties,
+ String separator,
+ StringBuilder csvResult
+ ) throws DefaultModeRequiredException {
+ ArrayList entries = new ArrayList<>();
+ entries.add(unitEntry);
+ entries = expandEntries(entries, pathElementArray, first + 1);
+ appendRows(entries, csvProperties, separator, unitPath, csvResult);
+ }
+
+ /**
+ * Skip the value that starts with the specified event
+ *
+ * @param parser The parser
+ * @param event The first event of the value
+ */
+ private static void skipValue(JsonParser parser, Event event) {
+ if (event == Event.START_OBJECT) {
+ parser.skipObject();
+ } else if (event == Event.START_ARRAY) {
+ parser.skipArray();
+ }
+ }
+
+ /**
+ * Node of the tree of the paths that a unit must keep, relative to the unit, case-insensitive, without array
+ * indexes: a node of the unit is kept when its path is a prefix of a needed path
+ */
+ private static final class PathNode {
+
+ private final Map children = new HashMap<>();
+ }
+
+ /**
+ * Build the tree of the paths that the units must keep for the entry key and the properties
+ *
+ * @param pathElementArray The elements of the entry key path
+ * @param start The index of the first element below the unit
+ * @param csvProperties The cleaned properties
+ * @return The root of the tree, or null when every node must be kept
+ */
+ private static PathNode neededPaths(String[] pathElementArray, int start, String[] csvProperties) {
+ StringBuilder entry = new StringBuilder();
+ for (int e = start; e < pathElementArray.length; e++) {
+ String pathElement = pathElementArray[e];
+ if (pathElement.isEmpty()) {
+ continue;
+ }
+ // A wildcard expands the children of the map: keep everything
+ if ("*".equals(pathElement)) {
+ return null;
+ }
+ entry.append('/').append("\\*".equals(pathElement) ? "*" : pathElement);
+ }
+
+ PathNode root = new PathNode();
+ if (!addNeededPath(root, entry.toString())) {
+ return null;
+ }
+ for (String property : csvProperties) {
+ String path = property.equals(".") ? entry.toString() : entry + "/" + property;
+ // Same resolution of ../ as for the rows; a reference above the unit is detected by appendRows()
+ while (path.contains("/../")) {
+ int pos2 = path.indexOf("/../");
+ int pos1 = path.lastIndexOf("/", pos2 - 1);
+ if (pos1 < 0) {
+ return null;
+ }
+ path = path.substring(0, pos1) + path.substring(pos2 + 3);
+ }
+ if (!addNeededPath(root, path)) {
+ return null;
+ }
+ }
+ return root;
+ }
+
+ /**
+ * Add a path, relative to the unit, to the tree of the needed paths
+ *
+ * @param root The root of the tree
+ * @param path The path, starting with "/" (or empty for the unit itself)
+ * @return false when the path cannot be compared safely (non-ASCII characters)
+ */
+ private static boolean addNeededPath(PathNode root, String path) {
+ PathNode node = root;
+ String[] segments = path.split("/", -1);
+ // The first segment is the empty string before the leading "/"
+ for (int s = 1; s < segments.length; s++) {
+ String name = indexFree(segments[s]);
+ if (name == null) {
+ return false;
+ }
+ PathNode child = node.children.get(name);
+ if (child == null) {
+ child = new PathNode();
+ node.children.put(name, child);
+ }
+ node = child;
+ }
+ return true;
+ }
+
+ /**
+ * Lower-cased path segment without its trailing array indexes
+ *
+ * @param segment The segment
+ * @return The normalized segment, null when it contains non-ASCII characters
+ */
+ private static String indexFree(String segment) {
+ for (int i = 0; i < segment.length(); i++) {
+ if (segment.charAt(i) > 127) {
+ return null;
+ }
+ }
+ String name = segment;
+ while (name.endsWith("]") && name.lastIndexOf('[') >= 0) {
+ String index = name.substring(name.lastIndexOf('[') + 1, name.length() - 1);
+ if (index.isEmpty() || !index.chars().allMatch(Character::isDigit)) {
+ break;
+ }
+ name = name.substring(0, name.lastIndexOf('['));
+ }
+ return name.toLowerCase(Locale.ROOT);
+ }
+
+ /**
+ * Flatten the value that starts with the specified event into the map of this unit, as navigateTree() does
+ * for a value of a parsed document
+ *
+ * @param parser The parser, positioned on the first event of the value
+ * @param event The first event of the value
+ * @param path The path of the value
+ * @param needed The node of the needed paths that matches the value, null to keep everything
+ * @throws DefaultModeRequiredException when an object has a duplicate key
+ */
+ private void flatten(JsonParser parser, Event event, String path, PathNode needed)
+ throws DefaultModeRequiredException {
+ switch (event) {
+ case START_OBJECT:
+ if (!removeNodes) {
+ map.put(path, "{object}");
+ }
+ Set keys = new HashSet<>();
+ Event child;
+ while ((child = parser.next()) != Event.END_OBJECT) {
+ String key = parser.getString();
+ // The default mode keeps the last value of a duplicate key
+ if (!keys.add(key)) {
+ throw new DefaultModeRequiredException();
+ }
+ child = parser.next();
+ PathNode childNeeded = null;
+ if (needed != null) {
+ // A key with a "/" or an index, or with non-ASCII characters, is kept entirely
+ String name = key.indexOf('/') >= 0 || key.indexOf('[') >= 0 || key.indexOf(']') >= 0
+ ? null
+ : indexFree(key);
+ if (name != null) {
+ childNeeded = needed.children.get(name);
+ if (childNeeded == null) {
+ skipValue(parser, child);
+ continue;
+ }
+ }
+ }
+ flatten(parser, child, path + "/" + key, childNeeded);
+ }
+ break;
+ case START_ARRAY:
+ if (!removeNodes) {
+ map.put(path, "{array}");
+ }
+ int i = 0;
+ Event element;
+ while ((element = parser.next()) != Event.END_ARRAY) {
+ // The elements of an array have the path of the array, without index
+ flatten(parser, element, path + "[" + i + "]", needed);
+ i++;
+ }
+ arrayPaths.add(path);
+ arrayLengths.add(i);
+ break;
+ case VALUE_STRING:
+ map.put(path, parser.getString());
+ break;
+ case VALUE_NUMBER:
+ // Same JsonNumber, hence the same string, as in the parsed document
+ map.put(path, parser.getValue().toString());
+ break;
+ case VALUE_TRUE:
+ map.put(path, JsonValue.ValueType.TRUE.toString());
+ break;
+ case VALUE_FALSE:
+ map.put(path, JsonValue.ValueType.FALSE.toString());
+ break;
+ case VALUE_NULL:
+ map.put(path, JsonValue.ValueType.NULL.toString());
+ break;
+ default:
+ break;
+ }
+ }
}
diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md
index 85a278f..98fe979 100644
--- a/src/site/markdown/index.md
+++ b/src/site/markdown/index.md
@@ -142,4 +142,27 @@ If you have a JSON property literally named `*`, escape it with a backslash:
```Java
// Refers to the literal property "*" under members, not a wildcard
jFlat.toCSV("/members/\\*", new String[] { "name" }, ";");
-```
\ No newline at end of file
+```
+
+# Event Parsing Mode
+
+By default, `parse()` keeps a map of every node of the document, which uses about 20 times the size of the document. For large lists (a REST API that returns thousands of items, for example), create the instance in event parsing mode:
+
+```Java
+JFlat jFlat = new JFlat(json, true);
+jFlat.parse();
+System.out.print(jFlat.toCSV("/items/status/conditions", new String[] { "type", "../../metadata/name" }, ";"));
+```
+
+In event parsing mode, the document is read with the event parser of `javax.json.stream` instead of being loaded as a tree. `parse()` only checks the document. `toCSV()` reads it again and flattens the value of the first element of the entry key (`items` above) one array element at a time, skipping the nodes that neither the entry key nor the properties can reach. The memory used is the document plus one element, and the result is the same as in the default mode, rows in the same order.
+
+The document itself is still kept in memory as a whole (a `Reader` is read entirely by `parse()`), and the CSV is still returned as a whole: the event parsing only changes how the document is read.
+
+The documents that this mode cannot process exactly are processed as in the default mode, with the same memory usage:
+
+* the document is an array, the entry key is `/`, or its first element is `*` or contains an index (`[0]`)
+* the first element of the entry key is present more than once in the root object (whatever the case), or a root key contains `/` or `[`
+* a property refers to a node above the array element (`../kind` from `/items`)
+* an object of an element has a duplicate key
+
+`getFlatTree()` builds the map of the whole document, as in the default mode.
diff --git a/src/test/java/org/metricshub/jflat/JFlatTest.java b/src/test/java/org/metricshub/jflat/JFlatTest.java
index 3b022a0..dc5e1ae 100644
--- a/src/test/java/org/metricshub/jflat/JFlatTest.java
+++ b/src/test/java/org/metricshub/jflat/JFlatTest.java
@@ -2,52 +2,70 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.lang.reflect.Field;
import java.text.ParseException;
+import java.util.Random;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
public class JFlatTest {
- @Test
- void flatMap() throws IllegalStateException, ParseException, IOException {
+ @ParameterizedTest
+ @ValueSource(booleans = { false, true })
+ void flatMap(boolean eventParsing) throws IllegalStateException, ParseException, IOException {
JFlat jFlat;
- jFlat = new JFlat(getResourceAsString("/simple.json"));
+ jFlat = new JFlat(getResourceAsString("/simple.json"), eventParsing);
jFlat.parse();
assertEquals(getResourceAsString("/simple-flatMap.txt"), jFlat.getFlatTree().toString());
- jFlat = new JFlat(getResourceAsString("/simple.json"));
+ jFlat = new JFlat(getResourceAsString("/simple.json"), eventParsing);
jFlat.parse(true);
assertEquals(getResourceAsString("/simple-flatMap-removeNodes.txt"), jFlat.getFlatTree().toString());
- jFlat = new JFlat(getResourceAsString("/complex.json"));
+ jFlat = new JFlat(getResourceAsString("/complex.json"), eventParsing);
jFlat.parse();
assertEquals(getResourceAsString("/complex-flatMap.txt"), jFlat.getFlatTree().toString());
- jFlat = new JFlat(getResourceAsString("/large.json"));
+ jFlat = new JFlat(getResourceAsString("/large.json"), eventParsing);
jFlat.parse();
assertEquals(getResourceAsString("/large-flatMap.txt"), jFlat.getFlatTree().toString());
- jFlat = new JFlat(getResourceAsString("/object-keys.json"));
+ jFlat = new JFlat(getResourceAsString("/object-keys.json"), eventParsing);
+ jFlat.parse();
+ assertEquals(getResourceAsString("/object-keys-flatMap.txt"), jFlat.getFlatTree().toString());
+
+ jFlat = new JFlat(new StringReader(getResourceAsString("/object-keys.json")), eventParsing);
jFlat.parse();
assertEquals(getResourceAsString("/object-keys-flatMap.txt"), jFlat.getFlatTree().toString());
}
- @Test
- void edgeCases() throws IllegalStateException, ParseException, IOException {
+ @ParameterizedTest
+ @ValueSource(booleans = { false, true })
+ void edgeCases(boolean eventParsing) throws IllegalStateException, ParseException, IOException {
// parse() not done
- JFlat simple = new JFlat(getResourceAsString("/simple.json"));
+ JFlat simple = new JFlat(getResourceAsString("/simple.json"), eventParsing);
assertThrows(
IllegalStateException.class,
() -> simple.getFlatTree(),
"Non-parsed JSON document should throw an IllegalStateException"
);
+ assertThrows(
+ IllegalStateException.class,
+ () -> simple.toCSV("/", null, null),
+ "Non-parsed JSON document should throw an IllegalStateException"
+ );
// empty JSON
- JFlat empty = new JFlat("");
+ JFlat empty = new JFlat("", eventParsing);
assertThrows(
ParseException.class,
() -> empty.parse(),
@@ -55,17 +73,35 @@ void edgeCases() throws IllegalStateException, ParseException, IOException {
);
// syntax error
- JFlat wrong = new JFlat("{ this: is a wrong JSON document");
+ JFlat wrong = new JFlat("{ this: is a wrong JSON document", eventParsing);
assertThrows(
ParseException.class,
() -> wrong.parse(),
"JSON document with syntax error should trigger a ParseException error"
);
+
+ // reader that fails: IOException, and the reader is closed
+ boolean[] closed = new boolean[1];
+ Reader failing = new Reader() {
+ @Override
+ public int read(char[] buffer, int offset, int length) throws IOException {
+ throw new IOException("Read failure");
+ }
+
+ @Override
+ public void close() {
+ closed[0] = true;
+ }
+ };
+ JFlat broken = new JFlat(failing, eventParsing);
+ assertThrows(IOException.class, () -> broken.parse(), "A reader that fails should trigger an IOException");
+ assertTrue(closed[0], "The reader should be closed");
}
- @Test
- void csv() throws IllegalStateException, ParseException, IOException {
- JFlat simple = new JFlat(getResourceAsString("/simple.json"));
+ @ParameterizedTest
+ @ValueSource(booleans = { false, true })
+ void csv(boolean eventParsing) throws IllegalStateException, ParseException, IOException {
+ JFlat simple = new JFlat(getResourceAsString("/simple.json"), eventParsing);
simple.parse();
assertEquals("[0];\n[1];\n", simple.toCSV("/", null, null).toString());
assertEquals("[0]/attribute1;\n[1]/attribute1;\n", simple.toCSV("/attribute1", null, null).toString());
@@ -85,9 +121,10 @@ void csv() throws IllegalStateException, ParseException, IOException {
assertEquals("", simple.toCSV("/nonexistent", null, null).toString());
}
- @Test
- void csvProperties() throws IllegalStateException, ParseException, IOException {
- JFlat simple = new JFlat(getResourceAsString("/simple.json"));
+ @ParameterizedTest
+ @ValueSource(booleans = { false, true })
+ void csvProperties(boolean eventParsing) throws IllegalStateException, ParseException, IOException {
+ JFlat simple = new JFlat(getResourceAsString("/simple.json"), eventParsing);
simple.parse();
assertEquals("[0];{object};\n[1];{object};\n", simple.toCSV("/", new String[] { "." }, null).toString());
assertEquals("[0];{array};\n[1];{array};\n", simple.toCSV("/", new String[] { "arrayA" }, null).toString());
@@ -111,9 +148,10 @@ void csvProperties() throws IllegalStateException, ParseException, IOException {
assertEquals("[0] {object} \n[1] {object} \n", simple.toCSV("/", new String[] { "." }, " ").toString());
}
- @Test
- void csvWildcard() throws IllegalStateException, ParseException, IOException {
- JFlat nodeDrives = new JFlat(getResourceAsString("/object-keys.json"));
+ @ParameterizedTest
+ @ValueSource(booleans = { false, true })
+ void csvWildcard(boolean eventParsing) throws IllegalStateException, ParseException, IOException {
+ JFlat nodeDrives = new JFlat(getResourceAsString("/object-keys.json"), eventParsing);
nodeDrives.parse();
// Wildcard to list all drive entries
@@ -146,9 +184,10 @@ void csvWildcard() throws IllegalStateException, ParseException, IOException {
);
}
- @Test
- void csvWildcardEscape() throws IllegalStateException, ParseException, IOException {
- JFlat jFlat = new JFlat(getResourceAsString("/wildcard-key.json"));
+ @ParameterizedTest
+ @ValueSource(booleans = { false, true })
+ void csvWildcardEscape(boolean eventParsing) throws IllegalStateException, ParseException, IOException {
+ JFlat jFlat = new JFlat(getResourceAsString("/wildcard-key.json"), eventParsing);
jFlat.parse();
// Wildcard "*" expands all children of "items"
@@ -161,6 +200,424 @@ void csvWildcardEscape() throws IllegalStateException, ParseException, IOExcepti
assertEquals("/items/*;1;star;\n", jFlat.toCSV("/items/\\*", new String[] { "id", "name" }, ";").toString());
}
+ @ParameterizedTest
+ @ValueSource(booleans = { false, true })
+ void csvList(boolean eventParsing) throws IllegalStateException, ParseException, IOException {
+ String list =
+ "{\"kind\":\"PodList\",\"items\":[" +
+ "{\"metadata\":{\"name\":\"a\",\"labels\":{\"app\":\"x\"}},\"status\":{\"conditions\":[" +
+ "{\"type\":\"Ready\",\"status\":\"True\"},{\"type\":\"Initialized\",\"status\":\"True\"}]}}," +
+ "{\"metadata\":{\"name\":\"b\"},\"status\":{\"conditions\":[]}}," +
+ "{\"metadata\":{\"name\":\"c\"},\"status\":{\"phase\":null,\"restarts\":1.50e1}}" +
+ "]}";
+ JFlat jFlat = new JFlat(list, eventParsing);
+ jFlat.parse();
+
+ // One row per element, in the order of the array
+ assertEquals(
+ "/items[0];a;x;\n/items[1];b;;\n/items[2];c;;\n",
+ jFlat.toCSV("/items", new String[] { "metadata/name", "metadata/labels/app" }, ";").toString()
+ );
+
+ // Entry key below the elements, with properties of the element
+ assertEquals(
+ "/items[0]/status/conditions[0];Ready;a;\n/items[0]/status/conditions[1];Initialized;a;\n" +
+ "/items[1]/status/conditions;;b;\n",
+ jFlat.toCSV("/items/status/conditions", new String[] { "type", "../../metadata/name" }, ";").toString()
+ );
+
+ // Markers, null and numbers as the default mode writes them
+ assertEquals(
+ "/items[0]/status;{object};;;\n/items[1]/status;{object};;;\n/items[2]/status;{object};NULL;15.0;\n",
+ jFlat.toCSV("items/status", new String[] { ".", "phase", "restarts" }, ";").toString()
+ );
+
+ // A property above the element
+ assertEquals(
+ "/items[0];a;PodList;\n/items[1];b;PodList;\n/items[2];c;PodList;\n",
+ jFlat.toCSV("/items", new String[] { "metadata/name", "../kind" }, ";").toString()
+ );
+
+ // Case-insensitive entry key and properties, the IDs are written as in the document
+ assertEquals(
+ "/items[0];a;\n/items[1];b;\n/items[2];c;\n",
+ jFlat.toCSV("/ITEMS", new String[] { "METADATA/NAME" }, ";").toString()
+ );
+ }
+
+ /**
+ * The documents that the event parsing mode must process as the default mode does
+ */
+ @Test
+ void eventParsingEdgeCases() {
+ // Key "" in an element
+ assertEventParsingMatchesDefault("{\"items\":[{\"\":{\"name\":\"e\"},\"name\":\"n\"}]}", "/items", "spec/..//name");
+ // Root key that contains "/" or "[", root key present twice
+ assertEventParsingMatchesDefault("{\"items\":{\"x\":{\"a\":1}},\"items/x\":{\"a\":2}}", "/items/x", "a");
+ assertEventParsingMatchesDefault("{\"items\":[{\"a\":1}],\"items[0]\":{\"a\":2}}", "/items", "a");
+ assertEventParsingMatchesDefault("{\"items\":[{\"a\":1}],\"Items\":[{\"a\":2}]}", "/items", "a");
+ // Duplicate key in an element
+ assertEventParsingMatchesDefault(
+ "{\"items\":[{\"a\":{\"b\":1},\"c\":3,\"a\":{\"d\":2}}]}",
+ "/items",
+ "a/b",
+ "a/d",
+ "c"
+ );
+ // Properties above the element and within the element
+ assertEventParsingMatchesDefault("{\"kind\":\"K\",\"items\":[{\"a\":1}]}", "/items", "a", "../kind");
+ assertEventParsingMatchesDefault(
+ "{\"items\":[{\"name\":\"a\",\"spec\":{\"c\":[{\"x\":1},{\"x\":2}]}}]}",
+ "/items/spec/c",
+ "x",
+ "../../name",
+ "../../../../x"
+ );
+ // Wildcard below the value, below an element (where the default mode passes the entry through)
+ assertEventParsingMatchesDefault(
+ "{\"items\":{\"spec\":{\"x\":{\"name\":\"a\"},\"y\":{\"name\":\"b\"}}}}",
+ "/items/spec/*",
+ "name"
+ );
+ assertEventParsingMatchesDefault(
+ "{\"items\":[{\"spec\":{\"x\":{\"name\":\"a\"},\"y\":{\"name\":\"b\"}}},[1,[2]]]}",
+ "/items/spec/*",
+ "name"
+ );
+ assertEventParsingMatchesDefault("{\"items\":[[1,2],[3],{\"a\":[4]}]}", "/items/*", ".");
+ assertEventParsingMatchesDefault("{\"items\":[[1,2],[3],{\"a\":[4]}]}", "/items/*/a", ".");
+ // Empty list, scalar, null, escaped wildcard
+ assertEventParsingMatchesDefault("{\"items\":[]}", "/items", ".");
+ assertEventParsingMatchesDefault("{\"items\":[]}", "/items/*", ".");
+ assertEventParsingMatchesDefault("{\"items\":\"x\"}", "/items", ".");
+ assertEventParsingMatchesDefault("{\"items\":null}", "items", ".");
+ assertEventParsingMatchesDefault("{\"*\":[{\"a\":1}],\"b\":2}", "/\\*", "a", "../b");
+ // Keys equal whatever the case, non-ASCII keys
+ assertEventParsingMatchesDefault(
+ "{\"items\":[{\"Name\":\"a\",\"name\":\"b\",\"K\":1,\"k\":2}]}",
+ "/ITEMS",
+ "NAME",
+ "k"
+ );
+ assertEventParsingMatchesDefault("{\"items\":[{\"été\":{\"a\":1},\"b\":2}]}", "/items", "ÉTÉ/a", "b");
+ // Numbers, trailing content, number out of range
+ assertEventParsingMatchesDefault(
+ "{\"items\":[{\"a\":1.50e1,\"b\":-0,\"c\":1E+2}]} trailing",
+ "/items",
+ "a",
+ "b",
+ "c"
+ );
+ assertEventParsingMatchesDefault("{\"items\":[{\"a\":1}],\"b\":1e99999999999}", "/items", "a");
+ }
+
+ /**
+ * Assert that the event parsing mode gives the same result as the default mode, with and without the nodes
+ * without value
+ *
+ * @param json The document
+ * @param entryKey The entry key
+ * @param properties The properties
+ */
+ private static void assertEventParsingMatchesDefault(String json, String entryKey, String... properties) {
+ boolean[] byEvents = new boolean[1];
+ for (boolean removeNodes : new boolean[] { false, true }) {
+ assertEquals(
+ run(new JFlat(json), removeNodes, entryKey, properties.clone(), ";", byEvents),
+ run(new JFlat(json, true), removeNodes, entryKey, properties.clone(), ";", byEvents),
+ json
+ );
+ }
+ }
+
+ /**
+ * The event parsing mode gives the same result as the default mode, or throws the same exception, on generated
+ * documents, entry keys and properties
+ */
+ @Test
+ void eventParsingMatchesDefault() {
+ Random random = new Random(20261008L);
+ int byEvents = 0;
+ int calls = 0;
+ for (int d = 0; d < 4000; d++) {
+ String json = randomDocument(random);
+ for (int k = 0; k < 6; k++) {
+ String entryKey = randomEntryKey(random);
+ String[] properties = randomProperties(random);
+ String separator = SEPARATORS[random.nextInt(SEPARATORS.length)];
+ boolean removeNodes = random.nextBoolean();
+
+ boolean[] byEventsCall = new boolean[1];
+ String expected = run(new JFlat(json), removeNodes, entryKey, properties.clone(), separator, byEventsCall);
+ String actual = run(new JFlat(json, true), removeNodes, entryKey, properties.clone(), separator, byEventsCall);
+ assertEquals(
+ expected,
+ actual,
+ "Document: " + json + "\nEntry key: " + entryKey + "\nProperties: " + String.join(", ", properties)
+ );
+
+ calls++;
+ if (byEventsCall[0]) {
+ byEvents++;
+ }
+ }
+ }
+
+ // Most calls must be read by events, not processed as in the default mode
+ assertTrue(byEvents > calls / 2, byEvents + " calls read by events out of " + calls);
+ }
+
+ /**
+ * Parse and convert a document twice, then dump the flat tree
+ *
+ * @param byEvents Set to whether the conversions were read by events (without the map of the whole document)
+ * @return The CSV and the flat tree, or the exception
+ */
+ private static String run(
+ JFlat jFlat,
+ boolean removeNodes,
+ String entryKey,
+ String[] properties,
+ String separator,
+ boolean[] byEvents
+ ) {
+ StringBuilder result = new StringBuilder();
+ byEvents[0] = false;
+ try {
+ jFlat.parse(removeNodes);
+ result.append(jFlat.toCSV(entryKey, properties, separator));
+ result.append(jFlat.toCSV(entryKey, properties, separator));
+ Field fullMap = JFlat.class.getDeclaredField("fullMap");
+ fullMap.setAccessible(true);
+ byEvents[0] = !(Boolean) fullMap.get(jFlat);
+ result.append("--\n").append(jFlat.getFlatTree());
+ } catch (Exception e) {
+ result.append(e.getClass().getName()).append(": ").append(e.getMessage());
+ }
+ return result.toString();
+ }
+
+ private static final String[] SEPARATORS = { null, ";", "," };
+
+ private static final String[] KEYS = {
+ "items",
+ "items",
+ "items",
+ "Items",
+ "metadata",
+ "name",
+ "Name",
+ "kind",
+ "spec",
+ "status",
+ "conditions",
+ "containers",
+ "id",
+ "*",
+ "",
+ "..",
+ ".",
+ "a/b",
+ "x[0]",
+ "x]",
+ "items/name",
+ "items[0]",
+ "été",
+ "K",
+ "K"
+ };
+
+ private static final String[] ROOT_KEYS = { "kind", "apiVersion", "metadata", "status" };
+
+ private static final String[] NUMBERS = {
+ "0",
+ "-0",
+ "1",
+ "-12",
+ "1.50",
+ "1e3",
+ "1E-2",
+ "2.0e+10",
+ "12345678901234567890",
+ "0.000"
+ };
+
+ private static final String[] STRINGS = {
+ "",
+ "a",
+ "B",
+ "true",
+ "{object}",
+ "x;y",
+ "line\\nbreak",
+ "\\u00e9",
+ "\\\"q\\\""
+ };
+
+ private static final String[] ENTRY_ELEMENTS = {
+ "items",
+ "items",
+ "items",
+ "Items",
+ "metadata",
+ "name",
+ "spec",
+ "status",
+ "conditions",
+ "containers",
+ "containers[0]",
+ "x[0]",
+ "*",
+ "\\*",
+ "..",
+ "",
+ "kind",
+ "été"
+ };
+
+ private static final String[] PROPERTIES = {
+ ".",
+ "name",
+ "NAME",
+ "id",
+ "kind",
+ "../kind",
+ "../../kind",
+ "../name",
+ "../../name",
+ "../../../x",
+ "spec/name",
+ "./name",
+ "/name",
+ "//name",
+ "nonexistent",
+ "a/b",
+ "*",
+ "..",
+ "",
+ "./",
+ "spec/../name",
+ "spec/..//name",
+ "containers[0]/name",
+ "conditions[1]",
+ "x[0]",
+ "metadata/name",
+ "status/conditions",
+ "été",
+ "k"
+ };
+
+ private static String randomDocument(Random random) {
+ switch (random.nextInt(20)) {
+ case 0:
+ return randomArray(random, 1);
+ case 1:
+ return randomValue(random, 4);
+ case 2:
+ return "{\"items\":[" + randomObject(random, 2) + "," + randomObject(random, 2);
+ default:
+ StringBuilder json = new StringBuilder("{");
+ int keys = random.nextInt(5);
+ for (int i = 0; i < keys; i++) {
+ json.append(randomRootKey(random)).append(":").append(randomValue(random, 1)).append(",");
+ }
+ json.append("\"items\":");
+ json.append(random.nextInt(4) == 0 ? randomValue(random, 1) : randomList(random));
+ keys = random.nextInt(3);
+ for (int i = 0; i < keys; i++) {
+ json.append(",").append(randomRootKey(random)).append(":").append(randomValue(random, 1));
+ }
+ return json.append('}').append(random.nextInt(10) == 0 ? " trailing" : "").toString();
+ }
+ }
+
+ private static String randomList(Random random) {
+ StringBuilder json = new StringBuilder("[");
+ int length = random.nextInt(5);
+ for (int i = 0; i < length; i++) {
+ if (i > 0) {
+ json.append(',');
+ }
+ json.append(random.nextInt(8) == 0 ? randomValue(random, 2) : randomObject(random, 2));
+ }
+ return json.append(']').toString();
+ }
+
+ private static String randomKey(Random random) {
+ return "\"" + KEYS[random.nextInt(KEYS.length)] + "\"";
+ }
+
+ private static String randomRootKey(Random random) {
+ return random.nextInt(6) == 0 ? randomKey(random) : "\"" + ROOT_KEYS[random.nextInt(ROOT_KEYS.length)] + "\"";
+ }
+
+ private static String randomValue(Random random, int depth) {
+ switch (random.nextInt(depth > 4 ? 6 : 9)) {
+ case 0:
+ return "\"" + STRINGS[random.nextInt(STRINGS.length)] + "\"";
+ case 1:
+ case 2:
+ return NUMBERS[random.nextInt(NUMBERS.length)];
+ case 3:
+ return "true";
+ case 4:
+ return "false";
+ case 5:
+ return "null";
+ case 6:
+ case 7:
+ return randomObject(random, depth + 1);
+ default:
+ return randomArray(random, depth + 1);
+ }
+ }
+
+ private static String randomObject(Random random, int depth) {
+ StringBuilder json = new StringBuilder("{");
+ int keys = random.nextInt(6);
+ for (int i = 0; i < keys; i++) {
+ if (i > 0) {
+ json.append(',');
+ }
+ json.append(randomKey(random)).append(':').append(randomValue(random, depth));
+ }
+ return json.append('}').toString();
+ }
+
+ private static String randomArray(Random random, int depth) {
+ StringBuilder json = new StringBuilder("[");
+ int length = random.nextInt(4);
+ for (int i = 0; i < length; i++) {
+ if (i > 0) {
+ json.append(',');
+ }
+ json.append(randomValue(random, depth));
+ }
+ return json.append(']').toString();
+ }
+
+ private static String randomEntryKey(Random random) {
+ StringBuilder entryKey = new StringBuilder(random.nextInt(5) == 0 ? "" : "/");
+ int length = random.nextInt(20) == 0 ? 0 : 1 + random.nextInt(4);
+ for (int i = 0; i < length; i++) {
+ if (i > 0) {
+ entryKey.append('/');
+ }
+ entryKey.append(
+ i == 0 && random.nextInt(4) > 0 ? "items" : ENTRY_ELEMENTS[random.nextInt(ENTRY_ELEMENTS.length)]
+ );
+ }
+ return entryKey.toString();
+ }
+
+ private static String[] randomProperties(Random random) {
+ String[] properties = new String[random.nextInt(5)];
+ for (int i = 0; i < properties.length; i++) {
+ properties[i] = PROPERTIES[random.nextInt(PROPERTIES.length)];
+ }
+ return properties;
+ }
+
/**
* Reads the specified resource file and returns its content as a String
*