From 0f4675828c494f48da88f063873577b0d70d7cf6 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Sat, 20 Sep 2025 08:17:58 +0200 Subject: [PATCH] Bump versions and update README --- Cargo.lock | 103 ++++++++++++++++++++++++++----------------------- Cargo.toml | 6 +-- README.md | 82 +++++++++++++++++++++++++++------------ pyproject.toml | 6 +++ 4 files changed, 120 insertions(+), 77 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e837da6..790da29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,9 +42,9 @@ dependencies = [ [[package]] name = "arrow" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bb018b6960c87fd9d025009820406f74e83281185a8bdcb44880d2aa5c9a87" +checksum = "c26b57282a08ae92f727497805122fec964c6245cfa0e13f0e75452eaf3bc41f" dependencies = [ "arrow-arith", "arrow-array", @@ -55,18 +55,18 @@ dependencies = [ "arrow-ipc", "arrow-json", "arrow-ord", + "arrow-pyarrow", "arrow-row", "arrow-schema", "arrow-select", "arrow-string", - "pyo3", ] [[package]] name = "arrow-arith" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44de76b51473aa888ecd6ad93ceb262fb8d40d1f1154a4df2f069b3590aa7575" +checksum = "cebf38ca279120ff522f4954b81a39527425b6e9f615e6b72842f4de1ffe02b8" dependencies = [ "arrow-array", "arrow-buffer", @@ -78,9 +78,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ed77e22744475a9a53d00026cf8e166fe73cf42d89c4c4ae63607ee1cfcc3f" +checksum = "744109142cdf8e7b02795e240e20756c2a782ac9180d4992802954a8f871c0de" dependencies = [ "ahash", "arrow-buffer", @@ -94,9 +94,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0391c96eb58bf7389171d1e103112d3fc3e5625ca6b372d606f2688f1ea4cce" +checksum = "601bb103c4c374bcd1f62c66bcea67b42a2ee91a690486c37d4c180236f11ccc" dependencies = [ "bytes", "half", @@ -105,9 +105,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f39e1d774ece9292697fcbe06b5584401b26bd34be1bec25c33edae65c2420ff" +checksum = "eed61d9d73eda8df9e3014843def37af3050b5080a9acbe108f045a316d5a0be" dependencies = [ "arrow-array", "arrow-buffer", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9055c972a07bf12c2a827debfd34f88d3b93da1941d36e1d9fee85eebe38a12a" +checksum = "fa95b96ce0c06b4d33ac958370db8c0d31e88e54f9d6e08b0353d18374d9f991" dependencies = [ "arrow-array", "arrow-cast", @@ -135,15 +135,14 @@ dependencies = [ "chrono", "csv", "csv-core", - "lazy_static", "regex", ] [[package]] name = "arrow-data" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf75ac27a08c7f48b88e5c923f267e980f27070147ab74615ad85b5c5f90473d" +checksum = "43407f2c6ba2367f64d85d4603d6fb9c4b92ed79d2ffd21021b37efa96523e12" dependencies = [ "arrow-buffer", "arrow-schema", @@ -153,22 +152,23 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a222f0d93772bd058d1268f4c28ea421a603d66f7979479048c429292fac7b2e" +checksum = "e4b0487c4d2ad121cbc42c4db204f1509f8618e589bc77e635e9c40b502e3b90" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", + "arrow-select", "flatbuffers", ] [[package]] name = "arrow-json" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9085342bbca0f75e8cb70513c0807cc7351f1fbf5cb98192a67d5e3044acb033" +checksum = "26d747573390905905a2dc4c5a61a96163fe2750457f90a04ee2a88680758c79" dependencies = [ "arrow-array", "arrow-buffer", @@ -188,9 +188,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2f1065a5cad7b9efa9e22ce5747ce826aa3855766755d4904535123ef431e7" +checksum = "c142a147dceb59d057bad82400f1693847c80dca870d008bf7b91caf902810ae" dependencies = [ "arrow-array", "arrow-buffer", @@ -199,11 +199,23 @@ dependencies = [ "arrow-select", ] +[[package]] +name = "arrow-pyarrow" +version = "56.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9038de599df1b81f63b42220e2b6cd6fd4f09af333858cd320db9bef5ac757" +dependencies = [ + "arrow-array", + "arrow-data", + "arrow-schema", + "pyo3", +] + [[package]] name = "arrow-row" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3703a0e3e92d23c3f756df73d2dc9476873f873a76ae63ef9d3de17fda83b2d8" +checksum = "dac6620667fccdab4204689ca173bd84a15de6bb6b756c3a8764d4d7d0c2fc04" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,18 +226,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" +checksum = "dfa93af9ff2bb80de539e6eb2c1c8764abd0f4b73ffb0d7c82bf1f9868785e66" dependencies = [ "bitflags", ] [[package]] name = "arrow-select" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b7b85575702b23b85272b01bc1c25a01c9b9852305e5d0078c79ba25d995d4" +checksum = "be8b2e0052cd20d36d64f32640b68a5ab54d805d24a473baee5d52017c85536c" dependencies = [ "ahash", "arrow-array", @@ -237,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "55.1.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9260fddf1cdf2799ace2b4c2fc0356a9789fa7551e0953e35435536fecefebbd" +checksum = "c2155e26e17f053c8975c546fc70cf19c00542f9abf43c23a88a46ef7204204f" dependencies = [ "arrow-array", "arrow-buffer", @@ -499,12 +511,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "lexical-core" version = "1.0.5" @@ -708,11 +714,10 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" dependencies = [ - "cfg-if", "indoc", "libc", "memoffset", @@ -726,9 +731,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" dependencies = [ "once_cell", "target-lexicon", @@ -736,9 +741,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" dependencies = [ "libc", "pyo3-build-config", @@ -746,9 +751,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -758,9 +763,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" dependencies = [ "heck", "proc-macro2", @@ -786,9 +791,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha", "rand_core", diff --git a/Cargo.toml b/Cargo.toml index 032cb29..c0bdda9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ name = "falsa" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.24.1", features = ["extension-module"] } -arrow = { version = "55.1.0", features = ["pyarrow"] } -rand = "0.9.1" +pyo3 = { version = "0.25.1", features = ["extension-module"] } +arrow = { version = "56.1.0", features = ["pyarrow"] } +rand = "0.9.2" rand_chacha = "0.9.0" diff --git a/README.md b/README.md index a8d8463..339544b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,53 @@ # falsa -falsa makes it easy to generate sample datasets. +An unofficial re-implementation of the [H2O DB-like OPS benchmark](https://github.com/duckdblabs/db-benchmark) datasets +generators but in Rust. While the original R scripts are great, they are memory intensive and does not provide a simple +way to run them. False provides a ready to use Python CLI app with a Rust backend and an ability of the out-of-core +generation. + +![PyPI - Downloads](https://img.shields.io/pypi/dm/falsa) ![PyPI - License](https://img.shields.io/pypi/l/falsa) ![PyPI - Version](https://img.shields.io/pypi/v/falsa) + +## Installation + +### Pip install + +```bash +pip install falsa +``` + +## Usage + +```bash +falsa --help +``` + +``` + Usage: falsa [OPTIONS] COMMAND [ARGS]... + + H2O db-like-benchmark data generation. + This implementation is unofficial! + For the official implementation please check https://github.com/duckdblabs/db-benchmark/tree/main/_data + + Available commands are: + - groupby: generate GroupBy dataset; + - join: generate three Join datasets (small, medium, big); + + + Author: github.com/SemyonSinchenko + Source code: https://github.com/mrpowers-io/falsa + +╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ --install-completion Install completion for the current shell. │ +│ --show-completion Show completion for the current shell, to copy it or customize the installation. │ +│ --help Show this message and exit. │ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ groupby Create H2O GroupBy Dataset │ +│ join Create three H2O join datasets │ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +## Simple example Here is how to generate a Parquet file with 100 million rows and 9 columns of data for example: @@ -26,33 +73,17 @@ Here are the first three rows of data in the file: With falsa, you can generate many sample datasets. -## Installation - -### Pip install - -In virtualenv with python 3.9+: - -```sh -pip install git+https://github.com/mrpowers-io/falsa.git@main -falsa --help -``` - -### Maturin build - -In virtualenv with python 3.9+: - -```sh -maturin develop --release -falsa --help -``` - ## h2o datasets -The h2o datasets are used to benchmark query engines on a single machine, [see here](https://duckdblabs.github.io/db-benchmark/). +The h2o datasets are used to benchmark query engines on a single +machine, [see here](https://duckdblabs.github.io/db-benchmark/). -Here are [the original R Scripts](https://github.com/duckdblabs/db-benchmark/tree/main/_data) to generate the sample datasets. These still work if you know how to run R (the large dataset generation can error out if you machine doesn't have sufficient memory). +Here are [the original R Scripts](https://github.com/duckdblabs/db-benchmark/tree/main/_data) to generate the sample +datasets. These still work if you know how to run R (the large dataset generation can error out if you machine doesn't +have sufficient memory). -falsa is good if you want to generate these datasets with a Python interface or if you are facing memory issues with the R scripts. +falsa is good if you want to generate these datasets with a Python interface or if you are facing memory issues with the +R scripts. ### h2o groupby dataset @@ -104,4 +135,5 @@ Here's the detailed description of the table: └────────────┴───────────┴───────────┴──────────────┴───────────┴───┴───────────────┴──────────┴───────────┴───────────┘ ``` -The h2o dataset is useful for group by benchmarks. For example, you can use id1 to do an aggregation on a low cardinality column and id3 to do an aggreation on a high cardinality column. +The h2o dataset is useful for group by benchmarks. For example, you can use id1 to do an aggregation on a low +cardinality column and id3 to do an aggreation on a high cardinality column. diff --git a/pyproject.toml b/pyproject.toml index c0677ed..dd9e942 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,12 @@ build-backend = "maturin" [project] name = "falsa" +license = "Apache 2.0" +description = "Unofficial Rust-based implementation of H2O benchmark dataset generation" +maintainers = [ + { name = "Sem", email = "ssinchenko@apache.org" }, +] +readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9" classifiers = [ "Programming Language :: Rust",