From 51ccb72e9019a7d890f52a98e4508be34446eef2 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 30 Oct 2026 16:59:58 +0100 Subject: [PATCH 1/3] Revert "admin cli: temporary command to backfill source/doc file counts" This reverts commit ca50bb9ce3590453e5c5a422d0cdc34eeef12c58. --- ...b7de93f3774d9656689424d3f0027c6370792.json | 15 -- ...849995fe271833cbbaa0484b46f4b842bdb87.json | 15 -- ...b733bb9908c9fd2eb606d541a0821a3abdea2.json | 86 ------- Cargo.lock | 1 - crates/bin/docs_rs_admin/Cargo.toml | 1 - .../docs_rs_admin/src/backfill_file_counts.rs | 221 ------------------ crates/bin/docs_rs_admin/src/main.rs | 9 - .../lib/docs_rs_storage/src/archive_index.rs | 8 - 8 files changed, 356 deletions(-) delete mode 100644 .sqlx/query-927868a0a1b429752a222d48685b7de93f3774d9656689424d3f0027c6370792.json delete mode 100644 .sqlx/query-a0bdebf73e290c17dd22cafd4e9849995fe271833cbbaa0484b46f4b842bdb87.json delete mode 100644 .sqlx/query-d829e851446e016e0c508980e02b733bb9908c9fd2eb606d541a0821a3abdea2.json delete mode 100644 crates/bin/docs_rs_admin/src/backfill_file_counts.rs diff --git a/.sqlx/query-927868a0a1b429752a222d48685b7de93f3774d9656689424d3f0027c6370792.json b/.sqlx/query-927868a0a1b429752a222d48685b7de93f3774d9656689424d3f0027c6370792.json deleted file mode 100644 index 97ceb9518d..0000000000 --- a/.sqlx/query-927868a0a1b429752a222d48685b7de93f3774d9656689424d3f0027c6370792.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE builds\n SET documentation_file_count = $2\n WHERE\n id = $1 AND\n documentation_file_count IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "927868a0a1b429752a222d48685b7de93f3774d9656689424d3f0027c6370792" -} diff --git a/.sqlx/query-a0bdebf73e290c17dd22cafd4e9849995fe271833cbbaa0484b46f4b842bdb87.json b/.sqlx/query-a0bdebf73e290c17dd22cafd4e9849995fe271833cbbaa0484b46f4b842bdb87.json deleted file mode 100644 index a96cace0c7..0000000000 --- a/.sqlx/query-a0bdebf73e290c17dd22cafd4e9849995fe271833cbbaa0484b46f4b842bdb87.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE releases\n SET source_file_count = $2\n WHERE\n id = $1 AND\n source_file_count IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "a0bdebf73e290c17dd22cafd4e9849995fe271833cbbaa0484b46f4b842bdb87" -} diff --git a/.sqlx/query-d829e851446e016e0c508980e02b733bb9908c9fd2eb606d541a0821a3abdea2.json b/.sqlx/query-d829e851446e016e0c508980e02b733bb9908c9fd2eb606d541a0821a3abdea2.json deleted file mode 100644 index af8029e94a..0000000000 --- a/.sqlx/query-d829e851446e016e0c508980e02b733bb9908c9fd2eb606d541a0821a3abdea2.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n r.id,\n c.name AS \"name: KrateName\",\n r.version AS \"version: Version\",\n r.source_file_count,\n latest_build.id AS \"latest_build_id?: BuildId\",\n latest_build.documentation_file_count\n FROM releases r\n JOIN crates c ON c.id = r.crate_id\n LEFT JOIN LATERAL (\n SELECT id, documentation_file_count\n FROM builds\n WHERE rid = r.id AND build_status = 'success'\n ORDER BY build_finished DESC, id DESC\n LIMIT 1\n ) latest_build ON true\n WHERE r.source_file_count IS NULL OR (\n latest_build.id IS NOT NULL AND\n latest_build.documentation_file_count IS NULL\n )\n ORDER BY r.id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int4", - "origin": { - "Table": { - "table": "releases", - "name": "id" - } - } - }, - { - "ordinal": 1, - "name": "name: KrateName", - "type_info": "Text", - "origin": { - "Table": { - "table": "crates", - "name": "name" - } - } - }, - { - "ordinal": 2, - "name": "version: Version", - "type_info": "Text", - "origin": { - "Table": { - "table": "releases", - "name": "version" - } - } - }, - { - "ordinal": 3, - "name": "source_file_count", - "type_info": "Int8", - "origin": { - "Table": { - "table": "releases", - "name": "source_file_count" - } - } - }, - { - "ordinal": 4, - "name": "latest_build_id?: BuildId", - "type_info": "Int4", - "origin": { - "Table": { - "table": "builds", - "name": "id" - } - } - }, - { - "ordinal": 5, - "name": "documentation_file_count", - "type_info": "Int8", - "origin": { - "Table": { - "table": "builds", - "name": "documentation_file_count" - } - } - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - true - ] - }, - "hash": "d829e851446e016e0c508980e02b733bb9908c9fd2eb606d541a0821a3abdea2" -} diff --git a/Cargo.lock b/Cargo.lock index dee9917b86..821740f7d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2007,7 +2007,6 @@ dependencies = [ "docs_rs_logging", "docs_rs_opentelemetry", "docs_rs_repository_stats", - "docs_rs_storage", "docs_rs_test_fakes", "docs_rs_types", "docs_rs_uri", diff --git a/crates/bin/docs_rs_admin/Cargo.toml b/crates/bin/docs_rs_admin/Cargo.toml index 4f9eafa65c..113e4800dc 100644 --- a/crates/bin/docs_rs_admin/Cargo.toml +++ b/crates/bin/docs_rs_admin/Cargo.toml @@ -18,7 +18,6 @@ docs_rs_fastly = { path = "../../lib/docs_rs_fastly" } docs_rs_headers = { path = "../../lib/docs_rs_headers" } docs_rs_logging = { path = "../../lib/docs_rs_logging" } docs_rs_repository_stats = { path = "../../lib/docs_rs_repository_stats" } -docs_rs_storage = { path = "../../lib/docs_rs_storage" } docs_rs_types = { path = "../../lib/docs_rs_types" } docs_rs_uri = { path = "../../lib/docs_rs_uri" } docs_rs_utils = { path = "../../lib/docs_rs_utils" } diff --git a/crates/bin/docs_rs_admin/src/backfill_file_counts.rs b/crates/bin/docs_rs_admin/src/backfill_file_counts.rs deleted file mode 100644 index d7ac70386d..0000000000 --- a/crates/bin/docs_rs_admin/src/backfill_file_counts.rs +++ /dev/null @@ -1,221 +0,0 @@ -use anyhow::{Context as _, Result}; -use docs_rs_context::Context; -use docs_rs_storage::{PathNotFoundError, rustdoc_archive_path, source_archive_path}; -use docs_rs_types::{BuildId, KrateName, Version}; -use futures_util::TryStreamExt as _; -use tracing::{info, warn}; - -pub(crate) async fn backfill_file_counts(ctx: &Context) -> Result<(u64, u64, u64, u64)> { - let mut list_conn = ctx.pool()?.get_async().await?; - let mut update_conn = ctx.pool()?.get_async().await?; - let storage = ctx.storage()?; - let mut sources = 0; - let mut documentation = 0; - let mut missing_sources = 0; - let mut missing_documentation = 0; - - let mut rows = sqlx::query!( - r#"SELECT - r.id, - c.name AS "name: KrateName", - r.version AS "version: Version", - r.source_file_count, - latest_build.id AS "latest_build_id?: BuildId", - latest_build.documentation_file_count - FROM releases r - JOIN crates c ON c.id = r.crate_id - LEFT JOIN LATERAL ( - SELECT id, documentation_file_count - FROM builds - WHERE rid = r.id AND build_status = 'success' - ORDER BY build_finished DESC, id DESC - LIMIT 1 - ) latest_build ON true - WHERE r.source_file_count IS NULL OR ( - latest_build.id IS NOT NULL AND - latest_build.documentation_file_count IS NULL - ) - ORDER BY r.id"#, - ) - .fetch(&mut *list_conn); - - while let Some(row) = rows.try_next().await? { - info!(%row.name, %row.version, "updating doc & source file counts"); - - if row.source_file_count.is_none() { - let archive = source_archive_path(&row.name, &row.version); - match storage - .find_archive_index(&archive, row.latest_build_id) - .await - { - Ok(mut index) => { - let count = i64::try_from(index.file_count().await?)?; - sources += sqlx::query!( - "UPDATE releases - SET source_file_count = $2 - WHERE - id = $1 AND - source_file_count IS NULL - ", - row.id, - count, - ) - .execute(&mut *update_conn) - .await? - .rows_affected(); - } - Err(err) if err.is::() => { - warn!(%archive, "source archive index is missing"); - missing_sources += 1; - } - Err(err) => { - return Err(err).with_context(|| format!("fetching index for {archive}")); - } - } - } - - // The archive path is shared by all builds of a release. Match the latest - // successful build, which is also the cache key used when serving rustdoc. - if let Some(build_id) = row.latest_build_id - && row.documentation_file_count.is_none() - { - let archive = rustdoc_archive_path(&row.name, &row.version); - match storage.find_archive_index(&archive, Some(build_id)).await { - Ok(mut index) => { - let count = i64::try_from(index.file_count().await?)?; - documentation += sqlx::query!( - "UPDATE builds - SET documentation_file_count = $2 - WHERE - id = $1 AND - documentation_file_count IS NULL - ", - build_id as _, - count, - ) - .execute(&mut *update_conn) - .await? - .rows_affected(); - } - Err(err) if err.is::() => { - warn!(%archive, %build_id, "documentation archive index is missing"); - missing_documentation += 1; - } - Err(err) => { - return Err(err).with_context(|| format!("fetching index for {archive}")); - } - } - } - } - - println!( - "Backfilled {sources} source and {documentation} documentation file counts; \ - skipped {missing_sources} missing source and {missing_documentation} missing documentation indexes" - ); - Ok(( - sources, - documentation, - missing_sources, - missing_documentation, - )) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::testing::TestEnvironment; - use docs_rs_test_fakes::FakeBuild; - use docs_rs_types::testing::{FOO, V1}; - - #[tokio::test(flavor = "multi_thread")] - async fn backfills_current_archives_without_changing_older_builds() -> Result<()> { - let env = TestEnvironment::new().await?; - let release_id = env - .fake_release() - .await - .name(&FOO) - .version(V1) - .builds(vec![ - FakeBuild::default(), - FakeBuild::default(), - FakeBuild::default().successful(false), - ]) - .create() - .await?; - let mut conn = env.async_conn().await?; - - let builds: Vec<(i32, Option)> = sqlx::query_as( - "SELECT id, documentation_file_count FROM builds WHERE rid = $1 ORDER BY id", - ) - .bind(release_id.0) - .fetch_all(&mut *conn) - .await?; - assert_eq!(builds.len(), 3); - let source_count = env - .storage()? - .find_archive_index(&source_archive_path(&FOO, &V1), Some(BuildId(builds[1].0))) - .await? - .file_count() - .await?; - let documentation_count = env - .storage()? - .find_archive_index(&rustdoc_archive_path(&FOO, &V1), Some(BuildId(builds[1].0))) - .await? - .file_count() - .await?; - - sqlx::query("UPDATE releases SET source_file_count = NULL WHERE id = $1") - .bind(release_id.0) - .execute(&mut *conn) - .await?; - sqlx::query("UPDATE builds SET documentation_file_count = NULL WHERE rid = $1") - .bind(release_id.0) - .execute(&mut *conn) - .await?; - sqlx::query("UPDATE builds SET documentation_size = NULL WHERE id = $1") - .bind(builds[1].0) - .execute(&mut *conn) - .await?; - drop(conn); - - assert_eq!(backfill_file_counts(env.context()).await?, (1, 1, 0, 0)); - assert_eq!(backfill_file_counts(env.context()).await?, (0, 0, 0, 0)); - - let mut conn = env.async_conn().await?; - sqlx::query("UPDATE releases SET source_file_count = NULL WHERE id = $1") - .bind(release_id.0) - .execute(&mut *conn) - .await?; - drop(conn); - assert_eq!(backfill_file_counts(env.context()).await?, (1, 0, 0, 0)); - - let mut conn = env.async_conn().await?; - sqlx::query("UPDATE builds SET documentation_file_count = NULL WHERE id = $1") - .bind(builds[1].0) - .execute(&mut *conn) - .await?; - drop(conn); - assert_eq!(backfill_file_counts(env.context()).await?, (0, 1, 0, 0)); - - let mut conn = env.async_conn().await?; - let restored_source_count: i64 = - sqlx::query_scalar("SELECT source_file_count FROM releases WHERE id = $1") - .bind(release_id.0) - .fetch_one(&mut *conn) - .await?; - let restored_builds: Vec<(i32, Option)> = sqlx::query_as( - "SELECT id, documentation_file_count FROM builds WHERE rid = $1 ORDER BY id", - ) - .bind(release_id.0) - .fetch_all(&mut *conn) - .await?; - assert_eq!(restored_source_count, i64::try_from(source_count)?); - assert_eq!(restored_builds[0].1, None); - assert_eq!( - restored_builds[1].1, - Some(i64::try_from(documentation_count)?) - ); - assert_eq!(restored_builds[2].1, None); - Ok(()) - } -} diff --git a/crates/bin/docs_rs_admin/src/main.rs b/crates/bin/docs_rs_admin/src/main.rs index 129bfdc69e..273b90a755 100644 --- a/crates/bin/docs_rs_admin/src/main.rs +++ b/crates/bin/docs_rs_admin/src/main.rs @@ -1,10 +1,8 @@ -mod backfill_file_counts; mod rebuilds; #[cfg(test)] pub(crate) mod testing; use anyhow::{Context as _, Result, bail}; -use backfill_file_counts::backfill_file_counts; use chrono::{NaiveDate, Utc}; use clap::{Parser, Subcommand}; use docs_rs_build_limits::{Overrides, blacklist}; @@ -383,9 +381,6 @@ enum DatabaseSubcommand { /// Backfill GitHub/GitLab stats for crates. BackfillRepositoryStats, - /// Backfill source and documentation file counts from archive indexes. - BackfillFileCounts, - /// Updates info for a crate from the registry's API UpdateCrateRegistryFields { #[arg(name = "CRATE")] @@ -457,10 +452,6 @@ impl DatabaseSubcommand { workspaces::rewrite_repository_stats(&mut conn).await?; } - Self::BackfillFileCounts => { - backfill_file_counts(&ctx).await?; - } - Self::UpdateCrateRegistryFields { name } => { let mut conn = ctx.pool()?.get_async().await?; let registry_data = ctx.registry_api()?.get_crate_data(&name).await?; diff --git a/crates/lib/docs_rs_storage/src/archive_index.rs b/crates/lib/docs_rs_storage/src/archive_index.rs index 167a8aa98f..be7788f54f 100644 --- a/crates/lib/docs_rs_storage/src/archive_index.rs +++ b/crates/lib/docs_rs_storage/src/archive_index.rs @@ -873,14 +873,6 @@ impl Index { Ok(Self { conn }) } - /// Number of files stored in the archive. - pub async fn file_count(&mut self) -> Result { - let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM files") - .fetch_one(&mut self.conn) - .await?; - Ok(count.try_into()?) - } - #[instrument(skip(self))] pub async fn find

(&mut self, search_for: P) -> Result> where From 0b77f2ddcc9b36822ffc63d5cfd5548219aa81eb Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 30 Oct 2026 17:15:09 +0100 Subject: [PATCH 2/3] show documentation stats on homepage --- ...ef1d749775d6032d0f29a5ccb108698d66337.json | 21 +++ ...0f3ea370cd374183c6a0e229aeada18bba533.json | 12 ++ ...5b2265b0586c95c965588322fb6b45b2ddf6a.json | 12 ++ crates/bin/cratesfyi/src/daemon.rs | 6 +- crates/bin/docs_rs_watcher/src/lib.rs | 15 ++ crates/bin/docs_rs_watcher/src/main.rs | 3 +- crates/bin/docs_rs_watcher/src/statistics.rs | 147 ++++++++++++++++++ .../bin/docs_rs_web/src/handlers/releases.rs | 62 +++++++- crates/bin/docs_rs_web/src/page/templates.rs | 24 +++ .../bin/docs_rs_web/templates/core/home.html | 35 +++-- .../docs_rs_web/templates/style/style.scss | 52 ++++++- .../src/service_config/mod.rs | 3 + .../src/service_config/statistics.rs | 33 ++++ crates/lib/docs_rs_test_fakes/src/legacy.rs | 11 +- 14 files changed, 417 insertions(+), 19 deletions(-) create mode 100644 .sqlx/query-6ba28b62ca39d499abd8ee447a5ef1d749775d6032d0f29a5ccb108698d66337.json create mode 100644 .sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json create mode 100644 .sqlx/query-f01332ffd2953142862fca68f095b2265b0586c95c965588322fb6b45b2ddf6a.json create mode 100644 crates/bin/docs_rs_watcher/src/statistics.rs create mode 100644 crates/lib/docs_rs_database/src/service_config/statistics.rs diff --git a/.sqlx/query-6ba28b62ca39d499abd8ee447a5ef1d749775d6032d0f29a5ccb108698d66337.json b/.sqlx/query-6ba28b62ca39d499abd8ee447a5ef1d749775d6032d0f29a5ccb108698d66337.json new file mode 100644 index 0000000000..7029105966 --- /dev/null +++ b/.sqlx/query-6ba28b62ca39d499abd8ee447a5ef1d749775d6032d0f29a5ccb108698d66337.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(SUM(b.documentation_file_count), 0)::BIGINT AS \"count!\"\n FROM releases r\n JOIN (\n SELECT DISTINCT ON (rid) rid, documentation_file_count\n FROM builds\n WHERE build_status = 'success'\n ORDER BY rid, id DESC\n ) b ON b.rid = r.id\n WHERE r.rustdoc_status = TRUE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "6ba28b62ca39d499abd8ee447a5ef1d749775d6032d0f29a5ccb108698d66337" +} diff --git a/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json b/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json new file mode 100644 index 0000000000..fb106d1b14 --- /dev/null +++ b/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "SET LOCAL lock_timeout = '1s'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533" +} diff --git a/.sqlx/query-f01332ffd2953142862fca68f095b2265b0586c95c965588322fb6b45b2ddf6a.json b/.sqlx/query-f01332ffd2953142862fca68f095b2265b0586c95c965588322fb6b45b2ddf6a.json new file mode 100644 index 0000000000..1c2270ea1d --- /dev/null +++ b/.sqlx/query-f01332ffd2953142862fca68f095b2265b0586c95c965588322fb6b45b2ddf6a.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "SET LOCAL statement_timeout = '15s'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "f01332ffd2953142862fca68f095b2265b0586c95c965588322fb6b45b2ddf6a" +} diff --git a/crates/bin/cratesfyi/src/daemon.rs b/crates/bin/cratesfyi/src/daemon.rs index 2403cffe1c..07712fd26c 100644 --- a/crates/bin/cratesfyi/src/daemon.rs +++ b/crates/bin/cratesfyi/src/daemon.rs @@ -4,7 +4,8 @@ use docs_rs_config::AppConfig as _; use docs_rs_context::Context; use docs_rs_watcher::{ start_background_queue_rebuild, start_background_repository_stats_updater, - start_background_service_metric_collector, watch_registry, + start_background_service_metric_collector, start_background_statistics_collector, + watch_registry, }; use docs_rs_web::run_web_server; use std::sync::Arc; @@ -67,8 +68,9 @@ pub fn start_daemon(context: Context) -> Result<(), Error> { start_background_queue_rebuild(watcher_config, &context.clone()).await?; // when people run the daemon, we assume the daemon is the one single process where - // we can collect the service metrics. + // we can collect the service metrics & statistics. start_background_service_metric_collector(&context).await?; + start_background_statistics_collector(&context).await?; Ok::<(), Error>(()) })?; diff --git a/crates/bin/docs_rs_watcher/src/lib.rs b/crates/bin/docs_rs_watcher/src/lib.rs index 60dc80b357..7255e366c4 100644 --- a/crates/bin/docs_rs_watcher/src/lib.rs +++ b/crates/bin/docs_rs_watcher/src/lib.rs @@ -6,6 +6,7 @@ pub mod index_watcher; mod metrics; mod rebuilds; mod service_metrics; +mod statistics; #[cfg(test)] mod testing; @@ -89,6 +90,20 @@ pub async fn start_background_service_metric_collector(context: &Context) -> Res Ok(()) } +pub async fn start_background_statistics_collector(context: &Context) -> Result<()> { + let pool = context.pool()?.clone(); + + start_async_cron("statistics collector", Duration::from_mins(5), move || { + let pool = pool.clone(); + async move { + let mut conn = pool.get_async().await?; + statistics::gather_documentation_statistics(&mut conn).await + } + }); + + Ok(()) +} + pub async fn start_background_queue_rebuild(config: Arc, context: &Context) -> Result<()> { let pool = context.pool()?.clone(); let build_queue = context.build_queue()?.clone(); diff --git a/crates/bin/docs_rs_watcher/src/main.rs b/crates/bin/docs_rs_watcher/src/main.rs index 4b95c08c3f..614aa6da70 100644 --- a/crates/bin/docs_rs_watcher/src/main.rs +++ b/crates/bin/docs_rs_watcher/src/main.rs @@ -78,9 +78,10 @@ impl CommandLine { docs_rs_watcher::start_background_queue_rebuild(config.clone(), &ctx).await?; } - // We assume that we can collect service metrics from the registry watcher, + // We assume that we can collect service metrics & statistics from the registry watcher, // which should only run once, and all the time. docs_rs_watcher::start_background_service_metric_collector(&ctx).await?; + docs_rs_watcher::start_background_statistics_collector(&ctx).await?; docs_rs_watcher::watch_registry(&config, &ctx).await?; } diff --git a/crates/bin/docs_rs_watcher/src/statistics.rs b/crates/bin/docs_rs_watcher/src/statistics.rs new file mode 100644 index 0000000000..d8b572371a --- /dev/null +++ b/crates/bin/docs_rs_watcher/src/statistics.rs @@ -0,0 +1,147 @@ +use anyhow::Result; +use chrono::{Duration, Utc}; +use docs_rs_database::service_config::{ + ConfigName, DocumentationStatistics, get_config, set_config, +}; +use sqlx::Connection as _; +use tracing::{info, instrument}; + +/// Refresh the cached documentation file count at most once every twelve hours. +#[instrument(skip_all)] +pub async fn gather_documentation_statistics(conn: &mut sqlx::PgConnection) -> Result<()> { + let cached: Option = + get_config(conn, ConfigName::DocumentationStatistics).await?; + + if cached.is_some_and(|stats| Utc::now() - stats.last_updated() < Duration::hours(12)) { + return Ok(()); + } + + info!("updating cached documentation statistics"); + + let mut tx = conn.begin().await?; + + // set a statement timeout, so this expensive query won't run wild. + sqlx::query!("SET LOCAL statement_timeout = '15s'") + .execute(&mut *tx) + .await?; + sqlx::query!("SET LOCAL lock_timeout = '1s'") + .execute(&mut *tx) + .await?; + + // tested on prod, on 2026-10-30, with ~3 mio releases: + // ~5 seconds + // + // more optimizations possible, we'll try again on the new RDS server. + let count = sqlx::query_scalar!( + r#"SELECT COALESCE(SUM(b.documentation_file_count), 0)::BIGINT AS "count!" + FROM releases r + JOIN ( + SELECT DISTINCT ON (rid) rid, documentation_file_count + FROM builds + WHERE build_status = 'success' + ORDER BY rid, id DESC + ) b ON b.rid = r.id + WHERE r.rustdoc_status = TRUE"#, + ) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + + set_config( + conn, + ConfigName::DocumentationStatistics, + DocumentationStatistics::new(count.try_into()?), + ) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::TestEnvironment; + use docs_rs_test_fakes::FakeBuild; + use docs_rs_types::testing::{BAR, BAZ, FOO, KRATE}; + use test_case::test_case; + + #[tokio::test(flavor = "multi_thread")] + async fn counts_latest_successful_build_per_documented_release() -> Result<()> { + let env = TestEnvironment::builder().build().await?; + env.fake_release() + .await + .name(&FOO) + .builds(vec![ + // older successful build, ignored + FakeBuild::default().documentation_file_count(Some(100)), + // last successful build, count will be used + FakeBuild::default().documentation_file_count(Some(7)), + // latest build = failed, count will be ignored (but shouldn't exist anyways) + FakeBuild::default() + .successful(false) + .documentation_file_count(Some(999)), + ]) + .create() + .await?; + let mut conn = env.async_conn().await?; + + // failed release, will be ignored + env.fake_release() + .await + .name(&BAZ) + .build_result_failed() + .create() + .await?; + + for (name, count) in [ + // release/build with count, will be used + (&BAR, Some(13_u64)), + // release/build without count, will be ignored + (&KRATE, None), + ] { + env.fake_release() + .await + .name(name) + .builds(vec![FakeBuild::default().documentation_file_count(count)]) + .create() + .await?; + } + + gather_documentation_statistics(&mut conn).await?; + + let stats: DocumentationStatistics = + get_config(&mut conn, ConfigName::DocumentationStatistics) + .await? + .unwrap(); + assert_eq!(stats.documentation_file_count(), 20); + Ok(()) + } + + #[test_case(None, 0; "initializes_empty_database")] + #[test_case(Some(1), 42; "preserves_fresh_cache")] + #[test_case(Some(12), 0; "refreshes_stale_cache")] + #[tokio::test(flavor = "multi_thread")] + async fn refreshes_only_when_needed(age_hours: Option, expected: u64) -> Result<()> { + let env = TestEnvironment::builder().build().await?; + let mut conn = env.async_conn().await?; + let cached = age_hours + .map(|hours| DocumentationStatistics::new_at(42, Utc::now() - Duration::hours(hours))); + + if let Some(cached) = &cached { + set_config(&mut conn, ConfigName::DocumentationStatistics, cached).await?; + } + + let before = Utc::now(); + gather_documentation_statistics(&mut conn).await?; + + let stats: DocumentationStatistics = + get_config(&mut conn, ConfigName::DocumentationStatistics) + .await? + .unwrap(); + assert_eq!(stats.documentation_file_count(), expected); + if expected == 42 { + assert_eq!(Some(stats), cached); + } else { + assert!(stats.last_updated() >= before); + } + Ok(()) + } +} diff --git a/crates/bin/docs_rs_web/src/handlers/releases.rs b/crates/bin/docs_rs_web/src/handlers/releases.rs index d1e6c2fbdb..8839ef89eb 100644 --- a/crates/bin/docs_rs_web/src/handlers/releases.rs +++ b/crates/bin/docs_rs_web/src/handlers/releases.rs @@ -22,6 +22,7 @@ use axum::{ use base64::{Engine, engine::general_purpose::STANDARD as b64}; use chrono::{DateTime, Utc}; use docs_rs_build_queue::{AsyncBuildQueue, PRIORITY_CONTINUOUS, QueuedCrate}; +use docs_rs_database::service_config::{ConfigName, DocumentationStatistics, get_config}; use docs_rs_registry_api::{self as registry_api, RegistryApi}; use docs_rs_types::{Duration, KrateName, ReqVersion, Version}; use docs_rs_uri::encode_url_path; @@ -260,6 +261,7 @@ async fn get_search_results( #[derive(Debug, Clone, PartialEq, Eq)] struct HomePage { recent_releases: Vec, + documentation_statistics: Option, } impl_axum_webpage! { @@ -270,8 +272,13 @@ impl_axum_webpage! { pub(crate) async fn home_page(mut conn: DbConnection) -> AxumResult { let recent_releases = get_releases(&mut conn, 1, RELEASES_IN_HOME, Order::ReleaseTime, true).await?; + let documentation_statistics: Option = + get_config(&mut conn, ConfigName::DocumentationStatistics).await?; - Ok(HomePage { recent_releases }) + Ok(HomePage { + recent_releases, + documentation_statistics, + }) } #[derive(Template)] @@ -912,6 +919,7 @@ mod tests { use docs_rs_database::releases::{ finish_build, initialize_build, initialize_crate, initialize_release, }; + use docs_rs_database::service_config::set_config; use docs_rs_registry_api::{CrateOwner, OwnerKind, SearchQuery, testing::TestRegistry}; use docs_rs_test_fakes::{FakeBuild, fake_release_that_failed_before_build}; use docs_rs_types::{ @@ -2070,6 +2078,7 @@ mod tests { fn home_page_description_title_escapes_quotes() { let description = r#"A "quoted" crate description"#; let mut page = HomePage { + documentation_statistics: None, recent_releases: vec![Release { name: FOO, version: V1, @@ -2094,6 +2103,57 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn home_page_shows_cached_documentation_statistics() -> Result<()> { + let env = TestEnvironment::new().await?; + let mut conn = env.async_conn().await?; + set_config( + &mut conn, + ConfigName::DocumentationStatistics, + DocumentationStatistics::new_at( + 1_234_567_890, + Utc.with_ymd_and_hms(2020, 1, 2, 3, 4, 5).unwrap(), + ), + ) + .await?; + drop(conn); + + let response = env.web_app().await.assert_success("/").await?; + let html = kuchikiki::parse_html().one(response.text().await?); + let statistics = html.select_first(".documentation-statistics").unwrap(); + assert_eq!( + statistics.attributes.borrow().get("title"), + Some("Last updated: Jan 02, 2020"), + ); + assert_eq!( + statistics + .as_node() + .select_first(".documentation-file-count") + .unwrap() + .text_contents(), + "1,234,567,890", + ); + assert_eq!( + statistics + .as_node() + .select_first(".documentation-statistics-label") + .unwrap() + .text_contents(), + "Documentation files hosted", + ); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn home_page_without_documentation_statistics() -> Result<()> { + let env = TestEnvironment::new().await?; + let response = env.web_app().await.assert_success("/").await?; + let html = kuchikiki::parse_html().one(response.text().await?); + assert!(html.select_first(".documentation-statistics").is_err()); + assert!(html.select_first(".landing-search-form").is_ok()); + Ok(()) + } + #[test] fn home_page_links() { async_wrapper(|env| async move { diff --git a/crates/bin/docs_rs_web/src/page/templates.rs b/crates/bin/docs_rs_web/src/page/templates.rs index c570920f69..44faf78dcd 100644 --- a/crates/bin/docs_rs_web/src/page/templates.rs +++ b/crates/bin/docs_rs_web/src/page/templates.rs @@ -96,6 +96,18 @@ pub mod filters { use std::fmt::Display; use std::time::Duration; + /// Format an integer with commas separating groups of three digits. + #[askama::filter_fn] + pub fn thousands(value: u64, _: &dyn Values) -> askama::Result { + let mut number = value.to_string(); + let mut separator = number.len(); + while separator > 3 { + separator -= 3; + number.insert(separator, ','); + } + Ok(number) + } + pub fn escape_html_inner(input: &str) -> askama::Result { if !input.chars().any(|c| "&<>\"'/".contains(c)) { return Ok(Cow::Borrowed(input)); @@ -300,6 +312,18 @@ mod tests { use std::{any::Any, collections::HashMap, time::Duration}; use test_case::test_case; + #[test_case(0 => "0"; "zero")] + #[test_case(999 => "999"; "no separator")] + #[test_case(1000 => "1,000"; "thousand")] + #[test_case(1_234_567_890 => "1,234,567,890"; "billion")] + #[test_case(u64::MAX => "18,446,744,073,709,551,615"; "maximum")] + fn test_thousands(value: u64) -> String { + let values: HashMap<&str, Box> = HashMap::new(); + filters::thousands::default() + .execute(value, &values) + .unwrap() + } + #[test_case(Duration::from_secs(0) => "0s"; "zero")] #[test_case(Duration::from_secs(1) => "1s"; "simple")] #[test_case(Duration::from_micros(2123456) => "2s"; "cuts microseconds")] diff --git a/crates/bin/docs_rs_web/templates/core/home.html b/crates/bin/docs_rs_web/templates/core/home.html index 32014cb0c8..fff09e5207 100644 --- a/crates/bin/docs_rs_web/templates/core/home.html +++ b/crates/bin/docs_rs_web/templates/core/home.html @@ -14,19 +14,30 @@

{{ crate::icons::IconCubes.render_solid(false, false, "") }} Docs.rs

-
-
- -
+
+ +
+ +
-
- - -
- +
+ + +
+ + {% if let Some(stats) = documentation_statistics %} +
+
+ {{ stats.documentation_file_count()|thousands }} + Documentation files hosted +
+ {{ crate::icons::IconFileLines.render_regular(false, false, "documentation-statistics-icon") }} +
+ {% endif %} +
diff --git a/crates/bin/docs_rs_web/templates/style/style.scss b/crates/bin/docs_rs_web/templates/style/style.scss index 70e2219fba..8f3e8195b3 100644 --- a/crates/bin/docs_rs_web/templates/style/style.scss +++ b/crates/bin/docs_rs_web/templates/style/style.scss @@ -241,8 +241,12 @@ div.landing { } form.landing-search-form { - max-width: 500px; - margin: 0 auto; + grid-column: 2; + box-sizing: border-box; + min-width: 0; + width: 100%; + max-width: 532px; + margin: 0; padding: 0.4em 1em; div.buttons { @@ -253,6 +257,50 @@ div.landing { input#search { margin: 0; } + + .landing-search-and-statistics { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 532px) minmax(0, 1fr); + align-items: start; + gap: 24px; + } + + .documentation-statistics { + grid-column: 3; + justify-self: end; + display: flex; + align-items: center; + gap: 20px; + flex-shrink: 0; + padding-top: 0.4em; + text-align: right; + + .documentation-statistics-label { + display: block; + margin-top: 6px; + font-size: 0.875em; + opacity: 0.75; + } + } + + .documentation-statistics-icon { + font-size: 3em; + color: var(--color-navbar-standard); + } + + .documentation-file-count { + display: block; + font-size: 2em; + line-height: 1.2; + } + + @media (max-width: 1100px) { + .landing-search-and-statistics { + display: flex; + flex-direction: column; + align-items: center; + } + } } #search-select-nav { diff --git a/crates/lib/docs_rs_database/src/service_config/mod.rs b/crates/lib/docs_rs_database/src/service_config/mod.rs index 0653639a96..ac716c3a07 100644 --- a/crates/lib/docs_rs_database/src/service_config/mod.rs +++ b/crates/lib/docs_rs_database/src/service_config/mod.rs @@ -1,9 +1,11 @@ mod abnormalities; +mod statistics; use anyhow::Result; use serde::{Serialize, de::DeserializeOwned}; pub use abnormalities::{Abnormality, Alert}; +pub use statistics::DocumentationStatistics; #[derive(strum::IntoStaticStr)] #[strum(serialize_all = "snake_case")] @@ -14,6 +16,7 @@ pub enum ConfigName { Toolchain, Abnormality, Alert, + DocumentationStatistics, } pub async fn set_config( diff --git a/crates/lib/docs_rs_database/src/service_config/statistics.rs b/crates/lib/docs_rs_database/src/service_config/statistics.rs new file mode 100644 index 0000000000..8edd20a51f --- /dev/null +++ b/crates/lib/docs_rs_database/src/service_config/statistics.rs @@ -0,0 +1,33 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DocumentationStatistics { + documentation_file_count: u64, + last_updated: DateTime, +} + +impl DocumentationStatistics { + pub fn documentation_file_count(&self) -> u64 { + self.documentation_file_count + } + + pub fn last_updated(&self) -> DateTime { + self.last_updated + } + + pub fn new(documentation_file_count: u64) -> Self { + Self { + documentation_file_count, + last_updated: Utc::now(), + } + } + + #[cfg(any(test, feature = "testing"))] + pub fn new_at(documentation_file_count: u64, last_updated: DateTime) -> Self { + Self { + documentation_file_count, + last_updated, + } + } +} diff --git a/crates/lib/docs_rs_test_fakes/src/legacy.rs b/crates/lib/docs_rs_test_fakes/src/legacy.rs index acf388d92f..8a42dd8e87 100644 --- a/crates/lib/docs_rs_test_fakes/src/legacy.rs +++ b/crates/lib/docs_rs_test_fakes/src/legacy.rs @@ -99,6 +99,7 @@ pub struct FakeBuild { /// new build logs: we have a record in the `builds_logs` table for each log, including a status /// old build logs: people have to run `s3 ls` with prefix to know which build logs exist legacy_build_logs: bool, + documentation_file_count: Option, } const DEFAULT_CONTENT: &[u8] = @@ -640,6 +641,13 @@ impl FakeBuild { } } + pub fn documentation_file_count(self, documentation_file_count: Option) -> Self { + Self { + documentation_file_count, + ..self + } + } + pub fn legacy_build_logs(self, legacy_build_logs: bool) -> Self { Self { legacy_build_logs, @@ -663,7 +671,7 @@ impl FakeBuild { &self.docsrs_version, self.build_status, Some(ByteSize::b(42)), - Some(33), + self.documentation_file_count, self.memory_peak, None::<&SimpleBuildError>, ) @@ -723,6 +731,7 @@ impl Default for FakeBuild { build_status: BuildStatus::Success, memory_peak: Some(23u64.into()), legacy_build_logs: false, + documentation_file_count: Some(33), } } } From 6968c8bc20336e48217fbc135668921e7c60ceb9 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 2 Nov 2026 16:57:05 +0100 Subject: [PATCH 3/3] Improve display of docs statistics on the index page --- .../bin/docs_rs_web/templates/core/home.html | 42 +++++++++---------- .../docs_rs_web/templates/style/style.scss | 34 +++------------ 2 files changed, 26 insertions(+), 50 deletions(-) diff --git a/crates/bin/docs_rs_web/templates/core/home.html b/crates/bin/docs_rs_web/templates/core/home.html index fff09e5207..0cffd4dc94 100644 --- a/crates/bin/docs_rs_web/templates/core/home.html +++ b/crates/bin/docs_rs_web/templates/core/home.html @@ -14,30 +14,28 @@

{{ crate::icons::IconCubes.render_solid(false, false, "") }} Docs.rs

-
-
-
- -
+ +
+ +
-
- - -
-
- {% if let Some(stats) = documentation_statistics %} -
-
- {{ stats.documentation_file_count()|thousands }} - Documentation files hosted -
- {{ crate::icons::IconFileLines.render_regular(false, false, "documentation-statistics-icon") }} +
+ + +
+ + {% if let Some(stats) = documentation_statistics %} +
+
+ {{ stats.documentation_file_count()|thousands }} + Documentation files hosted
- {% endif %} -
+ {{ crate::icons::IconFileLines.render_regular(false, false, "documentation-statistics-icon") }} +
+ {% endif %}
diff --git a/crates/bin/docs_rs_web/templates/style/style.scss b/crates/bin/docs_rs_web/templates/style/style.scss index 8f3e8195b3..a8198c4830 100644 --- a/crates/bin/docs_rs_web/templates/style/style.scss +++ b/crates/bin/docs_rs_web/templates/style/style.scss @@ -241,12 +241,8 @@ div.landing { } form.landing-search-form { - grid-column: 2; - box-sizing: border-box; - min-width: 0; - width: 100%; - max-width: 532px; - margin: 0; + max-width: 500px; + margin: 0 auto; padding: 0.4em 1em; div.buttons { @@ -258,28 +254,18 @@ div.landing { margin: 0; } - .landing-search-and-statistics { - display: grid; - grid-template-columns: minmax(0, 1fr) minmax(0, 532px) minmax(0, 1fr); - align-items: start; - gap: 24px; - } - .documentation-statistics { - grid-column: 3; - justify-self: end; display: flex; align-items: center; + justify-content: center; gap: 20px; - flex-shrink: 0; - padding-top: 0.4em; + margin-top: 30px; text-align: right; .documentation-statistics-label { display: block; - margin-top: 6px; - font-size: 0.875em; - opacity: 0.75; + font-size: 0.9em; + opacity: 0.8; } } @@ -293,14 +279,6 @@ div.landing { font-size: 2em; line-height: 1.2; } - - @media (max-width: 1100px) { - .landing-search-and-statistics { - display: flex; - flex-direction: column; - align-items: center; - } - } } #search-select-nav {