From a9d679478741523a9a994fed3807ee8c2766bc8a Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Tue, 13 Oct 2026 01:35:36 +0000 Subject: [PATCH 1/5] feat(mcp): MediaWiki client, text converters, and the wiki permit --- mcp/src/backends/mod.rs | 1 + mcp/src/backends/wiki.rs | 1287 +++++++++++++++++ mcp/src/config.rs | 54 + mcp/src/limits.rs | 65 +- mcp/tests/fixtures/wiki/README.md | 24 + .../fixtures/wiki/error_invalidtitle.json | 1 + .../fixtures/wiki/error_missingtitle.json | 1 + .../fixtures/wiki/error_pagecannotexist.json | 1 + .../wiki/parse_aberration_report.json | 1 + mcp/tests/fixtures/wiki/parse_about.json | 1 + ...arse_balancer_genotype_variant_report.json | 1 + .../fixtures/wiki/parse_batch_download.json | 1 + .../fixtures/wiki/parse_fbgn0000553.json | 1 + .../wiki/search_ftp_archive_wildcard.json | 1 + .../wiki/search_ftp_archive_zero.json | 1 + mcp/tests/fixtures/wiki/search_report.json | 1 + mcp/tests/fixtures/wiki/search_stock.json | 1 + .../fixtures/wiki/siteinfo_rightsinfo.json | 1 + mcp/tests/support/mod.rs | 2 + mcp/tests/wiki_client.rs | 501 +++++++ 20 files changed, 1943 insertions(+), 4 deletions(-) create mode 100644 mcp/src/backends/wiki.rs create mode 100644 mcp/tests/fixtures/wiki/README.md create mode 100644 mcp/tests/fixtures/wiki/error_invalidtitle.json create mode 100644 mcp/tests/fixtures/wiki/error_missingtitle.json create mode 100644 mcp/tests/fixtures/wiki/error_pagecannotexist.json create mode 100644 mcp/tests/fixtures/wiki/parse_aberration_report.json create mode 100644 mcp/tests/fixtures/wiki/parse_about.json create mode 100644 mcp/tests/fixtures/wiki/parse_balancer_genotype_variant_report.json create mode 100644 mcp/tests/fixtures/wiki/parse_batch_download.json create mode 100644 mcp/tests/fixtures/wiki/parse_fbgn0000553.json create mode 100644 mcp/tests/fixtures/wiki/search_ftp_archive_wildcard.json create mode 100644 mcp/tests/fixtures/wiki/search_ftp_archive_zero.json create mode 100644 mcp/tests/fixtures/wiki/search_report.json create mode 100644 mcp/tests/fixtures/wiki/search_stock.json create mode 100644 mcp/tests/fixtures/wiki/siteinfo_rightsinfo.json create mode 100644 mcp/tests/wiki_client.rs diff --git a/mcp/src/backends/mod.rs b/mcp/src/backends/mod.rs index 70b1d7f..27b09a2 100644 --- a/mcp/src/backends/mod.rs +++ b/mcp/src/backends/mod.rs @@ -6,3 +6,4 @@ pub mod diopt; pub mod seq; pub mod species; pub mod sql; +pub mod wiki; diff --git a/mcp/src/backends/wiki.rs b/mcp/src/backends/wiki.rs new file mode 100644 index 0000000..c348f3c --- /dev/null +++ b/mcp/src/backends/wiki.rs @@ -0,0 +1,1287 @@ +//! Read-only client for the FlyBase wiki's MediaWiki API (`api.php`), plus the converters that turn search +//! snippets (wikitext) and parsed pages (HTML) into plain text. Three GET requests only; the client cannot +//! write (writes need POST and a token it never fetches). No query or title text ever reaches a log: every +//! reqwest error is rendered with `without_url()`, and MediaWiki errors are logged by code alone. +use crate::error::ToolError; +use serde::Deserialize; +use serde::de::DeserializeOwned; +use std::time::Duration; +use url::Url; + +/// Largest API response the client reads (the biggest page today is about 1.03 MB of parse JSON). +pub const MAX_WIKI_BYTES: usize = 4 * 1024 * 1024; +/// Prefix of every public wiki link. Never derived from the private base URL. +pub const PUBLIC_WIKI: &str = "https://wiki.flybase.org/wiki/"; +/// Namespaces `search` covers: Help, Category, FlyBase. The main namespace is mostly redirect stubs. +pub const SEARCH_NAMESPACES: &str = "12|14|1000"; + +const TIMEOUT: Duration = Duration::from_secs(10); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(3); +const HEALTH_TIMEOUT: Duration = Duration::from_secs(2); + +#[derive(Deserialize, Debug, Clone, Default, PartialEq)] +pub struct SearchInfo { + #[serde(default)] + pub totalhits: u64, +} + +#[derive(Deserialize, Debug, Clone, PartialEq)] +pub struct SearchHit { + #[serde(default)] + pub ns: i32, + pub title: String, + #[serde(default)] + pub snippet: String, + #[serde(default)] + pub timestamp: String, +} + +#[derive(Deserialize, Debug, Clone, PartialEq)] +pub struct Redirect { + pub from: String, + pub to: String, +} + +#[derive(Deserialize, Debug, Clone, PartialEq)] +pub struct PageInfo { + /// Absent on an invalid title. + #[serde(default)] + pub ns: i32, + pub title: String, + #[serde(default)] + pub missing: bool, + #[serde(default)] + pub invalid: bool, + #[serde(default)] + pub touched: Option, +} + +/// `query` of the combined `list=search` + `generator=search&redirects=1&prop=info` request. +#[derive(Deserialize, Debug, Clone, Default, PartialEq)] +pub struct SearchResponse { + #[serde(default)] + pub searchinfo: SearchInfo, + #[serde(default)] + pub search: Vec, + #[serde(default)] + pub redirects: Vec, + #[serde(default)] + pub pages: Vec, +} + +#[derive(Deserialize, Debug, Clone, PartialEq)] +pub struct Section { + /// MediaWiki sends the level as a string (`"2"`). + pub level: String, + /// HTML-escaped heading text (`Gene Model & Products`). + pub line: String, + #[serde(default)] + pub anchor: String, +} + +/// `parse` of `action=parse&prop=text|sections|revid` (formatversion 2). +#[derive(Deserialize, Debug, Clone, PartialEq)] +pub struct ParseResponse { + /// Normalized, post-redirect title. + pub title: String, + #[serde(default)] + pub pageid: u64, + #[serde(default)] + pub revid: u64, + pub text: String, + #[serde(default)] + pub sections: Vec
, + #[serde(default)] + pub redirects: Vec, +} + +#[derive(Deserialize, Debug, Clone, Default, PartialEq)] +pub struct RightsInfo { + #[serde(default)] + pub url: String, + #[serde(default)] + pub text: String, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum WikiError { + /// `missingtitle` or `pagecannotexist`. + NotFound, + /// Any other MediaWiki error code (`invalidtitle`, `badvalue`, ...): the request itself is wrong. + Invalid(String), + /// The response is over MAX_WIKI_BYTES. + TooLarge, + /// Transport error, timeout, redirect, non-200, non-JSON, unexpected shape, or an outage code. + Unavailable(String), +} + +impl WikiError { + pub fn into_tool(self, tool: &'static str) -> ToolError { + match self { + Self::NotFound => ToolError::NotFound { + tool, + message: "no wiki page has that title; try search_wiki".into(), + }, + Self::Invalid(code) => { + tracing::info!(tool, code = %code, "wiki rejected the request"); + ToolError::InvalidInput { + tool, + arg: if tool == "search_wiki" { + "query" + } else { + "title" + }, + message: format!("the wiki rejected it ({code})"), + } + } + Self::TooLarge => ToolError::LimitExceeded { + tool, + message: "this wiki page is too large to return; read it on the wiki".into(), + }, + Self::Unavailable(m) => { + tracing::error!(tool, error = %m, "wiki unavailable"); + ToolError::BackendUnavailable { + tool, + message: "the FlyBase wiki is not reachable; try again in a few minutes".into(), + } + } + } + } +} + +/// A MediaWiki error code reduced to identifier characters, so nothing else the server sent is echoed. +fn sanitize_code(code: &str) -> String { + let c: String = code + .chars() + .filter(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '-') + .take(64) + .collect(); + if c.is_empty() { "unknown".into() } else { c } +} + +fn map_code(code: &str) -> WikiError { + let code = sanitize_code(code); + match code.as_str() { + "missingtitle" | "pagecannotexist" => WikiError::NotFound, + "readonly" | "maxlag" | "ratelimited" => { + WikiError::Unavailable(format!("MediaWiki error {code}")) + } + c if c.starts_with("internal_api_error") => { + WikiError::Unavailable(format!("MediaWiki error {code}")) + } + _ => WikiError::Invalid(code), + } +} + +fn transport(e: reqwest::Error) -> WikiError { + WikiError::Unavailable(e.without_url().to_string()) +} + +pub struct WikiClient { + http: reqwest::Client, + api: Url, + host: reqwest::header::HeaderValue, +} + +impl WikiClient { + /// `base` is the wiki root (`http:///`); requests go to `api.php`. Only + /// `http://` is accepted: this build has no TLS backend. `host_header` is sent as `Host` on every + /// request so the origin answers as `wiki.flybase.org`. + pub fn new(mut base: Url, host_header: String) -> anyhow::Result { + if base.scheme() != "http" { + anyhow::bail!("WIKI_BASE_URL must be http://; this build has no TLS backend"); + } + if !base.path().ends_with('/') { + let p = format!("{}/", base.path()); + base.set_path(&p); + } + base.set_query(None); + base.set_fragment(None); + let api = base.join("api.php")?; + let host = reqwest::header::HeaderValue::from_str(host_header.trim()) + .map_err(|_| anyhow::anyhow!("WIKI_HOST_HEADER is not a valid header value"))?; + if host.is_empty() { + anyhow::bail!("WIKI_HOST_HEADER is empty"); + } + let http = reqwest::Client::builder() + .timeout(TIMEOUT) + .connect_timeout(CONNECT_TIMEOUT) + // The origin serves the wiki directly; a redirect means something is wrong, and following one + // could fetch an arbitrary reachable URL. + .redirect(reqwest::redirect::Policy::none()) + .user_agent(concat!("flybase-mcp/", env!("CARGO_PKG_VERSION"))) + .build()?; + Ok(Self { http, api, host }) + } + + /// `api.php` with `format=json&formatversion=2` and then `pairs`, every pair added through + /// `append_pair` so user text is always one percent-encoded value. + fn url(&self, pairs: &[(&str, &str)]) -> Url { + let mut u = self.api.clone(); + { + let mut q = u.query_pairs_mut(); + for (k, v) in pairs { + q.append_pair(k, v); + } + q.append_pair("format", "json"); + q.append_pair("formatversion", "2"); + } + u + } + + /// GET `url`, read at most MAX_WIKI_BYTES, decode the JSON, map a MediaWiki `error`, and deserialize + /// the `key` member. + async fn get( + &self, + url: Url, + key: &str, + timeout: Option, + ) -> Result { + let mut rb = self + .http + .get(url) + .header(reqwest::header::HOST, self.host.clone()); + if let Some(t) = timeout { + rb = rb.timeout(t); + } + let mut resp = rb.send().await.map_err(transport)?; + let status = resp.status().as_u16(); + if (300..400).contains(&status) { + return Err(WikiError::Unavailable(format!( + "unexpected redirect (HTTP {status})" + ))); + } + if status != 200 { + return Err(WikiError::Unavailable(format!("HTTP {status}"))); + } + if let Some(n) = resp.content_length() + && n > MAX_WIKI_BYTES as u64 + { + tracing::warn!(bytes = n, "wiki response over the 4 MiB cap"); + return Err(WikiError::TooLarge); + } + let mut buf: Vec = Vec::new(); + while let Some(chunk) = resp.chunk().await.map_err(transport)? { + if buf.len() + chunk.len() > MAX_WIKI_BYTES { + tracing::warn!( + bytes = buf.len() + chunk.len(), + "wiki response over the 4 MiB cap" + ); + return Err(WikiError::TooLarge); + } + buf.extend_from_slice(&chunk); + } + let mut v: serde_json::Value = serde_json::from_slice(&buf) + .map_err(|_| WikiError::Unavailable("non-JSON response".into()))?; + if let Some(err) = v.get("error") { + let code = err.get("code").and_then(|c| c.as_str()).unwrap_or(""); + return Err(map_code(code)); + } + let member = v + .get_mut(key) + .map(serde_json::Value::take) + .unwrap_or(serde_json::Value::Null); + // The serde message can quote response values, so it is not kept. + serde_json::from_value(member) + .map_err(|_| WikiError::Unavailable(format!("unexpected response shape ({key})"))) + } + + /// One combined request: `list=search` for titles, snippets and timestamps, plus + /// `generator=search&redirects=1&prop=info` for the redirect map and the resolved pages. + pub async fn search( + &self, + query: &str, + limit: u32, + offset: u32, + ) -> Result { + let limit = limit.to_string(); + let offset = offset.to_string(); + let url = self.url(&[ + ("action", "query"), + ("list", "search"), + ("srsearch", query), + ("srnamespace", SEARCH_NAMESPACES), + ("srlimit", &limit), + ("sroffset", &offset), + ("srprop", "snippet|timestamp"), + ("srinfo", "totalhits"), + ("generator", "search"), + ("gsrsearch", query), + ("gsrnamespace", SEARCH_NAMESPACES), + ("gsrlimit", &limit), + ("gsroffset", &offset), + ("redirects", "1"), + ("prop", "info"), + ]); + // MediaWiki omits `query` entirely in some empty cases; that is an empty result. + let q: Option = self.get(url, "query", None).await?; + Ok(q.unwrap_or_default()) + } + + /// The parsed page (following redirects), without edit links, TOC, or the limit report. + pub async fn page(&self, title: &str) -> Result { + let url = self.url(&[ + ("action", "parse"), + ("page", title), + ("redirects", "1"), + ("prop", "text|sections|revid"), + ("disablelimitreport", "1"), + ("disableeditsection", "1"), + ("disabletoc", "1"), + ]); + self.get(url, "parse", None).await + } + + fn siteinfo_url(&self) -> Url { + self.url(&[ + ("action", "query"), + ("meta", "siteinfo"), + ("siprop", "rightsinfo"), + ]) + } + + /// The wiki's licence (`$wgRightsText`, `$wgRightsUrl`); both empty when unset. + pub async fn rights(&self) -> Result { + #[derive(Deserialize)] + struct Q { + #[serde(default)] + rightsinfo: RightsInfo, + } + let q: Q = self.get(self.siteinfo_url(), "query", None).await?; + Ok(q.rightsinfo) + } + + /// True when the siteinfo request answers with valid JSON within 2 s. + pub async fn healthy(&self) -> bool { + #[derive(Deserialize)] + struct Q {} + self.get::(self.siteinfo_url(), "query", Some(HEALTH_TIMEOUT)) + .await + .is_ok() + } +} + +// --------------------------------------------------------------------------------------------------------- +// Entities + +/// Decode HTML entities once, left to right: `amp lt gt quot apos nbsp`, decimal and hex numeric references. +/// ` ` and ` ` become a plain space. Unknown or malformed references are kept as written, so +/// `&lt;` gives `<`. +pub fn decode_entities(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find('&') { + out.push_str(&rest[..i]); + rest = &rest[i..]; + // An entity is at most 10 bytes before its ';' (`􏿿`). + let end = rest.as_bytes().iter().take(12).position(|&b| b == b';'); + let decoded = end.and_then(|e| decode_one(&rest[1..e]).map(|c| (c, e))); + match decoded { + Some((c, e)) => { + out.push(c); + rest = &rest[e + 1..]; + } + None => { + out.push('&'); + rest = &rest[1..]; + } + } + } + out.push_str(rest); + out +} + +fn decode_one(name: &str) -> Option { + let c = match name { + "amp" => '&', + "lt" => '<', + "gt" => '>', + "quot" => '"', + "apos" => '\'', + "nbsp" => ' ', + _ => { + let num = name.strip_prefix('#')?; + let n = match num.strip_prefix(['x', 'X']) { + Some(h) if !h.is_empty() && h.bytes().all(|b| b.is_ascii_hexdigit()) => { + u32::from_str_radix(h, 16).ok()? + } + Some(_) => return None, + None if !num.is_empty() && num.bytes().all(|b| b.is_ascii_digit()) => { + num.parse().ok()? + } + None => return None, + }; + if n == 160 { ' ' } else { char::from_u32(n)? } + } + }; + Some(c) +} + +// --------------------------------------------------------------------------------------------------------- +// Search snippets (wikitext) + +/// Remove every `<...>` tag (the `searchmatch` spans and anything else). +fn strip_tags(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_tag = false; + for c in s.chars() { + match (in_tag, c) { + (false, '<') => in_tag = true, + (true, '>') => in_tag = false, + (false, c) => out.push(c), + (true, _) => {} + } + } + out +} + +/// `[[target|label]]` to `label`, `[[target]]` to `target`. +fn replace_internal_links(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find("[[") { + out.push_str(&rest[..i]); + let after = &rest[i + 2..]; + match after.find("]]") { + Some(j) => { + let inner = &after[..j]; + // A nested `[[` means the outer link's close is further on; keep the outer marker as text. + if inner.contains("[[") { + out.push_str("[["); + rest = after; + continue; + } + out.push_str(inner.rsplit('|').next().unwrap_or(inner).trim()); + rest = &after[j + 2..]; + } + None => { + // A link cut off at the snippet's edge: keep its target text. + out.push_str(after.split('|').next_back().unwrap_or(after)); + rest = ""; + } + } + } + out.push_str(rest); + out +} + +/// `[http://x label]` to `label`, `[http://x]` to `http://x`. Brackets without a URL scheme stay. +fn replace_external_links(s: &str) -> String { + const SCHEMES: [&str; 5] = ["http://", "https://", "ftp://", "//", "mailto:"]; + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find('[') { + out.push_str(&rest[..i]); + let after = &rest[i + 1..]; + let close = after.find(']'); + match close { + Some(j) if SCHEMES.iter().any(|p| after.starts_with(p)) => { + let inner = &after[..j]; + match inner.split_once(char::is_whitespace) { + Some((_, label)) if !label.trim().is_empty() => out.push_str(label.trim()), + _ => out.push_str(inner.trim()), + } + rest = &after[j + 1..]; + } + _ => { + out.push('['); + rest = after; + } + } + } + out.push_str(rest); + out +} + +/// Drop `{{...}}` innermost first, then any unmatched `{{` or `}}` left by a template cut at the edge. +fn drop_templates(s: &str) -> String { + let mut t = s.to_string(); + while let Some(close) = t.find("}}") { + match t[..close].rfind("{{") { + Some(open) => t.replace_range(open..close + 2, ""), + None => t.replace_range(close..close + 2, ""), + } + } + t.replace("{{", "") +} + +/// Strip one snippet line's leading block markup and its table cell separators. +fn clean_line(line: &str) -> String { + let mut l = line.trim(); + loop { + let before = l; + for p in ["#REDIRECT", "#redirect", "#Redirect"] { + if let Some(r) = l.strip_prefix(p) { + l = r.trim_start(); + } + } + for p in ["{|", "|}", "|-", "||", "|", "!", "*", "#", ":", ";"] { + if let Some(r) = l.strip_prefix(p) { + l = r.trim_start(); + } + } + if l == before { + break; + } + } + // `== Heading ==` + let l = l.trim_start_matches('=').trim_end_matches('='); + let l = l.replace("||", " | ").replace("!!", " | "); + l.split_whitespace().collect::>().join(" ") +} + +/// A search snippet (raw wikitext lines with `searchmatch` spans and escaped entities) as one line of plain +/// text: tags removed, entities decoded once, links reduced to their labels, templates, bold and italic +/// quotes, table and list markup, heading markers and `#REDIRECT` dropped, lines joined with one space. +pub fn clean_snippet(s: &str) -> String { + let t = decode_entities(&strip_tags(s)); + let t = replace_internal_links(&t); + let t = replace_external_links(&t); + let t = drop_templates(&t); + let t = t.replace("'''", "").replace("''", ""); + t.lines() + .map(clean_line) + .filter(|l| !l.is_empty()) + .collect::>() + .join(" ") +} + +// --------------------------------------------------------------------------------------------------------- +// Parsed pages (HTML) + +const BLOCK: [&str; 16] = [ + "p", + "div", + "dl", + "dt", + "dd", + "ul", + "ol", + "li", + "table", + "tr", + "pre", + "blockquote", + "caption", + "center", + "section", + "hr", +]; +const VOID: [&str; 12] = [ + "br", "hr", "img", "meta", "link", "input", "wbr", "col", "source", "area", "base", "track", +]; + +struct Tag<'a> { + name: String, + closing: bool, + self_closing: bool, + attrs: &'a str, +} + +/// The value of attribute `want` in a raw attribute string (quoted or not), names matched case-insensitively. +fn attr(attrs: &str, want: &str) -> Option { + let b = attrs.as_bytes(); + let mut i = 0; + while i < b.len() { + while i < b.len() && (b[i].is_ascii_whitespace() || b[i] == b'/') { + i += 1; + } + let ns = i; + while i < b.len() && !b[i].is_ascii_whitespace() && b[i] != b'=' && b[i] != b'/' { + i += 1; + } + let name = &attrs[ns..i]; + while i < b.len() && b[i].is_ascii_whitespace() { + i += 1; + } + let mut value = ""; + if i < b.len() && b[i] == b'=' { + i += 1; + while i < b.len() && b[i].is_ascii_whitespace() { + i += 1; + } + if i < b.len() && (b[i] == b'"' || b[i] == b'\'') { + let q = b[i]; + let vs = i + 1; + i = vs; + while i < b.len() && b[i] != q { + i += 1; + } + value = &attrs[vs..i]; + i += 1; + } else { + let vs = i; + while i < b.len() && !b[i].is_ascii_whitespace() { + i += 1; + } + value = &attrs[vs..i]; + } + } + if name.eq_ignore_ascii_case(want) { + return Some(value.to_string()); + } + if name.is_empty() && i < b.len() { + i += 1; + } + } + None +} + +fn has_class(attrs: &str, class: &str) -> bool { + attr(attrs, "class").is_some_and(|c| c.split_whitespace().any(|x| x == class)) +} + +/// Plain-text builder: collapses whitespace outside `
`, at most one blank line in a row, and turns line
+/// breaks inside a table cell into spaces.
+#[derive(Default)]
+struct Out {
+    buf: String,
+    space: bool,
+    pre: usize,
+    cell: usize,
+    cells_in_row: usize,
+    list: usize,
+    dd: usize,
+    links: Vec<(Option, usize)>,
+}
+
+impl Out {
+    fn at_break(&self) -> bool {
+        self.buf.is_empty() || self.buf.ends_with(['\n', '\t', ' '])
+    }
+
+    fn trim_end_spaces(&mut self) {
+        while self.buf.ends_with(' ') {
+            self.buf.pop();
+        }
+    }
+
+    fn text(&mut self, raw: &str) {
+        let t = decode_entities(raw);
+        if self.pre > 0 {
+            self.buf.push_str(&t);
+            self.space = false;
+            return;
+        }
+        for c in t.chars() {
+            if c.is_whitespace() {
+                self.space = true;
+            } else {
+                if self.space && !self.at_break() {
+                    self.buf.push(' ');
+                }
+                self.space = false;
+                self.buf.push(c);
+            }
+        }
+    }
+
+    fn push_raw(&mut self, s: &str) {
+        if self.space && !self.at_break() {
+            self.buf.push(' ');
+        }
+        self.space = false;
+        self.buf.push_str(s);
+    }
+
+    /// End the current line (a space inside a table cell). Several breaks in a row give one line end.
+    fn newline(&mut self) {
+        if self.cell > 0 {
+            self.space = true;
+            return;
+        }
+        self.space = false;
+        if self.pre == 0 {
+            self.trim_end_spaces();
+        }
+        if self.buf.is_empty() || self.buf.ends_with('\n') {
+            return;
+        }
+        self.buf.push('\n');
+    }
+
+    fn blank_line(&mut self) {
+        self.newline();
+        if self.cell == 0 && !self.buf.is_empty() && !self.buf.ends_with("\n\n") {
+            self.buf.push('\n');
+        }
+    }
+
+    fn finish(mut self) -> String {
+        self.trim_end_spaces();
+        let lines: Vec<&str> = self.buf.lines().collect();
+        let mut out = String::with_capacity(self.buf.len());
+        let mut blank = 0;
+        for l in lines {
+            if l.trim().is_empty() {
+                blank += 1;
+                if blank > 1 {
+                    continue;
+                }
+                out.push('\n');
+            } else {
+                blank = 0;
+                out.push_str(l);
+                out.push('\n');
+            }
+        }
+        out.trim_matches('\n').to_string()
+    }
+}
+
+/// Parse a tag starting at `s[0] == '<'`. Returns the tag and its byte length, or None when `<` does not
+/// begin a tag. A `>` inside a quoted attribute value does not end the tag.
+fn parse_tag(s: &str) -> Option<(Tag<'_>, usize)> {
+    let b = s.as_bytes();
+    let mut i = 1;
+    let closing = b.get(1) == Some(&b'/');
+    if closing {
+        i += 1;
+    }
+    if !b.get(i).is_some_and(|c| c.is_ascii_alphabetic()) {
+        return None;
+    }
+    let ns = i;
+    while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'-' || b[i] == b':') {
+        i += 1;
+    }
+    let name = s[ns..i].to_ascii_lowercase();
+    let as_ = i;
+    let mut quote: Option = None;
+    while i < b.len() {
+        match (quote, b[i]) {
+            (None, b'"' | b'\'') => quote = Some(b[i]),
+            (Some(q), c) if c == q => quote = None,
+            (None, b'>') => break,
+            _ => {}
+        }
+        i += 1;
+    }
+    if i >= b.len() {
+        return None;
+    }
+    let attrs = &s[as_..i];
+    let self_closing = attrs.trim_end().ends_with('/');
+    Some((
+        Tag {
+            name,
+            closing,
+            self_closing,
+            attrs,
+        },
+        i + 1,
+    ))
+}
+
+/// Find the end of a raw-text element (`script`, `style`): the byte just after ``.
+fn raw_text_end(s: &str, name: &str) -> usize {
+    let lower = s.to_ascii_lowercase();
+    let needle = format!(" s[i..].find('>').map_or(s.len(), |j| i + j + 1),
+        None => s.len(),
+    }
+}
+
+fn heading_level(name: &str) -> Option {
+    match name.as_bytes() {
+        [b'h', d @ b'1'..=b'6'] => Some((d - b'0') as usize),
+        _ => None,
+    }
+}
+
+/// A parsed MediaWiki page as plain text. Paragraphs, lists, definition lists and headings keep their lines
+/// (`- ` for list items, two spaces per `
` level, `== Title ==` for headings, one `=` per level); a +/// table row is one line with tab-separated cells; `
` keeps its whitespace; external links read
+/// `text (href)`; scripts, styles, comments, reference markers, edit links and the TOC are dropped.
+/// Entities are decoded once.
+pub fn html_to_text(html: &str) -> String {
+    let mut out = Out::default();
+    // (tag name, depth) of an element being skipped with everything inside it.
+    let mut skip: Option<(String, usize)> = None;
+    let mut i = 0;
+    while i < html.len() {
+        let rest = &html[i..];
+        let Some(lt) = rest.find('<') else {
+            if skip.is_none() {
+                out.text(rest);
+            }
+            break;
+        };
+        if lt > 0 {
+            if skip.is_none() {
+                out.text(&rest[..lt]);
+            }
+            i += lt;
+            continue;
+        }
+        if rest.starts_with("").map_or(rest.len(), |j| j + 3);
+            continue;
+        }
+        if rest.starts_with("').map_or(rest.len(), |j| j + 1);
+            continue;
+        }
+        let Some((tag, len)) = parse_tag(rest) else {
+            if skip.is_none() {
+                out.text("<");
+            }
+            i += 1;
+            continue;
+        };
+        i += len;
+        let name = tag.name.as_str();
+        let void = VOID.contains(&name) || tag.self_closing;
+        if !tag.closing && (name == "script" || name == "style") {
+            if !tag.self_closing {
+                i += raw_text_end(&html[i..], name);
+            }
+            continue;
+        }
+        if let Some((sname, depth)) = skip.as_mut() {
+            if sname == name {
+                if tag.closing {
+                    *depth -= 1;
+                    if *depth == 0 {
+                        skip = None;
+                    }
+                } else if !void {
+                    *depth += 1;
+                }
+            }
+            continue;
+        }
+        if !tag.closing
+            && ((name == "sup" && has_class(tag.attrs, "reference"))
+                || has_class(tag.attrs, "mw-editsection")
+                || attr(tag.attrs, "id").is_some_and(|v| v == "toc"))
+        {
+            if !void {
+                skip = Some((tag.name.clone(), 1));
+            }
+            continue;
+        }
+        handle_tag(&mut out, &tag);
+    }
+    out.finish()
+}
+
+fn handle_tag(out: &mut Out, tag: &Tag<'_>) {
+    let name = tag.name.as_str();
+    if name == "br" {
+        out.newline();
+        return;
+    }
+    if let Some(level) = heading_level(name) {
+        let marks = "=".repeat(level.max(2));
+        if tag.closing {
+            out.trim_end_spaces();
+            out.space = false;
+            out.buf.push(' ');
+            out.buf.push_str(&marks);
+            out.blank_line();
+        } else {
+            out.blank_line();
+            out.push_raw(&marks);
+            out.buf.push(' ');
+        }
+        return;
+    }
+    match (name, tag.closing) {
+        ("td" | "th", false) => {
+            if out.cell == 0 {
+                if out.cells_in_row > 0 {
+                    out.trim_end_spaces();
+                    out.space = false;
+                    out.buf.push('\t');
+                }
+                out.cells_in_row += 1;
+            } else {
+                out.space = true;
+            }
+            out.cell += 1;
+        }
+        ("td" | "th", true) => {
+            out.cell = out.cell.saturating_sub(1);
+            out.space = out.cell > 0;
+        }
+        ("tr", _) => {
+            out.newline();
+            if out.cell == 0 {
+                out.cells_in_row = 0;
+            }
+        }
+        ("pre", false) => {
+            out.newline();
+            out.pre += 1;
+        }
+        ("pre", true) => {
+            // Ends the line without trimming the block's own trailing spaces.
+            out.newline();
+            out.pre = out.pre.saturating_sub(1);
+        }
+        ("ul" | "ol", false) => {
+            out.newline();
+            out.list += 1;
+        }
+        ("ul" | "ol", true) => {
+            out.list = out.list.saturating_sub(1);
+            out.newline();
+        }
+        ("li", false) => {
+            out.newline();
+            if out.cell == 0 {
+                let indent = "  ".repeat(out.list.saturating_sub(1));
+                out.buf.push_str(&indent);
+                out.buf.push_str("- ");
+            }
+        }
+        ("dd", false) => {
+            out.newline();
+            out.dd += 1;
+            if out.cell == 0 {
+                out.buf.push_str(&"  ".repeat(out.dd));
+            }
+        }
+        ("dd", true) => {
+            out.dd = out.dd.saturating_sub(1);
+            out.newline();
+        }
+        ("a", false) => {
+            let href = if has_class(tag.attrs, "external") {
+                attr(tag.attrs, "href").map(|h| decode_entities(&h))
+            } else {
+                None
+            };
+            out.links.push((href, out.buf.len()));
+        }
+        ("a", true) => {
+            if let Some((Some(href), start)) = out.links.pop() {
+                let text = out.buf.get(start..).unwrap_or("").trim().to_string();
+                if !text.is_empty() && text != href {
+                    out.push_raw(&format!(" ({href})"));
+                } else if text.is_empty() {
+                    out.push_raw(&href);
+                }
+            }
+        }
+        (n, _) if BLOCK.contains(&n) => out.newline(),
+        _ => {}
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn fixture_text(name: &str) -> String {
+        let path = format!("{}/tests/fixtures/wiki/{name}", env!("CARGO_MANIFEST_DIR"));
+        let v: serde_json::Value =
+            serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
+        v["parse"]["text"].as_str().unwrap().to_string()
+    }
+
+    fn report_snippets() -> Vec<(String, String)> {
+        let path = format!(
+            "{}/tests/fixtures/wiki/search_report.json",
+            env!("CARGO_MANIFEST_DIR")
+        );
+        let v: serde_json::Value =
+            serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
+        v["query"]["search"]
+            .as_array()
+            .unwrap()
+            .iter()
+            .map(|h| {
+                (
+                    h["title"].as_str().unwrap().to_string(),
+                    h["snippet"].as_str().unwrap().to_string(),
+                )
+            })
+            .collect()
+    }
+
+    fn snippet(title: &str) -> String {
+        report_snippets()
+            .into_iter()
+            .find(|(t, _)| t == title)
+            .unwrap_or_else(|| panic!("{title} not in search_report.json"))
+            .1
+    }
+
+    #[test]
+    fn about_address_block_stays_multi_line() {
+        let t = html_to_text(&fixture_text("parse_about.json"));
+        assert!(
+            t.contains(
+                "  FlyBase-Harvard\n  Biological Laboratories, Harvard University\n  16 Divinity Avenue\n  Cambridge, Massachusetts 02138\n  USA"
+            ),
+            "address block not found"
+        );
+        // 
 keeps its leading spaces and line breaks; non-ASCII names survive.
+        assert!(t.contains("   Norbert Perrimon (PI)\n   Gil dos Santos\n"));
+        assert!(t.contains("   Katja Röper (Co-PI)\n"));
+        // External link: text (href).
+        assert!(t.contains("contact FlyBase (http://flybase.org/contact/email) form"));
+        assert!(t.contains("== FlyBase Consortium =="));
+        for tag in ["a   b\nc

  x\n    y  

z

"), + "a b c\n x\n y \nz" + ); + assert_eq!( + html_to_text( + "
one\ntwothree
four
Hv
" + ), + "one two\tthree four\nH\tv" + ); + } + + #[test] + fn lists_headings_and_links() { + assert_eq!( + html_to_text("
  • one
  • two
    • deep
"), + "- one\n- two\n - deep" + ); + assert_eq!(html_to_text("
  1. first
"), "- first"); + assert_eq!( + html_to_text( + "

x

Title

Sub

y

" + ), + "x\n\n== Title ==\n\n=== Sub ===\n\ny" + ); + assert_eq!( + html_to_text( + "see the site or Foo" + ), + "see the site (http://x.org/?a=1&b=2) or Foo" + ); + assert_eq!( + html_to_text("http://x.org"), + "http://x.org" + ); + } + + #[test] + fn entities_decode_once() { + assert_eq!( + html_to_text("a b & c AA &lt; <tag> "q" '"), + "a b & c AA < \"q\" '" + ); + assert_eq!(decode_entities("&lt;"), "<"); + assert_eq!( + decode_entities("é &bogus; & &#; &#x;"), + "é &bogus; & &#; &#x;" + ); + assert_eq!(decode_entities("�"), "�"); + } + + #[test] + fn dropped_elements() { + let html = "

keep one

\ + \ + \ +

text[1] two

\ +

Head[edit]

\ +
  • Contents
\ +

three

"; + assert_eq!( + html_to_text(html), + "keep one\ntext two\n\n== Head ==\n\nthree" + ); + } + + #[test] + fn uppercase_tags_and_gt_in_attributes() { + assert_eq!( + html_to_text( + "

a

  • y\">b
cd
" + ), + "a\n- b\nc\td" + ); + assert_eq!(html_to_text("1 < 2 and 3 > 2"), "1 < 2 and 3 > 2"); + } + + #[test] + fn broken_redirect_page_text() { + let t = html_to_text(&fixture_text("parse_fbgn0000553.json")); + assert_eq!(t, "- REDIRECT [[Dmel\\e(dpv)]]"); + } + + #[test] + fn real_report_snippets() { + assert_eq!( + clean_snippet(&snippet("FlyBase:Report Help")), + "Aberration Report Allele Report" + ); + let combo = clean_snippet(&snippet("FlyBase:Combination Report")); + assert!( + combo.contains("Feature type | The type of feature described in the report."), + "{combo}" + ); + assert!(combo.starts_with("The Split System Combination report displays")); + assert!(combo.contains("\"split system combination\""), "{combo}"); + let ins = clean_snippet(&snippet("FlyBase:Insertion Report")); + assert!( + ins.ends_with("information in the Insertion Report."), + "{ins}" + ); + assert_eq!( + clean_snippet(&snippet("FlyBase:Pathway Report")), + "FlyBase:Signaling Pathway Report" + ); + for (title, s) in report_snippets() { + let c = clean_snippet(&s); + for bad in ["<", "[[", "]]", "'''", "'", "\n", "#REDIRECT"] { + assert!(!c.contains(bad), "{title}: {bad:?} in {c:?}"); + } + } + } + + #[test] + fn snippet_markup() { + assert_eq!( + clean_snippet("{{Note|hidden}} after {{a|{{b}}}} x"), + "after x" + ); + assert_eq!(clean_snippet("cut {{Templ|arg"), "cut Templ|arg"); + assert_eq!( + clean_snippet("#REDIRECT [[FlyBase:Stocks]]"), + "FlyBase:Stocks" + ); + assert_eq!( + clean_snippet("== Heading ==\n* item\n# num\n: indent"), + "Heading item num indent" + ); + assert_eq!( + clean_snippet("{|\n|-\n! A !! B\n| a || b\n|}"), + "A | B a | b" + ); + assert_eq!( + clean_snippet( + "[http://flybase.org/x Stock Collections]\n" + ), + "Stock Collections" + ); + assert_eq!( + clean_snippet("see [1] and [http://x.org]"), + "see [1] and http://x.org" + ); + assert_eq!( + clean_snippet("''italic'' and '''bold'''"), + "italic and bold" + ); + assert_eq!(clean_snippet("[[FlyBase:Cut off|lab"), "lab"); + assert_eq!(clean_snippet(""), ""); + } + + #[test] + fn error_codes_map_per_table() { + assert_eq!(map_code("missingtitle"), WikiError::NotFound); + assert_eq!(map_code("pagecannotexist"), WikiError::NotFound); + for c in [ + "internal_api_error_DBQueryError", + "readonly", + "maxlag", + "ratelimited", + ] { + assert!(matches!(map_code(c), WikiError::Unavailable(_)), "{c}"); + } + assert_eq!( + map_code("invalidtitle"), + WikiError::Invalid("invalidtitle".into()) + ); + assert_eq!(map_code("badvalue"), WikiError::Invalid("badvalue".into())); + assert_eq!( + map_code("odd code "), + WikiError::Invalid("oddcodex".into()), + "codes are reduced to identifier characters" + ); + } + + #[test] + fn errors_map_to_tool_classes() { + assert_eq!( + WikiError::NotFound.into_tool("get_wiki_page").code(), + "not_found" + ); + assert_eq!( + WikiError::TooLarge.into_tool("get_wiki_page").code(), + "limit_exceeded" + ); + assert_eq!( + WikiError::Unavailable("x".into()) + .into_tool("search_wiki") + .code(), + "backend_unavailable" + ); + match WikiError::Invalid("badvalue".into()).into_tool("search_wiki") { + ToolError::InvalidInput { arg, message, .. } => { + assert_eq!(arg, "query"); + assert!(message.contains("badvalue")); + } + e => panic!("{e:?}"), + } + match WikiError::Invalid("invalidtitle".into()).into_tool("get_wiki_page") { + ToolError::InvalidInput { arg, .. } => assert_eq!(arg, "title"), + e => panic!("{e:?}"), + } + } + + #[test] + fn https_base_is_rejected() { + match WikiClient::new( + "https://wiki.example/".parse().unwrap(), + "wiki.flybase.org".into(), + ) { + Ok(_) => panic!("https base must be rejected"), + Err(e) => assert!(e.to_string().contains("http://"), "{e}"), + } + assert!(WikiClient::new("http://127.0.0.1/".parse().unwrap(), "bad\nhost".into()).is_err()); + } + + #[test] + fn api_url_keeps_base_path() { + let c = + WikiClient::new("http://127.0.0.1:9/mediawiki".parse().unwrap(), "h".into()).unwrap(); + assert_eq!( + c.url(&[("action", "query")]).as_str(), + "http://127.0.0.1:9/mediawiki/api.php?action=query&format=json&formatversion=2" + ); + let c = WikiClient::new("http://127.0.0.1:9".parse().unwrap(), "h".into()).unwrap(); + assert!( + c.url(&[]) + .as_str() + .starts_with("http://127.0.0.1:9/api.php?") + ); + } +} diff --git a/mcp/src/config.rs b/mcp/src/config.rs index bd34705..0519d2f 100644 --- a/mcp/src/config.rs +++ b/mcp/src/config.rs @@ -2,6 +2,8 @@ use anyhow::Context; use std::net::SocketAddr; use url::Url; +pub const DEFAULT_WIKI_HOST: &str = "wiki.flybase.org"; + #[derive(Clone)] pub struct Config { pub bind: SocketAddr, @@ -11,6 +13,10 @@ pub struct Config { pub pool_size: usize, pub blast_base_url: Option, pub blast_token: Option, + /// Private wiki origin (`WIKI_BASE_URL`, `http://` only). Unset: the wiki tools answer `backend_unavailable`. + pub wiki_base_url: Option, + /// `Host` sent to the wiki origin (`WIKI_HOST_HEADER`, default `wiki.flybase.org`). + pub wiki_host_header: String, /// Response cache bound in bytes (`FLYBASE_MCP_RESPONSE_CACHE_MB`, default 256 MiB; 0 disables it). pub response_cache_bytes: u64, } @@ -41,6 +47,16 @@ impl Config { .transpose() .context("BLAST_BASE_URL")?, blast_token: get("BLAST_SERVICE_TOKEN"), + // An empty value (a compose file passing `${WIKI_BASE_URL:-}`) means unset. + wiki_base_url: get("WIKI_BASE_URL") + .filter(|v| !v.trim().is_empty()) + .map(|v| v.trim().parse()) + .transpose() + .context("WIKI_BASE_URL")?, + wiki_host_header: get("WIKI_HOST_HEADER") + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .unwrap_or_else(|| DEFAULT_WIKI_HOST.into()), response_cache_bytes: get("FLYBASE_MCP_RESPONSE_CACHE_MB") .map(|v| v.parse::()) .transpose() @@ -89,6 +105,8 @@ impl std::fmt::Debug for Config { .field("pool_size", &self.pool_size) .field("blast_base_url", &self.blast_base_url) .field("blast_token", &self.blast_token.as_ref().map(|_| "***")) + .field("wiki_base_url", &self.wiki_base_url) + .field("wiki_host_header", &self.wiki_host_header) .field("response_cache_bytes", &self.response_cache_bytes) .finish() } @@ -113,6 +131,42 @@ mod tests { assert_eq!(c.response_cache_bytes, 256 * 1024 * 1024); assert!(c.allowed_hosts().contains(&"mcp.flybase.org".to_string())); assert_eq!(c.allowed_origins(), vec!["https://mcp.flybase.org"]); + assert_eq!(c.wiki_base_url, None); + assert_eq!(c.wiki_host_header, "wiki.flybase.org"); + } + #[test] + fn wiki_env_vars() { + let c = Config::from_lookup(|k| match k { + "WIKI_BASE_URL" => Some("http://192.0.2.10/".into()), + "WIKI_HOST_HEADER" => Some("wiki.example.org".into()), + _ => env(k), + }) + .unwrap(); + assert_eq!( + c.wiki_base_url.as_ref().map(|u| u.as_str()), + Some("http://192.0.2.10/") + ); + assert_eq!(c.wiki_host_header, "wiki.example.org"); + let d = format!("{c:?}"); + assert!( + d.contains("wiki_base_url") && d.contains("192.0.2.10"), + "{d}" + ); + assert!(d.contains("wiki.example.org"), "{d}"); + let c = Config::from_lookup(|k| match k { + "WIKI_BASE_URL" => Some(" ".into()), + "WIKI_HOST_HEADER" => Some("".into()), + _ => env(k), + }) + .unwrap(); + assert_eq!(c.wiki_base_url, None, "an empty WIKI_BASE_URL is unset"); + assert_eq!(c.wiki_host_header, "wiki.flybase.org"); + let e = Config::from_lookup(|k| match k { + "WIKI_BASE_URL" => Some("not a url".into()), + _ => env(k), + }) + .unwrap_err(); + assert!(e.to_string().contains("WIKI_BASE_URL"), "{e}"); } #[test] fn debug_redacts_secrets() { diff --git a/mcp/src/limits.rs b/mcp/src/limits.rs index 4b3c4d6..c34e786 100644 --- a/mcp/src/limits.rs +++ b/mcp/src/limits.rs @@ -8,6 +8,8 @@ pub const HEAVY_PERMITS: usize = 6; pub const BLAST_PERMITS: usize = 2; /// Permits for blast_status and blast_results, which wait on HTTP and must never hold a chado permit. pub const BLAST_POLL_PERMITS: usize = 8; +/// Permits for the wiki tools, which wait on HTTP and must never hold a chado permit. +pub const WIKI_PERMITS: usize = 4; pub const MAX_JSON_BYTES: usize = 256 * 1024; pub const MAX_FASTA_BYTES: usize = 512 * 1024; pub const MAX_ROWS: i64 = 1000; @@ -22,6 +24,7 @@ pub struct Limits { heavy: Arc, blast: Arc, blast_poll: Arc, + wiki: Arc, } impl Limits { @@ -31,6 +34,7 @@ impl Limits { heavy: Arc::new(Semaphore::new(HEAVY_PERMITS)), blast: Arc::new(Semaphore::new(BLAST_PERMITS)), blast_poll: Arc::new(Semaphore::new(BLAST_POLL_PERMITS)), + wiki: Arc::new(Semaphore::new(WIKI_PERMITS)), } } pub async fn light(&self, tool: &'static str) -> ToolResult { @@ -45,13 +49,17 @@ impl Limits { pub async fn blast_poll(&self, tool: &'static str) -> ToolResult { acquire(&self.blast_poll, tool).await } - /// (light, heavy, blast, blast_poll) permits currently free. - pub fn available(&self) -> (usize, usize, usize, usize) { + pub async fn wiki(&self, tool: &'static str) -> ToolResult { + acquire(&self.wiki, tool).await + } + /// (light, heavy, blast, blast_poll, wiki) permits currently free. + pub fn available(&self) -> (usize, usize, usize, usize, usize) { ( self.light.available_permits(), self.heavy.available_permits(), self.blast.available_permits(), self.blast_poll.available_permits(), + self.wiki.available_permits(), ) } } @@ -93,7 +101,13 @@ mod tests { } assert_eq!( l.available(), - (LIGHT_PERMITS, 0, BLAST_PERMITS, BLAST_POLL_PERMITS) + ( + LIGHT_PERMITS, + 0, + BLAST_PERMITS, + BLAST_POLL_PERMITS, + WIKI_PERMITS + ) ); let got = tokio::time::timeout(Duration::from_millis(50), l.light("t")).await; assert!( @@ -104,9 +118,27 @@ mod tests { let _submit = l.blast("blast_submit").await.unwrap(); assert_eq!( l.available(), - (LIGHT_PERMITS, 0, BLAST_PERMITS - 1, BLAST_POLL_PERMITS - 1), + ( + LIGHT_PERMITS, + 0, + BLAST_PERMITS - 1, + BLAST_POLL_PERMITS - 1, + WIKI_PERMITS + ), "blast and blast_poll are separate pools" ); + let _wiki = l.wiki("get_wiki_page").await.unwrap(); + assert_eq!( + l.available(), + ( + LIGHT_PERMITS, + 0, + BLAST_PERMITS - 1, + BLAST_POLL_PERMITS - 1, + WIKI_PERMITS - 1 + ), + "the wiki pool is separate from the chado and BLAST pools" + ); } #[tokio::test(start_paused = true)] async fn exhausted_semaphore_returns_busy_after_pool_wait() { @@ -127,6 +159,31 @@ mod tests { drop(held); assert!(l.blast("blast_submit").await.is_ok()); } + #[tokio::test(start_paused = true)] + async fn wiki_permit_times_out_after_pool_wait() { + let l = Limits::new(); + let mut held = Vec::new(); + for _ in 0..WIKI_PERMITS { + held.push(l.wiki("search_wiki").await.unwrap()); + } + let started = tokio::time::Instant::now(); + let e = l.wiki("search_wiki").await.unwrap_err(); + assert_eq!(e.code(), "backend_unavailable"); + assert!(e.to_string().starts_with("search_wiki: server busy"), "{e}"); + let waited = started.elapsed(); + assert!( + waited >= POOL_WAIT && waited < POOL_WAIT + Duration::from_secs(1), + "waited {waited:?}" + ); + assert!( + tokio::time::timeout(Duration::from_millis(10), l.light("search")) + .await + .is_ok_and(|p| p.is_ok()), + "an exhausted wiki pool leaves chado permits free" + ); + drop(held); + assert!(l.wiki("search_wiki").await.is_ok()); + } #[test] fn clamp() { assert_eq!(clamp_limit(None), 25); diff --git a/mcp/tests/fixtures/wiki/README.md b/mcp/tests/fixtures/wiki/README.md new file mode 100644 index 0000000..1f0ba5e --- /dev/null +++ b/mcp/tests/fixtures/wiki/README.md @@ -0,0 +1,24 @@ +# Wiki API fixtures + +Full responses from the public FlyBase wiki API, captured with anonymous GET requests on 2026-10-13 (UTC) +and saved unedited. Tests serve them through wiremock; nothing here is hand-written. + +Every URL carries the same parameters the client sends (`format=json&formatversion=2` last). + +| File | Content | URL | +|---|---|---| +| `parse_aberration_report.json` | `parse` of FlyBase:Aberration Report (report-guide tables) | | +| `parse_about.json` | `parse` of FlyBase:About (`dl/dd` address blocks, `
`, external link) |  |
+| `parse_batch_download.json` | `parse` of FlyBase:Batch Download |  |
+| `parse_balancer_genotype_variant_report.json` | `parse` of FlyBase:Balancer/Genotype variant Report (`/` in the title) |  |
+| `parse_fbgn0000553.json` | `parse` of FBgn0000553, a broken main-namespace redirect (text `
  1. REDIRECT [[...`) | | +| `search_stock.json` | combined search for `stock` (one redirect, FlyBase:Stock Collections to FlyBase:Stocks) | | +| `search_report.json` | combined search for `report` (wikitext snippets, a `#REDIRECT` snippet) | | +| `siteinfo_rightsinfo.json` | `siteinfo` `rightsinfo` (both strings empty) | | +| `error_missingtitle.json` | `parse` of a page that does not exist: `missingtitle` | | +| `error_invalidtitle.json` | `parse` of `Foo`: `invalidtitle` | | +| `error_pagecannotexist.json` | `parse` of `Special:RecentChanges`: `pagecannotexist` | | +| `search_ftp_archive_zero.json` | combined search for `FTP archive`: zero hits (input to the wildcard retry) | | +| `search_ftp_archive_wildcard.json` | combined search for `ftp* archive*`: the wildcard retry's hit | | + +To refresh one, fetch its URL again and overwrite the file; keep the bytes as served. diff --git a/mcp/tests/fixtures/wiki/error_invalidtitle.json b/mcp/tests/fixtures/wiki/error_invalidtitle.json new file mode 100644 index 0000000..f04fe87 --- /dev/null +++ b/mcp/tests/fixtures/wiki/error_invalidtitle.json @@ -0,0 +1 @@ +{"error":{"code":"invalidtitle","info":"Bad title \"Foo\".","docref":"See https://wiki.flybase.org/mediawiki/api.php for API usage. Subscribe to the mediawiki-api-announce mailing list at <https://lists.wikimedia.org/postorius/lists/mediawiki-api-announce.lists.wikimedia.org/> for notice of API deprecations and breaking changes."}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/error_missingtitle.json b/mcp/tests/fixtures/wiki/error_missingtitle.json new file mode 100644 index 0000000..274a03e --- /dev/null +++ b/mcp/tests/fixtures/wiki/error_missingtitle.json @@ -0,0 +1 @@ +{"error":{"code":"missingtitle","info":"The page you specified doesn't exist.","docref":"See https://wiki.flybase.org/mediawiki/api.php for API usage. Subscribe to the mediawiki-api-announce mailing list at <https://lists.wikimedia.org/postorius/lists/mediawiki-api-announce.lists.wikimedia.org/> for notice of API deprecations and breaking changes."}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/error_pagecannotexist.json b/mcp/tests/fixtures/wiki/error_pagecannotexist.json new file mode 100644 index 0000000..73fb16e --- /dev/null +++ b/mcp/tests/fixtures/wiki/error_pagecannotexist.json @@ -0,0 +1 @@ +{"error":{"code":"pagecannotexist","info":"Namespace doesn't allow actual pages.","docref":"See https://wiki.flybase.org/mediawiki/api.php for API usage. Subscribe to the mediawiki-api-announce mailing list at <https://lists.wikimedia.org/postorius/lists/mediawiki-api-announce.lists.wikimedia.org/> for notice of API deprecations and breaking changes."}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/parse_aberration_report.json b/mcp/tests/fixtures/wiki/parse_aberration_report.json new file mode 100644 index 0000000..9ffa13e --- /dev/null +++ b/mcp/tests/fixtures/wiki/parse_aberration_report.json @@ -0,0 +1 @@ +{"parse":{"title":"FlyBase:Aberration Report","pageid":100775,"revid":173638,"redirects":[],"text":"

    The Aberration Report provides information on individual chromosomal aberrations. Aberrations are chromosomal rearrangements that differ from the wild-type arrangement of chromosomes.\n

    This is a field-by-field guide to the information that is provided in the Aberration Report.\n

    FlyBase attributes data to the publication that reported it, so that users can easily refer back to the original publication if they wish. Thus, where possible in the fields below, the publication(s) that are the source of the information are listed, typically in parentheses to the right of the data. The exception in the Aberration Report is the General Information section which contains a summary of the identity of the aberration and a summary of cytological data.\n

    \n\n

    General Information

    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    SymbolThe valid symbol that is used in FlyBase for the aberration.\n

    The first part of the symbol (before the '\\') is the standard prefix for the species (from the Species Abbreviations list). For species other than D.melanogaster, the species prefix is displayed wherever the aberration symbol is used throughout FlyBase. For D.melanogaster aberrations, the species prefix is only displayed in the General Information section at the top of a Report.\n

    See the General principles for naming aberrations section of the FlyBase Nomenclature document for a detailed description of how aberration symbols are constructed.\n

    \n
    NameThe valid full name that is used in FlyBase for the aberration.\n
    Feature typeA list of controlled vocabulary terms from the Sequence Ontology that describe the nature of the chromosomal aberration relative to the wild-type chromosomal order.\n
    Also Known AsA list of up to 10 commonly used symbol synonyms for the aberration. (This list is made computationally, based on the frequency of alternative symbols that have been curated from the literature.). This field is only displayed on an individual Aberration Report if it contains data.\n
    SpeciesThe organism that the aberration originates from, with the initial letter of the genus and the full species name listed.\n
    FlyBase IDThe Primary FlyBase identifier number of the aberration, used to uniquely identify the aberration in the database.\n

    An aberration may also have any number of Secondary FlyBase identifier numbers, which are listed in the Secondary FlyBase IDs section of the Aberration Report.\n

    \n
    Computed Cytological Breakpoints (relative to progenitor)Computed cytological breakpoints for the aberration, relative to the progenitor chromosome (if given), based upon data from the literature.\n

    See Computed cytological data in FlyBase for a detailed description of how these computed cytological breakpoints are calculated.\n

    \n
    Deleted segmentThe maximal cytological range of a segment that is deleted in the aberration.\n

    The statement reflects either observed cytology or cytology estimated from sequence coordinates.\n

    \n
    Duplicated segment\tThe maximal cytological range of a segment that is duplicated in the aberration.\n

    The statement reflects either observed cytology or cytology estimated from sequence coordinates.\n

    \n
    Genomic MapsA JBrowse thumbnail image showing the location of the deleted and/or duplicated segments on the genome. This field is only displayed if segments have been mapped to the genome.\n

    To the left of the JBrowse thumbnail, links to JBrowse are provided.\n

    \n
    Sequenced Breakpoints (relative to progenitor)The coordinates of aberration breakpoints relative to the progenitor chromosome (if given), as reported by authors or inferred by FlyBase.\n
    Member of large scale dataset(s)If the aberration is a member of large scale dataset, the name of the dataset is listed here (hyperlinked to the relevant Large Dataset Metadata Report).\n

    A description of the attributes of the dataset is also present.\n

    \n
    \n

    Nature of the Aberration

    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    Cytological OrderA description of the new cytological order of the chromosomal aberration.\n

    The sequence of each chromosome involved in the chromosomal aberration is specified from one end to the other according to salivary gland chromosome band terminology. Each chromosome is separated by a semicolon. Points of breakage and reunion are indicated by vertical bars, and segments between these points are designated by the most extreme band known to be present at each end, separated by a dash. If the order of a particular segment is not known, the segment is included within parentheses, and hierarchies of ambiguities are represented by parentheses within parentheses.\n

    See the New order section of the FlyBase Nomenclature document for an example.\n

    \n
    ProgenitorThe mutant chromosome(s) on which the aberration was induced, where the progenitor is relevant to the derivative aberration. The progenitor is usually an aberration, a mutant allele or a transposable element insertion.\n

    Clicking on the symbol of the progenitor will take you to the relevant Report page.\n

    \n
    MutagenA list of controlled vocabulary terms from the FlyBase controlled vocabulary that describe the origin of the chromosomal aberration, typically the mutagen(s) used to induce the chromosomal aberration.\n
    Class of aberration (relative to wild type)A list of controlled vocabulary terms from the Sequence Ontology that describe the nature of the chromosomal aberration relative to the progenitor chromosome.\n
    Class of aberration (relative to progenitor)A list of controlled vocabulary terms from the Sequence Ontology that describe the nature of the chromosomal aberration relative to the progenitor chromosome.\n
    Cytological Breakpoints (relative to progenitor)A list of reported breakpoints for the aberration, relative to the progenitor chromosome (if given).\n

    See the Cytogenetic descriptions section of the FlyBase Nomenclature document for an explanation of the notation used by FlyBase for cytological breakpoints.\n

    \n
    Causes allelesA list of alleles caused by the aberration, due to a breakpoint within the locus.\n

    Clicking on an allele symbol will take you to the relevant Allele Report.\n

    \n
    Carries allelesA list of alleles that are carried on the aberration but are separable from its breakpoints.\n

    Clicking on an allele symbol will take you to the relevant Allele Report.\n

    \n
    Transposon InsertionsA list of insertions of transgenic constructs or natural transposons that are carried on the aberration. The insertions listed can be either separable or inseparable from any of the breakpoints of the aberration.\n

    Clicking on an insertion symbol will take you to the relevant Insertion Report.\n

    \n
    Formalized genetic dataA formalised summary listing breakpoints of the aberration (\"bk1\", \"bk2\" etc.) and their position relative to certain genes carried on the chromosome (these are not necessarily the genes that immediately flank the breakpoints). Each gene or breakpoint is listed in the order they are positioned along the chromsome, separated by the symbol \"<<\", which should be read as \"lies to the left of\". If a breakpoint is known to hit a certain gene, this is listed as for example \"bk1 hits Sep1\".\n
    Genetic mapping informationFree text comments relating to genetic mapping information for the aberration.\n
    CommentsFree text comments about the molecular nature of the aberration.\n
    Comments on CytologyFree text comments about the cytology of the aberration.\n
    Sequence CrossreferencesA table of DDBJ/EMBL/GenBank sequence accession numbers relevant to the aberration.\n

    Clicking on the accession number will take you to the appropriate entry in the GenBank database.\n

    \n
    \n

    Gene Deletion and Duplication Data

    \n

    Genes Deleted/Disrupted

    \n

    Complementation Data

    \n\n\n\n\n\n\n\n
    Completely deleted / disruptedA list of genes that have been reported to be completely deleted/disrupted by the aberration, as determined by genetic complementation analysis. This generally means that the aberration behaves as a null mutation for each gene in the list.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    Partially deleted / disruptedA list of genes that have been reported to be partially deleted/disrupted by the aberration, as determined by genetic complementation analysis. This usually means that the aberration behaves as a hypomorphic mutation for each gene in the list.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    \n

    Molecular Data

    \n\n\n\n\n\n\n\n
    Completely deletedA list of genes that have been reported to be completely deleted by the aberration, as determined by molecular mapping.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    Partially deletedA list of genes that have been reported to be partially deleted by the aberration, as determined by molecular mapping.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    \n

    Genes NOT Deleted/Disrupted

    \n\n\n\n\n\n\n\n
    Complementation DataA list of genes that have been reported not to be removed or broken by the aberration, as determined by genetic complementation analysis.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    Molecular DataA list of genes that have been reported not to be removed or broken by the aberration, as determined by molecular mapping.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    \n

    Genes Duplicated

    \n

    Complementation Data

    \n\n\n\n\n\n\n\n
    Completely duplicatedA list of genes that have been reported to be fully duplicated within the aberration, as determined by genetic complementation analysis. This usually means that the the duplication behaves as a wild type allele for each gene in the list.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    Partially duplicatedA list of genes that have been reported to be partially duplicated within the aberration, as determined by genetic complementation analysis. This usually means that the the duplication provides some function for each gene in the list, but behaves as a hypomorphic mutation in each case.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    \n

    Molecular Data

    \n\n\n\n\n\n\n\n
    Completely duplicatedA list of genes that have been reported to be fully duplicated within the aberration, as determined by molecular mapping.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    Partially duplicatedA list of genes that have been reported to be partially duplicated within the aberration, as determined by molecular mapping.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    \n

    Genes NOT Duplicated

    \n\n\n\n\n\n\n\n\n
    Complementation DataA list of genes that have been reported not to be duplicated within the aberration, as determined by genetic complementation analysis.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    Molecular DataA list of genes that have been reported not to be duplicated within the aberration, as determined by molecular mapping.\n

    Clicking on a gene symbol will take you to the relevant Gene Report.\n

    This field is only displayed in an individual Aberration Report if it contains data.\n

    \n
    \n

    Related Comments

    \n

    Free text comments relating to both complementation analysis and molecular mapping information concerning the aberration.\n

    \n

    Phenotypic Data

    \n\n\n\n\n\n\n\n
    In combination with other aberrationsFree text description of the reported mutant phenotype of the aberration when in combination with other aberrations.\n
    NOT in combination with other aberrationsFree text description of the reported mutant phenotype of the aberration (while not in combination with other aberrations), when the mutant phenotype cannot be attributed to individual genes that are broken or deleted by the aberration.\n
    \n

    Stocks

    \n

    Clicking on the \"Search for Stocks\" button allows you to search for stocks that contain the aberration.\n

    \n

    Notes on Origin

    \n\n\n\n
    DiscovererA list of the individuals who identified or generated the aberration.\n
    \n

    Free text comments about the origin of the aberration.\n

    For example, information that it was simultaneously induced with another mutation/aberration, or information about the genotype of the progenitor which is irrelevant to the derivative.\n

    \n

    Balancer/ Genotype Variants of the Aberration

    \n

    A list of balancer and/or genotype variants of the aberration. Genotype variants carry the same chromosomal rearrangement as the aberration, but differ genetically, e.g. carrying a different constellation of alleles and/or transposable element insertions. A genotype variant chromosome may also be a Balancer, which is a multiply inverted chromosome that suppresses recombination with the homologous chromosome, allowing the homologous chromosome (or part of it) to be maintained intact in a cross or stock.\n

    Clicking on the balancer/genotype variant symbol will take you to the relevant Balancer/Genotype variant Report.\n

    \n

    Separable Components

    \n

    A list of separable components of the aberration. These are chromosomal rearrangements that are derived from the parent aberration by non-mutagenic means. These component aberrations have properties that differ from those of the intact parent aberration.\n

    For example, many transpositions can be separated into a deleted-chromosome component and a duplicated-chromosome component.\n

    Clicking on the separable component symbol will take you to the relevant Aberration Report.\n

    \n

    Other Comments

    \n

    Miscellaneous free text comments about the aberration.\n

    \n

    Synonyms and Secondary IDs

    \n

    Reported As

    \n\n\n\n\n\n\n\n
    Symbol SynonymA list of symbols that have been used in the literature, or by FlyBase, to describe the aberration.\n
    Name SynonymA list of names that have been used in the literature, or by FlyBase, to describe the aberration.\n
    \n

    Secondary FlyBase IDs

    \n

    A list of Secondary FlyBase identifier numbers of the aberration.\n

    If an aberration has a secondary identifier number, it generally indicates that at some point it has been merged with or split from other entries in the database. See the FlyBase identifier numbers section for some examples of the cases where identifier numbers are made secondary.\n

    \n

    References

    \n

    A list of publications that discuss the aberration, subdivided into fields by type of publication. Only those fields containing data are displayed in an individual Aberration Report.\n

    ","sections":[{"toclevel":1,"level":"2","line":"General Information","number":"1","index":"1","fromtitle":"FlyBase:Aberration_Report","byteoffset":774,"anchor":"General_Information","linkAnchor":"General_Information"},{"toclevel":1,"level":"2","line":"Nature of the Aberration","number":"2","index":"2","fromtitle":"FlyBase:Aberration_Report","byteoffset":4457,"anchor":"Nature_of_the_Aberration","linkAnchor":"Nature_of_the_Aberration"},{"toclevel":1,"level":"2","line":"Gene Deletion and Duplication Data","number":"3","index":"3","fromtitle":"FlyBase:Aberration_Report","byteoffset":8599,"anchor":"Gene_Deletion_and_Duplication_Data","linkAnchor":"Gene_Deletion_and_Duplication_Data"},{"toclevel":2,"level":"3","line":"Genes Deleted/Disrupted","number":"3.1","index":"4","fromtitle":"FlyBase:Aberration_Report","byteoffset":8639,"anchor":"Genes_Deleted/Disrupted","linkAnchor":"Genes_Deleted/Disrupted"},{"toclevel":3,"level":"4","line":"Complementation Data","number":"3.1.1","index":"5","fromtitle":"FlyBase:Aberration_Report","byteoffset":8670,"anchor":"Complementation_Data","linkAnchor":"Complementation_Data"},{"toclevel":3,"level":"4","line":"Molecular Data","number":"3.1.2","index":"6","fromtitle":"FlyBase:Aberration_Report","byteoffset":9699,"anchor":"Molecular_Data","linkAnchor":"Molecular_Data"},{"toclevel":2,"level":"3","line":"Genes NOT Deleted/Disrupted","number":"3.2","index":"7","fromtitle":"FlyBase:Aberration_Report","byteoffset":10453,"anchor":"Genes_NOT_Deleted/Disrupted","linkAnchor":"Genes_NOT_Deleted/Disrupted"},{"toclevel":2,"level":"3","line":"Genes Duplicated","number":"3.3","index":"8","fromtitle":"FlyBase:Aberration_Report","byteoffset":11263,"anchor":"Genes_Duplicated","linkAnchor":"Genes_Duplicated"},{"toclevel":3,"level":"4","line":"Complementation Data","number":"3.3.1","index":"9","fromtitle":"FlyBase:Aberration_Report","byteoffset":11287,"anchor":"Complementation_Data_2","linkAnchor":"Complementation_Data_2"},{"toclevel":3,"level":"4","line":"Molecular Data","number":"3.3.2","index":"10","fromtitle":"FlyBase:Aberration_Report","byteoffset":12339,"anchor":"Molecular_Data_2","linkAnchor":"Molecular_Data_2"},{"toclevel":2,"level":"3","line":"Genes NOT Duplicated","number":"3.4","index":"11","fromtitle":"FlyBase:Aberration_Report","byteoffset":13107,"anchor":"Genes_NOT_Duplicated","linkAnchor":"Genes_NOT_Duplicated"},{"toclevel":2,"level":"3","line":"Related Comments","number":"3.5","index":"12","fromtitle":"FlyBase:Aberration_Report","byteoffset":13884,"anchor":"Related_Comments","linkAnchor":"Related_Comments"},{"toclevel":1,"level":"2","line":"Phenotypic Data","number":"4","index":"13","fromtitle":"FlyBase:Aberration_Report","byteoffset":14030,"anchor":"Phenotypic_Data","linkAnchor":"Phenotypic_Data"},{"toclevel":1,"level":"2","line":"Stocks","number":"5","index":"14","fromtitle":"FlyBase:Aberration_Report","byteoffset":14581,"anchor":"Stocks","linkAnchor":"Stocks"},{"toclevel":1,"level":"2","line":"Notes on Origin","number":"6","index":"15","fromtitle":"FlyBase:Aberration_Report","byteoffset":14697,"anchor":"Notes_on_Origin","linkAnchor":"Notes_on_Origin"},{"toclevel":1,"level":"2","line":"Balancer/ Genotype Variants of the Aberration","number":"7","index":"16","fromtitle":"FlyBase:Aberration_Report","byteoffset":15120,"anchor":"Balancer/_Genotype_Variants_of_the_Aberration","linkAnchor":"Balancer/_Genotype_Variants_of_the_Aberration"},{"toclevel":1,"level":"2","line":"Separable Components","number":"8","index":"17","fromtitle":"FlyBase:Aberration_Report","byteoffset":15839,"anchor":"Separable_Components","linkAnchor":"Separable_Components"},{"toclevel":1,"level":"2","line":"Other Comments","number":"9","index":"18","fromtitle":"FlyBase:Aberration_Report","byteoffset":16340,"anchor":"Other_Comments","linkAnchor":"Other_Comments"},{"toclevel":1,"level":"2","line":"Synonyms and Secondary IDs","number":"10","index":"19","fromtitle":"FlyBase:Aberration_Report","byteoffset":16415,"anchor":"Synonyms_and_Secondary_IDs","linkAnchor":"Synonyms_and_Secondary_IDs"},{"toclevel":2,"level":"3","line":"Reported As","number":"10.1","index":"20","fromtitle":"FlyBase:Aberration_Report","byteoffset":16447,"anchor":"Reported_As","linkAnchor":"Reported_As"},{"toclevel":2,"level":"3","line":"Secondary FlyBase IDs","number":"10.2","index":"21","fromtitle":"FlyBase:Aberration_Report","byteoffset":16788,"anchor":"Secondary_FlyBase_IDs","linkAnchor":"Secondary_FlyBase_IDs"},{"toclevel":1,"level":"2","line":"References","number":"11","index":"22","fromtitle":"FlyBase:Aberration_Report","byteoffset":17167,"anchor":"References","linkAnchor":"References"}],"showtoc":true}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/parse_about.json b/mcp/tests/fixtures/wiki/parse_about.json new file mode 100644 index 0000000..fd14d07 --- /dev/null +++ b/mcp/tests/fixtures/wiki/parse_about.json @@ -0,0 +1 @@ +{"parse":{"title":"FlyBase:About","pageid":95404,"revid":173662,"redirects":[],"text":"
    \n

    FlyBase Consortium

    \n

    The FlyBase project is carried out by a consortium of Drosophila researchers and computer scientists at: Harvard University, University of Cambridge (UK), Indiana University and the University of New Mexico.\n

    Please use the contact FlyBase form to write to us.\n

    \n
    FlyBase-Harvard
    \n
    Biological Laboratories, Harvard University
    \n
    16 Divinity Avenue
    \n
    Cambridge, Massachusetts 02138
    \n
    USA
    \n
       Norbert Perrimon (PI)\n   Gil dos Santos\n   L. Sian Gramates\n   Ian Longden\n   Jolene Seme\n   Christopher J. Tabone\n   Mark Zytkovicz\n
    \n
    FlyBase-Cambridge
    \n
    Department of Physiology, Development and Neuroscience, University of Cambridge
    \n
    Downing Street, Cambridge CB2 3DY
    \n
    UK
    \n
       Katja Röper (Co-PI)\n   Helen Attrill\n   Aoife Larkin\n   Steven Marygold\n   Alex McLachlan\n   Gillian Millburn\n   Arzu Öztürk-Çolak\n   Clare Pilgrim\n   Rossana Zaru\n
    \n
    FlyBase-Indiana
    \n
    Department of Biology, Indiana University
    \n
    1001 East 3rd Street
    \n
    Bloomington, Indiana 47405-7005
    \n
    USA
    \n
       Brian Calvi (Co-PI)\n   Seth Campbell\n   Victor Strelets\n   Jim Thurmond\n
    \n
    FlyBase-New Mexico
    \n
    Department of Biology, Castetter Hall - Room 200
    \n
    University of New Mexico,
    \n
    Albuquerque, New Mexico 87131
    \n
    USA
    \n
       Richard Cripps (Co-PI)\n   TyAnna Lovato\n
    \n

    FlyBase Publications

    \n
    2026
    \n
    Wood V, Bruford EA, Marygold SJ, O’Donnell VB, Röper K, Davies JA, Orengo C, Nurse P, Hall N (2026)
    \n
    Invisible Infrastructure at Risk: Funding Crises Threaten Global Bioscience Research.
    \n
    Genetics iyag274. DOI:10.1093/genetics/iyag274
    \n


    \n

    \n
    Hall N, Röper K, Wood V, Nurse P (2026)
    \n
    Treat biological databases as infrastructure, not projects.
    \n
    Nature 656(8128):786. DOI:10.1038/d41586-026-02575-5
    \n


    \n

    \n
    Garapati PV, Zaru R, Attrill H, dos Santos G, Goodman J, Thurmond J, Marygold SJ (2026)
    \n
    Comprehensive annotation of the enzymes of Drosophila melanogaster.
    \n
    G3 (Bethesda) 16(2):jkaf294. DOI:10.1093/g3journal/jkaf294 FBrf0264581
    \n


    \n

    \n
    The Gene Ontology Consortium (2026)
    \n
    The Gene Ontology knowledgebase in 2026.
    \n
    Nucleic Acids Res. 54(D1):D1779–D1792 DOI:10.1093/nar/gkaf1292
    \n


    \n

    \n
    The RNAcentral Consortium (2026)
    \n
    RNAcentral in 2026: Genes and literature integration.
    \n
    Nucleic Acids Res. 54(D1):D303–D313 DOI:10.1093/nar/gkaf1329
    \n


    \n

    \n
    Green A, Ontiveros-Palacios N, Jandalala I, Panni S, Wood V, Antonazzo G, Attrill H, Bateman A, Sweeney B (2026)
    \n
    GOFlowLLM-curating miRNA literature with large language models and flowcharts.
    \n
    Bioinformatics. 42(1):btaf683 DOI:10.1093/bioinformatics/btaf683
    \n


    \n

    \n
    2025
    \n
    Matentzoglu N, Bello SM, Stefancsik R, Alghamdi SM, Anagnostopoulos AV, Balhoff JP, Balk MA, Bradford YM, Bridges Y, Callahan TJ, Caufield H, Cuzick A, Carmody LC, Caron AR, de Souza V, Engel SR, Fey P, Fisher M, Gehrke S, Grove C, Hansen P, Harris NL, Harris MA, Harris L, Ibrahim A, Jacobsen JOB, Köhler S, McMurry JA, Munoz-Fuentes V, Munoz-Torres MC, Parkinson H, Pendlington ZM, Pilgrim C, Robb SMC, Robinson PN, Seager J, Segerdell E, Smedley D, Sollis E, Toro S, Vasilevsky N, Wood V, Haendel MA, Mungall CJ, McLaughlin JA, Osumi-Sutherland D. (2025)
    \n
    The Unified Phenotype Ontology : a framework for cross-species integrative phenomics.
    \n
    Genetics 229(3):iyaf027. DOI:10.1093/genetics/iyaf027
    \n


    \n

    \n
    Feuermann M, Mi H, Gaudet P, Muruganujan A, Lewis SE, Ebert D, Mushayahama T, Gene Ontology Consortium and Thomas PD (2025)
    \n
    A compendium of human gene functions derived from evolutionary modelling.
    \n
    Nature 640(8057):146-154 DOI:10.1038/s41586-025-08592-0
    \n


    \n

    \n
    Zaru R and Marygold SJ (2025)
    \n
    In silico identification and analysis of paralogs encoding enzymes of carbohydrate metabolism in Drosophila melanogaster.
    \n
    MicroPubl Biol. 001425 DOI:10.17912/micropub.biology.001425 FBrf0261725
    \n


    \n

    \n
    2024
    \n
    Antonazzo G, Gaudet P, Lovering RC, Attrill H (2024)
    \n
    Representation of non-coding RNA-mediated regulation of gene expression using the Gene Ontology.
    \n
    RNA Biol. 21(1) 36–48 DOI:10.1080/15476286.2024.2408523
    \n


    \n

    \n
    Marygold SJ (2024)
    \n
    The alpha-ketoacid dehydrogenase complexes of Drosophila melanogaster.
    \n
    MicroPubl Biol. 001209 DOI:10.17912/micropub.biology.001209 FBrf0259508
    \n


    \n

    \n
    Alliance of Genome Resources Consortium (2024)
    \n
    Updates to the Alliance of Genome Resources Central Infrastructure.
    \n
    Genetics 227(1) iyae049 DOI:10.1093/genetics/iyae049 FBrf0259417
    \n


    \n

    \n
    Sklias A, Cruciani S, Marchand V, Spagnuolo M, Lavergne G, Bourguignon V, Brambilla A, Dreos R, Marygold SJ, Novoa EM, Motorin Y, Roignant JY (2024)
    \n
    Comprehensive map of ribosomal 2′-O-methylation and C/D box snoRNAs in Drosophila melanogaster.
    \n
    Nucleic Acids Res 52(6) 2848-2864 DOI:10.1093/nar/gkae139 FBrf0259352
    \n


    \n

    \n
    Öztürk-Çolak A, Marygold SJ, Antonazzo G, Attrill H, Goutte-Gattat D, Jenkins VK, Matthews BB, Millburn G, dos Santos G, Tabone CJ, FlyBase Consortium (2024)
    \n
    FlyBase: updates to the Drosophila genes and genomes database.
    \n
    Genetics 227(1) iyad211 DOI:10.1093/genetics/iyad211 FBrf0259457
    \n


    \n

    \n
    Attrill H, Antonazzo G, Goodman JL, Thurmond J, Strelets VB, Brown NH; FlyBase Consortium (2024)
    \n
    A new experimental evidence-weighted signaling pathway resource in FlyBase.
    \n
    Development 151(3) dev202255 DOI:10.1242/dev.202255 FBrf0258673
    \n


    \n

    \n
    George N, Fexova S, Fuentes AM, Madrigal P, Bi Y, Iqbal H, Kumbham U, Nolte NF, Zhao L, Thanki AS, Yu ID, Marugan Calles JC, Erdos K, Vilmovsky L, Kurri SR, Vathrakokoili-Pournara A, Osumi-Sutherland D, Prakash A, Wang S, Tello-Ruiz MK, Kumari S, Ware D, Goutte-Gattat D, Hu Y, Brown N, Perrimon N, Vizcaíno JA, Burdett T, Teichmann S, Brazma A, Papatheodorou I (2024)
    \n
    Expression Atlas update: insights from sequencing data at both bulk and single cell level.
    \n
    Nucleic Acids Res. 52(D1):D107-D114 DOI:10.1093/nar/gkad1021
    \n


    \n

    \n
    2023
    \n
    Attrill H (2023)
    \n
    Comparing the history of signalling pathway research using the research publication record of representative genes.
    \n
    MicroPubl Biol. 000998 DOI:10.17912/micropub.biology.000998 FBrf0258056
    \n


    \n

    \n
    The Gene Ontology Consortium (2023)
    \n
    The Gene Ontology knowledgebase in 2023.
    \n
    Genetics 224(1) iyad031. DOI:10.1093/genetics/iyad031
    \n


    \n

    \n
    Hu Y, Comjean A, Attrill H, Antonazzo G, Thurmond J, Chen W, Li F, Chao T, Mohr SE, Brown NH, Perrimon N. (2023)
    \n
    PANGEA: a new gene set enrichment tool for Drosophila and common research organisms.
    \n
    Nucleic Acids Res 51(W1) W419--WW426. DOI:10.1093/nar/gkad331 FBrf0256969
    \n


    \n

    \n
    Marygold SJ, FlyBase Consortium (2023)
    \n
    Exploring FlyBase Data Using QuickSearch (Updated protocol)
    \n
    Current Protocols 3:e731. DOI:10.1002/cpz1.731 FBrf0256232
    \n


    \n

    \n
    Court R, Costa M, Pilgrim C, Millburn G, Holmes A, McLachlan A, Larkin A, Matentzoglu N, Kir H, Parkinson H, Brown NH, O'Kane CJ, Armstrong JD, Jefferis GSXE, Osumi-Sutherland D. (2023)
    \n
    Virtual Fly Brain-An interactive atlas of the Drosophila nervous system.
    \n
    Front Physiol. 14:1076533. DOI:10.3389/fphys.2023.1076533 FBrf0255719
    \n


    \n

    \n
    2022
    \n
    Jenkins VK, Larkin A, Thurmond J; FlyBase Consortium. (2022)
    \n
    Using FlyBase: A Database of Drosophila Genes and Genetics.
    \n
    Methods Mol Biol. 2540:1-34. DOI:10.1007/978-1-0716-2541-5_1 FBrf0254317
    \n


    \n

    \n
    Trovisco V (2022)
    \n
    Featured Resource: FlyBase.
    \n
    The Node: Resources
    \n


    \n

    \n
    Marygold SJ, Chan PP, Lowe TM (2022)
    \n
    Systematic identification of tRNA genes in Drosophila melanogaster.
    \n
    MicroPubl Biol. 000560 DOI:10.17912/micropub.biology.000560 FBrf0253514
    \n


    \n

    \n
    Gramates LS, Agapite J, Attrill H, Calvi BR, Crosby MA, dos Santos G, Goodman JL, Goutte-Gattat D, Jenkins VK, Kaufman T, Larkin A, Matthews BB, Millburn G, Strelets VB, FlyBase Consortium (2022)
    \n
    FlyBase: A guided tour of highlighted features.
    \n
    Genetics 220(4) iyac035 DOI:10.1093/genetics/iyac035 FBrf0253076
    \n


    \n

    \n
    Alliance of Genome Resources Consortium (2022)
    \n
    Harmonizing model organism data in the Alliance of Genome Resources.
    \n
    Genetics 220(4) iyac022 DOI:10.1093/genetics/iyac022 FBrf0253213
    \n


    \n

    \n
    Liu Y, Li JSS, Rodiger J, Comjean A, Attrill H, Antonazzo G, Brown NH, Hu Y, Perrimon N (2022)
    \n
    FlyPhoneDB: An integrated web-based resource for cell-cell communication prediction in Drosophila.
    \n
    Genetics 220(3) iyab235 DOI:10.1093/genetics/iyab235 FBrf0252867
    \n


    \n

    \n
    2021
    \n
    Meyer H, Buhr A, Callaerts P, Schiemann R, Wolfner MF, Marygold SJ (2021)
    \n
    Identification and bioinformatic analysis of neprilysin and neprilysin-like metalloendopeptidases in Drosophila melanogaster.
    \n
    MicroPubl Biol. 000410 DOI:10.17912/micropub.biology.000410 FBrf0249415
    \n


    \n

    \n
    Hu Y, Tattikota SG, Liu Y, Comjean A, Gao Y, Forman C, Kim G, Rodiger J, Papatheodorou I, Dos Santos G, Mohr SE, Perrimon N (2021)
    \n
    DRscDB: A single-cell RNA-seq resource for data mining and data comparison across species.
    \n
    Comput Struct Biotechnol J 19:2018-2026. DOI:10.1016/j.csbj.2021.04.021FBrf0249044
    \n


    \n

    \n
    Ahn SJ and Marygold SJ (2021)
    \n
    The UDP-Glycosyltransferase Family in Drosophila melanogaster: Nomenclature Update, Gene Expression and Phylogenetic Analysis.
    \n
    Front Physiol 12 648481 DOI:10.3389/fphys.2021.648481FBrf0248570
    \n


    \n

    \n
    The Gene Ontology Consortium (2021)
    \n
    The Gene Ontology resource: enriching a GOld mine.
    \n
    Nucleic Acids Res. 49(D1) D325–D334 DOI:10.1093/nar/gkaa1113 FBrf0247877
    \n


    \n

    \n
    Larkin A, Marygold SJ, Antonazzo G, Attrill H, dos Santos G, Garapati PV, Goodman JL, Gramates LS, Millburn G, Strelets VB, Tabone CJ, Thurmond J, FlyBase Consortium (2021)
    \n
    FlyBase: updates to the Drosophila melanogaster knowledge base.
    \n
    Nucleic Acids Res. 49(D1) D899–D907 DOI:10.1093/nar/gkaa1026 FBrf0247700
    \n


    \n

    \n
    The RNAcentral Consortium (2021)
    \n
    RNAcentral 2021: secondary structure integration, improved sequence search and new member databases.
    \n
    Nucleic Acids Res. 49(D1) D212–D220 DOI:10.1093/nar/gkaa921
    \n


    \n

    \n
    2020
    \n
    Marygold SJ, Alic N, Gilmour DS, Grewal SS (2020)
    \n
    In silico identification of Drosophila melanogaster genes encoding RNA polymerase subunits.
    \n
    MicroPubl Biol. 000320 DOI:10.17912/micropub.biology.000320 FBrf0247389
    \n


    \n

    \n
    Wood V, Carbon S, Harris MA, Lock A, Engel SR, Hill DP, Van Auken K, Attrill H, Feuermann M, Gaudet P, Lovering RC, Poux S, Rutherford KM and Mungall CJ (2020)
    \n
    Term Matrix: A novel Gene Ontology annotation quality control system based on ontology term co-annotation patterns.
    \n
    Open Biol 10(9) 200149 DOI:10.1098/rsob.200149
    \n


    \n

    \n
    Kishore R, Arnaboldi V, Van Slyke CE, Chan J, Nash RS, Urbano JM, Dolan ME, Engel SR, Shimoyama M, Sternberg PW, the Alliance of Genome Resources (2020)
    \n
    Automated generation of gene summaries at the Alliance of Genome Resources.
    \n
    Database 2020 baaa037 DOI:10.1093/database/baaa037
    \n


    \n

    \n
    Antonazzo G, Urbano JM, Marygold SJ, Millburn GH and Brown NH (2020)
    \n
    Building a pipeline to solicit expert knowledge from the community to aid gene summary curation.
    \n
    Database 2020 baz152 DOI:10.1093/database/baz152 FBrf0244633
    \n


    \n

    \n
    Marygold SJ, Attrill H, Speretta E, Warner K, Magrane M, Berloco M, Cotterill S, McVey M, Rong Y and Yamaguchi M (2020)
    \n
    The DNA polymerases of Drosophila melanogaster.
    \n
    Fly 14(1-4) 49-61 DOI:10.1080/19336934.2019.1710076 FBrf0245686
    \n


    \n

    \n
    Alliance of Genome Resources Consortium (2020)
    \n
    Alliance of Genome Resources Portal: unified model organism research platform.
    \n
    Nucleic Acids Res 48(D1) D650–D658 DOI:10.1093/nar/gkz813 FBrf0245774
    \n


    \n

    \n
    2019
    \n
    Alliance of Genome Resources Consortium (2019)
    \n
    The Alliance of Genome Resources: Building a Modern Data Ecosystem for Model Organism Databases.
    \n
    Genetics 213(4):1189-1196 DOI:10.1534/genetics.119.302523 FBrf0244239
    \n


    \n

    \n
    Attrill H, Gaudet P, Huntley RP, Lovering RC, Engel SR, Poux S, Van Auken KM, Georghiou G, Chibucos MC, Berardini TZ, Wood V, Drabkin H, Fey P, Garmiri P, Harris MA, Sawford T, Reiser L, Tauber R, Toro S; Gene Ontology Consortium. (2019)
    \n
    Annotation of gene product function from high-throughput studies using the Gene Ontology.
    \n
    Database 2019 baz007 DOI:10.1093/database/baz007
    \n


    \n

    \n
    Garapati PV, Zhang J, Rey AJ and Marygold SJ (2019)
    \n
    Towards comprehensive annotation of Drosophila melanogaster enzymes in FlyBase
    \n
    Database 2019 bay144 DOI:10.1093/database/bay144 FBrf0241253
    \n


    \n

    \n
    Thurmond J, Goodman JL, Strelets VB, Attrill H, Gramates LS, Marygold SJ, Matthews BB, Millburn G, Antonazzo G, Trovisco V, Kaufman TC, Calvi BR, FlyBase Consortium. (2019)
    \n
    FlyBase 2.0: the next generation.
    \n
    Nucleic Acids Res. 47(D1) D759–D765 DOI:10.1093/nar/gky1003 FBrf0241170
    \n


    \n

    \n
    The RNAcentral Consortium (2019)
    \n
    RNAcentral: a hub of information for non-coding RNA sequences.
    \n
    Nucleic Acids Res. 47(D1) D221–D229 DOI:10.1093/nar/gky1034
    \n


    \n

    \n
    The Gene Ontology Consortium (2019)
    \n
    The Gene Ontology Resource: 20 years and still GOing strong.
    \n
    Nucleic Acids Res. 47(D1) D330–D338 DOI:10.1093/nar/gky1055
    \n


    \n

    \n
    2018
    \n
    Howe DG, Blake JA, Bradford YM, Bult CJ, Calvi BR, Engel SR, Kadin JA, Kaufman TC, Kishore R, Laulederkind SJF, Lewis SE, Moxon SAT, Richardson JE and Smith C. (2018)
    \n
    Model organism data evolving in support of translational medicine.
    \n
    Lab Anim (NY) 47:277-289 DOI:10.1038/s41684-018-0150-4 FBrf0241113
    \n


    \n

    \n
    Rey AJ, Attrill H, Marygold SJ, FlyBase Consortium. (2018)
    \n
    Using FlyBase to Find Functionally Related Drosophila Genes.
    \n
    Eukaryotic Genomic Databases: Methods and Protocols (M. Kollmar, ed.) pp. 493-512. Springer, New York DOI:10.1007%2F978-1-4939-7737-6_16 FBrf0238904
    \n


    \n

    \n
    2017
    \n
    Hu Y, Comjean A, Mohr SE; FlyBase Consortium, Perrimon N. (2017)
    \n
    Gene2Function: An Integrated Online Resource for Gene Function Discovery.
    \n
    G3 (Bethesda) 7(8):2855-2858 DOI:10.1534/g3.117.043885 FBrf0236248
    \n


    \n

    \n
    The Gene Ontology Consortium. (2017)
    \n
    Expansion of the Gene Ontology knowledgebase and resources
    \n
    Nucleic Acids Res. 45(D1):D331-D338 DOI:10.1093/nar/gkw1108
    \n


    \n

    \n
    Gramates LS, Marygold SJ, dos Santos G, Urbano J-M, Antonazzo G, Matthews BB, Rey AJ, Tabone CJ, Crosby MA, Emmert DB, Falls K, Goodman JL, Hu Y, Ponting L, Schroeder AJ, Strelets VB, Thurmond J, Zhou P, FlyBase Consortium. (2017)
    \n
    FlyBase at 25: looking to the future.
    \n
    Nucleic Acids Res. 45(D1):D663-D671 DOI:10.1093/nar/gkw1016 FBrf0234415
    \n


    \n

    \n
    Marygold SJ, Attrill H, Lasko P. (2017)
    \n
    The Translation Factors of Drosophila melanogaster.
    \n
    Fly 11(1):65-74 DOI:10.1080/19336934.2016.1220464 FBrf0233552
    \n


    \n

    \n
    2016
    \n
    Marygold SJ, Antonazzo G, Attrill H, Costa M, Crosby MA, Dos Santos G, Goodman JL, Gramates LS, Matthews BB, Rey AJ, Thurmond J; FlyBase Consortium. (2016)
    \n
    Exploring FlyBase Data Using QuickSearch.
    \n
    Curr. Protoc. Bioinformatics 56:1.31.1-1.31.23 DOI:10.1002/cpbi.19 FBrf0234167
    \n


    \n

    \n
    Marygold SJ, Crosby MA, Goodman JL, FlyBase Consortium. (2016)
    \n
    Using FlyBase, a Database of Drosophila Genes and Genomes.
    \n
    Drosophila: Methods and Protocols, Second edition, vol. 1478 (C. Dahmann, ed.) pp. 1-31. Springer, New York DOI:10.1007/978-1-4939-6371-3_1 FBrf0233766)
    \n


    \n

    \n
    Kahsai L, Millburn GH, Cook KR. (2016)
    \n
    Phenotypes Associated with Second Chromosome P Element Insertions in Drosophila melanogaster.
    \n
    G3 (Bethesda). 6(8):2665-2670 DOI:10.1534/g3.116.030940 FBrf0233125
    \n


    \n

    \n
    Millburn GH, Crosby MA, Gramates LS, Tweedie S; FlyBase Consortium. (2016)
    \n
    FlyBase portals to human disease research using Drosophila models.
    \n
    Dis. Model Mech. 9(3):245-252 DOI:10.1242/dmm.023317 FBrf0231138
    \n


    \n

    \n
    Attrill H, Falls K, Goodman JL, Millburn GH, Antonazzo G, Rey AJ, Marygold SJ; FlyBase Consortium. (2016)
    \n
    FlyBase: establishing a Gene Group resource for Drosophila melanogaster.
    \n
    Nucleic Acids Res. 44(D1):D786-D792 DOI:10.1093/nar/gkv1046 FBrf0230602
    \n


    \n

    \n
    2015
    \n
    Lu J, Marygold SJ, Gharib WH and Suter B. (2015)
    \n
    The aminoacyl-tRNA synthetases of Drosophila melanogaster.
    \n
    Fly 9(2):53-61 DOI:10.1080/19336934.2015.1101196 FBrf0230668
    \n


    \n

    \n
    Matthews BB, dos Santos G, Crosby MA, Emmert DB, St Pierre SE, Gramates LS, Zhou P, Schroeder AJ, Falls K, Strelets V, Russo SM, Gelbart WM, FlyBase Consortium. (2015)
    \n
    Gene Model Annotations for Drosophila melanogaster: Impact of High-Throughput Data.
    \n
    G3 (Bethesda) 5:1721-1736 DOI:10.1534/g3.115.018929 FBrf0229216
    \n


    \n

    \n
    Crosby MA, Gramates LS, dos Santos G, Matthews BB, St Pierre SE, Zhou P, Schroeder AJ, Falls K, Emmert DB, Russo SM, Gelbart WM, FlyBase Consortium. (2015)
    \n
    Gene Model Annotations for Drosophila melanogaster: The Rule-Benders.
    \n
    G3 (Bethesda) 5:1737-1749 DOI:10.1534/g3.115.018937 FBrf0229217
    \n


    \n

    \n
    The Gene Ontology Consortium (2015)
    \n
    Gene Ontology Consortium: going forward.
    \n
    Nucleic Acids Res. 43(Database issue):D1049-56 DOI:10.1093/nar/gku1179
    \n


    \n

    \n
    Dos Santos G, Schroeder AJ, Goodman JL, Strelets VB, Crosby MA, Thurmond J, Emmert DB, Gelbart WM; FlyBase Consortium. (2015)
    \n
    FlyBase: introduction of the Drosophila melanogaster Release 6 reference genome assembly and large-scale migration of genome annotations.
    \n
    Nucleic Acids Res. 43(Database issue): D690-D697 DOI:10.1093/nar/gku1099 FBrf0227324
    \n


    \n

    \n
    2014
    \n
    Alam-Faruque Y, Hill DP, Dimmer EC, Harris MA, Foulger RE, Tweedie S, Attrill H, Howe DG, Thomas SR, Davidson D, Woolf AS, Blake JA, Mungall CJ, O'Donovan C, Apweiler R, Huntley RP. (2014)
    \n
    Representing kidney development using the gene ontology.
    \n
    PLoS One. 9(6):e99864 DOI:10.1371/journal.pone.0099864
    \n


    \n

    \n
    Mao Y, Van Auken K, Li D, Arighi CN, McQuilton P, Hayman GT, Tweedie S, Schaeffer ML, Laulederkind SJ, Wang SJ, Gobeill J, Ruch P, Luu AT, Kim JJ, Chiang JH, Chen YD, Yang CJ, Liu H, Zhu D, Li Y, Yu H, Emadzadeh E, Gonzalez G, Chen JM, Dai HJ, Lu Z. (2014)
    \n
    Overview of the gene ontology task at BioCreative IV.
    \n
    Database 2014. bau086 DOI:10.1093/database/bau086
    \n


    \n

    \n
    Van Auken K, Schaeffer ML, McQuilton P, Laulederkind SJ, Li D, Wang SJ, Hayman GT, Tweedie S, Arighi CN, Done J, Müller HM, Sternberg PW, Mao Y, Wei CH, Lu Z (2014)
    \n
    BC4GO: a full-text corpus for the BioCreative IV GO Task.
    \n
    Database 2014. bau074 DOI:10.1093/database/bau074
    \n


    \n

    \n
    Cejuela, JM, McQuilton P, Ponting L, Marygold, SJ, Stefancsik R, Millburn GH, Rost B, FlyBase Consortium (2014)
    \n
    tagtog: interactive and text-mining-assisted annotation of gene mentions in PLOS full-text articles.
    \n
    Database 2014. bau033 DOI:10.1093/database/bau033
    \n


    \n

    \n
    St. Pierre SE, Ponting L, Stefancsik R, McQuilton P, FlyBase Consortium (2014)
    \n
    FlyBase 102 - advanced approaches to interrogating FlyBase.
    \n
    Nucleic Acids Res. 42(Database issue):D780-8 DOI:10.1093/nar/gkt1092 FBrf0223749
    \n


    \n

    \n
    2013
    \n
    Costa M, Reeve S, Grumbling G, Osumi-Sutherland D. (2013)
    \n
    The Drosophila anatomy ontology.
    \n
    J. Biomed. Semantics 4(1):32 DOI:10.1186/2041-1480-4-32 FBrf0223849
    \n


    \n

    \n
    Osumi-Sutherland D, Marygold SJ, Millburn GH, McQuilton PA, Ponting L, Stefancsik R, Falls K, Brown NH, Gkoutos GV. (2013)
    \n
    The Drosophila phenotype ontology.
    \n
    J. Biomed. Semantics 4(1):30 DOI:10.1186/2041-1480-4-30 FBrf0223876
    \n


    \n

    \n
    Vasilevsky NA, Brush MH, Paddock H, Ponting L, Tripathy SJ, Larocca GM, Haendel MA (2013)
    \n
    On the reproducibility of science: unique identification of research resources in the biomedical literature.
    \n
    PeerJ. 1:e148 DOI:10.7717/peerj.148 FBrf0222712
    \n


    \n

    \n
    Marygold SJ, Leyland PC, Seal RL, Goodman JL, Thurmond J, Strelets VB, Wilson RJ; FlyBase consortium (2013)
    \n
    FlyBase: improvements to the bibliography.
    \n
    Nucleic Acids Res. 41(Database issue):D751-D757 DOI:10.1093/nar/gks1024 FBrf0220350
    \n


    \n

    \n
    Hoehndorf R, Hardy NW, Osumi-Sutherland D, Tweedie S, Schofield PN, Gkoutos GV. (2013)
    \n
    Systematic analysis of experimental phenotype data reveals gene functions.
    \n
    PLoS One. 8(4):e60847 DOI:10.1371/journal.pone.0060847 FBrf0221415
    \n


    \n

    \n
    The Gene Ontology Consortium (2013)
    \n
    Gene Ontology annotations and resources.
    \n
    Nucleic Acids Res. 41(Database issue):D530-5. DOI:10.1093/nar/gks1050
    \n


    \n

    \n
    2012
    \n
    McQuilton P and FlyBase Consortium (2012)
    \n
    Opportunities for text mining in the FlyBase genetic literature curation workflow.
    \n
    Database 2012:bas039 DOI:10.1093/database/bas039 FBrf0219955
    \n


    \n

    \n
    Bunt SM, Grumbling GB, Field HI, Marygold SJ, Brown NH, Millburn GH, FlyBase Consortium (2012)
    \n
    Directly e-mailing authors of newly published papers encourages community curation.
    \n
    Database 2012:bas024 DOI:10.1093/database/bas024 FBrf0218522
    \n


    \n

    \n
    McQuilton P, St Pierre SE, Thurmond J; FlyBase Consortium (2012)
    \n
    FlyBase 101 – the basics of navigating FlyBase.
    \n
    Nucleic Acids Res. 40 (Database issue):D706-14 DOI:10.1093/nar/gkr1030 FBrf0217108
    \n


    \n

    \n
    Fang R, Schindelman G, Van Auken K, Fernandes J, Chen W, Wang X, Davis P, Tuli MA, Marygold S, Millburn G, Matthews B, Zhang H, Brown N, Gelbart WM, Sternberg PW (2012)
    \n
    Automatic categorization of diverse experimental information in the bioscience literature.
    \n
    BMC Bioinformatics. 13(1):16 DOI:10.1186/1471-2105-13-1 FBrf0218533
    \n


    \n

    \n
    The Gene Ontology Consortium (2012)
    \n
    The Gene Ontology: enhancements for 2011.
    \n
    Nucleic Acids Res. 40(Database issue):D559-64 DOI:10.1093/nar/gkr1028
    \n


    \n

    \n
    2011
    \n
    Rangarajan A, Schedl T, Yook K, Chan J, Haenel S, Otis L, Faelten S, DePellegrin‑Connelly T, Isaacson R, Skrzypek MS, Marygold SJ, Stefancsik R, Cherry JM, Sternberg PW and Müller H-M (2011)
    \n
    Toward an interactive article: integrating journals and biological databases.
    \n
    BMC Bioinformatics 12:175 DOI:10.1186/1471-2105-12-175 FBrf0214821
    \n


    \n

    \n
    Daines, B., Wang, H., Wang, L., Li, Y., Han, Y., Emmert, D., Gelbart, W., Wang, X., Li, W., Gibbs, R., Chen, R. (2011).
    \n
    The Drosophila melanogaster transcriptome by paired-end RNA sequencing.
    \n
    Genome Res. 21(2): 315--324 DOI:10.1101/gr.107854.110 FBrf0213064
    \n


    \n

    \n
    Khodiyar VK, Hill DP, Howe D, Berardini TZ, Tweedie S, Talmud PJ, Breckenridge R, Bhattarcharya S, Riley P, Scambler P, Lovering RC (2011)
    \n
    The representation of heart development in the gene ontology.
    \n
    Dev Biol. 354(1):9-17 DOI:10.1016/j.ydbio.2011.03.011
    \n


    \n

    \n
    2010
    \n
    The Gene Ontology Consortium (2010)
    \n
    The Gene Ontology in 2010: extensions and refinements.
    \n
    Nucleic Acids Res. 38(Database issue):D331-5 DOI:10.1093/nar/gkp1018
    \n


    \n

    \n
    Pfreundt U, James DP, Tweedie S, Wilson D, Teichmann SA, Adryan B (2010)
    \n
    FlyTF: improved annotation and enhanced functionality of the Drosophila transcription factor database.
    \n
    Nucleic Acids Res. 38(Database issue):D443-7 DOI:10.1093/nar/gkp910 FBrf0209611
    \n


    \n

    \n
    2009
    \n
    St. Pierre S and McQuilton P (2009)
    \n
    Inside FlyBase (Biocuration as a career).
    \n
    Fly 3(1):112-114 DOI:10.4161/fly.3.1.7769 FBrf0213427
    \n


    \n

    \n
    Reference Genome Group of the Gene Ontology Consortium (2009)
    \n
    The Gene Ontology's Reference Genome Project: a unified framework for functional annotation across species.
    \n
    PLoS Comput Biol. 5(7):e1000431 DOI:10.1371/journal.pcbi.1000431
    \n


    \n

    \n
    Tweedie S, Ashburner M, Falls K, Leyland P, McQuilton P, Marygold S, Millburn G, Osumi‑Sutherland D, Schroeder A, Seal R, Zhang H; FlyBase Consortium (2009)
    \n
    FlyBase: enhancing Drosophila Gene Ontology annotations.
    \n
    Nucleic Acids Res. 37(Database issue):D555-D559 DOI:10.1093/nar/gkn788 FBrf0206621
    \n


    \n

    \n
    Loevenich SN, Brunner E, King NL, Deutsch EW, Stein SE; FlyBase Consortium, Aebersold R, Hafen E (2009)
    \n
    The Drosophila melanogaster PeptideAtlas facilitates the use of peptide data for improved fly proteomics and genome annotation.
    \n
    BMC Bioinformatics. 10:59 DOI:10.1186/1471-2105-10-59 FBrf0207486
    \n


    \n

    \n
    2008
    \n
    Drysdale R, FlyBase Consortium, (2008)
    \n
    FlyBase : a database for the Drosophila research community.
    \n
    Methods Mol. Biol. 420: 45-59 DOI:10.1007/978-1-59745-583-1_3 FBrf0205804
    \n


    \n

    \n
    Howe D, Costanzo M, Fey P, Gojobori T, Hannick L, Hide W, Hill DP, Kania R, Schaeffer M, St Pierre S, Twigger S, White O, Rhee SY (2008)
    \n
    Big data: The future of biocuration.
    \n
    Nature 455(7209):47-50 DOI:10.1038/455047a
    \n


    \n

    \n
    Karamanis N, Seal R, Lewin I, McQuilton P, Vlachos A, Gasperin C, Drysdale R, Briscoe T, (2008)
    \n
    Natural language processing in aid of FlyBase curators.
    \n
    BMC Bioinformatics 9:193 DOI:10.1186/1471-2105-9-193 FBrf0206381
    \n


    \n

    \n
    Wilson RJ, Goodman JL, Strelets VB, FlyBase Consortium (2008)
    \n
    FlyBase: integration and improvements to query tools.
    \n
    Nucleic Acids Res. 36:D588-D593 DOI:10.1093/nar/gkm930 FBrf0203012
    \n


    \n

    \n
    2007
    \n
    Marygold SJ, Roote J, Reuter G, Lambertsson A, Ashburner M, Millburn GH, Harrison PM, Yu Z, Kenmochi N, Kaufman TC, Leevers SJ, Cook KR (2007)
    \n
    The ribosomal protein genes and Minute loci of Drosophila melanogaster.
    \n
    Genome Biol. 2007;8(10):R216. DOI:10.1186/gb-2007-8-10-r216 FBrf0205398
    \n


    \n

    \n
    Crosby MA, Goodman JL, Strelets VB, Zhang P, Gelbart WM, FlyBase Consortium (2007)
    \n
    FlyBase: genomes by the dozen.
    \n
    Nucleic Acids Res. 35:D486-D491 DOI:10.1093/nar/gkl827 FBrf0194786
    \n


    \n

    \n
    Drosophila 12 Genomes Consortium. (2007)
    \n
    Evolution of genes and genomes on the Drosophila phylogeny.
    \n
    Nature 450(7167):203-218 DOI:10.1038/nature06341 FBrf0200326
    \n


    \n

    \n
    Karamanis N, Lewin L, Seal R, Drysdale R, Briscoe E (2007)
    \n
    Integrating natural language processing with FlyBase curation.
    \n
    Pac Symp Biocomput. 2007:245-56 FBrf0206379
    \n


    \n

    \n
    Lin MF, Carlson JW, Crosby MA, Matthews BB, Yu C, Park S, Wan KH, Schroeder AJ, Gramates LS, St Pierre SE, Roark M, Wiley KL Jr, Kulathinal RJ, Zhang P, Myrick KV, Antone JV, Celniker SE, Gelbart WM, Kellis M.. (2007)
    \n
    Revisiting the protein-coding gene catalog of Drosophila melanogaster using 12 fly genomes.
    \n
    Genome Res. 17(12):1823-1836 DOI:10.1101/gr.6679507 FBrf0202906
    \n


    \n

    \n
    Mungall CJ, Emmert DB, FlyBase Consortium (2007)
    \n
    A Chado case study: an ontology-based modular schema for representing genome-associated biological information.
    \n
    Bioinformatics 23(13):i337-346 DOI:10.1093/bioinformatics/btm189 FBrf0201050
    \n


    \n

    \n
    Stark A, Lin MF, Kheradpour P, Pedersen JS, Parts L, Carlson JW, Crosby MA, Rasmussen MD, Roy S, Deoras AN, Ruby JG, Brennecke J; Harvard FlyBase curators; Berkeley Drosophila Genome Project, Hodges E, Hinrichs AS, Caspi A, Paten B, Park SW, Han MV, Maeder ML, Polansky BJ, Robson BE, Aerts S, van Helden J, Hassan B, Gilbert DG, Eastman DA, Rice M, Weir M, Hahn MW, Park Y, Dewey CN, Pachter L, Kent WJ, Haussler D, Lai EC, Bartel DP, Hannon GJ, Kaufman TC, Eisen MB, Clark AG, Smith D, Celniker SE, Gelbart WM, Kellis M (2007)
    \n
    Discovery of functional elements in 12 Drosophila genomes using evolutionary signatures.
    \n
    Nature 450(7167):219-232 DOI:10.1038/nature06340 FBrf0200674
    \n


    \n

    \n
    2006
    \n
    Zhou P, Emmert D, Zhang P (2006)
    \n
    Using Chado to store genome annotation data.
    \n
    Curr Protoc Bioinformatics 12:9.6.1-9.6.28. DOI:10.1002/0471250953.bi0906s12 FBrf0206380
    \n


    \n

    \n
    Grumbling G, Strelets V, FlyBase Consortium (2006)
    \n
    FlyBase: anatomical data, images and queries.
    \n
    Nucleic Acids Res. 34:D484-D488 DOI:10.1093/nar/gkj068 FBrf0191339
    \n


    \n

    \n
    2005
    \n
    Drysdale RA, Crosby MA, FlyBase Consortium (2005)
    \n
    FlyBase : Genes and gene models.
    \n
    Nucleic Acids Res., 33: D390-D395 DOI:10.1093/nar/gki046 FBrf0184221
    \n


    \n

    \n
    Richards S, Liu Y, Bettencourt BR, Hradecky P, Letovsky S, Nielsen R, Thornton K, Hubisz MJ, Chen R, Meisel RP, Couronne O, Hua S, Smith MA, Zhang P, Liu J, Bussemaker HJ, van Batenburg MF, Howells SL, Scherer SE, Sodergren E, Matthews BB, Crosby MA, Schroeder AJ, Ortiz-Barrientos D, Rives CM, Metzker ML, Muzny DM, Scott G, Steffen D, Wheeler DA, Worley KC, Havlak P, Durbin KJ, Egan A, Gill R, Hume J, Morgan MB, Miner G, Hamilton C, Huang Y, Waldron L, Verduzco D, Clerc-Blankenburg KP, Dubchak I, Noor MA, Anderson W, White KP, Clark AG, Schaeffer SW, Gelbart W, Weinstock GM, Gibbs RA. (2005)
    \n
    Comparative genome sequencing of Drosophila pseudoobscura: chromosomal, gene, and cis-element evolution.
    \n
    Genome Res. 15(1):1-18 DOI:10.1101/gr.3059305 FBrf0184012
    \n


    \n

    \n
    2003
    \n
    FlyBase Consortium (2003)
    \n
    The FlyBase Database of the Drosophila Genome Projects and community literature.
    \n
    Nucleic Acids Res. 31:172-175 DOI:10.1093/nar/gkg094 FBrf0159266
    \n


    \n

    \n
    Drysdale R (2003)
    \n
    The Drosophila melanogaster genome sequencing and annotation projects: A status report.
    \n
    Briefings in Functional Genomics and Proteomics 2(2):128--134 DOI:10.1093/bfgp/2.2.128 FBrf0167361
    \n


    \n

    \n
    2002
    \n
    Mungall CJ, Misra S, Berman BP, Carlson J, Frise E, Harris N, Marshall B, Shu S, Kaminker JS, Prochnik SE, Smith CD, Smith E, Tupy JL, Wiel C, Rubin GM and Lewis SE. (2002)
    \n
    An integrated computational pipeline and database to support whole-genome sequence annotation.
    \n
    Genome Biol. 3:research0081 DOI:10.1186/gb-2002-3-12-research0081 FBrf0155825
    \n


    \n

    \n
    Lewis SE, Searle SMJ, Harris N, Gibson M, Iyer V, Richter J, Wiel C, Bayraktaroglu L, Birney E, Crosby MA, Kaminker JS, Matthews BB, Prochnik SE, Smith CD, Tupy JL, Rubin GM, Misra S, Mungall CJ and Clamp ME. (2002)
    \n
    Apollo: a sequence annotation editor.
    \n
    Genome Biol. 3(12):research0082 DOI:10.1186/gb-2002-3-12-research0082 FBrf0155826
    \n


    \n

    \n
    Misra S, Crosby MA, Mungall CJ, Matthews BB, Campbell KS, Hradecky P, Huang Y, Kaminker JS, Millburn GH, Prochnik SE, Smith CD, Tupy JL, Whitfield EJ, Bayraktaroglu L, Berman BP, Bettencourt BR, Celniker SE, de Grey ADNJ, Drysdale RA, Harris NL, Richter J, Russo S, Schroeder AJ, Shu SQ, Stapleton M, Yamada C, Ashburner M, Gelbart WM, Rubin GM and Lewis SE. (2002)
    \n
    Annotation of the Drosophila melanogaster euchromatic genome: a systematic review.
    \n
    Genome Biol. 3:research0083 DOI:10.1186/gb-2002-3-12-research0083 FBrf0155827
    \n


    \n

    \n
    FlyBase Consortium (2002)
    \n
    The FlyBase Database of the Drosophila Genome Projects and community literature.
    \n
    Nucleic Acids Res. 30:106-108 DOI:10.1093/nar/30.1.106 FBrf0141697
    \n


    \n

    \n
    2001
    \n
    Matthews KA and Cook KR (2001)
    \n
    Genetic Information Online: A Brief Introduction to FlyBase and Other Organismal Databases.
    \n
    In Encyclopedia of Genetics (E.C.R. Reeve, ed.):839-841. FBrf0185967
    \n


    \n

    \n
    Drysdale R (2001)
    \n
    Phenotypic data in FlyBase.
    \n
    Brief. Bioinform. 2(1):68-80 DOI:10.1093/bib/2.1.68 FBrf0137187
    \n


    \n

    \n
    1999
    \n
    FlyBase Consortium (1999)
    \n
    FlyBase: Genomic and post-genomic viewpoints.
    \n
    In Bioinformatics: Databases and Systems (S. Letovsky, ed.):141-150 FBrf0111589
    \n


    \n

    \n
    FlyBase Consortium (1999)
    \n
    The FlyBase Database of the Drosophila Genome Projects and community literature.
    \n
    Nucleic Acids Res. 27:85-88 DOI:10.1093/nar/27.1.85 FBrf0107356
    \n


    \n

    \n
    1998
    \n
    FlyBase Consortium (1998)
    \n
    FlyBase - A Drosophila database.
    \n
    Nucleic Acids Res. 26:85-88 DOI:10.1093/nar/26.1.85 FBrf0100346
    \n


    \n

    \n
    1997
    \n
    FlyBase Consortium (1997)
    \n
    FlyBase - A Drosophila database.
    \n
    Nucleic Acids Res. 25:63-66 DOI:10.1093/nar/25.1.63 FBrf0093096
    \n


    \n

    \n
    FlyBase Consortium (1997)
    \n
    Genetic nomenclature guidelines; Drosophila anatomical terms; Bibliography of Drosophila: 1994-1996; 1982-1993.
    \n
    D. I. S. 79:1-399 FBrf0099661
    \n


    \n

    \n
    FlyBase Consortium (1997)
    \n
    Gene-order maps; Transposon and vector maps.
    \n
    D. I. S. 78:1-544 FBrf0099660
    \n


    \n

    \n
    1996
    \n
    FlyBase Consortium (1996)
    \n
    FlyBase - The Drosophila database.
    \n
    Nucleic Acids Res. 24:53-56 DOI:10.1093/nar/24.1.53 FBrf0085043
    \n


    \n

    \n
    FlyBase Consortium (1996)
    \n
    Laboratory Stock Lists on FlyBase - Your Contribution is Needed.
    \n
    D. I. S. 77:156-157 FBrf0091458
    \n


    \n

    \n
    1995
    \n
    FlyBase Consortium (1995)
    \n
    Drosophila melanogaster.
    \n
    Trends Genet. 11(Genetic Nomenclature Guide supplement):26-29 FBrf0078422
    \n


    \n

    \n
    1994
    \n
    FlyBase Consortium (1994)
    \n
    FlyBase - The Drosophila database.
    \n
    Nucleic Acids Res. 22(17):3456-3458 DOI:10.1093/nar/22.17.3456 FBrf0077015
    \n


    \n

    \n
    Ashburner M and Drysdale R (1994)
    \n
    FlyBase - The Drosophila genetic database.
    \n
    Development 120:2077-2079 FBrf0072526
    \n


    \n

    \n
    FlyBase Consortium (1994)
    \n
    The bibliography of Drosophila 1982-1993.
    \n
    D. I. S. 74:1-675 FBrf0078421
    \n


    \n

    \n
    FlyBase Consortium (1994)
    \n
    Genetic loci.
    \n
    D. I. S. 73:1-617 FBrf0078420
    \n


    \n

    \n
    1993
    \n
    Ashburner M (1993)
    \n
    FlyBase.
    \n
    GNome News 13:19-20 FBrf0058642
    \n


    \n

    \n
    FlyBase Consortium (1993)
    \n
    FlyBase - a Drosophila genetic database, release 9301.
    \n
    D. I. S. 72:185-195 FBrf0081999
    \n

    FlyBase Mission Statement

    \n

    The aim of the FlyBase project is to provide an openly accessible centralized resource for Drosophila genetic, genomic and functional data.\n

    \n

    FlyBase: a very brief history

    \n

    FlyBase grew from the long-held principles of free dissemination of information and reagents by the Drosophila genetics community. Resources such as the Drosophila Information Service, a community newsletter publishing new mutants and stock lists and “The Red Book\" (an exhaustive compendium of Drosophila melanogaster genes, mutations, phenotypes and chromosomal aberrations), were central to making this principle work. During the 1980s, it became clear that these printed resources could not keep pace with or accommodate the vast amount of genetic and genomic information being generated.\n

    In 1989, at the 30th Annual Drosophila Research Conference, community members began discussing a computer-based resource that could house such information in a scalable and sustainable manner. In 1992, the National Center for Human Genome Research (NIH, US) together with the Medical Research Council (UK) funded the establishment of FlyBase, a database of genetic and molecular data for the Drosophila research community. The FlyBase project was a multisite collaboration headed by William Gelbart (PI; Harvard, US), Thom Kaufman and Kathy Matthews (Indiana, US), John Merriam (UCLA, US) and Michael Ashburner (Cambridge, UK).\n

    Since then, FlyBase has grown from a simple relational database with flat files accessible via FTP into a comprehensive knowledgebase to meet the growing and evolving needs of the fly research community. This community, with its exemplary culture of sharing, has in turn strongly supported FlyBase. Looking to the future, FlyBase is working with other key knowledgebases to help construct an integrated model organism resource, the Alliance of Genome Resources.\n

    \n

    FlyBase Licenses and Availability

    \n

    FlyBase comprises a series of electronic documents and information processing software, the reference copy of which currently resides at the Internet address <https://s3ftp.flybase.org/index.html>.\n

    FlyBase software is made available under either the MIT License (Expat) or the BSD 3-Clause. \n

    All annotations and data produced by FlyBase that are accessible from flybase.org are distributed under a Creative Commons Attribution 4.0 International License (CC BY 4.0). Different licensing and copyright terms may apply for data provided by these resources that are not distributed by FlyBase.\n

    FlyBase adheres to the principle of full and unfettered data access to FlyBase data for the entire scientific community, and is a member of FAIRsharing. FlyBase is also is registered at the re3data.org repository and is an ELIXIR-UK service.\n

    We are now members of the Global Biodata Coalition and our application is under review with Core Trust Seal.\n

    The primary means of disseminating FlyBase data is through our web site (https://flybase.org) which provides an interface with many options for browsing, interrogating and manipulating FlyBase data according to the needs of the user. In addition, the web site links to an S3 ftp site containing all FlyBase data in a variety of formats, including various sequence formats, archival data, and the entire FlyBase Chado database.\n

    All FlyBase data are available for download to other resources without restriction. In addition to our web site, FlyBase shares data with other major genetic / genomic resources. Most notable among these are NCBI and UniProt. FlyBase is the authority source for data on D. melanogaster and provides files (including whole genome annotation GenBank submissions to NCBI) that are used to generate reference files such as RefSeq. D. melanogaster GenBank/RefSeq submissions are updated annually, and power users are encouraged to use these stable annotation sets in their analyses.\n

    \n

    Warranty

    \n

    FlyBase and the Genetics Society of America make no warranty of any kind with respect to the subject matter included herein. FlyBase and the Genetics Society of America specifically disclaim all warranties, expressed, implied or otherwise, including without limitation, all warranties of merchantability, fitness for a particular purpose, or non-infringement.\n

    The information contained in this publication is subject to change without notice.\n

    By viewing or using this information the user agrees that in no event shall FlyBase or the Genetics Society of America be liable for any special, incidental, indirect, or consequential damages of any kind, or any damages whatsoever, whether or not advised of the possibility of damage, arising out of or in connection with the use or performance of this information.\n

    \n

    Disclaimer

    \n

    Information in FlyBase is curated from the scientific literature, including unrefereed sources such as abstracts, reviews, and personal communications. With the partial exception of map data, statements of fact are not vetted by FlyBase. Users should remember that published information is not always correct information and are encouraged to use FlyBase as a guide to the literature rather than as a substitute for it.\n

    \n

    Herskowitz's Bibliography Copyright

    \n

    The following statements are with respect to the copyright of Parts 5 and 6 of Herskowitz's bibliography:\n

    \"Bibliography on the genetics of Drosophila: Part 5, by Irwin H. Herskowitz is reproduced with the permission of Macmillan Publishing Company. Copyright ©1969 by Macmillan Publishing Company. \"\n

    \"Bibliography on the genetics of Drosophila: Part 6, by Irwin H. Herskowitz is reproduced with the permission of Macmillan Publishing Company. Copyright ©1974 by Macmillan Publishing Company.\" \n

    \n

    Lindsley and Zimm (1992)

    \n

    Lindsley and Zimm (1992) - The New Redbook\nThe Genome of Drosophila melanogaster\nDan L. Lindsley & Georgianna G. Zimm\nCopyright 1992 By Academic Press, Inc.\nAll Rights Reserved.\n

    ISBN 0-12-450990-8\nAcademic Press, Inc.\n1250 Sixth Avenue, San Diego, California 92101-4311\n

    FlyBase is based on the information contained in Lindsley and Zimm's The Genome of Drosophila melanogaster (Academic Press, 1992), however we no longer redistribute the complete text of the book.\n

    \n

    FlyBase Data Sources

    \n

    The Red Book\n

    Lindsley and Zimm (1992) The Genome of Drosophila melanogaster\n

    Bibliographies\n

    The major sources that have been incorporated into the FlyBase Bibliography:\n

    \n
    Bibliography from The Genetics of Drosophila. T. H. Morgan, C. B. Bridges and A. H. Sturtevant, 1925. Bibliographica Genetica II: 1-262. 'S-Gravenhage, Martinus Nijhoff.
    \n
    \n
    Bibliography on the Genetics of Drosophila. [Part One.] H. J. Muller, 1939. Imperial Bureau of Animal Breeding and Genetics. Edinburgh: Oliver and Boyd.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Two. Irwin H. Herskowitz, 1952. Commonwealth Agricultural Bureaux. Slough: Farnham Royal.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Three. Irwin H. Herskowitz, 1958. Indiana University Publications.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Four. Irwin H. Herskowitz, 1963. McGraw-Hill Book Company, Inc.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Five. Irwin H. Herskowitz, 1969. Macmillan Publishing Company, Inc.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Six. Irwin H. Herskowitz, 1974. Macmillan Publishing Company, Inc.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Seven A. Irwin H. Herskowitz, 1976. Drosophila Information Service 51: 159-193.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Seven B. Irwin H. Herskowitz, 1977. Drosophila Information Service 52:186-226.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Seven C. Irwin H. Herskowitz, 1978. Drosophila Information Service 53: 219-244.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Parts Seven D(I) and D(II). Irwin H. Herskowitz, 1980. Drosophila Information Service 55: 218-262.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Seven E. Irwin H. Herskowitz, 1981. Drosophila Information Service 56: 199-210.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Eight A. Irwin H. Herskowitz, 1982. Drosophila Information Service 58: 227-270.
    \n
    \n
    Bibliography on the Genetics of Drosophila. Part Eight B. Irwin H. Herskowitz, 1983. Drosophila Information Service 59: 163-207.
    \n
    \n
    Contents pages of Drosophila Information Service: volume 50 (1975) to volume 75 (1994).
    \n
    \n
    Bibliography of the biochemical genetics of Drosophila. Morton S. Fuchs, 1965. Drosophila Information Service 40: 105-126.
    \n
    \n
    Bibliography of Hawaiian Drosophilidae. Hampton L. Carson and Linden T. Teramoto, 1982. Drosophila Information Service: 58: 215-226.
    \n
    \n
    Bibliography on Drosophila hydei. F. M. A. van Breugel, 1987. Drosophila Information Service 66: 198-208.
    \n
    \n
    Bibliography on the Drosophila nasuta subgroup. W.-E. Kalisch, 1991. Drosophila Information Service 70: 110-117.
    \n
    \n
    Bibliography on Drosophila ananassae. B. N. Singh, 1991. Drosophila Information Service 70: 205-211.
    \n
    \n
    The Bibliography of Theoretical Population Genetics. J. Felsenstein, 1988. Dowden, Hutchinson and Ross Publications, Inc.
    \n
    \n
    Bibliography: longevity, ageing and parental age effects in Drosophila (1907-86). M. Hani Soliman, Frédéric A. Lints, Cécile V. Lints and Pol Bullens. Pages 241-293. In: Drosophila as a Model System for Ageing Studies. F.A. Lints and M.H. Soliman (Eds.), 1988. Blackie Publications. Glasgow, Scotland.
    \n
    \n
    End of Chapter references from Drosophila: A laboratory handbook and Drosophila: A laboratory manual. Michael Ashburner, 1989. Cold Spring Harbor Laboratory Press.
    \n
    \n
    A bibliography on Drosophila compiled and provided by Dr. G. Baechli (University of Zurich). January 1993 with regular subsequent updates.
    \n
    \n
    References from Drosophila ananassae. Genetical and biological aspects. Y.N. Tobari (Editor). 1993.
    \n
    \n
    Japan Scientific Societies Press and Karger, Tokyo and Basel.
    \n
    \n
    References from the Environmental Mutagen Information Center (EMIC) database.
    \n
    \n
    MEDLINE (National Library of Medicine, Washington). Searching all fields with the text word DROSOPHIL* or MELANOGASTER. Searched from 1966 to date.
    \n
    \n
    BIOSIS records searched on the keyword DROSOPHIL from 1982 to 2008.
    \n
    \n
    References from all Drosophila entries in the GenBank, NCBI, EMBL and DDBJ nucleic acid sequence databases and SWISS-PROT and PIR protein sequence databases.
    \n
    \n
    References from John Merriam's Genevent database of molecular data.
    \n
    \n
    Drosophila records from the Zoological Record, volume 115 (1978) to 2008.
    \n
    \n
    Jeff Hall's bibliography on neurogenetics of Drosophila.
    \n

    External Databases\n

    Nucleic Acid Sequence Databases (GenBank/EMBL/DDBJ)\nUniProtKB\nInterPro\n

    Journal Articles and Books\n

    Publications relevant to Drosophila from various journals and books.\n

    Drosophila Conference Abstracts\t\n

    The following abstracts were curated until 2008:\nGSA Annual Drosophila Research Conference Abstracts\nEuropean Drosophila Research Conference Abstracts\n

    Personal Communications to FlyBase\n

    Information and observations provided by the Fly Community\n

    Direct Submission of High-Throughput Data\n

    \n
    Proteomics data
    \n
    \n
    Center for Model Organism Proteomes --
    \n
    (SystemsX and Research Priority Project of the University of Zurich, Switzerland)
    \n
    \n
    Microarrays (Oligos, cDNAs)
    \n
    \n
    Affymetrix, Santa Clara, CA (Affy v1, Affy v2)
    \n
    \n
    Drosophila Genomics Resource Center, Bloomington, IN (DGRC-1, DGRC-2)
    \n
    \n
    Tiling BACs
    \n
    \n
    Berkeley Drosophila Genome Project
    \n
    \n

    Insertion Collection Data\n

    \n
    Transgene insertion data
    \n
    Gene Disruption Project (2005 - present)(Bellen et. al., 2004)
    \n
    \n
    Berkeley Drosophila Genome Project (1998 - 2004)(Spradling et. al., 1999)
    \n
    \n
    DrosDel Project, Cambridge, UK (2003 - 2004)(Ryder et. al., 2004)
    \n
    \n
    P{wHy} Project, Gelbart laboratory, Harvard (2002 - present)(Huet et. al., 2002)
    \n
    \n
    P{GawB} Project, GETDB, Japan (2002 - 2004)
    \n
    \n
    NCI lethals (2002 - 2004)(Oh et. al., 2003)
    \n
    \n
    P{GT1} Project, Ed Ryder, Cambridge, UK (2004)
    \n
    \n
    Schaefer lines, Goettingen, Germany (2004)(Beinert et. al., 2004)
    \n
    \n
    Natural transposon insertion data
    \n
    \n
    Kaminker J. et al. 2002
    \n
    \n
    Quesneville H. et al. 2005
    \n

    Stock Lists Submitted to FlyBase\n

    Bloomington Drosophila Stock Center\n

    Drosophila Genetic Resource Center\n

    UC San Diego Drosophila Species Stock Center\n

    Exelixis Drosophila Stock Collection at Harvard Medical School\n

    Nusslein-Volhard laboratory\n

    Garcia-Bellido laboratory\n

    Carpenter laboratory\n

    \n

    FlyBase Alumni

    \n

    FlyBase thanks these former members of the FlyBase consortium for their contribution:\n

    \n
       Boris Adryan\n   Julie Agapite\n   Manish Anand\n   Giulia Antonazzo\n   Michael Ashburner\n   Phill Baker\n   Leyla Bayraktaroglu\n   Ben Berman\n   Brian Bettencourt\n   Lee Bitsoi\n   Pete Brokstein\n   Kris Broll\n   Nick Brown\n   Stephanie Bunt\n   Heather Butler\n   Kathy Campbell\n   Russell Collins\n   Kim Cook\n   Marta Costa\n   Lynn Crosby\n   Bryon Czoch\n   Anthony DeAngelo\n   Aubrey de Grey\n   Rachel Drysdale\n   David Emmert\n   Kathleen Falls\n   Silvie Fexova\n   Helen Field\n   Rebecca Foulger\n   Erwin Frise\n   Phani Garapati\n   William Gelbart\n   Mark Gibson\n   Don Gilbert\n   Josh Goodman\n   Damien Goutte-Gattat\n   Gary Grumbling\n   Janet Gulotta\n   Cyrus Harmon\n   Nomi Harris\n   Gregg Helt\n   Alex Holmes\n   Pavel Hradecky\n   Yanmei Huang\n   Victoria Jenkins\n   Tamsin Jones\n   Josh Kaminker\n   Thomas Kaufman\n   Kerry Knight\n   Sophia Kossida\n   Pravija Krishna\n   Rob Kulathinal\n   Julia Lamenza\n   Joe Lemaire\n   Stan Letovsky\n   Suzanna Lewis\n   Paul Leyland\n   Guo-chun Liao\n   Anne Loraine\n   Beverley Matthews\n   Kathy Matthews\n   Glenn McAlpine\n   Peter McQuilton\n   John Merriam\n   Sima Misra\n   Emanuel Mongin\n   Chris Mungall\n   Tatiana Murnikova\n   Catherine Nelson\n   David Osumi-Sutherland\n   Dawn Palmer\n   Harriet Platero\n   Laura Ponting\n   Sinzi Pop\n   Simon Prochnik\n   Simon Reeve\n   Alix Rey\n   John Richter\n   Wayne Rindone\n   Margo Roark\n   Gerry Rubin\n   Susan Russo Gelbart\n   Susan St. Pierre\n   Andrew Schroeder\n   Ruth Seal\n   Jolene Seme\n   Carolyn Shemen\n   Hardik Sheth\n   Nihar Sheth\n   Shengqiang Shu\n   David Simas\n   Chris Smith\n   Frank Smutniak\n   Nicole Staudt\n   Ray Stefancsik\n   Carol Sutherland\n   Vitor Trovisco\n   Jon Tupy\n   Susan Tweedie\n   Jose Maria Urbano\n   Edwin Wang\n   Eddy Wellbourne\n   Maggie Werner-Washburne\n   Eleanor Whitfield\n   Colin Wiel\n   Mark Williams\n   Robert Wilson\n   J.D. Wong\n   Chihiro Yamada\n   Mark Yandell\n   Haiyan Zhang\n   Peili Zhang\n   Pinglei Zhou\n
    \n

    Funding Sources Supporting FlyBase

    \n

    1. \"FlyBase: A Drosophila Genomic and Genetic Database\"\nNHGRI Award #: U24HG013300\n

    2. \"Emergency fund to rescue FlyBase Curation\"\nWellcome Discretionary Award\n

    3. \"GO annotation: maximizing the potential of Drosophila research to benefit human health\"\nUK Medical Research Council Award #: MR/W024233/1\n

    4. “Virtual Fly Brain”\nWellcome Trust Award #: PLM13398\n

    5. \"Alliance Central: A Platform for sustainable development of next generation genome knowledgebases\"\nNHGRI Award #: U24HG010859\n

    6. \"Investigating metabolic responses to high sugar diets and the onset of diabetic phenotypes\"\nNIDDK Award #: 1R01DK136945-01\n

    7. Generous donations and contributions from FlyBase users all over the world\n

    ","sections":[{"toclevel":1,"level":"2","line":"FlyBase Consortium","number":"1","index":"1","fromtitle":"FlyBase:About","byteoffset":0,"anchor":"FlyBase_Consortium","linkAnchor":"FlyBase_Consortium"},{"toclevel":1,"level":"2","line":"FlyBase Publications","number":"2","index":"2","fromtitle":"FlyBase:About","byteoffset":1309,"anchor":"FlyBase_Publications","linkAnchor":"FlyBase_Publications"},{"toclevel":1,"level":"2","line":"FlyBase Mission Statement","number":"3","index":"3","fromtitle":"FlyBase:About","byteoffset":42313,"anchor":"FlyBase_Mission_Statement","linkAnchor":"FlyBase_Mission_Statement"},{"toclevel":1,"level":"2","line":"FlyBase: a very brief history","number":"4","index":"4","fromtitle":"FlyBase:About","byteoffset":42484,"anchor":"FlyBase:_a_very_brief_history","linkAnchor":"FlyBase:_a_very_brief_history"},{"toclevel":1,"level":"2","line":"FlyBase Licenses and Availability","number":"5","index":"5","fromtitle":"FlyBase:About","byteoffset":44456,"anchor":"FlyBase_Licenses_and_Availability","linkAnchor":"FlyBase_Licenses_and_Availability"},{"toclevel":2,"level":"3","line":"Warranty","number":"5.1","index":"6","fromtitle":"FlyBase:About","byteoffset":46831,"anchor":"Warranty","linkAnchor":"Warranty"},{"toclevel":2,"level":"3","line":"Disclaimer","number":"5.2","index":"7","fromtitle":"FlyBase:About","byteoffset":47660,"anchor":"Disclaimer","linkAnchor":"Disclaimer"},{"toclevel":2,"level":"3","line":"Herskowitz's Bibliography Copyright","number":"5.3","index":"8","fromtitle":"FlyBase:About","byteoffset":48099,"anchor":"Herskowitz's_Bibliography_Copyright","linkAnchor":"Herskowitz's_Bibliography_Copyright"},{"toclevel":2,"level":"3","line":"Lindsley and Zimm (1992)","number":"5.4","index":"9","fromtitle":"FlyBase:About","byteoffset":48638,"anchor":"Lindsley_and_Zimm_(1992)","linkAnchor":"Lindsley_and_Zimm_(1992)"},{"toclevel":1,"level":"2","line":"FlyBase Data Sources","number":"6","index":"10","fromtitle":"FlyBase:About","byteoffset":49138,"anchor":"FlyBase_Data_Sources","linkAnchor":"FlyBase_Data_Sources"},{"toclevel":1,"level":"2","line":"FlyBase Alumni","number":"7","index":"11","fromtitle":"FlyBase:About","byteoffset":55479,"anchor":"FlyBase_Alumni","linkAnchor":"FlyBase_Alumni"},{"toclevel":1,"level":"2","line":"Funding Sources Supporting FlyBase","number":"8","index":"12","fromtitle":"FlyBase:About","byteoffset":57634,"anchor":"Funding_Sources_Supporting_FlyBase","linkAnchor":"Funding_Sources_Supporting_FlyBase"}],"showtoc":true}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/parse_balancer_genotype_variant_report.json b/mcp/tests/fixtures/wiki/parse_balancer_genotype_variant_report.json new file mode 100644 index 0000000..e306204 --- /dev/null +++ b/mcp/tests/fixtures/wiki/parse_balancer_genotype_variant_report.json @@ -0,0 +1 @@ +{"parse":{"title":"FlyBase:Balancer/Genotype variant Report","pageid":100776,"revid":173641,"redirects":[],"text":"

    Last Updated: 11th December 2017\n

    The Balancer/Genotype Variant Report contains the information for an individual balancer/genotype variant. Genotype variants are chromosomal rearrangements which carry the same rearrangement as their parent aberration (listed in the Parent aberration field of the report), but they differ genetically from the parent aberration, e.g. carrying a different constellation of alleles and/or transposable element insertions.\n

    A genotype variant chromosome may also be a Balancer, which is a multiply inverted chromosome that suppresses recombination with the homologous chromosome. This allows the homologous chromosome (or part of it) to be maintained intact in a cross or stock. Balancers also typically carry a dominant marker and are often lethal or sterile when homozygous. This can increase their usefulness to researchers.\n

    If an individual genotype variant chromosome has been reported to be usable as a balancer this is indicated by the presence of the Described as Balancer field in the Report.\n

    This is a field-by-field guide to the information provided in the Balancer/Genotype Variant Report. FlyBase attributes data to the publication that reported it, so that users can easily refer back to the original publication if they wish. Thus, where possible in the fields below, the publication(s) that are the source of the information are listed, typically in parentheses to the right of the data. The exception in the Genotype Variant/Balancer Report is the General Information section which contains a summary of the identity of the genotype variant/balancer.\n

    \n\n

    General Information

    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    SymbolThe valid symbol that is used in FlyBase for the balancer/genotype variant.\n

    -\nThe first part of the symbol (before the '\\') is the standard prefix for the species (from the Species Abbreviations list). For species other than D.melanogaster, the species prefix is displayed wherever the balancer/genotype variant symbol is used throughout FlyBase. For D.melanogaster balancer/genotype variants, the species prefix is only displayed in the General Information section at the top of a Report.\n

    \n
    SpeciesThe organism that the balancer/genotype variant originates from, with the initial letter of the genus and the full species name listed.\n
    FlyBase IDThe Primary FlyBase identifier number of the balancer/genotype variant, used to uniquely identify the balancer/genotype variant in the database.\n

    A balancer/genotype variant may also have any number of Secondary FlyBase identifier numbers, which are listed in the Secondary FlyBase IDs section of the Balancer/Genotype Variant Report.\n

    \n
    Short GenotypeThis combines the symbol of a classical balancer with new allele, aberration or transposable element insertion symbols to define a unique balancer variant.\n
    Parent AberrationThe parent aberration of the balancer/genotype variant. The balancer/genotype variant carries the same rearrangement as the parent aberration, but differs genetically, e.g. carrying a different constellation of alleles and/or transposable element insertions.\n
    Parent CytologyA description of the new cytological order of the parent aberration (by definition the same as that of the balancer/genotype variant).\n

    The sequence of each chromosome involved in the chromosomal aberration is specified from one end to the other according to salivary gland chromosome band terminology. Each chromosome is separated by a semicolon. Points of breakage and reunion are indicated by vertical bars, and segments between these points are designated by the most extreme band known to be present at each end, separated by a dash. If the order of a particular segment is not known, the segment is included within parentheses, and hierarchies of ambiguities are represented by parentheses within parentheses.\n

    \n
    ProgenitorThe mutant chromosome on which the balancer/genotype variant was induced. This is either a chromosomal aberration or another balancer/genotype variant.\n
    Described as BalancerThis field is only displayed on an individual report if the genotype variant chromosome has been reported as being usable as a balancer. The publication(s) that reported this information are listed, where available.\n
    Carries AllelesAllele(s) that are carried on the balancer/genotype variant. (Note that any alleles caused by an aberration breakpoint are not included here but are listed in the 'Causes Alleles' field of the relevant parent aberration report.)\n

    Clicking on an allele symbol will take you to the relevant Allele Report.\n

    \n
    Transposon InsertionsInsertion(s) (of transgenic constructs or natural transposons) that are carried on the balancer/genotype variant.\n

    Clicking on an insertion symbol will take you to the relevant Insertion Report.\n

    \n
    DiscovererThe individual(s) who first identified or generated the balancer/genotype variant.\n
    \n

    Stocks

    \n

    A list of the stocks from the public Stock Centers that contain this balancer/genotype variant; for each Stock Center, stock number and genotype are displayed. Clicking on the genotype will take you to the appropriate Stock Report. If there are a large number of stocks, the list on the page may be truncated, but clicking on the \"More stocks available...\" link will take you to a hit list of all available stocks containing the balancer/genotype variant.\n

    \n

    Other Information

    \n

    Miscellaneous free text comments about the balancer/genotype variant.\n

    \n

    Synonyms & Secondary IDs

    \n

    Reported As

    \n\n\n\n\n\n\n\n
    Symbol SynonymA list of symbols that have been used in the literature, or by FlyBase, to describe the balancer/genotype variant.\n
    Name SynonymA list of names that have been used in the literature, or by FlyBase, to describe the balancer/genotype variant.\n
    \n

    Secondary FlyBase IDs

    \n

    A list of Secondary FlyBase identifier numbers of the balancer/genotype variant.\n

    If a balancer/genotype variant has a secondary identifier number, it generally indicates that at some point it has been merged with or split from other entries in the database. See the FlyBase identifier numbers section for some examples of the cases where identifier numbers are made secondary.\n

    \n

    References

    \n

    A list of publications that discuss the balancer/genotype variant, organised by type of publication.\n

    ","sections":[{"toclevel":1,"level":"2","line":"General Information","number":"1","index":"1","fromtitle":"FlyBase:Balancer/Genotype_variant_Report","byteoffset":1632,"anchor":"General_Information","linkAnchor":"General_Information"},{"toclevel":1,"level":"2","line":"Stocks","number":"2","index":"2","fromtitle":"FlyBase:Balancer/Genotype_variant_Report","byteoffset":5105,"anchor":"Stocks","linkAnchor":"Stocks"},{"toclevel":1,"level":"2","line":"Other Information","number":"3","index":"3","fromtitle":"FlyBase:Balancer/Genotype_variant_Report","byteoffset":5574,"anchor":"Other_Information","linkAnchor":"Other_Information"},{"toclevel":1,"level":"2","line":"Synonyms & Secondary IDs","number":"4","index":"4","fromtitle":"FlyBase:Balancer/Genotype_variant_Report","byteoffset":5668,"anchor":"Synonyms_&_Secondary_IDs","linkAnchor":"Synonyms_&_Secondary_IDs"},{"toclevel":2,"level":"3","line":"Reported As","number":"4.1","index":"5","fromtitle":"FlyBase:Balancer/Genotype_variant_Report","byteoffset":5698,"anchor":"Reported_As","linkAnchor":"Reported_As"},{"toclevel":2,"level":"3","line":"Secondary FlyBase IDs","number":"4.2","index":"6","fromtitle":"FlyBase:Balancer/Genotype_variant_Report","byteoffset":6046,"anchor":"Secondary_FlyBase_IDs","linkAnchor":"Secondary_FlyBase_IDs"},{"toclevel":1,"level":"2","line":"References","number":"5","index":"7","fromtitle":"FlyBase:Balancer/Genotype_variant_Report","byteoffset":6460,"anchor":"References","linkAnchor":"References"}],"showtoc":true}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/parse_batch_download.json b/mcp/tests/fixtures/wiki/parse_batch_download.json new file mode 100644 index 0000000..fae090a --- /dev/null +++ b/mcp/tests/fixtures/wiki/parse_batch_download.json @@ -0,0 +1 @@ +{"parse":{"title":"FlyBase:Batch Download","pageid":126621,"revid":170730,"redirects":[],"text":"
    \n

    Batch Download Overview

    \n

    The Batch Download tool provides access to data from a variety of specific fields in FlyBase reports for a specified list of symbols or IDs. The data from multiple fields can be accessed at once. The specified list of IDs can be large (e.g. all genes) or small (e.g. one gene). Results can be downloaded or viewed online.\n

    \n

    Validating FlyBase symbols and IDs

    \n

    Batch Download does not work with invalid or out of date (secondary) IDs. We strongly suggest that you first validate your list of symbols/IDs by using our ID Validator tool. Once validated you can then export them back to Batch Download.\n

    \n

    Quick Instructions

    \n
    1. The Batch Download tool is organized into three main parts: a dropdown menu for the selection of Output Format, a dropdown menu for selection of Output Options, and a box for inputting symbols or IDs.
    2. \n
    3. Select the output format which can be either an HTML table or a tab-separated file. There is also an option in this tab called “From precomputed files”, which allows you to select fields from the precomputed files in FlyBase.
    4. \n
    5. Choose to display the results in a browser or to save them as a tab-delimited file to your computer or device.
    6. \n
    7. Enter your list of IDs in the box. The list of IDs can be separated by commas, tabs, return characters or spaces, and can be entered manually in the box or by uploading the information from your computer. To upload a file of IDs, click on “Browse” and select a file to upload. You can also populate Batch Download with IDs by choosing “Batch Download\" from the “Export” menu in any hit list.
    8. \n
    9. Click on “Continue to Select Fields”. A new window will appear with fields specific to the data type of the IDs you entered. Select your field(s) of interest.
    10. \n
    11. Click on one of the two “Get Field Data” buttons located at the top right and bottom right corners of the form.
    \n

    Additional Information and more detailed Instructions

    \n
    • \"Field Data\" includes all the data that is presented in various FlyBase reports. For example, for a list of genes, you can retrieve transcript expression data, GO: Molecular Function terms, lists of cDNA clones, etc. For a list of alleles, you can retrieve phenotypic class information, lists of stocks, etc. Note that the Human Disease Model, Gene Groups, Physical Interaction, and Strain reports are not supported by Batch Download. To retrieve cell line data, enter Cell line IDs, not symbols.
    • \n
    • There are three output options: As HTML Table, As Tab Separated File, and From Precomputed Files. Use the first option if you are going to view your results in the browser window or print them out directly, since the output will be nicely formatted. However, if you'd like to save your output in a file or open it in a spreadsheet, choose to see the data tab separated. Finally, the option to see data from Precomputed Files allows you to obtain the data available in the FlyBase precomputed files for your list of IDs. For example, if you have a list of genes, you can get the overlapping affy oligos from the \"Genes: fbgn_exons2affy1_overlaps.tsv\" file. There are many other options available from precomputed files, so it's worth taking a look at the list. See the Downloads Overview page for more information on the precomputed files.
    • \n
    • It is important to note that Batch Download can only retrieve information about a single dataset at one time. For example, you cannot simultaneously retrieve field data for genes and alleles. If you input a list that contains a mix of gene and allele symbols, whichever data type has more entries will be recognized and the other data type will be ignored. If your list contains equal numbers, the identity of the first entry on the list will be recognized.
    • \n
    • Once you have entered your list of IDs, click \"Continue to Select Fields\". This opens a new window that includes a menu of all the available fields for your dataset of interest. The fields are organized in a similar fashion to the report pages. Multiple fields can be selected at once by checking more than one check box, or by using the Check Section or Check All buttons.
    • \n
    • Click \"Get field data\" to obtain your results.
    • \n
    • Important tip: If you entered Batch Download via a Hits List (i.e. you exported your hits from the Hits List to Batch Download), your data will be entered as FlyBase IDs (e.g. FBgn, FBal, etc.). If you want to see the associated symbols in your output, be sure to select the field \"Symbol: symbol\".
    ","sections":[{"toclevel":1,"level":"1","line":"Batch Download Overview","number":"1","index":"1","fromtitle":"FlyBase:Batch_Download","byteoffset":0,"anchor":"Batch_Download_Overview","linkAnchor":"Batch_Download_Overview"},{"toclevel":1,"level":"1","line":"Validating FlyBase symbols and IDs","number":"2","index":"2","fromtitle":"FlyBase:Batch_Download","byteoffset":349,"anchor":"Validating_FlyBase_symbols_and_IDs","linkAnchor":"Validating_FlyBase_symbols_and_IDs"},{"toclevel":1,"level":"1","line":"Quick Instructions","number":"3","index":"3","fromtitle":"FlyBase:Batch_Download","byteoffset":660,"anchor":"Quick_Instructions","linkAnchor":"Quick_Instructions"},{"toclevel":1,"level":"1","line":"Additional Information and more detailed Instructions","number":"4","index":"4","fromtitle":"FlyBase:Batch_Download","byteoffset":1889,"anchor":"Additional_Information_and_more_detailed_Instructions","linkAnchor":"Additional_Information_and_more_detailed_Instructions"}],"showtoc":true}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/parse_fbgn0000553.json b/mcp/tests/fixtures/wiki/parse_fbgn0000553.json new file mode 100644 index 0000000..22a966a --- /dev/null +++ b/mcp/tests/fixtures/wiki/parse_fbgn0000553.json @@ -0,0 +1 @@ +{"parse":{"title":"FBgn0000553","pageid":34051,"revid":52203,"redirects":[],"text":"
    1. REDIRECT [[Dmel\\e(dp<up>v</up>)]]
    ","sections":[],"showtoc":false}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/search_ftp_archive_wildcard.json b/mcp/tests/fixtures/wiki/search_ftp_archive_wildcard.json new file mode 100644 index 0000000..65551ca --- /dev/null +++ b/mcp/tests/fixtures/wiki/search_ftp_archive_wildcard.json @@ -0,0 +1 @@ +{"batchcomplete":true,"query":{"pages":[{"pageid":126832,"ns":1000,"title":"FlyBase:Using FTP Archives","index":1,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2025-07-31T15:55:23Z","lastrevid":173369,"length":2937}],"searchinfo":{"totalhits":1},"search":[{"ns":1000,"title":"FlyBase:Using FTP Archives","pageid":126832,"snippet":"...g/contact/email Contact FlyBase form] if you are experiencing issues. Some archives are not available while we complete this process. 30 Jun 2025.\n==Genomes archives ==\n","timestamp":"2025-07-31T15:55:23Z"}]}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/search_ftp_archive_zero.json b/mcp/tests/fixtures/wiki/search_ftp_archive_zero.json new file mode 100644 index 0000000..7a1536a --- /dev/null +++ b/mcp/tests/fixtures/wiki/search_ftp_archive_zero.json @@ -0,0 +1 @@ +{"batchcomplete":true,"query":{"searchinfo":{"totalhits":0},"search":[]}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/search_report.json b/mcp/tests/fixtures/wiki/search_report.json new file mode 100644 index 0000000..d3441a2 --- /dev/null +++ b/mcp/tests/fixtures/wiki/search_report.json @@ -0,0 +1 @@ +{"batchcomplete":true,"continue":{"sroffset":10,"gsroffset":10,"continue":"gsroffset||"},"query":{"redirects":[{"ns":1000,"title":"FlyBase:Pathway Report","pageid":126856,"index":2,"from":"FlyBase:Pathway Report","to":"FlyBase:Signaling Pathway Report"}],"pages":[{"pageid":100773,"ns":1000,"title":"FlyBase:Report Help","index":1,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2021-10-13T21:55:22Z","lastrevid":171466,"length":1086},{"pageid":100778,"ns":1000,"title":"FlyBase:Clone Report","index":3,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2017-12-13T21:00:11Z","lastrevid":169186,"length":7165},{"pageid":100779,"ns":1000,"title":"FlyBase:Insertion Report","index":5,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2022-12-20T17:46:55Z","lastrevid":171937,"length":16344},{"pageid":100782,"ns":1000,"title":"FlyBase:Strain Report","index":9,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2024-12-13T09:54:00Z","lastrevid":173112,"length":7167},{"pageid":100783,"ns":1000,"title":"FlyBase:Reference Report","index":10,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2024-12-13T11:06:00Z","lastrevid":173113,"length":6760},{"pageid":100791,"ns":1000,"title":"FlyBase:Stock Report","index":6,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2017-12-13T19:35:45Z","lastrevid":169175,"length":2955},{"pageid":100792,"ns":1000,"title":"FlyBase:Transcript Report","index":7,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2024-12-16T14:36:21Z","lastrevid":173146,"length":8937},{"pageid":100793,"ns":1000,"title":"FlyBase:Interaction Report","index":8,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2018-01-03T20:11:18Z","lastrevid":169640,"length":5571},{"pageid":126684,"ns":1000,"title":"FlyBase:Signaling Pathway Report","index":2,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2025-04-23T08:30:59Z","lastrevid":173283,"length":18936},{"pageid":126857,"ns":1000,"title":"FlyBase:Combination Report","index":4,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2025-09-05T14:35:49Z","lastrevid":173440,"length":2925}],"searchinfo":{"totalhits":29},"search":[{"ns":1000,"title":"FlyBase:Report Help","pageid":100773,"snippet":"[[FlyBase:Aberration Report|Aberration Report]]\n[[FlyBase:Allele Report|Allele Report]]\n","timestamp":"2021-10-13T21:55:22Z"},{"ns":1000,"title":"FlyBase:Pathway Report","pageid":126856,"snippet":"#REDIRECT [[FlyBase:Signaling Pathway Report]]\n","timestamp":"2024-12-05T12:27:18Z"},{"ns":1000,"title":"FlyBase:Clone Report","pageid":100778,"snippet":"The '''Clone Report''' provides information on clones. The report includes information on the type of clone, the sequence of the clone insert\n...refix is only displayed in the GENERAL INFORMATION section at the top of a Report.\n","timestamp":"2017-12-13T21:00:11Z"},{"ns":1000,"title":"FlyBase:Combination Report","pageid":126857,"snippet":"The '''Split System Combination''' report displays curated information for split driver lines, where two (or more) he\n|'''Feature type'''\t || The type of feature described in the report. In this case, "split system combination".\n","timestamp":"2025-09-05T14:35:49Z"},{"ns":1000,"title":"FlyBase:Insertion Report","pageid":100779,"snippet":"The insertion report is used to present data on insertions of transgenic constructs, insertions \nThis is a field-by-field guide to the information in the '''Insertion Report'''.\n","timestamp":"2022-12-20T17:46:55Z"},{"ns":1000,"title":"FlyBase:Stock Report","pageid":100791,"snippet":"The '''Stock Report''' provides information on Drosophila stocks in stock centers and laborator\n...a field-by-field guide to the information that is provided in the '''Stock Report'''.\n","timestamp":"2017-12-13T19:35:45Z"},{"ns":1000,"title":"FlyBase:Transcript Report","pageid":100792,"snippet":"...to annotated transcripts. Curated transcript data is found on the '''Gene Report''', subsection '''Transcript Data''', field '''Reported transcript sizes'''\n...is a field-by-field guide to the information provided in the '''Transcript Report'''.\n","timestamp":"2024-12-16T14:36:21Z"},{"ns":1000,"title":"FlyBase:Interaction Report","pageid":100793,"snippet":"The Interaction Report contains information for the physical interaction of two gene products (pro\nThis is a field-by-field guide to the information provided in the Interaction Report.\n","timestamp":"2018-01-03T20:11:18Z"},{"ns":1000,"title":"FlyBase:Strain Report","pageid":100782,"snippet":"The '''Strain Report''' contains the information for an individual strain. \nThis is a field-by-field guide to the information provided in the '''Strain Report'''. FlyBase attributes data to the publication that reported it, so that us\n","timestamp":"2024-12-13T09:54:00Z"},{"ns":1000,"title":"FlyBase:Reference Report","pageid":100783,"snippet":"...ct, external IDs (PubMed, PubMed Central, DOI), and links to other FlyBase Report pages for any objects curated from the reference.\n...ny references in FlyBase that are directly linked to the current Reference Report. Common examples of relationship types are:\n","timestamp":"2024-12-13T11:06:00Z"}]}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/search_stock.json b/mcp/tests/fixtures/wiki/search_stock.json new file mode 100644 index 0000000..21ccce7 --- /dev/null +++ b/mcp/tests/fixtures/wiki/search_stock.json @@ -0,0 +1 @@ +{"batchcomplete":true,"query":{"redirects":[{"ns":1000,"title":"FlyBase:Stock Collections","pageid":95470,"index":2,"from":"FlyBase:Stock Collections","to":"FlyBase:Stocks"}],"pages":[{"pageid":94924,"ns":14,"title":"Category:Stock Collections","index":1,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2016-10-04T17:36:52Z","lastrevid":118998,"length":86},{"pageid":100791,"ns":1000,"title":"FlyBase:Stock Report","index":3,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2017-12-13T19:35:45Z","lastrevid":169175,"length":2955},{"pageid":100837,"ns":1000,"title":"FlyBase:Stocks","index":2,"contentmodel":"wikitext","pagelanguage":"en","pagelanguagehtmlcode":"en","pagelanguagedir":"ltr","touched":"2025-04-03T19:23:36Z","lastrevid":173263,"length":12678}],"searchinfo":{"totalhits":3},"search":[{"ns":14,"title":"Category:Stock Collections","pageid":94924,"snippet":"[http://flybase.org/static_pages/allied-data/stock_collections.html Stock Collections]\n","timestamp":"2013-03-01T21:05:40Z"},{"ns":1000,"title":"FlyBase:Stock Collections","pageid":95470,"snippet":"","timestamp":"2018-03-20T16:01:46Z"},{"ns":1000,"title":"FlyBase:Stock Report","pageid":100791,"snippet":"..., when available, provides links to a collection's web site for additional stock information.\n...her stock-related communications should be directed to the provider of the stock rather than to FlyBase.\n","timestamp":"2017-12-13T19:35:45Z"}]}} \ No newline at end of file diff --git a/mcp/tests/fixtures/wiki/siteinfo_rightsinfo.json b/mcp/tests/fixtures/wiki/siteinfo_rightsinfo.json new file mode 100644 index 0000000..53929f8 --- /dev/null +++ b/mcp/tests/fixtures/wiki/siteinfo_rightsinfo.json @@ -0,0 +1 @@ +{"batchcomplete":true,"query":{"rightsinfo":{"url":"","text":""}}} \ No newline at end of file diff --git a/mcp/tests/support/mod.rs b/mcp/tests/support/mod.rs index ca23f06..26b44fb 100644 --- a/mcp/tests/support/mod.rs +++ b/mcp/tests/support/mod.rs @@ -12,6 +12,8 @@ pub fn test_config() -> Config { })), "CHADO_POOL_SIZE" => Some("4".into()), "FLYBASE_MCP_RESPONSE_CACHE_MB" => std::env::var(k).ok(), + // The wiki stays unconfigured (as does BLAST); wiki tests set `wiki_base_url` to their own mock. + "WIKI_BASE_URL" | "WIKI_HOST_HEADER" => None, _ => None, }) .unwrap() diff --git a/mcp/tests/wiki_client.rs b/mcp/tests/wiki_client.rs new file mode 100644 index 0000000..44ac7ec --- /dev/null +++ b/mcp/tests/wiki_client.rs @@ -0,0 +1,501 @@ +//! The MediaWiki client against wiremock, serving the captured responses in `tests/fixtures/wiki/`. +//! No database is needed. +use flybase_mcp::backends::wiki::{MAX_WIKI_BYTES, WikiClient, WikiError}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wiremock::matchers::{header, method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const HOST: &str = "wiki.flybase.org"; + +fn fixture(name: &str) -> String { + std::fs::read_to_string(format!( + "{}/tests/fixtures/wiki/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() +} + +fn json(name: &str) -> ResponseTemplate { + ResponseTemplate::new(200).set_body_raw(fixture(name), "application/json; charset=utf-8") +} + +fn client(mock: &MockServer) -> WikiClient { + WikiClient::new(mock.uri().parse().unwrap(), HOST.into()).unwrap() +} + +/// The decoded query pairs of the only request the mock received. +async fn only_request_pairs(mock: &MockServer) -> Vec<(String, String)> { + let reqs = mock.received_requests().await.unwrap(); + assert_eq!(reqs.len(), 1); + reqs[0] + .url + .query_pairs() + .map(|(k, v)| (k.into_owned(), v.into_owned())) + .collect() +} + +fn pairs(v: &[(&str, &str)]) -> Vec<(String, String)> { + v.iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[tokio::test] +async fn search_sends_host_and_exact_pairs_and_decodes_the_fixture() { + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api.php")) + .and(header("host", HOST)) + .respond_with(json("search_stock.json")) + .expect(1) + .mount(&mock) + .await; + let r = client(&mock).search("stock", 10, 0).await.unwrap(); + assert_eq!( + only_request_pairs(&mock).await, + pairs(&[ + ("action", "query"), + ("list", "search"), + ("srsearch", "stock"), + ("srnamespace", "12|14|1000"), + ("srlimit", "10"), + ("sroffset", "0"), + ("srprop", "snippet|timestamp"), + ("srinfo", "totalhits"), + ("generator", "search"), + ("gsrsearch", "stock"), + ("gsrnamespace", "12|14|1000"), + ("gsrlimit", "10"), + ("gsroffset", "0"), + ("redirects", "1"), + ("prop", "info"), + ("format", "json"), + ("formatversion", "2"), + ]) + ); + assert_eq!(r.searchinfo.totalhits, 3); + assert_eq!(r.search.len(), 3); + assert_eq!(r.search[1].title, "FlyBase:Stock Collections"); + assert_eq!(r.search[1].snippet, ""); + assert_eq!(r.redirects.len(), 1); + assert_eq!(r.redirects[0].from, "FlyBase:Stock Collections"); + assert_eq!(r.redirects[0].to, "FlyBase:Stocks"); + assert_eq!(r.pages.len(), 3); + assert!(r.pages.iter().all(|p| !p.missing && !p.invalid)); + let stocks = r + .pages + .iter() + .find(|p| p.title == "FlyBase:Stocks") + .unwrap(); + assert_eq!(stocks.ns, 1000); + assert_eq!(stocks.touched.as_deref(), Some("2025-04-03T19:23:36Z")); +} + +#[tokio::test] +async fn zero_hit_and_wildcard_fixtures_decode() { + let mock = MockServer::start().await; + Mock::given(query_param("srsearch", "FTP archive")) + .respond_with(json("search_ftp_archive_zero.json")) + .mount(&mock) + .await; + Mock::given(query_param("srsearch", "ftp* archive*")) + .respond_with(json("search_ftp_archive_wildcard.json")) + .mount(&mock) + .await; + let c = client(&mock); + let zero = c.search("FTP archive", 10, 0).await.unwrap(); + assert_eq!(zero.searchinfo.totalhits, 0); + assert!(zero.search.is_empty() && zero.redirects.is_empty() && zero.pages.is_empty()); + let wild = c.search("ftp* archive*", 10, 0).await.unwrap(); + assert_eq!(wild.search[0].title, "FlyBase:Using FTP Archives"); +} + +#[tokio::test] +async fn missing_query_member_is_an_empty_search() { + let mock = MockServer::start().await; + Mock::given(path("/api.php")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw(r#"{"batchcomplete":true}"#, "application/json"), + ) + .mount(&mock) + .await; + let r = client(&mock).search("x", 10, 0).await.unwrap(); + assert_eq!(r.searchinfo.totalhits, 0); + assert!(r.search.is_empty()); +} + +#[tokio::test] +async fn hostile_query_is_one_srsearch_value() { + let mock = MockServer::start().await; + Mock::given(path("/api.php")) + .respond_with(json("search_ftp_archive_zero.json")) + .mount(&mock) + .await; + let q = "a&action=edit|b#c%d=e é"; + client(&mock).search(q, 5, 20).await.unwrap(); + let got = only_request_pairs(&mock).await; + let sr: Vec<&str> = got + .iter() + .filter(|(k, _)| k == "srsearch") + .map(|(_, v)| v.as_str()) + .collect(); + assert_eq!(sr, vec![q]); + assert_eq!(sr[0].as_bytes(), q.as_bytes(), "byte-exact"); + let gsr: Vec<&str> = got + .iter() + .filter(|(k, _)| k == "gsrsearch") + .map(|(_, v)| v.as_str()) + .collect(); + assert_eq!(gsr, vec![q]); + let actions: Vec<&str> = got + .iter() + .filter(|(k, _)| k == "action") + .map(|(_, v)| v.as_str()) + .collect(); + assert_eq!(actions, vec!["query"], "no injected action pair"); + assert!(got.contains(&("srlimit".into(), "5".into()))); + assert!(got.contains(&("gsroffset".into(), "20".into()))); + assert_eq!(got.len(), 17, "no extra pairs: {got:?}"); +} + +#[tokio::test] +async fn page_sends_exact_pairs_and_decodes_every_parse_fixture() { + for (title, file) in [ + ("FlyBase:Aberration Report", "parse_aberration_report.json"), + ("FlyBase:About", "parse_about.json"), + ("FlyBase:Batch Download", "parse_batch_download.json"), + ( + "FlyBase:Balancer/Genotype variant Report", + "parse_balancer_genotype_variant_report.json", + ), + ("FBgn0000553", "parse_fbgn0000553.json"), + ] { + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api.php")) + .and(header("host", HOST)) + .respond_with(json(file)) + .expect(1) + .mount(&mock) + .await; + let p = client(&mock).page(title).await.unwrap(); + assert_eq!( + only_request_pairs(&mock).await, + pairs(&[ + ("action", "parse"), + ("page", title), + ("redirects", "1"), + ("prop", "text|sections|revid"), + ("disablelimitreport", "1"), + ("disableeditsection", "1"), + ("disabletoc", "1"), + ("format", "json"), + ("formatversion", "2"), + ]), + "{title}" + ); + assert_eq!(p.title, title); + assert!(p.redirects.is_empty(), "{title}: redirects decode as empty"); + assert!(p.revid > 0 && p.pageid > 0, "{title}"); + assert!(!p.text.is_empty(), "{title}"); + if title == "FBgn0000553" { + assert!(p.sections.is_empty()); + } else { + assert!(!p.sections.is_empty(), "{title}"); + assert!( + p.sections.iter().all(|s| s.level.parse::().is_ok()), + "{title}: level is a numeric string" + ); + } + } +} + +#[tokio::test] +async fn aberration_sections_keep_string_levels() { + let mock = MockServer::start().await; + Mock::given(path("/api.php")) + .respond_with(json("parse_aberration_report.json")) + .mount(&mock) + .await; + let p = client(&mock) + .page("FlyBase:Aberration Report") + .await + .unwrap(); + assert_eq!(p.sections[0].level, "2"); + assert_eq!(p.sections[0].line, "General Information"); + assert_eq!(p.sections[0].anchor, "General_Information"); +} + +#[tokio::test] +async fn rights_and_health_use_siteinfo() { + let mock = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api.php")) + .and(header("host", HOST)) + .respond_with(json("siteinfo_rightsinfo.json")) + .expect(2) + .mount(&mock) + .await; + let c = client(&mock); + let r = c.rights().await.unwrap(); + assert_eq!((r.url.as_str(), r.text.as_str()), ("", "")); + assert_eq!( + only_request_pairs(&mock).await, + pairs(&[ + ("action", "query"), + ("meta", "siteinfo"), + ("siprop", "rightsinfo"), + ("format", "json"), + ("formatversion", "2"), + ]) + ); + assert!(c.healthy().await); +} + +#[tokio::test] +async fn healthy_is_false_on_error_or_outage() { + let mock = MockServer::start().await; + Mock::given(path("/api.php")) + .respond_with(ResponseTemplate::new(503)) + .mount(&mock) + .await; + assert!(!client(&mock).healthy().await); + let refused = WikiClient::new( + format!("http://127.0.0.1:{}/", free_port().await) + .parse() + .unwrap(), + HOST.into(), + ) + .unwrap(); + assert!(!refused.healthy().await); +} + +#[tokio::test] +async fn error_codes_map_per_table() { + let mock = MockServer::start().await; + for (title, file) in [ + ("Missing", "error_missingtitle.json"), + ("Foo", "error_invalidtitle.json"), + ("Special:RecentChanges", "error_pagecannotexist.json"), + ] { + Mock::given(query_param("page", title)) + .respond_with(json(file)) + .mount(&mock) + .await; + } + for code in [ + "internal_api_error_DBQueryError", + "readonly", + "maxlag", + "ratelimited", + "somethingnew", + ] { + Mock::given(query_param("page", code)) + .respond_with(ResponseTemplate::new(200).set_body_raw( + format!(r#"{{"error":{{"code":"{code}","info":"x"}}}}"#), + "application/json", + )) + .mount(&mock) + .await; + } + let c = client(&mock); + assert_eq!(c.page("Missing").await.unwrap_err(), WikiError::NotFound); + assert_eq!( + c.page("Special:RecentChanges").await.unwrap_err(), + WikiError::NotFound + ); + assert_eq!( + c.page("Foo").await.unwrap_err(), + WikiError::Invalid("invalidtitle".into()) + ); + for code in [ + "internal_api_error_DBQueryError", + "readonly", + "maxlag", + "ratelimited", + ] { + match c.page(code).await.unwrap_err() { + WikiError::Unavailable(m) => assert!(m.contains(code), "{m}"), + e => panic!("{code}: {e:?}"), + } + } + assert_eq!( + c.page("somethingnew").await.unwrap_err(), + WikiError::Invalid("somethingnew".into()) + ); + // The tool-level classes. + assert_eq!( + c.page("Missing") + .await + .unwrap_err() + .into_tool("get_wiki_page") + .code(), + "not_found" + ); + assert_eq!( + c.page("Foo") + .await + .unwrap_err() + .into_tool("get_wiki_page") + .code(), + "invalid_input" + ); + assert_eq!( + c.page("readonly") + .await + .unwrap_err() + .into_tool("get_wiki_page") + .code(), + "backend_unavailable" + ); +} + +#[tokio::test] +async fn redirect_non_200_and_non_json_are_unavailable() { + let mock = MockServer::start().await; + Mock::given(query_param("page", "redirect")) + .respond_with( + ResponseTemplate::new(301).insert_header("location", "http://elsewhere.example/"), + ) + .mount(&mock) + .await; + Mock::given(query_param("page", "html")) + .respond_with(ResponseTemplate::new(200).set_body_raw("login", "text/html")) + .mount(&mock) + .await; + Mock::given(query_param("page", "500")) + .respond_with(ResponseTemplate::new(500)) + .mount(&mock) + .await; + Mock::given(query_param("page", "shape")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw(r#"{"parse":{"title":"secretquery"}}"#, "application/json"), + ) + .mount(&mock) + .await; + let c = client(&mock); + match c.page("redirect").await.unwrap_err() { + WikiError::Unavailable(m) => assert!(m.contains("redirect") && m.contains("301"), "{m}"), + e => panic!("{e:?}"), + } + assert_eq!( + mock.received_requests().await.unwrap().len(), + 1, + "redirect not followed" + ); + assert!(matches!( + c.page("html").await, + Err(WikiError::Unavailable(_)) + )); + assert!(matches!(c.page("500").await, Err(WikiError::Unavailable(m)) if m == "HTTP 500")); + match c.page("shape").await.unwrap_err() { + WikiError::Unavailable(m) => assert!(!m.contains("secretquery"), "{m}"), + e => panic!("{e:?}"), + } +} + +#[tokio::test] +async fn body_over_the_cap_is_too_large() { + let mock = MockServer::start().await; + Mock::given(path("/api.php")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw(vec![b' '; MAX_WIKI_BYTES + 1], "application/json"), + ) + .mount(&mock) + .await; + let e = client(&mock) + .page("FlyBase:Fly Lab List") + .await + .unwrap_err(); + assert_eq!(e, WikiError::TooLarge); + assert_eq!(e.into_tool("get_wiki_page").code(), "limit_exceeded"); +} + +#[tokio::test] +async fn body_at_the_cap_is_read() { + let mock = MockServer::start().await; + let body = fixture("siteinfo_rightsinfo.json"); + let mut padded = body.into_bytes(); + padded.resize(MAX_WIKI_BYTES, b' '); + Mock::given(path("/api.php")) + .respond_with(ResponseTemplate::new(200).set_body_raw(padded, "application/json")) + .mount(&mock) + .await; + assert!(client(&mock).rights().await.is_ok()); +} + +/// A server that answers with a chunked body (no Content-Length) longer than the cap, so the running total +/// is what stops the read. +#[tokio::test] +async fn chunked_body_over_the_cap_is_too_large() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut req = [0u8; 4096]; + let _ = sock.read(&mut req).await; + let head = "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ntransfer-encoding: chunked\r\n\r\n"; + if sock.write_all(head.as_bytes()).await.is_err() { + return; + } + let chunk = vec![b' '; 64 * 1024]; + let framed = [ + format!("{:x}\r\n", chunk.len()).into_bytes(), + chunk, + b"\r\n".to_vec(), + ] + .concat(); + for _ in 0..(MAX_WIKI_BYTES / (64 * 1024) + 2) { + if sock.write_all(&framed).await.is_err() { + return; + } + } + let _ = sock.write_all(b"0\r\n\r\n").await; + }); + let c = WikiClient::new( + format!("http://127.0.0.1:{port}/").parse().unwrap(), + HOST.into(), + ) + .unwrap(); + assert_eq!(c.page("big").await.unwrap_err(), WikiError::TooLarge); +} + +async fn free_port() -> u16 { + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + l.local_addr().unwrap().port() +} + +#[tokio::test] +async fn transport_errors_never_carry_the_query() { + let c = WikiClient::new( + format!("http://127.0.0.1:{}/", free_port().await) + .parse() + .unwrap(), + HOST.into(), + ) + .unwrap(); + for e in [ + c.search("secretquery", 10, 0).await.unwrap_err(), + c.page("secretquery").await.unwrap_err(), + ] { + let WikiError::Unavailable(m) = &e else { + panic!("{e:?}") + }; + assert!(!m.contains("secretquery"), "{m}"); + assert!(!format!("{e:?}").contains("secretquery")); + let t = e.into_tool("search_wiki"); + assert_eq!(t.code(), "backend_unavailable"); + assert!(!t.to_string().contains("secretquery")); + } +} + +#[test] +fn https_base_is_rejected_at_new() { + let e = WikiClient::new("https://wiki.flybase.org/".parse().unwrap(), HOST.into()) + .err() + .expect("https must be rejected"); + assert!(e.to_string().contains("http://"), "{e}"); +} From 6a7b093e574566a8263415fe52d508d04e152e91 Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Tue, 13 Oct 2026 01:48:42 +0000 Subject: [PATCH 2/5] fix(mcp): bound html_to_text indentation, tag lookahead, and link nesting --- mcp/src/backends/wiki.rs | 136 ++++++++++++++++++++++++++++++++++----- 1 file changed, 120 insertions(+), 16 deletions(-) diff --git a/mcp/src/backends/wiki.rs b/mcp/src/backends/wiki.rs index c348f3c..be2de41 100644 --- a/mcp/src/backends/wiki.rs +++ b/mcp/src/backends/wiki.rs @@ -411,7 +411,12 @@ fn decode_one(name: &str) -> Option { } None => return None, }; - if n == 160 { ' ' } else { char::from_u32(n)? } + match n { + 160 => ' ', + // Control characters other than tab, newline and carriage return stay as written. + 0..=8 | 11 | 12 | 14..=31 | 127..=159 => return None, + _ => char::from_u32(n)?, + } } }; Some(c) @@ -567,6 +572,14 @@ const BLOCK: [&str; 16] = [ "section", "hr", ]; +/// Deepest list or `
    ` nesting that adds indentation; deeper levels indent no further, so output +/// stays linear in the input however deep the nesting. +const MAX_INDENT_DEPTH: usize = 8; +/// Longest tag the scanner reads; a `<` with no `>` within this many bytes is text. +const MAX_TAG_BYTES: usize = 4096; +/// Nested external links tracked for `text (href)`; deeper ones are plain text. +const MAX_LINK_DEPTH: usize = 16; + const VOID: [&str; 12] = [ "br", "hr", "img", "meta", "link", "input", "wbr", "col", "source", "area", "base", "track", ]; @@ -643,6 +656,8 @@ struct Out { list: usize, dd: usize, links: Vec<(Option, usize)>, + /// `` elements opened beyond MAX_LINK_DEPTH. + untracked_links: usize, } impl Out { @@ -730,7 +745,8 @@ impl Out { } /// Parse a tag starting at `s[0] == '<'`. Returns the tag and its byte length, or None when `<` does not -/// begin a tag. A `>` inside a quoted attribute value does not end the tag. +/// begin a tag. A `>` inside a quoted attribute value does not end the tag. The scan is bounded: an +/// unquoted `<` or MAX_TAG_BYTES without a `>` means this `<` is text, so unclosed tags cost linear time. fn parse_tag(s: &str) -> Option<(Tag<'_>, usize)> { let b = s.as_bytes(); let mut i = 1; @@ -748,16 +764,18 @@ fn parse_tag(s: &str) -> Option<(Tag<'_>, usize)> { let name = s[ns..i].to_ascii_lowercase(); let as_ = i; let mut quote: Option = None; - while i < b.len() { + let limit = b.len().min(MAX_TAG_BYTES); + while i < limit { match (quote, b[i]) { (None, b'"' | b'\'') => quote = Some(b[i]), (Some(q), c) if c == q => quote = None, (None, b'>') => break, + (None, b'<') => return None, _ => {} } i += 1; } - if i >= b.len() { + if i >= limit { return None; } let attrs = &s[as_..i]; @@ -774,13 +792,20 @@ fn parse_tag(s: &str) -> Option<(Tag<'_>, usize)> { } /// Find the end of a raw-text element (`script`, `style`): the byte just after ``. +/// The search is case-insensitive and allocates nothing. fn raw_text_end(s: &str, name: &str) -> usize { - let lower = s.to_ascii_lowercase(); - let needle = format!(" s[i..].find('>').map_or(s.len(), |j| i + j + 1), - None => s.len(), + let b = s.as_bytes(); + let n = name.as_bytes(); + let mut from = 0; + while let Some(off) = s[from..].find("= n.len() && tail[..n.len()].eq_ignore_ascii_case(n) { + return s[i..].find('>').map_or(s.len(), |j| i + j + 1); + } + from = i + 2; } + s.len() } fn heading_level(name: &str) -> Option { @@ -797,7 +822,9 @@ fn heading_level(name: &str) -> Option { /// Entities are decoded once. pub fn html_to_text(html: &str) -> String { let mut out = Out::default(); - // (tag name, depth) of an element being skipped with everything inside it. + // (tag name, depth) of an element being skipped with everything inside it. Depth counts only that tag + // name, so an unclosed same-name element inside drops the rest of the page; acceptable because the API + // flags already remove the TOC and edit links, and references always close. let mut skip: Option<(String, usize)> = None; let mut i = 0; while i < html.len() { @@ -932,7 +959,7 @@ fn handle_tag(out: &mut Out, tag: &Tag<'_>) { ("li", false) => { out.newline(); if out.cell == 0 { - let indent = " ".repeat(out.list.saturating_sub(1)); + let indent = " ".repeat(out.list.clamp(1, MAX_INDENT_DEPTH) - 1); out.buf.push_str(&indent); out.buf.push_str("- "); } @@ -941,7 +968,7 @@ fn handle_tag(out: &mut Out, tag: &Tag<'_>) { out.newline(); out.dd += 1; if out.cell == 0 { - out.buf.push_str(&" ".repeat(out.dd)); + out.buf.push_str(&" ".repeat(out.dd.min(MAX_INDENT_DEPTH))); } } ("dd", true) => { @@ -949,6 +976,10 @@ fn handle_tag(out: &mut Out, tag: &Tag<'_>) { out.newline(); } ("a", false) => { + if out.links.len() >= MAX_LINK_DEPTH { + out.untracked_links += 1; + return; + } let href = if has_class(tag.attrs, "external") { attr(tag.attrs, "href").map(|h| decode_entities(&h)) } else { @@ -957,12 +988,24 @@ fn handle_tag(out: &mut Out, tag: &Tag<'_>) { out.links.push((href, out.buf.len())); } ("a", true) => { + if out.untracked_links > 0 { + out.untracked_links -= 1; + return; + } if let Some((Some(href), start)) = out.links.pop() { - let text = out.buf.get(start..).unwrap_or("").trim().to_string(); - if !text.is_empty() && text != href { - out.push_raw(&format!(" ({href})")); - } else if text.is_empty() { + let text = out.buf.get(start..).unwrap_or("").trim(); + let (empty, same) = (text.is_empty(), text == href); + if empty { out.push_raw(&href); + } else if !same { + // Whitespace at the end of the link text stays pending for whatever follows. + let pending = out.space; + out.space = false; + out.trim_end_spaces(); + out.buf.push_str(" ("); + out.buf.push_str(&href); + out.buf.push(')'); + out.space = pending; } } } @@ -1131,6 +1174,67 @@ mod tests { assert_eq!(html_to_text("1 < 2 and 3 > 2"), "1 < 2 and 3 > 2"); } + #[test] + fn link_text_ending_in_space_and_control_references() { + assert_eq!( + html_to_text("x y"), + "x (h) y" + ); + assert_eq!( + decode_entities("a�bcŸ d"), + "a�bcŸ\td" + ); + } + + /// Output size and time stay linear on hostile nesting and unclosed markup (each case is about 1 MB, + /// or 50,000 levels deep). Uncapped list indentation gave 400 MB for 160 KB of input. The time bound is generous for debug builds. + #[test] + fn hostile_html_is_linear() { + let cases: Vec<(&str, String)> = vec![ + ("nested ul", "