diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e0d54435..c414abba42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ ## Enhancements + * [phoenix.js] Long polling retries interrupted requests without rejoining + channels while its server session remains alive. Messages are retained until + acknowledged and retried pushes are deduplicated. New `socket.onSuspend` and + `socket.onResume` callbacks report interruptions and recovery without a + channel error. Older servers retain the existing reconnect behavior. During + a rolling deploy, a session that reaches a node with an older server falls + back to that behavior for the rest of the session. * The longpoll session token is now sent in a `x-phoenix-longpoll-token` header instead of the query string. If you do rolling deploys where both old and new nodes are active at the same time, ensure that you deploy diff --git a/assets/js/phoenix/connection.js b/assets/js/phoenix/connection.js index 3364df6761..6c1ea5d3fc 100644 --- a/assets/js/phoenix/connection.js +++ b/assets/js/phoenix/connection.js @@ -32,9 +32,12 @@ export default class Connection { this.channelsErrored = false // set once the close event was handled, as LongPoll can emit it more than once this.closeHandled = false + this.suspended = false transport.onopen = () => { if(this.isCurrent()){ socket.onConnOpen() } } transport.onerror = error => { if(this.isCurrent()){ socket.onConnError(error) } } transport.onmessage = event => { if(this.isCurrent()){ socket.onConnMessage(event) } } + transport.onsuspend = () => { if(this.isCurrent()){ socket.onConnSuspend(this) } } + transport.onresume = () => { if(this.isCurrent()){ socket.onConnResume(this) } } transport.onclose = event => { if(this.isCurrent() && !this.closeHandled){ this.closeHandled = true diff --git a/assets/js/phoenix/index.js b/assets/js/phoenix/index.js index 1c93968260..33e47172bc 100644 --- a/assets/js/phoenix/index.js +++ b/assets/js/phoenix/index.js @@ -87,6 +87,22 @@ * socket.onClose( () => console.log("the connection dropped") ) * ``` * + * LongPoll automatically retries interrupted requests while the server keeps + * its session alive. During recovery, channels stay joined and outgoing messages + * are buffered. Use `onSuspend` and `onResume` to show connection status without + * resetting channel state: + * + * ``` + * socket.onSuspend( () => console.log("recovering the connection") ) + * socket.onResume( () => console.log("the connection recovered") ) + * ``` + * + * Recovery requires a server that supports acknowledged long-poll batches. + * With an older server, and for the rest of a session that reached a node + * with an older server during a rolling upgrade, failed requests close the + * socket and channels rejoin. If the session expires, the usual error/close events + * fire and channels rejoin. Push timeouts continue to apply while suspended. + * * * ## Channel Hooks * diff --git a/assets/js/phoenix/longpoll.js b/assets/js/phoenix/longpoll.js index 392fe43e7f..f63fb163a7 100644 --- a/assets/js/phoenix/longpoll.js +++ b/assets/js/phoenix/longpoll.js @@ -7,6 +7,9 @@ import { import Ajax from "./ajax" +// how long we wait before retrying a failed request of a session we can resume +const RETRY_DELAYS = [100, 500, 1000, 2000, 5000] + let arrayBufferToBase64 = (buffer) => { let binary = "" let bytes = new Uint8Array(buffer) @@ -33,10 +36,23 @@ export default class LongPoll { this.currentBatch = null this.currentBatchTimer = null this.batchBuffer = [] + // set once the server tells us that it keeps the messages until we acknowledge them, + // so we can retry a failed request instead of starting a new session + this.resumable = false + this.degraded = false + // the last batch we received, and the seq of the last message we sent + this.ack = 0 + this.seq = 0 + this.retries = {GET: 0, POST: 0} + this.retryTimers = new Set() + // set once a retry failed as well, until GET and POST have both recovered + this.interrupted = false this.onopen = function (){ } // noop this.onerror = function (){ } // noop this.onmessage = function (){ } // noop this.onclose = function (){ } // noop + this.onsuspend = function (){ } // noop + this.onresume = function (){ } // noop this.pollEndpoint = this.normalizeEndpoint(endPoint) this.readyState = SOCKET_STATES.connecting // we must wait for the caller to finish setting up our callbacks and timeout properties, @@ -51,8 +67,8 @@ export default class LongPoll { .replace(new RegExp("(.*)\/" + TRANSPORTS.websocket), "$1/" + TRANSPORTS.longpoll)) } - endpointURL(){ - return this.pollEndpoint + endpointURL(params = {}){ + return Ajax.appendParams(this.pollEndpoint, params) } closeAndRetry(error, code, reason, wasClean){ @@ -79,9 +95,10 @@ export default class LongPoll { if(this.authToken){ headers["X-Phoenix-AuthToken"] = this.authToken } - this.ajax("GET", headers, null, () => this.ontimeout(), resp => { + const params = this.resumable ? {ack: this.ack} : {} + this.ajax("GET", headers, null, params, () => this.retryOr("GET", () => this.poll(), () => this.ontimeout()), resp => { if(resp){ - var {status, token, messages} = resp + var {status, token, messages, batch} = resp if(status === 410 && this.token !== null){ // In case we already have a token, this means that our existing session // is gone. We fail so that the client rejoins its channels. @@ -89,12 +106,25 @@ export default class LongPoll { return } this.token = token + if(batch !== undefined){ + this.resumable = !this.degraded + } else if(this.resumable && (status === 200 || status === 204)){ + // TODO: Remove in Phoenix 1.10, when no cluster mixes 1.8 and newer nodes. + // + // The poll went through a node with an older server, as during a rolling + // deploy, which forwards it without our ack. The session then dropped the + // messages we did not acknowledge yet, like it does for older clients. + // We got them unless this poll is a retry, as they were in the response we lost. + if(!this.degrade()){ return } + } } else { status = 0 } - switch(status){ case 200: + // A late poll may send a batch we already received. Batches are immutable. + if(batch !== undefined && batch <= this.ack){ messages = [] } + if(batch !== undefined){ this.ack = Math.max(this.ack, batch) } messages.forEach(msg => { // Tasks are what things like event handlers, setTimeout callbacks, // promise resolves and more are run within. @@ -117,15 +147,17 @@ export default class LongPoll { // // We might stop before that, e.g. when a POST times out or a message // handler disconnects, which cancels the remaining messages. - const timer = setTimeout(() => { - this.messageTimers.delete(timer) - this.onmessage({data: msg}) - }, 0) - this.messageTimers.add(timer) + this.scheduleMessage(() => this.onmessage({data: msg})) + }) + // Restore only after delivering the batch, so hooks see its updates. + // Closing from a message callback cancels this task as well. + this.scheduleMessage(() => { + this.succeeded("GET") + this.poll() }) - this.poll() break case 204: + this.succeeded("GET") this.poll() break case 410: @@ -138,6 +170,9 @@ export default class LongPoll { this.close(1008, "forbidden", false) break case 0: + // the connection failed, but the server still has our session and messages + this.retryOr("GET", () => this.poll(), () => this.closeAndRetry(500, 1011, "internal server error", false)) + break case 500: this.closeAndRetry(500, 1011, "internal server error", false) break @@ -146,44 +181,129 @@ export default class LongPoll { }) } + scheduleMessage(callback){ + const timer = setTimeout(() => { + this.messageTimers.delete(timer) + callback() + }, 0) + this.messageTimers.add(timer) + } + // we collect all pushes within the current event loop by // setTimeout 0, which optimizes back-to-back procedural // pushes against an empty buffer send(body){ if(typeof(body) !== "string"){ body = arrayBufferToBase64(body) } + // messages are numbered, so the server drops the ones it received when we send + // them again. A batch is sent with the seq of its first message. + this.seq++ if(this.currentBatch){ this.currentBatch.push(body) } else if(this.awaitingBatchAck){ + if(this.batchBuffer.length === 0){ this.batchBufferSeq = this.seq } this.batchBuffer.push(body) } else { this.currentBatch = [body] + const seq = this.seq this.currentBatchTimer = setTimeout(() => { - this.batchSend(this.currentBatch) + this.batchSend(this.currentBatch, seq) this.currentBatch = null }, 0) } } - batchSend(messages, offset = 0){ + batchSend(messages, seq, offset = 0){ + if(!this.isActive()){ return } this.awaitingBatchAck = true const next = offset + MAX_LONGPOLL_BATCH_SIZE const batch = messages.slice(offset, next) - this.ajax("POST", {"Content-Type": "application/x-ndjson"}, batch.join("\n"), () => this.ontimeout(), resp => { - if(!resp || resp.status !== 200){ + const retry = () => this.batchSend(messages, seq, offset) + // servers that don't announce batches don't know about seq either + const params = this.resumable ? {seq: seq + offset} : {} + this.ajax("POST", {"Content-Type": "application/x-ndjson"}, batch.join("\n"), params, () => this.retryOr("POST", retry, () => this.ontimeout()), resp => { + if(!resp){ + this.retryOr("POST", retry, () => this.closeAndRetry(null, 1011, "internal server error", false)) + } else if(resp.status !== 200){ // this calls stop which already clears awaitingBatchAck this.closeAndRetry(resp && resp.status, 1011, "internal server error", false) - } else if(next < messages.length){ - this.batchSend(messages, next) - } else if(this.batchBuffer.length > 0){ - this.batchSend(this.batchBuffer) - this.batchBuffer = [] } else { - this.awaitingBatchAck = false + // a node with an older server forwards the request without our seq, so the + // session does not drop the messages we send again + // + // TODO: Remove in Phoenix 1.10, together with degrade() + if(this.resumable && resp.seq === undefined && !this.degrade()){ return } + this.succeeded("POST") + if(next < messages.length){ + this.batchSend(messages, seq, next) + } else if(this.batchBuffer.length > 0){ + this.batchSend(this.batchBuffer, this.batchBufferSeq) + this.batchBuffer = [] + } else { + this.awaitingBatchAck = false + } } }) } + /** + * @private + * + * Stops resuming for the rest of the session. A node with an older server, as + * during a rolling deploy, does not forward our ack and seq. Retrying through it + * could lose or duplicate messages, so we fail like we do with an older server. + * + * TODO: Remove in Phoenix 1.10, when no cluster mixes 1.8 and newer nodes. + */ + degrade(){ + this.degraded = true + this.resumable = false + // A request already being retried may have been processed. Neither replaying + // it without deduplication nor accepting a gap in received messages is safe. + if(this.retries.GET > 0 || this.retries.POST > 0){ + this.closeAndRetry(0, 1011, "session no longer resumable", false) + return false + } + return true + } + + /** + * @private + * + * Retries a failed request with backoff when the server keeps our session, + * otherwise fails. + */ + retryOr(method, retry, fail){ + if(!this.resumable){ return fail() } + const delay = RETRY_DELAYS[Math.min(this.retries[method], RETRY_DELAYS.length - 1)] + this.retries[method]++ + // a single failed request is retried quickly, so we only report the interruption + // once its retry failed as well + if(this.retries[method] > 1 && !this.interrupted){ + this.interrupted = true + this.onsuspend() + } + if(!this.isActive()){ return } + const timer = setTimeout(() => { + this.retryTimers.delete(timer) + if(!this.resumable){ return fail() } + retry() + }, delay) + this.retryTimers.add(timer) + } + + /** + * @private + */ + succeeded(method){ + this.retries[method] = 0 + // A successful poll must not hide an outstanding failed POST, or vice versa. + if(this.interrupted && this.retries.GET === 0 && this.retries.POST === 0){ + this.interrupted = false + this.onresume() + } + } + close(code, reason, wasClean){ this.stop(SOCKET_STATES.closed) this.emitClose(code, reason, wasClean) @@ -201,6 +321,9 @@ export default class LongPoll { for(let {req} of reqs){ req && req.abort() } for(let timer of this.messageTimers){ clearTimeout(timer) } this.messageTimers.clear() + for(let timer of this.retryTimers){ clearTimeout(timer) } + this.retryTimers.clear() + this.interrupted = false this.batchBuffer = [] this.awaitingBatchAck = false clearTimeout(this.currentBatchTimer) @@ -221,7 +344,7 @@ export default class LongPoll { } } - ajax(method, headers, body, onCallerTimeout, callback){ + ajax(method, headers, body, params, onCallerTimeout, callback){ // the request is ours until it completes or we stop, and the callbacks of a request // that is not ours anymore are ignored. It is ours before it starts, as a request can // complete while it starts. @@ -233,7 +356,7 @@ export default class LongPoll { if(this.token !== null){ headers = Object.assign({}, headers, {"X-Phoenix-Longpoll-Token": this.token}) } - request.req = Ajax.request(method, this.endpointURL(), headers, body, this.timeout, ontimeout, resp => { + request.req = Ajax.request(method, this.endpointURL(params), headers, body, this.timeout, ontimeout, resp => { if(this.reqs.delete(request)){ callback(resp) } }) } diff --git a/assets/js/phoenix/socket.js b/assets/js/phoenix/socket.js index ebd5e85d0b..c8826621ba 100644 --- a/assets/js/phoenix/socket.js +++ b/assets/js/phoenix/socket.js @@ -113,7 +113,7 @@ import Connection from "./connection" */ export default class Socket { constructor(endPoint, opts = {}){ - this.stateChangeCallbacks = {open: [], close: [], error: [], message: []} + this.stateChangeCallbacks = {open: [], close: [], error: [], message: [], suspend: [], resume: []} this.channels = [] this.sendBuffer = [] this.ref = 0 @@ -385,6 +385,28 @@ export default class Socket { return ref } + /** + * Registers a callback when the transport is interrupted but keeps its session + * and retries. Channels remain joined and pushes are buffered by the transport. + * Currently supported by LongPoll. A lost session instead emits error/close. + * @param {Function} callback + */ + onSuspend(callback){ + let ref = this.makeRef() + this.stateChangeCallbacks.suspend.push([ref, callback]) + return ref + } + + /** + * Registers a callback when a suspended transport recovers the same session. + * @param {Function} callback + */ + onResume(callback){ + let ref = this.makeRef() + this.stateChangeCallbacks.resume.push([ref, callback]) + return ref + } + /** * Registers callbacks for connection message events * @param {Function} callback @@ -691,6 +713,28 @@ export default class Socket { this.triggerChanError("connection_error", connection) } + /** @private */ + onConnSuspend(connection){ + if(!this.stillUses(connection) || !connection.isOpen() || connection.suspended){ return } + connection.suspended = true + if(this.hasLogger()) this.log("transport", "suspended") + for(let [, callback] of this.stateChangeCallbacks.suspend.slice()){ + if(!this.stillUses(connection) || !connection.isOpen()){ break } + callback() + } + } + + /** @private */ + onConnResume(connection){ + if(!this.stillUses(connection) || !connection.isOpen() || !connection.suspended){ return } + connection.suspended = false + if(this.hasLogger()) this.log("transport", "resumed") + for(let [, callback] of this.stateChangeCallbacks.resume.slice()){ + if(!this.stillUses(connection) || !connection.isOpen()){ break } + callback() + } + } + /** * @private * diff --git a/assets/test/longpoll_test.js b/assets/test/longpoll_test.js index 98f7d009d6..b151e91237 100644 --- a/assets/test/longpoll_test.js +++ b/assets/test/longpoll_test.js @@ -656,6 +656,335 @@ describe("LongPoll", () => { }) }) +describe("LongPoll resuming", () => { + let longpoll, events + + const requests = method => Ajax.request.mock.calls.filter(([m]) => m === method) + const last = method => requests(method).at(-1) + const params = call => Object.fromEntries(new URL(call[1]).searchParams) + const respond = (call, resp) => call[6](resp) + + // the server keeps the messages until we acknowledge them when it announces a batch + const open = (resp = {status: 410, token: "token", messages: [], batch: 0}) => { + longpoll = new LongPoll("http://localhost/socket/longpoll") + events = [] + longpoll.onopen = () => events.push("open") + longpoll.onmessage = ({data}) => events.push(data) + longpoll.onerror = () => events.push("error") + longpoll.onclose = () => events.push("close") + longpoll.onsuspend = () => events.push("suspend") + longpoll.onresume = () => events.push("resume") + jest.advanceTimersByTime(0) + respond(last("GET"), resp) + } + + beforeEach(() => { + jest.useFakeTimers() + jest.spyOn(Ajax, "request").mockImplementation(() => ({abort: jest.fn()})) + }) + + afterEach(() => { + longpoll.close() + jest.useRealTimers() + jest.restoreAllMocks() + }) + + it("acknowledges the last batch it received with the next poll", () => { + open() + expect(params(last("GET")).ack).toBe("0") + + respond(last("GET"), {status: 200, token: "token", messages: ["a"], batch: 1}) + jest.advanceTimersByTime(0) + expect(params(last("GET")).ack).toBe("1") + + // a poll without new messages keeps the ack + respond(last("GET"), {status: 204, token: "token", batch: 1}) + expect(params(last("GET")).ack).toBe("1") + }) + + it("does not acknowledge when the server does not announce batches", () => { + open({status: 410, token: "token", messages: []}) + expect(params(last("GET")).ack).toBeUndefined() + }) + + it.each([ + ["gets no response", call => respond(call, null)], + ["times out", call => call[5]()] + ])("polls again with the same ack when a poll %s", (_, fail) => { + open() + respond(last("GET"), {status: 200, token: "token", messages: ["a"], batch: 1}) + jest.advanceTimersByTime(0) + const polls = requests("GET").length + + fail(last("GET")) + jest.advanceTimersByTime(99) + expect(requests("GET")).toHaveLength(polls) + jest.advanceTimersByTime(1) + + expect(requests("GET")).toHaveLength(polls + 1) + expect(params(last("GET")).ack).toBe("1") + expect(events).not.toContain("error") + expect(events).not.toContain("close") + }) + + it("backs off while requests keep failing", () => { + open() + const delays = [] + for(let i = 0; i < 6; i++){ + const polls = requests("GET").length + respond(last("GET"), null) + let waited = 0 + while(requests("GET").length === polls){ + jest.advanceTimersByTime(100) + waited += 100 + } + delays.push(waited) + } + + expect(delays).toEqual([100, 500, 1000, 2000, 5000, 5000]) + }) + + it("reports suspension once a retry failed as well, and resumption once requests succeed", () => { + open() + respond(last("GET"), null) + jest.advanceTimersByTime(100) + expect(events).toEqual(["open"]) + + respond(last("GET"), null) + jest.advanceTimersByTime(500) + respond(last("GET"), null) + jest.advanceTimersByTime(1000) + expect(events).toEqual(["open", "suspend"]) + + respond(last("GET"), {status: 204, token: "token", batch: 0}) + expect(events).toEqual(["open", "suspend", "resume"]) + + // a single failure after that is not an interruption yet + respond(last("GET"), null) + jest.advanceTimersByTime(100) + expect(events).toEqual(["open", "suspend", "resume"]) + }) + + it("waits for both GET and POST to recover", () => { + open() + longpoll.send("a") + jest.advanceTimersByTime(0) + respond(last("GET"), null) + respond(last("POST"), null) + jest.advanceTimersByTime(100) + respond(last("GET"), null) + respond(last("POST"), null) + jest.advanceTimersByTime(500) + expect(events).toEqual(["open", "suspend"]) + + respond(last("GET"), {status: 204, token: "token", batch: 0}) + expect(events).toEqual(["open", "suspend"]) + respond(last("POST"), {status: 200, seq: 1}) + expect(events).toEqual(["open", "suspend", "resume"]) + }) + + it("delivers the recovered batch before reporting resumption", () => { + open() + respond(last("GET"), null) + jest.advanceTimersByTime(100) + respond(last("GET"), null) + jest.advanceTimersByTime(500) + respond(last("GET"), {status: 200, token: "token", messages: ["a", "b"], batch: 1}) + expect(events).toEqual(["open", "suspend"]) + jest.advanceTimersByTime(0) + expect(events).toEqual(["open", "suspend", "a", "b", "resume"]) + expect(params(last("GET")).ack).toBe("1") + }) + + it("does not deliver an already received batch again", () => { + open() + respond(last("GET"), {status: 200, token: "token", messages: ["a"], batch: 1}) + jest.advanceTimersByTime(0) + respond(last("GET"), {status: 200, token: "token", messages: ["a"], batch: 1}) + jest.advanceTimersByTime(0) + expect(events).toEqual(["open", "a"]) + }) + + it("does not retry when a suspension callback closes the transport", () => { + open() + longpoll.onsuspend = () => longpoll.close() + respond(last("GET"), null) + jest.advanceTimersByTime(100) + respond(last("GET"), null) + const polls = requests("GET").length + jest.advanceTimersByTime(10_000) + expect(requests("GET")).toHaveLength(polls) + expect(longpoll.retryTimers.size).toBe(0) + }) + + it.each([403, 500])("does not report resumption before failing with status %s", status => { + open() + respond(last("GET"), null) + jest.advanceTimersByTime(100) + respond(last("GET"), null) + jest.advanceTimersByTime(500) + respond(last("GET"), {status, token: "token"}) + expect(events).toEqual(["open", "suspend", "error", "close"]) + }) + + it("closes when the session is gone", () => { + open() + respond(last("GET"), {status: 410, token: "new-token", messages: [], batch: 0}) + + expect(events).toEqual(["open", "error", "close"]) + }) + + it("does not retry after it was closed", () => { + open() + respond(last("GET"), null) + longpoll.close() + const polls = requests("GET").length + jest.advanceTimersByTime(10_000) + + expect(requests("GET")).toHaveLength(polls) + }) + + it("numbers the messages it sends and sends a batch again when it gets no response", () => { + open() + longpoll.send("a") + longpoll.send("b") + jest.advanceTimersByTime(0) + expect(params(last("POST")).seq).toBe("1") + expect(last("POST")[3]).toBe("a\nb") + + longpoll.send("c") + respond(last("POST"), null) + jest.advanceTimersByTime(100) + expect(requests("POST")).toHaveLength(2) + expect(params(last("POST")).seq).toBe("1") + expect(last("POST")[3]).toBe("a\nb") + + respond(last("POST"), {status: 200, seq: 1}) + expect(params(last("POST")).seq).toBe("3") + expect(last("POST")[3]).toBe("c") + }) + + it("does not number the messages when the server does not announce batches", () => { + open({status: 410, token: "token", messages: []}) + longpoll.send("a") + jest.advanceTimersByTime(0) + + expect(params(last("POST")).seq).toBeUndefined() + }) + + it("sends messages pushed by a resume callback after a POST recovers", () => { + open() + longpoll.onresume = () => longpoll.send("b") + longpoll.send("a") + jest.advanceTimersByTime(0) + respond(last("POST"), null) + jest.advanceTimersByTime(100) + respond(last("POST"), null) + jest.advanceTimersByTime(500) + respond(last("POST"), {status: 200, seq: 1}) + jest.advanceTimersByTime(0) + expect(last("POST")[3]).toBe("b") + expect(params(last("POST")).seq).toBe("2") + }) + + // During a rolling deploy, a node with an older server forwards our requests + // without ack and seq, so the session treats us like an older client. + describe("with a node of an older server", () => { + it("delivers the messages of a poll and stops resuming", () => { + open() + respond(last("GET"), {status: 200, token: "token", messages: ["a"]}) + jest.advanceTimersByTime(0) + expect(events).toEqual(["open", "a"]) + expect(params(last("GET")).ack).toBeUndefined() + + // the next failure reconnects, as the session no longer keeps messages for us + respond(last("GET"), null) + expect(events).toEqual(["open", "a", "error", "close"]) + }) + + it.each([ + ["with messages", {status: 200, token: "token", messages: ["a"]}], + ["without messages", {status: 204, token: "token"}] + ])("reconnects when a retried poll %s was answered by it", (_, resp) => { + open() + respond(last("GET"), null) + jest.advanceTimersByTime(100) + // the session dropped the batch of the response we lost + respond(last("GET"), resp) + jest.advanceTimersByTime(0) + expect(events).toEqual(["open", "error", "close"]) + }) + + it("stops resuming after it answered a POST", () => { + open() + longpoll.send("a") + jest.advanceTimersByTime(0) + respond(last("POST"), {status: 200}) + longpoll.send("b") + jest.advanceTimersByTime(0) + expect(params(last("POST")).seq).toBeUndefined() + + respond(last("POST"), null) + expect(events).toEqual(["open", "error", "close"]) + }) + + it("keeps the downgrade when an outstanding modern poll completes", () => { + open() + longpoll.send("a") + jest.advanceTimersByTime(0) + respond(last("POST"), {status: 200}) + respond(last("GET"), {status: 204, token: "token", batch: 0}) + expect(params(last("GET")).ack).toBeUndefined() + + longpoll.send("b") + jest.advanceTimersByTime(0) + expect(params(last("POST")).seq).toBeUndefined() + respond(last("GET"), null) + expect(events).toEqual(["open", "error", "close"]) + }) + + it("fails a pending POST retry when a legacy poll downgrades the session", () => { + open() + longpoll.send("a") + jest.advanceTimersByTime(0) + respond(last("POST"), null) + respond(last("GET"), {status: 204, token: "token"}) + jest.advanceTimersByTime(1000) + + expect(requests("POST")).toHaveLength(1) + expect(longpoll.retryTimers.size).toBe(0) + expect(events).toEqual(["open", "error", "close"]) + }) + + it("fails a pending GET retry when a legacy POST downgrades the session", () => { + open() + longpoll.send("a") + jest.advanceTimersByTime(0) + respond(last("GET"), null) + const polls = requests("GET").length + respond(last("POST"), {status: 200}) + jest.advanceTimersByTime(1000) + + expect(requests("GET")).toHaveLength(polls) + expect(longpoll.retryTimers.size).toBe(0) + expect(events).toEqual(["open", "error", "close"]) + }) + + it("keeps resuming while POSTs are answered with seq", () => { + open() + longpoll.send("a") + jest.advanceTimersByTime(0) + respond(last("POST"), {status: 200, seq: 1}) + longpoll.send("b") + jest.advanceTimersByTime(0) + expect(params(last("POST")).seq).toBe("2") + + respond(last("POST"), null) + expect(events).toEqual(["open"]) + }) + }) +}) + describe("Socket with LongPoll", () => { describe("transportConnect", () => { it("should initialize with undefined protocols when no auth token", () => { diff --git a/assets/test/socket_test.js b/assets/test/socket_test.js index d2c9a58f92..c5716a23d7 100644 --- a/assets/test/socket_test.js +++ b/assets/test/socket_test.js @@ -60,7 +60,7 @@ describe("with transports", function (){ expect(socket.sendBuffer.length).toBe(0) expect(socket.ref).toBe(0) expect(socket.endPoint).toBe("/socket/websocket") - expect(socket.stateChangeCallbacks).toEqual({open: [], close: [], error: [], message: []}) + expect(socket.stateChangeCallbacks).toEqual({open: [], close: [], error: [], message: [], suspend: [], resume: []}) expect(socket.transport).toBe(WebSocket) expect(socket.timeout).toBe(10000) expect(socket.longpollerTimeout).toBe(20000) diff --git a/assets/test/suspend_test.js b/assets/test/suspend_test.js new file mode 100644 index 0000000000..cc42337aca --- /dev/null +++ b/assets/test/suspend_test.js @@ -0,0 +1,84 @@ +import {jest} from "@jest/globals" +import {Socket} from "../js/phoenix" +import {SOCKET_STATES} from "../js/phoenix/constants" + +class Transport { + constructor(){ this.readyState = SOCKET_STATES.connecting } + send(){} + close(){ this.readyState = SOCKET_STATES.closed } + open(){ this.readyState = SOCKET_STATES.open; this.onopen() } +} + +describe("transport suspension", () => { + let socket, channel, transport, events + + beforeEach(() => { + jest.useFakeTimers() + events = [] + socket = new Socket("/socket", {transport: Transport}) + socket.onSuspend(() => events.push("suspend")) + socket.onResume(() => events.push("resume")) + socket.connect() + transport = socket.conn + transport.open() + channel = socket.channel("room") + channel.onError(() => events.push("error")) + channel.join() + transport.onmessage({data: JSON.stringify([ + channel.joinRef(), channel.joinRef(), "room", "phx_reply", {status: "ok", response: {}} + ])}) + }) + + afterEach(() => { + socket.disconnect() + jest.useRealTimers() + }) + + it("preserves the join and delivers one notification per transition", () => { + const joinRef = channel.joinRef() + transport.onsuspend() + transport.onsuspend() + expect(events).toEqual(["suspend"]) + expect(channel.canPush()).toBe(true) + transport.onresume() + transport.onresume() + expect(events).toEqual(["suspend", "resume"]) + expect(channel.joinRef()).toBe(joinRef) + }) + + it("errors the channel if the suspended transport loses its session", () => { + transport.onsuspend() + transport.close() + transport.onclose({code: 3410}) + transport.onresume() + expect(events).toEqual(["suspend", "error"]) + expect(channel.isErrored()).toBe(true) + }) + + it("ignores events from a replaced transport", () => { + transport.onsuspend() + socket.disconnect() + socket.connect() + socket.conn.open() + events = [] + transport.onsuspend() + transport.onresume() + expect(events).toEqual([]) + }) + + it("allows removing callbacks with off", () => { + const callback = jest.fn() + socket.off([socket.onSuspend(callback), socket.onResume(callback)]) + transport.onsuspend() + transport.onresume() + expect(callback).not.toHaveBeenCalled() + }) + + it("stops notifying when a callback disconnects", () => { + socket.onSuspend(() => socket.disconnect()) + const callback = jest.fn() + socket.onSuspend(callback) + transport.onsuspend() + expect(callback).not.toHaveBeenCalled() + }) +}) diff --git a/lib/phoenix/transports/long_poll.ex b/lib/phoenix/transports/long_poll.ex index 4085e1abf7..95b800639f 100644 --- a/lib/phoenix/transports/long_poll.ex +++ b/lib/phoenix/transports/long_poll.ex @@ -82,27 +82,39 @@ defmodule Phoenix.Transports.LongPoll do case read_body(conn, []) do {:ok, body, conn} -> # We need to match on both v1 and v2 protocol, as well as wrap for backwards compat - status = + {status, seq} = case get_req_header(conn, "content-type") do ["application/x-ndjson"] -> - body - |> String.splitter(["\n", "\r\n"]) - |> Stream.take(@max_poll_batch_size) - |> Enum.find_value(fn part -> - msg = - case part do - "[" <> _ = txt -> {txt, :text} - base64 -> {safe_decode64!(base64), :binary} - end - - transport_dispatch(endpoint, server_ref, msg, opts) - end) + # the seq of the first message, sent by clients that resend messages + # + # TODO: Require seq in Phoenix 1.10. Phoenix 1.8 clients don't send it. + # Phoenix 1.9 clients only omit it after their session reached a 1.8 node. + seq = int_param(conn, "seq") + + status = + body + |> String.splitter(["\n", "\r\n"]) + |> Stream.take(@max_poll_batch_size) + |> Stream.with_index() + |> Enum.find_value(fn {part, index} -> + msg = + case part do + "[" <> _ = txt -> {txt, :text} + base64 -> {safe_decode64!(base64), :binary} + end + + transport_dispatch(endpoint, server_ref, msg, seq && seq + index, opts) + end) + + {status, seq} _ -> - transport_dispatch(endpoint, server_ref, {body, :text}, opts) + {transport_dispatch(endpoint, server_ref, {body, :text}, nil, opts), nil} end - conn |> put_status(status || :ok) |> status_json() + # the seq tells clients that we drop the messages they send again + extra = if seq, do: %{"seq" => seq}, else: %{} + conn |> put_status(status || :ok) |> status_json(extra) _ -> raise Plug.BadRequestError @@ -117,9 +129,16 @@ defmodule Phoenix.Transports.LongPoll do end end - defp transport_dispatch(endpoint, server_ref, body, opts) do + defp transport_dispatch(endpoint, server_ref, body, seq, opts) do ref = make_ref() - broadcast_from!(endpoint, server_ref, {:dispatch, client_ref(server_ref), body, ref}) + + # TODO: Always send seq in Phoenix 1.10, see publish/4 + message = + if seq, + do: {:dispatch, client_ref(server_ref), body, ref, seq}, + else: {:dispatch, client_ref(server_ref), body, ref} + + broadcast_from!(endpoint, server_ref, message) receive do {:ok, ^ref} -> nil @@ -154,39 +173,71 @@ defmodule Phoenix.Transports.LongPoll do {:ok, server_pid} -> data = {:v1, endpoint.config(:endpoint_id), server_pid, priv_topic} token = sign_token(endpoint, data, opts) - conn |> put_status(:gone) |> status_token_messages_json(token, []) + # the batch tells clients that we keep messages until they acknowledge them + conn |> put_status(:gone) |> status_token_messages_json(token, [], 0) end end defp listen(conn, server_ref, token, endpoint, opts) do ref = make_ref() client_ref = client_ref(server_ref) - broadcast_from!(endpoint, server_ref, {:flush, client_ref, ref}) - {status, messages} = + # the last batch the client received, sent by clients that acknowledge + ack = int_param(conn, "ack") + + flush = + case ack do + # TODO: Remove the nil case in Phoenix 1.10. Phoenix 1.8 clients don't send + # an ack. Phoenix 1.9 clients only omit it after their session reached a 1.8 node. + nil -> {:flush, client_ref, ref} + ack -> {:flush, client_ref, ref, ack} + end + + broadcast_from!(endpoint, server_ref, flush) + + {status, messages, batch} = receive do + # Phoenix 1.8 servers, and newer servers answering a flush without ack, + # reply without batch. + # + # TODO: Remove the clauses without batch in Phoenix 1.10 {:messages, messages, ^ref} -> - {:ok, messages} + {:ok, messages, nil} + + {:messages, messages, ^ref, batch} -> + {:ok, messages, batch} {:now_available, ^ref} -> - broadcast_from!(endpoint, server_ref, {:flush, client_ref, ref}) + broadcast_from!(endpoint, server_ref, flush) receive do - {:messages, messages, ^ref} -> {:ok, messages} + {:messages, messages, ^ref} -> {:ok, messages, nil} + {:messages, messages, ^ref, batch} -> {:ok, messages, batch} after opts[:window_ms] -> broadcast_from!(endpoint, server_ref, {:expired, client_ref, ref}) - {:no_content, []} + {:no_content, [], ack} end after opts[:window_ms] -> + # Without messages, the batch stays the one the client acknowledged. It tells + # the client that we keep the messages, like we do with messages. broadcast_from!(endpoint, server_ref, {:expired, client_ref, ref}) - {:no_content, []} + {:no_content, [], ack} end conn |> put_status(status) - |> status_token_messages_json(token, messages) + |> status_token_messages_json(token, messages, batch) + end + + defp int_param(conn, key) do + with value when is_binary(value) <- conn.params[key], + {int, ""} <- Integer.parse(value) do + int + else + _ -> nil + end end # Accept the token from either location so that newer clients can use the @@ -286,14 +337,24 @@ defmodule Phoenix.Transports.LongPoll do defp maybe_auth_token_from_header(conn, _), do: conn - defp status_json(conn) do - send_json(conn, %{"status" => conn.status || 200}) + defp status_json(conn, extra \\ %{}) do + send_json(conn, Map.put(extra, "status", conn.status || 200)) end - defp status_token_messages_json(conn, token, messages) do + # TODO: Remove in Phoenix 1.10, when every poll has a batch + defp status_token_messages_json(conn, token, messages, nil) do send_json(conn, %{"status" => conn.status || 200, "token" => token, "messages" => messages}) end + defp status_token_messages_json(conn, token, messages, batch) do + send_json(conn, %{ + "status" => conn.status || 200, + "token" => token, + "messages" => messages, + "batch" => batch + }) + end + defp send_json(conn, data) do conn |> put_resp_header("content-type", "application/json; charset=utf-8") diff --git a/lib/phoenix/transports/long_poll_server.ex b/lib/phoenix/transports/long_poll_server.ex index 6875efa4b6..78b3e1f706 100644 --- a/lib/phoenix/transports/long_poll_server.ex +++ b/lib/phoenix/transports/long_poll_server.ex @@ -24,7 +24,16 @@ defmodule Phoenix.Transports.LongPoll.Server do {:ok, handler_state} = handler.init(handler_state) state = %{ - buffer: [], + # the messages we did not send yet, newest first + pending: [], + # the messages we sent with the last batch that the client did not acknowledge yet, + # newest first, and when we first sent them + sent: [], + sent_at: nil, + # the number of the last batch we sent + batch: 0, + # the seq of the last message we received, to drop the ones the client resends + last_in: 0, handler: {handler, handler_state}, window_ms: trunc(window_ms * 1.5), pubsub_server: endpoint.config(:pubsub_server), @@ -45,6 +54,19 @@ defmodule Phoenix.Transports.LongPoll.Server do end end + # The client sends the same message again when it did not get the response to its POST. + def handle_info({:dispatch, client_ref, _body, ref, seq}, %{last_in: last_in} = state) + when seq <= last_in do + broadcast_from!(state, client_ref, {:ok, ref}) + {:noreply, state} + end + + def handle_info({:dispatch, client_ref, body, ref, seq}, state) do + handle_info({:dispatch, client_ref, body, ref}, %{state | last_in: seq}) + end + + # TODO: Merge into the clause above in Phoenix 1.10. Phoenix 1.8 nodes and + # clients dispatch without seq. def handle_info({:dispatch, client_ref, {body, opcode}, ref}, state) do %{handler: {handler, handler_state}} = state @@ -72,14 +94,61 @@ defmodule Phoenix.Transports.LongPoll.Server do {:noreply, state} end + # The client acknowledges the last batch it received with each poll, so we send the + # messages again until it did, as the response of a poll can get lost. + def handle_info({:flush, client_ref, ref, ack}, state) do + state = %{state | last_client_poll: now_ms()} + + state = + if ack == state.batch do + # The client received the last batch, so we can drop it. + %{state | sent: [], sent_at: nil} + else + # The response with the last batch got lost, + # so we send its messages again. + state + end + + case state do + %{sent: [_ | _] = sent} -> + # A timed-out poll can still reach us after its retry. Keep each batch + # immutable so that acknowledging an earlier response drops exactly + # the messages the client received, even when requests arrive late. + broadcast_from!(state, client_ref, {:messages, Enum.reverse(sent), ref, state.batch}) + {:noreply, %{state | client_ref: nil}} + + %{pending: []} -> + {:noreply, %{state | client_ref: {client_ref, ref}}} + + %{pending: sent} -> + batch = state.batch + 1 + broadcast_from!(state, client_ref, {:messages, Enum.reverse(sent), ref, batch}) + + {:noreply, + %{ + state + | sent: sent, + sent_at: now_ms(), + pending: [], + batch: batch, + client_ref: nil + }} + end + end + + # Backwards compat: clients that don't acknowledge get each message once. + # + # TODO: Remove in Phoenix 1.10. Phoenix 1.8 nodes and clients flush without ack. def handle_info({:flush, client_ref, ref}, state) do - case state.buffer do + state = %{state | sent: [], sent_at: nil, last_client_poll: now_ms()} + + case state.pending do [] -> - {:noreply, %{state | client_ref: {client_ref, ref}, last_client_poll: now_ms()}} + {:noreply, %{state | client_ref: {client_ref, ref}}} - buffer -> - broadcast_from!(state, client_ref, {:messages, Enum.reverse(buffer), ref}) - {:noreply, %{state | client_ref: nil, last_client_poll: now_ms(), buffer: []}} + pending -> + broadcast_from!(state, client_ref, {:messages, Enum.reverse(pending), ref}) + {:noreply, %{state | client_ref: nil, pending: []}} end end @@ -94,11 +163,16 @@ defmodule Phoenix.Transports.LongPoll.Server do end def handle_info(:shutdown_if_inactive, state) do - if now_ms() - state.last_client_poll > state.window_ms do - {:stop, {:shutdown, :inactive}, state} - else - schedule_inactive_shutdown(state.window_ms) - {:noreply, state} + cond do + now_ms() - state.last_client_poll > state.window_ms -> + {:stop, {:shutdown, :inactive}, state} + + unacknowledged?(state) -> + {:stop, {:shutdown, :unacknowledged}, state} + + true -> + schedule_inactive_shutdown(state.window_ms) + {:noreply, state} end end @@ -146,10 +220,20 @@ defmodule Phoenix.Transports.LongPoll.Server do end defp publish_reply(state, reply) do - notify_client_now_available(state) - {:noreply, update_in(state.buffer, &[IO.iodata_to_binary(reply) | &1])} + # A client that does not acknowledge must not be able to + # grow the buffer indefinitely, so we close when it did not acknowledge + # a batch within the window. + if unacknowledged?(state) do + {:stop, {:shutdown, :unacknowledged}, state} + else + notify_client_now_available(state) + {:noreply, %{state | pending: [IO.iodata_to_binary(reply) | state.pending]}} + end end + defp unacknowledged?(%{sent_at: nil}), do: false + defp unacknowledged?(state), do: now_ms() - state.sent_at > state.window_ms + defp notify_client_now_available(state) do case state.client_ref do {client_ref, ref} -> broadcast_from!(state, client_ref, {:now_available, ref}) @@ -157,7 +241,7 @@ defmodule Phoenix.Transports.LongPoll.Server do end end - defp now_ms, do: System.system_time(:millisecond) + defp now_ms, do: System.monotonic_time(:millisecond) defp schedule_inactive_shutdown(window_ms) do Process.send_after(self(), :shutdown_if_inactive, window_ms) diff --git a/test/phoenix/integration/long_poll_channels_test.exs b/test/phoenix/integration/long_poll_channels_test.exs index 2513f61b66..3230b5aa54 100644 --- a/test/phoenix/integration/long_poll_channels_test.exs +++ b/test/phoenix/integration/long_poll_channels_test.exs @@ -656,10 +656,135 @@ defmodule Phoenix.Integration.LongPollChannelsTest do assert_receive %Broadcast{event: "new_msg", payload: %{"body" => "hi!"}} end + test "#{@mode}: accepts legacy flushes without waiting for acknowledgements" do + vsn = "2.0.0" + session = join("/ws", "room:lobby", vsn, "1", @mode) + resp = poll(:get, "/ws", vsn, Map.put(session, "ack", "0")) + assert resp.body["status"] == 200 + + {:ok, {:v1, _id, server, _topic}} = + Phoenix.Token.verify(Endpoint, Atom.to_string(__MODULE__), session["token"]) + + # an older plug flushes without acknowledging and expects the reply without batch + Endpoint.broadcast!("room:lobby", "new_msg", %{"body" => "hi"}) + Process.sleep(50) + ref = make_ref() + send(server, {:flush, self(), ref}) + assert_receive {:messages, [_new_msg], ^ref} + + # it does not acknowledge the messages we sent before, which we drop + # like we drop the ones we send to it, so we don't wait for an ack + assert %{sent: [], sent_at: nil} = :sys.get_state(server) + end + + test "#{@mode}: resends messages until the client acknowledges them" do + vsn = "2.0.0" + + # new sessions tell clients that they keep messages + resp = poll(:get, "/ws", vsn, %{}) + assert resp.body["status"] == 410 + assert resp.body["batch"] == 0 + + session = join("/ws", "room:lobby", vsn, "1", @mode) + resp = poll(:get, "/ws", vsn, Map.put(session, "ack", "0")) + assert resp.body["status"] == 200 + assert [_reply, _user_entered, _joined] = messages = resp.body["messages"] + assert batch = resp.body["batch"] + + # the response got lost, so the client polls with the previous ack + resp = poll(:get, "/ws", vsn, Map.put(session, "ack", "0")) + assert resp.body["status"] == 200 + assert resp.body["messages"] == messages + assert resp.body["batch"] == batch + + # A lost response is replayed unchanged, even when new messages arrive. + Endpoint.broadcast("room:lobby", "new_msg", %{"body" => "hi"}) + Process.sleep(50) + resp = poll(:get, "/ws", vsn, Map.put(session, "ack", "0")) + assert resp.body["status"] == 200 + assert resp.body["messages"] == messages + assert resp.body["batch"] == batch + + resp = poll(:get, "/ws", vsn, Map.put(session, "ack", to_string(batch))) + assert [%Message{event: "new_msg"}] = resp.body["messages"] + assert resp.body["batch"] == batch + 1 + + # without messages, the batch stays the acknowledged one + resp = poll(:get, "/ws", vsn, Map.put(session, "ack", to_string(batch + 1))) + assert resp.body["status"] == 204 + assert resp.body["batch"] == batch + 1 + end + + test "#{@mode}: a delayed poll does not expand an already received batch" do + vsn = "2.0.0" + session = join("/ws", "room:lobby", vsn, "1", @mode) + received = poll(:get, "/ws", vsn, Map.put(session, "ack", "0")).body + batch = received["batch"] + + {:ok, {:v1, _id, server, _topic}} = + Phoenix.Token.verify(Endpoint, Atom.to_string(__MODULE__), session["token"]) + + # A new frame arrives, followed by an old request whose response the + # client already stopped waiting for. The next poll acknowledges the + # retry's response and must receive only the new frame. + send(server, {:socket_push, :text, "[null,null,\"room:lobby\",\"tick\",{}]"}) + ref = make_ref() + send(server, {:flush, self(), ref, 0}) + assert_receive {:messages, _messages, ^ref, ^batch} + + resp = poll(:get, "/ws", vsn, Map.put(session, "ack", to_string(batch))) + assert [%Message{event: "tick"}] = resp.body["messages"] + assert resp.body["batch"] == batch + 1 + end + + test "#{@mode}: closes when the client keeps polling without acknowledging" do + vsn = "2.0.0" + session = join("/ws", "room:lobby", vsn, "1", @mode) + + # the window is 200ms, so the client must acknowledge a batch within 300ms + statuses = + for _ <- 1..8 do + Process.sleep(100) + poll(:get, "/ws", vsn, Map.put(session, "ack", "0")).body["status"] + end + + assert [200, 200 | _] = statuses + assert List.last(statuses) == 410 + end + + test "#{@mode}: drops messages the client sends again" do + vsn = "2.0.0" + Phoenix.PubSub.subscribe(__MODULE__, "room:lobby") + session = join("/ws", "room:lobby", vsn, "1", @mode) + + message = fn body -> + %{ + "topic" => "room:lobby", + "event" => "new_msg", + "ref" => body, + "payload" => %{"body" => body} + } + end + + resp = poll(:post, "/ws", vsn, Map.put(session, "seq", "2"), [message.("a")]) + # the seq tells the client that we drop messages it sends again + assert resp.body["status"] == 200 + assert resp.body["seq"] == 2 + assert_receive %Broadcast{event: "new_msg", payload: %{"body" => "a"}} + + # the response got lost, so the client sends the message again with the next one + batch = [message.("a"), message.("b")] + resp = poll(:post, "/ws", vsn, Map.put(session, "seq", "2"), batch) + assert resp.body["status"] == 200 + assert_receive %Broadcast{event: "new_msg", payload: %{"body" => "b"}} + refute_received %Broadcast{event: "new_msg", payload: %{"body" => "a"}} + end + test "#{@mode}: lonpoll publishing batch events on v2 protocol" do vsn = "2.0.0" Phoenix.PubSub.subscribe(__MODULE__, "room:lobby") session = join("/ws", "room:lobby", vsn, "1", @mode) + # Publish successfully resp = poll(:post, "/ws", vsn, session, [ @@ -686,7 +811,9 @@ defmodule Phoenix.Integration.LongPollChannelsTest do } ]) + # clients that don't number their messages get no seq assert resp.body["status"] == 200 + refute Map.has_key?(resp.body, "seq") assert_receive %Broadcast{event: "new_msg", payload: %{"body" => "hi1"}} assert_receive %Broadcast{event: "new_msg", payload: %{"body" => "hi2"}} refute_receive %Broadcast{event: "new_msg", payload: %{"body" => "stale"}}