Redesign requests to server to prevent errors.

javascript/utils.js

@@ -1,6 +1,6 @@
 /** Collators used for sorting. */
-const INT_COLLATOR = new Intl.Collator([], {numeric: true});
-const STR_COLLATOR = new Intl.Collator("en", {numeric: true, sensitivity: "base"});
+const INT_COLLATOR = new Intl.Collator([], { numeric: true });
+const STR_COLLATOR = new Intl.Collator("en", { numeric: true, sensitivity: "base" });
 
 /** Helper functions for checking types and simplifying logging/error handling. */
 function isNumber(x) {
@@ -344,7 +344,7 @@ function waitForValueInObject(o, timeout_ms) {
                 return reject(`timed out waiting for value: ${o.k}: ${o.v}`);
             }, timeout_ms);
         }
-        waitForKeyInObject({k: o.k, obj: o.obj}, timeout_ms).then(() => {
+        waitForKeyInObject({ k: o.k, obj: o.obj }, timeout_ms).then(() => {
             (function _waitForValueInObject() {
 
                 if (o.k in o.obj && o.obj[o.k] == o.v) {
@@ -360,14 +360,155 @@ function waitForValueInObject(o, timeout_ms) {
 
 /** Requests */
 
+class FetchError extends Error {
+    constructor(...args) {
+        super(...args);
+        this.name = this.constructor.name;
+    }
+}
+
+class Fetch4xxError extends FetchError {
+    constructor(...args) { super(...args); }
+}
+
+class Fetch5xxError extends FetchError {
+    constructor(...args) { super(...args); }
+}
+
+class FetchRetryLimitError extends FetchError {
+    constructor(...args) { super(...args); }
+}
+
+class FetchTimeoutError extends FetchError {
+    constructor(...args) { super(...args); }
+}
+
+class FetchWithRetryAndBackoffTimeoutError extends FetchError {
+    constructor(...args) { super(...args); }
+}
+
+async function fetchWithRetryAndBackoff(url, data, opts = {}) {
+    /** Wrapper around `fetch` with retries, backoff, and timeout.
+     *
+     *  Uses a Decorrelated jitter backoff strategy.
+     *  https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
+     *
+     *  Args:
+     *      url:                    Primary URL to fetch.
+     *      data:                   Data to append to the URL when making the request.
+     *      opts:
+     *          method:             The HTTP request method to use.
+     *          timeout_ms:         Max allowed time before this function fails.
+     *          fetch_timeout_ms:   Max allowed time for individual `fetch` calls.
+     *          min_delay_ms:       Min time that a delay between requests can be.
+     *          max_delay_ms:       Max time that a delay between reqeusts can be.
+     *          response_handler:   A callback function that returns a promise.
+     *              This function is sent the response from the `fetch` call.
+     *              If not specified, all status codes >= 400 are handled as errors.
+     *              This is useful for handling requests whose responses from the server
+     *              are erronious but the HTTP status is 200.
+     */
+    opts.method = opts.method || "GET";
+    opts.timeout_ms = opts.timeout_ms || 30000;
+    opts.min_delay_ms = opts.min_delay_ms || 100;
+    opts.max_delay_ms = opts.max_delay_ms || 3000;
+    opts.fetch_timeout_ms = opts.fetch_timeout_ms || 10000;
+    // The default response handler function for `fetch` call responses.
+    const response_handler = (response) => new Promise(async (resolve, reject) => {
+        if (response.ok) {
+            const json = await response.json();
+            return resolve(json);
+        }
+        if (response.status >= 400 && response.status < 500) {
+            throw new Fetch4xxError("client error:", response);
+        }
+        if (response.status >= 500 && response.status < 600) {
+            throw new Fetch5xxError("server error:", response);
+        }
+        return reject(response);
+    });
+    opts.response_handler = opts.response_handler || response_handler;
+
+    const args = Object.entries(data).map(([k, v]) => {
+        return `${encodeURIComponent(k)}=${encodeURIComponent(v)}`;
+    }).join("&");
+    url = `${url}?${args}`;
+
+    let controller;
+    let retry = true;
+
+    const randrange = (min, max) => {
+        return Math.floor(Math.random() * (max - min + 1) + min);
+    }
+    const get_jitter = (base, max, prev) => {
+        return Math.min(max, randrange(base, prev * 3));
+    }
+
+    const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+    // Timeout for individual `fetch` calls.
+    const fetch_timeout = (ms, promise) => {
+        return new Promise((resolve, reject) => {
+            setTimeout(() => {
+                return reject(new FetchTimeoutError("Fetch timed out."));
+            }, ms);
+            return promise.then(resolve, reject);
+        });
+    };
+
+    // Timeout for all retries.
+    const run_timeout = (ms, promise) => {
+        return new Promise((resolve, reject) => {
+            setTimeout(() => {
+                retry = false;
+                controller.abort();
+                return reject(new FetchWithRetryAndBackoffTimeoutError("Request timed out."));
+            }, ms);
+            return promise.then(resolve, reject);
+        });
+    };
+
+    const run = async (delay_ms) => {
+        if (!retry) {
+            // Retry is controlled externally via `run_timeout`. This function's promise
+            // is also handled via that timeout so we can just return here.
+            return;
+        }
+        try {
+            controller = new AbortController();
+            const fetch_opts = { method: opts.method, signal: controller.signal };
+            const response = await fetch_timeout(opts.fetch_timeout_ms, fetch(url, fetch_opts));
+            return await opts.response_handler(response);
+        } catch (error) {
+            controller.abort();
+            // dont bother with anything else if told to not retry.
+            if (!retry) {
+                return;
+            }
+
+            if (error instanceof Fetch4xxError) {
+                throw error;
+            }
+            if (error instanceof Fetch5xxError) {
+                throw error;
+            }
+            // Any other errors mean we need to retry the request.
+            delay_ms = get_jitter(opts.min_delay_ms, opts.max_delay_ms, delay_ms);
+            await delay(delay_ms);
+            return await run(delay_ms);
+        }
+    }
+    return await run_timeout(opts.timeout_ms, run(opts.min_delay_ms));
+}
+
 function requestGet(url, data, handler, errorHandler) {
     var xhr = new XMLHttpRequest();
-    var args = Object.keys(data).map(function(k) {
+    var args = Object.keys(data).map(function (k) {
         return encodeURIComponent(k) + '=' + encodeURIComponent(data[k]);
     }).join('&');
     xhr.open("GET", url + "?" + args, true);
 
-    xhr.onreadystatechange = function() {
+    xhr.onreadystatechange = function () {
         if (xhr.readyState === 4) {
             if (xhr.status === 200) {
                 try {
@@ -404,18 +545,18 @@ function requestGetPromise(url, data, timeout_ms) {
 
         xhr.onload = () => {
             if (xhr.status >= 200 && xhr.status < 300) {
-                return resolve({status: xhr.status, response: JSON.parse(xhr.responseText)});
+                return resolve({ status: xhr.status, response: JSON.parse(xhr.responseText) });
             } else {
-                return reject({status: xhr.status, response: JSON.parse(xhr.responseText)});
+                return reject({ status: xhr.status, response: JSON.parse(xhr.responseText) });
             }
         };
 
         xhr.onerror = () => {
-            return reject({status: xhr.status, response: JSON.parse(xhr.responseText)});
+            return reject({ status: xhr.status, response: JSON.parse(xhr.responseText) });
         };
 
         xhr.ontimeout = () => {
-            return reject({status: 408, response: {detail: `Request timeout: ${url}`}});
+            return reject({ status: 408, response: { detail: `Request timeout: ${url}` } });
         };
 
         const payload = JSON.stringify(data);