From 8b27a00d4534cdd574dfa4a5766d049f9ed5868a Mon Sep 17 00:00:00 2001 From: Sj-Si Date: Fri, 26 Apr 2024 14:00:00 -0400 Subject: [PATCH] Redesign requests to server to prevent errors. --- javascript/extraNetworks.js | 207 +++++++++++++++++------------------ javascript/utils.js | 159 +++++++++++++++++++++++++-- modules/ui_extra_networks.py | 57 ++++------ 3 files changed, 271 insertions(+), 152 deletions(-) diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 49e90d982..3694af871 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -6,7 +6,7 @@ isString, isElement, isElementThrowError, - requestGetPromise, + fetchWithRetryAndBackoff, isElementLogError, isNumber, waitForKeyInObject, @@ -18,9 +18,9 @@ /*eslint no-undef: "error"*/ const SEARCH_INPUT_DEBOUNCE_TIME_MS = 250; -const EXTRA_NETWORKS_GET_PAGE_READY_MAX_ATTEMPTS = 10; -const EXTRA_NETWORKS_WAIT_FOR_PAGE_READY_TIMEOUT_MS = 1000; -const EXTRA_NETWORKS_REQUEST_GET_TIMEOUT_MS = 1000; +const EXTRA_NETWORKS_WAIT_FOR_PAGE_READY_TIMEOUT_MS = 30000; +const EXTRA_NETWORKS_INIT_DATA_TIMEOUT_MS = 60000; +const EXTRA_NETWORKS_FETCH_DATA_TIMEOUT_MS = 30000; const EXTRA_NETWORKS_REFRESH_INTERNAL_DEBOUNCE_TIMEOUT_MS = 200; const re_extranet = /<([^:^>]+:[^:]+):[\d.]+>(.*)/; @@ -36,7 +36,7 @@ var extra_networks_refresh_internal_debounce_timer; /** Boolean flags used along with utils.js::waitForBool(). */ // Set true when we first load the UI options. -const initialUiOptionsLoaded = {state: false}; +const initialUiOptionsLoaded = { state: false }; const _debounce = (handler, timeout_ms) => { /** Debounces a function call. @@ -65,6 +65,20 @@ const _debounce = (handler, timeout_ms) => { }; }; +class ExtraNetworksError extends Error { + constructor(...args) { + super(...args); + this.name = this.constructor.name; + } +} +class ExtraNetworksPageReadyError extends Error { + constructor(...args) { super(...args); } +} + +class ExtraNetworksDataReadyError extends Error { + constructor(...args) { super(...args); } +} + class ExtraNetworksTab { tabname; extra_networks_tabname; @@ -89,7 +103,7 @@ class ExtraNetworksTab { show_neg_prompt = true; compact_prompt_en = false; refresh_in_progress = false; - constructor({tabname, extra_networks_tabname}) { + constructor({ tabname, extra_networks_tabname }) { this.tabname = tabname; this.extra_networks_tabname = extra_networks_tabname; this.tabname_full = `${tabname}_${extra_networks_tabname}`; @@ -270,6 +284,12 @@ class ExtraNetworksTab { } async #refresh() { + try { + await this.waitForServerPageReady(); + } catch (error) { + console.error(`refresh error: ${error.message}`); + return; + } const btn_dirs_view = this.controls_elem.querySelector(".extra-network-control--dirs-view"); const btn_tree_view = this.controls_elem.querySelector(".extra-network-control--tree-view"); const div_dirs = this.container_elem.querySelector(".extra-network-content--dirs-view"); @@ -330,7 +350,7 @@ class ExtraNetworksTab { this.setFilterStr(filter_str); } - async waitForServerPageReady(max_attempts = EXTRA_NETWORKS_GET_PAGE_READY_MAX_ATTEMPTS) { + async waitForServerPageReady(timeout_ms) { /** Waits for a page on the server to be ready. * * We need to wait for the page to be ready before we can fetch any data. @@ -343,139 +363,118 @@ class ExtraNetworksTab { * max_attempts [int]: The max number of requests that will be attempted * before giving up. If set to 0, will attempt forever. */ - const err_prefix = `error waiting for server page (${this.tabname_full})`; - return new Promise((resolve, reject) => { - let attempt = 0; - const loop = () => { - console.log(`waitForServerPageReady: iter ${attempt}`); - let retry_delay_ms = 1000; - setTimeout(async() => { - try { - await requestGetPromise( - "./sd_extra_networks/page-is-ready", - {extra_networks_tabname: this.extra_networks_tabname}, - EXTRA_NETWORKS_WAIT_FOR_PAGE_READY_TIMEOUT_MS, - ); - console.warn("PAGE READY:", this.extra_networks_tabname); - return resolve(); - } catch (error) { - // If we get anything other than a timeout error, reject. - // Otherwise, fall through to retry request. - if (error.status !== 408 && error.status !== 404) { - return reject(`${err_prefix}: uncaught exception: ${JSON.stringify(error)}`); - } - if (error.status === 404) { - retry_delay_ms = 1000; - } - if (error.status === 408) { - retry_delay_ms = 0; - } - console.log("other error:", error.status, error); - } - - if (max_attempts !== 0 && attempt++ >= max_attempts) { - return reject(`${err_prefix}: max attempts exceeded`); - } else { - // small delay since our request has a timeout. - console.log("retrying:", attempt); - setTimeout(loop, retry_delay_ms); - } - }, 0); - }; - return loop(); + timeout_ms = timeout_ms || EXTRA_NETWORKS_WAIT_FOR_PAGE_READY_TIMEOUT_MS; + const response_handler = (response) => new Promise(async (resolve, reject) => { + if (!response.ok) { + return reject(response); + } + const json = await response.json(); + if (!json.ready) { + return reject(`page not ready: ${this.extra_networks_tabname}`); + } + return resolve(json); }); + + const url = "./sd_extra_networks/page-is-ready"; + const payload = { extra_networks_tabname: this.extra_networks_tabname }; + const opts = { timeout_ms: timeout_ms, response_handler: response_handler }; + return await fetchWithRetryAndBackoff(url, payload, opts); } async onInitCardsData() { - console.log("onInitCardsData"); try { await this.waitForServerPageReady(); } catch (error) { - console.error(JSON.stringify(error)); + console.error(`onInitCardsData error: ${error.message}`); return {}; } + const response_handler = (response) => new Promise(async (resolve, reject) => { + if (!response.ok) { + return reject(response); + } + const json = await response.json(); + if (!json.ready) { + return reject(`data not ready: ${this.extra_networks_tabname}`); + } + return resolve(json); + }); + const url = "./sd_extra_networks/init-cards-data"; - const payload = {tabname: this.tabname, extra_networks_tabname: this.extra_networks_tabname}; - const timeout = EXTRA_NETWORKS_REQUEST_GET_TIMEOUT_MS; + const payload = { tabname: this.tabname, extra_networks_tabname: this.extra_networks_tabname }; + const timeout_ms = EXTRA_NETWORKS_INIT_DATA_TIMEOUT_MS; + const opts = { timeout_ms: timeout_ms, response_handler: response_handler }; try { - const response = await requestGetPromise(url, payload, timeout); - return response.response; + const response = await fetchWithRetryAndBackoff(url, payload, opts); + return response.data; } catch (error) { - console.error(JSON.stringify(error)); + console.error(`onInitCardsData error: ${error.message}`); return {}; } } async onInitTreeData() { - console.log("onInitTreeData"); try { await this.waitForServerPageReady(); } catch (error) { - console.error(JSON.stringify(error)); + console.error(`onInitTreeData error: ${error.message}`); return {}; } + const response_handler = (response) => new Promise(async (resolve, reject) => { + if (!response.ok) { + return reject(response); + } + const json = await response.json(); + if (!json.ready) { + return reject(`data not ready: ${this.extra_networks_tabname}`); + } + return resolve(json); + }); + const url = "./sd_extra_networks/init-tree-data"; - const payload = {tabname: this.tabname, extra_networks_tabname: this.extra_networks_tabname}; - const timeout = EXTRA_NETWORKS_REQUEST_GET_TIMEOUT_MS; + const payload = { tabname: this.tabname, extra_networks_tabname: this.extra_networks_tabname }; + const timeout_ms = EXTRA_NETWORKS_INIT_DATA_TIMEOUT_MS; + const opts = { timeout_ms: timeout_ms, response_handler: response_handler } try { - const response = await requestGetPromise(url, payload, timeout); - return response.response; + const response = await fetchWithRetryAndBackoff(url, payload, opts); + return response.data; } catch (error) { - console.error(JSON.stringify(error)); + console.error(`onInitTreeData error: ${error.message}`); return {}; } } async onFetchCardsData(div_ids) { - console.log("onFetchCardsData:", div_ids); - /* - try { - await this.waitForServerPageReady(); - } catch (error) { - console.error(JSON.stringify(error)); - return {}; - } - */ - const url = "./sd_extra_networks/fetch-cards-data"; - const payload = {extra_networks_tabname: this.extra_networks_tabname, div_ids: div_ids}; - const timeout = EXTRA_NETWORKS_REQUEST_GET_TIMEOUT_MS; + const payload = { extra_networks_tabname: this.extra_networks_tabname, div_ids: div_ids }; + const timeout_ms = EXTRA_NETWORKS_FETCH_DATA_TIMEOUT_MS; + const opts = { timeout_ms: timeout_ms }; try { - const response = await requestGetPromise(url, payload, timeout); - if (response.response.missing_div_ids.length) { - console.warn(`Failed to fetch multiple div_ids: ${response.response.missing_div_ids}`); + const response = await fetchWithRetryAndBackoff(url, payload, opts); + if (response.missing_div_ids.length) { + console.warn(`Failed to fetch multiple div_ids: ${response.missing_div_ids}`); } - return response.response.data; + return response.data; } catch (error) { - console.error(JSON.stringify(error)); + console.error(`onFetchCardsData error: ${error.message}`); return {}; } } async onFetchTreeData(div_ids) { - console.log("onFetchTreeData:", div_ids); - /* - try { - await this.waitForServerPageReady(); - } catch (error) { - console.error(JSON.stringify(error)); - return {}; - } - */ - const url = "./sd_extra_networks/fetch-tree-data"; - const payload = {extra_networks_tabname: this.extra_networks_tabname, div_ids: div_ids}; - const timeout = EXTRA_NETWORKS_REQUEST_GET_TIMEOUT_MS; + const payload = { extra_networks_tabname: this.extra_networks_tabname, div_ids: div_ids }; + const timeout_ms = EXTRA_NETWORKS_FETCH_DATA_TIMEOUT_MS; + const opts = { timeout_ms: timeout_ms }; try { - const response = await requestGetPromise(url, payload, timeout); - if (response.response.missing_div_ids.length) { - console.warn(`Failed to fetch multiple div_ids: ${response.response.missing_div_ids}`); + const response = await fetchWithRetryAndBackoff(url, payload, opts); + if (response.missing_div_ids.length) { + console.warn(`Failed to fetch multiple div_ids: ${response.missing_div_ids}`); } - return response.response.data; + return response.data; } catch (error) { - console.error(JSON.stringify(error)); + console.error(`onFetchTreeData error: ${error.message}`); return {}; } } @@ -814,8 +813,8 @@ function extraNetworksFetchMetadata(extra_networks_tabname, card_name) { requestGet( "./sd_extra_networks/metadata", - {extra_networks_tabname: extra_networks_tabname, item: card_name}, - function(data) { + { extra_networks_tabname: extra_networks_tabname, item: card_name }, + function (data) { if (data && data.metadata) { extraNetworksShowMetadata(data.metadata); } else { @@ -842,7 +841,7 @@ function extraNetworksUnrelatedTabSelected(tabname) { async function extraNetworksTabSelected(tabname_full, show_prompt, show_neg_prompt) { /** called from python when user selects an extra networks tab */ - await waitForKeyInObject({obj: extra_networks_tabs, k: tabname_full}); + await waitForKeyInObject({ obj: extra_networks_tabs, k: tabname_full }); for (const [k, v] of Object.entries(extra_networks_tabs)) { if (k === tabname_full) { v.load(show_prompt, show_neg_prompt); @@ -874,7 +873,7 @@ function extraNetworksControlSearchClearOnClick(event, tabname_full) { txt_search_elem.dispatchEvent( new CustomEvent( "extra-network-control--search-clear", - {bubbles: true, detail: {tabname_full: tabname_full}}, + { bubbles: true, detail: { tabname_full: tabname_full } }, ) ); @@ -1001,16 +1000,6 @@ function extraNetworksControlRefreshOnClick(event, tabname_full) { clearTimeout(extra_networks_refresh_internal_debounce_timer); extra_networks_refresh_internal_debounce_timer = setTimeout(async () => { const tab = extra_networks_tabs[tabname_full]; - try { - await requestGetPromise( - "./sd_extra_networks/clear-page-data", - {extra_networks_tabname: tab.extra_networks_tabname}, - 5000, - ); - console.log("cleared page data:", tab.extra_networks_tabname); - } catch (error) { - console.error("error clearing page data:", error); - } // We want to reset tab lists on refresh click so that the viewing area // shows that it is loading new data. tab.tree_list.clear(); diff --git a/javascript/utils.js b/javascript/utils.js index 1fbd104de..bd780bc47 100644 --- a/javascript/utils.js +++ b/javascript/utils.js @@ -1,6 +1,6 @@ /** Collators used for sorting. */ -const INT_COLLATOR = new Intl.Collator([], {numeric: true}); -const STR_COLLATOR = new Intl.Collator("en", {numeric: true, sensitivity: "base"}); +const INT_COLLATOR = new Intl.Collator([], { numeric: true }); +const STR_COLLATOR = new Intl.Collator("en", { numeric: true, sensitivity: "base" }); /** Helper functions for checking types and simplifying logging/error handling. */ function isNumber(x) { @@ -344,7 +344,7 @@ function waitForValueInObject(o, timeout_ms) { return reject(`timed out waiting for value: ${o.k}: ${o.v}`); }, timeout_ms); } - waitForKeyInObject({k: o.k, obj: o.obj}, timeout_ms).then(() => { + waitForKeyInObject({ k: o.k, obj: o.obj }, timeout_ms).then(() => { (function _waitForValueInObject() { if (o.k in o.obj && o.obj[o.k] == o.v) { @@ -360,14 +360,155 @@ function waitForValueInObject(o, timeout_ms) { /** Requests */ +class FetchError extends Error { + constructor(...args) { + super(...args); + this.name = this.constructor.name; + } +} + +class Fetch4xxError extends FetchError { + constructor(...args) { super(...args); } +} + +class Fetch5xxError extends FetchError { + constructor(...args) { super(...args); } +} + +class FetchRetryLimitError extends FetchError { + constructor(...args) { super(...args); } +} + +class FetchTimeoutError extends FetchError { + constructor(...args) { super(...args); } +} + +class FetchWithRetryAndBackoffTimeoutError extends FetchError { + constructor(...args) { super(...args); } +} + +async function fetchWithRetryAndBackoff(url, data, opts = {}) { + /** Wrapper around `fetch` with retries, backoff, and timeout. + * + * Uses a Decorrelated jitter backoff strategy. + * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + * + * Args: + * url: Primary URL to fetch. + * data: Data to append to the URL when making the request. + * opts: + * method: The HTTP request method to use. + * timeout_ms: Max allowed time before this function fails. + * fetch_timeout_ms: Max allowed time for individual `fetch` calls. + * min_delay_ms: Min time that a delay between requests can be. + * max_delay_ms: Max time that a delay between reqeusts can be. + * response_handler: A callback function that returns a promise. + * This function is sent the response from the `fetch` call. + * If not specified, all status codes >= 400 are handled as errors. + * This is useful for handling requests whose responses from the server + * are erronious but the HTTP status is 200. + */ + opts.method = opts.method || "GET"; + opts.timeout_ms = opts.timeout_ms || 30000; + opts.min_delay_ms = opts.min_delay_ms || 100; + opts.max_delay_ms = opts.max_delay_ms || 3000; + opts.fetch_timeout_ms = opts.fetch_timeout_ms || 10000; + // The default response handler function for `fetch` call responses. + const response_handler = (response) => new Promise(async (resolve, reject) => { + if (response.ok) { + const json = await response.json(); + return resolve(json); + } + if (response.status >= 400 && response.status < 500) { + throw new Fetch4xxError("client error:", response); + } + if (response.status >= 500 && response.status < 600) { + throw new Fetch5xxError("server error:", response); + } + return reject(response); + }); + opts.response_handler = opts.response_handler || response_handler; + + const args = Object.entries(data).map(([k, v]) => { + return `${encodeURIComponent(k)}=${encodeURIComponent(v)}`; + }).join("&"); + url = `${url}?${args}`; + + let controller; + let retry = true; + + const randrange = (min, max) => { + return Math.floor(Math.random() * (max - min + 1) + min); + } + const get_jitter = (base, max, prev) => { + return Math.min(max, randrange(base, prev * 3)); + } + + const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + + // Timeout for individual `fetch` calls. + const fetch_timeout = (ms, promise) => { + return new Promise((resolve, reject) => { + setTimeout(() => { + return reject(new FetchTimeoutError("Fetch timed out.")); + }, ms); + return promise.then(resolve, reject); + }); + }; + + // Timeout for all retries. + const run_timeout = (ms, promise) => { + return new Promise((resolve, reject) => { + setTimeout(() => { + retry = false; + controller.abort(); + return reject(new FetchWithRetryAndBackoffTimeoutError("Request timed out.")); + }, ms); + return promise.then(resolve, reject); + }); + }; + + const run = async (delay_ms) => { + if (!retry) { + // Retry is controlled externally via `run_timeout`. This function's promise + // is also handled via that timeout so we can just return here. + return; + } + try { + controller = new AbortController(); + const fetch_opts = { method: opts.method, signal: controller.signal }; + const response = await fetch_timeout(opts.fetch_timeout_ms, fetch(url, fetch_opts)); + return await opts.response_handler(response); + } catch (error) { + controller.abort(); + // dont bother with anything else if told to not retry. + if (!retry) { + return; + } + + if (error instanceof Fetch4xxError) { + throw error; + } + if (error instanceof Fetch5xxError) { + throw error; + } + // Any other errors mean we need to retry the request. + delay_ms = get_jitter(opts.min_delay_ms, opts.max_delay_ms, delay_ms); + await delay(delay_ms); + return await run(delay_ms); + } + } + return await run_timeout(opts.timeout_ms, run(opts.min_delay_ms)); +} + function requestGet(url, data, handler, errorHandler) { var xhr = new XMLHttpRequest(); - var args = Object.keys(data).map(function(k) { + var args = Object.keys(data).map(function (k) { return encodeURIComponent(k) + '=' + encodeURIComponent(data[k]); }).join('&'); xhr.open("GET", url + "?" + args, true); - xhr.onreadystatechange = function() { + xhr.onreadystatechange = function () { if (xhr.readyState === 4) { if (xhr.status === 200) { try { @@ -404,18 +545,18 @@ function requestGetPromise(url, data, timeout_ms) { xhr.onload = () => { if (xhr.status >= 200 && xhr.status < 300) { - return resolve({status: xhr.status, response: JSON.parse(xhr.responseText)}); + return resolve({ status: xhr.status, response: JSON.parse(xhr.responseText) }); } else { - return reject({status: xhr.status, response: JSON.parse(xhr.responseText)}); + return reject({ status: xhr.status, response: JSON.parse(xhr.responseText) }); } }; xhr.onerror = () => { - return reject({status: xhr.status, response: JSON.parse(xhr.responseText)}); + return reject({ status: xhr.status, response: JSON.parse(xhr.responseText) }); }; xhr.ontimeout = () => { - return reject({status: 408, response: {detail: `Request timeout: ${url}`}}); + return reject({ status: 408, response: { detail: `Request timeout: ${url}` } }); }; const payload = JSON.stringify(data); diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index dae5dc99e..677cd5252 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -240,7 +240,6 @@ class ExtraNetworksPage: self.btn_dirs_view_item_tpl = shared.html("extra-networks-btn-dirs-view-item.html") def clear_data(self) -> None: - print(f"clearing data: {self.extra_networks_tabname}") self.is_ready = False self.metadata = {} self.items = {} @@ -250,9 +249,7 @@ class ExtraNetworksPage: self.nodes = {} def refresh(self) -> None: - print(f"refreshing: {self.extra_networks_tabname}") # Whenever we refresh, we want to build our datasets from scratch. - self.is_ready = False self.clear_data() def read_user_metadata(self, item, use_cache=True): @@ -936,10 +933,23 @@ def init_tree_data(tabname: str = "", extra_networks_tabname: str = "") -> JSONR data = page.generate_tree_view_data(tabname) - if data is None: - raise HTTPException(status_code=404, detail=f"data not ready: {extra_networks_tabname}") + return JSONResponse({"data": data, "ready": data is not None}) - return JSONResponse(data) + +def init_cards_data(tabname: str = "", extra_networks_tabname: str = "") -> JSONResponse: + """Generates the initial Cards View data and returns a simplified dataset. + + The data returned does not contain any HTML strings. + + Status Codes: + 200 on success + 404 if data isn't ready or tabname doesn't exist. + """ + page = get_page_by_name(extra_networks_tabname) + + data = page.generate_cards_view_data(tabname) + + return JSONResponse({"data": data, "ready": data is not None}) def fetch_tree_data( @@ -990,29 +1000,9 @@ def fetch_cards_data( res[div_id] = page.cards[div_id].html else: missed.append(div_id) - return JSONResponse({"data": res, "missing_div_ids": missed}) -def init_cards_data(tabname: str = "", extra_networks_tabname: str = "") -> JSONResponse: - """Generates the initial Cards View data and returns a simplified dataset. - - The data returned does not contain any HTML strings. - - Status Codes: - 200 on success - 404 if data isn't ready or tabname doesn't exist. - """ - page = get_page_by_name(extra_networks_tabname) - - data = page.generate_cards_view_data(tabname) - - if data is None: - raise HTTPException(status_code=404, detail=f"data not ready: {extra_networks_tabname}") - - return JSONResponse(data) - - def clear_page_data(extra_networks_tabname: str = "") -> JSONResponse: """Returns whether the specified page is ready for fetching data. @@ -1024,22 +1014,19 @@ def clear_page_data(extra_networks_tabname: str = "") -> JSONResponse: page.clear_data() - return JSONResponse({}, status_code=200) + return JSONResponse({}) + def page_is_ready(extra_networks_tabname: str = "") -> JSONResponse: """Returns whether the specified page is ready for fetching data. Status Codes: - 200 if page is ready - 404 if page isn't ready or tabname doesnt exist. + 200 on success. response contains ready state. + 404 if tabname doesnt exist. """ page = get_page_by_name(extra_networks_tabname) - print(f"page_is_ready: {extra_networks_tabname} => {page.is_ready}, {len(page.items)}, {len(list(page.list_items()))}") - if len(page.items) == len(list(page.list_items())): - return JSONResponse({}, status_code=200) - else: - raise HTTPException(status_code=404, detail=f"page not ready: {extra_networks_tabname}") + return JSONResponse({"ready": page.is_ready}) def get_metadata(extra_networks_tabname: str = "", item: str = "") -> JSONResponse: @@ -1216,7 +1203,9 @@ def create_ui(interface: gr.Blocks, unrelated_tabs, tabname): def create_html(): for tabname_full, page in ui.stored_extra_pages.items(): + page.is_ready = False ui.pages_contents[tabname_full] = page.create_html(ui.tabname) + page.is_ready = True def pages_html(): if not ui.pages_contents: