From bd1dbf21d05ffa4fcc2c71f165e13cae045d071a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kiripolszky=20K=C3=A1roly?= Date: Fri, 27 Apr 2018 11:57:52 +0200 Subject: [PATCH 1/3] initial implementation of firmware cache --- manifest.json | 3 +- package.json | 1 + src/js/FirmwareCache.js | 183 ++++++++++++++++++++++++++++++++ src/js/tabs/firmware_flasher.js | 101 +++++++++++------- src/main.html | 2 + 5 files changed, 249 insertions(+), 41 deletions(-) create mode 100644 src/js/FirmwareCache.js diff --git a/manifest.json b/manifest.json index ecc1fbdf..baec36a1 100755 --- a/manifest.json +++ b/manifest.json @@ -34,7 +34,8 @@ {"usbDevices": [ {"vendorId": 1155, "productId": 57105} ]}, - "webview" + "webview", + "unlimitedStorage" ], "sockets": { "tcp": { diff --git a/package.json b/package.json index c7e3cd4f..e86c38fd 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "dependencies": { "i18next": "^10.3.0", "i18next-xhr-backend": "^1.5.1", + "lru_map": "^0.3.3", "marked": "^0.3.12" }, "devDependencies": { diff --git a/src/js/FirmwareCache.js b/src/js/FirmwareCache.js new file mode 100644 index 00000000..cfa6afa3 --- /dev/null +++ b/src/js/FirmwareCache.js @@ -0,0 +1,183 @@ +'use strict'; + +/** + * Caching of previously downloaded firmwares and release descriptions + * + * Depends on LRUMap for which the docs can be found here: + * https://github.com/rsms/js-lru + */ + +/** + * @typedef {object} Descriptor Release descriptor object + * @property {string} releaseUrl + * @property {string} name + * @property {string} version + * @property {string} url + * @property {string} file + * @property {string} target + * @property {string} date + * @property {string} notes + * @property {string} status + * @see buildBoardOptions() in {@link release_checker.js} + */ + +/** + * @typedef {object} CacheItem + * @property {Descriptor} release + * @property {string} [hexdata] + */ + + /** + * Manages caching of downloaded firmware files + */ +let FirmwareCache = (function() { + + let MetadataStorage = (function() { + let CACHEKEY = "firmware-cache-metadata"; + + /** + * @param {Array} data LRU key-value pairs + */ + function persist(data) { + let obj = {}; + obj[CACHEKEY] = data; + chrome.storage.local.set(obj); + } + + /** + * @param {Function} callback + */ + function load(callback) { + chrome.storage.local.get(CACHEKEY, obj => { + let entries = typeof obj === "object" && obj.hasOwnProperty(CACHEKEY) + ? obj[CACHEKEY] + : []; + callback(entries); + }); + } + + return { + persist: persist, + load: load, + }; + })(); + + let metadataCache = new LRUMap(100); + let metadataLoaded = false; + + metadataCache.shift = function() { + // remove hexdata for oldest release + let oldest = LRUMap.prototype.shift.call(this); + if (oldest !== undefined) { + /** @type {CacheItem} */ + let cached = oldest[1]; + let hexdataKey = withHexdataPrefix(keyOf(cached.release)); + chrome.storage.local.remove(hexdataKey, + () => console.debug("Hex data removed: " + hexdataKey)); + } + return oldest; + }; + + /** + * @param {Descriptor} release + * @returns {string} A key used for caching the metadata for a release + */ + function keyOf(release) { + return release.file; + } + + /** + * @param {string} key + * @returns {string} A key for storing the hex data for a release + */ + function withHexdataPrefix(key) { + return "hex:" + key; + } + + /** + * @param {Descriptor} release + * @returns {boolean} + */ + function has(release) { + if (!metadataLoaded) { + console.warn("Cache not yet loaded"); + return false; + } + return metadataCache.has(keyOf(release)); + } + + /** + * @param {Descriptor} release + * @param {string} hexdata + */ + function put(release, hexdata) { + if (!metadataLoaded) { + console.warn("Cache not yet loaded"); + return; + } + if (has(release)) { + console.debug("Firmware is already cached: " + keyOf(release)); + return; + } + let key = keyOf(release); + let hexdataKey = withHexdataPrefix(key); + metadataCache.set(key, { + release: release, + }); + MetadataStorage.persist(metadataCache.toJSON()); + let obj = {}; + obj[hexdataKey] = hexdata; + chrome.storage.local.set(obj); + } + + /** + * @param {Descriptor} release + * @param {Function} callback + * @returns {(CacheItem|undefined)} + */ + function get(release, callback) { + if (!metadataLoaded) { + console.warn("Cache not yet loaded"); + return undefined; + } + let key = keyOf(release); + /** @type {CacheItem} */ + let cached = metadataCache.get(key); + if (cached !== undefined) { + let hexdataKey = withHexdataPrefix(key); + chrome.storage.local.get(hexdataKey, function(obj) { + cached.hexdata = typeof obj === "object" && obj.hasOwnProperty(hexdataKey) + ? obj[hexdataKey] + : null; + callback(cached); + }); + } + return cached; + } + + /** + * @param {Array} entries + */ + function onEntriesLoaded(entries) { + let pairs = []; + for (let entry of entries) { + pairs.push([entry.key, entry.value]); + } + metadataCache.assign(pairs); + metadataLoaded = true; + console.info("Firmware cache loaded; number of entries: " + entries.length); + } + + return { + has: has, + put: put, + get: get, + load: () => { + MetadataStorage.load(onEntriesLoaded); + }, + flush: () => { + MetadataStorage.persist(metadataCache.toJSON()); + metadataCache.clear(); + }, + }; +})(); diff --git a/src/js/tabs/firmware_flasher.js b/src/js/tabs/firmware_flasher.js index c178ca86..25390330 100755 --- a/src/js/tabs/firmware_flasher.js +++ b/src/js/tabs/firmware_flasher.js @@ -17,6 +17,8 @@ TABS.firmware_flasher.initialize = function (callback) { parsed_hex = false; // parsed raw hex in array format $('#content').load("./tabs/firmware_flasher.html", function () { + FirmwareCache.load(); + function parse_hex(str, callback) { // parsing hex in different thread var worker = new Worker('./js/workers/hex_parser.js'); @@ -30,6 +32,54 @@ TABS.firmware_flasher.initialize = function (callback) { worker.postMessage(str); } + function process_hex(data, summary) { + intel_hex = data; + + parse_hex(intel_hex, function (data) { + parsed_hex = data; + + if (parsed_hex) { + if (!FirmwareCache.has(summary)) { + FirmwareCache.put(summary, intel_hex); + console.info("Release put to cache: " + summary.file); + } + + var url; + + $('span.progressLabel').html('Loaded Online Firmware: (' + parsed_hex.bytes_total + ' bytes)'); + + $('a.flash_firmware').removeClass('disabled'); + + $('div.release_info .target').text(summary.target); + $('div.release_info .name').text(summary.version).prop('href', summary.releaseUrl); + $('div.release_info .date').text(summary.date); + $('div.release_info .status').text(summary.status); + $('div.release_info .file').text(summary.file).prop('href', summary.url); + + var formattedNotes = summary.notes.replace(/#(\d+)/g, '[#$1](https://github.com/betaflight/betaflight/pull/$1)'); + formattedNotes = marked(formattedNotes); + $('div.release_info .notes').html(formattedNotes); + $('div.release_info .notes').find('a').each(function() { + $(this).attr('target', '_blank'); + }); + + $('div.release_info').slideDown(); + + } else { + $('span.progressLabel').text(i18n.getMessage('firmwareFlasherHexCorrupted')); + } + }); + } + + function onLoadSuccess(data, summary) { + summary = typeof summary === "object" + ? summary + : $('select[name="firmware_version"] option:selected').data('summary'); + process_hex(data, summary); + $("a.load_remote_file").removeClass('disabled'); + $("a.load_remote_file").text(i18n.getMessage('firmwareFlasherButtonLoadOnline')); + }; + function buildBoardOptions(releaseData) { if (!releaseData) { $('select[name="board"]').empty().append(''); @@ -226,7 +276,15 @@ TABS.firmware_flasher.initialize = function (callback) { $('select[name="firmware_version"]').change(function(evt){ $('div.release_info').slideUp(); $('a.flash_firmware').addClass('disabled'); - if (evt.target.value=="0") { + let release = $("option:selected", evt.target).data("summary"); + let isCached = FirmwareCache.has(release); + if (evt.target.value=="0" || isCached) { + if (isCached) { + FirmwareCache.get(release, cached => { + console.info("Release found in cache: " + release.file); + onLoadSuccess(cached.hexdata, release); + }); + } $("a.load_remote_file").addClass('disabled'); } else { @@ -241,40 +299,6 @@ TABS.firmware_flasher.initialize = function (callback) { return; } - function process_hex(data, summary) { - intel_hex = data; - - parse_hex(intel_hex, function (data) { - parsed_hex = data; - - if (parsed_hex) { - var url; - - $('span.progressLabel').html('Loaded Online Firmware: (' + parsed_hex.bytes_total + ' bytes)'); - - $('a.flash_firmware').removeClass('disabled'); - - $('div.release_info .target').text(summary.target); - $('div.release_info .name').text(summary.version).prop('href', summary.releaseUrl); - $('div.release_info .date').text(summary.date); - $('div.release_info .status').text(summary.status); - $('div.release_info .file').text(summary.file).prop('href', summary.url); - - var formattedNotes = summary.notes.replace(/#(\d+)/g, '[#$1](https://github.com/betaflight/betaflight/pull/$1)'); - formattedNotes = marked(formattedNotes); - $('div.release_info .notes').html(formattedNotes); - $('div.release_info .notes').find('a').each(function() { - $(this).attr('target', '_blank'); - }); - - $('div.release_info').slideDown(); - - } else { - $('span.progressLabel').text(i18n.getMessage('firmwareFlasherHexCorrupted')); - } - }); - } - function failed_to_load() { $('span.progressLabel').text(i18n.getMessage('firmwareFlasherFailedToLoadOnlineFirmware')); $('a.flash_firmware').addClass('disabled'); @@ -286,11 +310,7 @@ TABS.firmware_flasher.initialize = function (callback) { if (summary) { // undefined while list is loading or while running offline $("a.load_remote_file").text(i18n.getMessage('firmwareFlasherButtonDownloading')); $("a.load_remote_file").addClass('disabled'); - $.get(summary.url, function (data) { - process_hex(data, summary); - $("a.load_remote_file").removeClass('disabled'); - $("a.load_remote_file").text(i18n.getMessage('firmwareFlasherButtonLoadOnline')); - }).fail(failed_to_load); + $.get(summary.url, onLoadSuccess).fail(failed_to_load); } else { $('span.progressLabel').text(i18n.getMessage('firmwareFlasherFailedToLoadOnlineFirmware')); } @@ -507,6 +527,7 @@ TABS.firmware_flasher.initialize = function (callback) { TABS.firmware_flasher.cleanup = function (callback) { PortHandler.flush_callbacks(); + FirmwareCache.flush(); // unbind "global" events $(document).unbind('keypress'); diff --git a/src/main.html b/src/main.html index 5fdc6bf0..2507775f 100755 --- a/src/main.html +++ b/src/main.html @@ -35,6 +35,7 @@ + @@ -96,6 +97,7 @@ + From caef8dbf3c7f2392a33d32d9de035eaacdaa10f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kiripolszky=20K=C3=A1roly?= Date: Tue, 1 May 2018 11:30:36 +0200 Subject: [PATCH 2/3] bold label for downloaded firmwares --- src/js/tabs/firmware_flasher.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/js/tabs/firmware_flasher.js b/src/js/tabs/firmware_flasher.js index 25390330..05cbd35f 100755 --- a/src/js/tabs/firmware_flasher.js +++ b/src/js/tabs/firmware_flasher.js @@ -210,7 +210,12 @@ TABS.firmware_flasher.initialize = function (callback) { descriptor.target, descriptor.date, descriptor.status - )).data('summary', descriptor); + )) + .css("font-weight", FirmwareCache.has(descriptor) + ? "bold" + : null + ) + .data('summary', descriptor); versions_e.append(select_e); }); From 54bdd2da144aa7697180c62e659446778666c79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kiripolszky=20K=C3=A1roly?= Date: Wed, 2 May 2018 14:47:41 +0200 Subject: [PATCH 3/3] FirmwareCache: memory footprint and UI improvements --- src/js/FirmwareCache.js | 147 +++++++++++++++++++------------- src/js/tabs/firmware_flasher.js | 17 +++- 2 files changed, 105 insertions(+), 59 deletions(-) diff --git a/src/js/FirmwareCache.js b/src/js/FirmwareCache.js index cfa6afa3..dda1b8f1 100644 --- a/src/js/FirmwareCache.js +++ b/src/js/FirmwareCache.js @@ -24,16 +24,19 @@ /** * @typedef {object} CacheItem * @property {Descriptor} release - * @property {string} [hexdata] + * @property {string} hexdata */ - /** - * Manages caching of downloaded firmware files - */ -let FirmwareCache = (function() { +/** + * Manages caching of downloaded firmware files + */ +let FirmwareCache = (function () { - let MetadataStorage = (function() { - let CACHEKEY = "firmware-cache-metadata"; + let onPutToCacheCallback, + onRemoveFromCacheCallback; + + let JournalStorage = (function () { + let CACHEKEY = "firmware-cache-journal"; /** * @param {Array} data LRU key-value pairs @@ -49,7 +52,7 @@ let FirmwareCache = (function() { */ function load(callback) { chrome.storage.local.get(CACHEKEY, obj => { - let entries = typeof obj === "object" && obj.hasOwnProperty(CACHEKEY) + let entries = typeof obj === "object" && obj.hasOwnProperty(CACHEKEY) ? obj[CACHEKEY] : []; callback(entries); @@ -62,25 +65,33 @@ let FirmwareCache = (function() { }; })(); - let metadataCache = new LRUMap(100); - let metadataLoaded = false; + let journal = new LRUMap(100), + journalLoaded = false; - metadataCache.shift = function() { - // remove hexdata for oldest release + journal.shift = function () { + // remove cached data for oldest release let oldest = LRUMap.prototype.shift.call(this); - if (oldest !== undefined) { - /** @type {CacheItem} */ - let cached = oldest[1]; - let hexdataKey = withHexdataPrefix(keyOf(cached.release)); - chrome.storage.local.remove(hexdataKey, - () => console.debug("Hex data removed: " + hexdataKey)); + if (oldest === undefined) { + return undefined; } + let key = oldest[0]; + let cacheKey = withCachePrefix(key); + chrome.storage.local.get(cacheKey, obj => { + /** @type {CacheItem} */ + let cached = typeof obj === "object" && obj.hasOwnProperty(cacheKey) + ? obj[cacheKey] + : null; + chrome.storage.local.remove(cacheKey, () => { + onRemoveFromCache(cached.release); + console.debug("Cache data removed: " + cacheKey); + }); + }); return oldest; }; /** * @param {Descriptor} release - * @returns {string} A key used for caching the metadata for a release + * @returns {string} A key used to store a release in the journal */ function keyOf(release) { return release.file; @@ -88,10 +99,10 @@ let FirmwareCache = (function() { /** * @param {string} key - * @returns {string} A key for storing the hex data for a release + * @returns {string} A key for storing cached data for a release */ - function withHexdataPrefix(key) { - return "hex:" + key; + function withCachePrefix(key) { + return "cache:" + key; } /** @@ -99,11 +110,11 @@ let FirmwareCache = (function() { * @returns {boolean} */ function has(release) { - if (!metadataLoaded) { + if (!journalLoaded) { console.warn("Cache not yet loaded"); return false; } - return metadataCache.has(keyOf(release)); + return journal.has(keyOf(release)); } /** @@ -111,48 +122,68 @@ let FirmwareCache = (function() { * @param {string} hexdata */ function put(release, hexdata) { - if (!metadataLoaded) { - console.warn("Cache not yet loaded"); - return; - } - if (has(release)) { - console.debug("Firmware is already cached: " + keyOf(release)); + if (!journalLoaded) { + console.warn("Cache journal not yet loaded"); return; } let key = keyOf(release); - let hexdataKey = withHexdataPrefix(key); - metadataCache.set(key, { - release: release, - }); - MetadataStorage.persist(metadataCache.toJSON()); + if (has(release)) { + console.debug("Firmware is already cached: " + key); + return; + } + journal.set(key, true); + JournalStorage.persist(journal.toJSON()); let obj = {}; - obj[hexdataKey] = hexdata; - chrome.storage.local.set(obj); + obj[withCachePrefix(key)] = { + release: release, + hexdata: hexdata, + }; + chrome.storage.local.set(obj, () => { + console.info("Release put to cache: " + key); + onPutToCache(release); + }); } /** * @param {Descriptor} release * @param {Function} callback - * @returns {(CacheItem|undefined)} */ function get(release, callback) { - if (!metadataLoaded) { - console.warn("Cache not yet loaded"); + if (!journalLoaded) { + console.warn("Cache journal not yet loaded"); return undefined; } let key = keyOf(release); - /** @type {CacheItem} */ - let cached = metadataCache.get(key); - if (cached !== undefined) { - let hexdataKey = withHexdataPrefix(key); - chrome.storage.local.get(hexdataKey, function(obj) { - cached.hexdata = typeof obj === "object" && obj.hasOwnProperty(hexdataKey) - ? obj[hexdataKey] - : null; - callback(cached); - }); + if (!has(release)) { + console.debug("Firmware is not cached: " + key); + return; + } + let cacheKey = withCachePrefix(key); + chrome.storage.local.get(cacheKey, obj => { + /** @type {CacheItem} */ + let cached = typeof obj === "object" && obj.hasOwnProperty(cacheKey) + ? obj[cacheKey] + : null; + callback(cached); + }); + } + + /** + * @param {Descriptor} release + */ + function onPutToCache(release) { + if (typeof onPutToCacheCallback === "function") { + onPutToCacheCallback(release); + } + } + + /** + * @param {Descriptor} release + */ + function onRemoveFromCache(release) { + if (typeof onRemoveFromCacheCallback === "function") { + onRemoveFromCacheCallback(release); } - return cached; } /** @@ -163,21 +194,23 @@ let FirmwareCache = (function() { for (let entry of entries) { pairs.push([entry.key, entry.value]); } - metadataCache.assign(pairs); - metadataLoaded = true; - console.info("Firmware cache loaded; number of entries: " + entries.length); + journal.assign(pairs); + journalLoaded = true; + console.info("Firmware cache journal loaded; number of entries: " + entries.length); } return { has: has, put: put, get: get, + onPutToCache: callback => onPutToCacheCallback = callback, + onRemoveFromCache: callback => onRemoveFromCacheCallback = callback, load: () => { - MetadataStorage.load(onEntriesLoaded); + JournalStorage.load(onEntriesLoaded); }, flush: () => { - MetadataStorage.persist(metadataCache.toJSON()); - metadataCache.clear(); + JournalStorage.persist(journal.toJSON()); + journal.clear(); }, }; })(); diff --git a/src/js/tabs/firmware_flasher.js b/src/js/tabs/firmware_flasher.js index 05cbd35f..92812c15 100755 --- a/src/js/tabs/firmware_flasher.js +++ b/src/js/tabs/firmware_flasher.js @@ -16,8 +16,22 @@ TABS.firmware_flasher.initialize = function (callback) { var intel_hex = false, // standard intel hex in string format parsed_hex = false; // parsed raw hex in array format + /** + * Change boldness of firmware option depending on cache status + * + * @param {Descriptor} release + */ + function onFirmwareCacheUpdate(release) { + $("option[value='{0}']".format(release.version)) + .css("font-weight", FirmwareCache.has(release) + ? "bold" + : "normal"); + } + $('#content').load("./tabs/firmware_flasher.html", function () { FirmwareCache.load(); + FirmwareCache.onPutToCache(onFirmwareCacheUpdate); + FirmwareCache.onRemoveFromCache(onFirmwareCacheUpdate); function parse_hex(str, callback) { // parsing hex in different thread @@ -41,7 +55,6 @@ TABS.firmware_flasher.initialize = function (callback) { if (parsed_hex) { if (!FirmwareCache.has(summary)) { FirmwareCache.put(summary, intel_hex); - console.info("Release put to cache: " + summary.file); } var url; @@ -213,7 +226,7 @@ TABS.firmware_flasher.initialize = function (callback) { )) .css("font-weight", FirmwareCache.has(descriptor) ? "bold" - : null + : "normal" ) .data('summary', descriptor);