Add runtime Lua script pre-compilation and general .luac file support (#3318) (#4119)

* [simu] simpgmspace: Populate file date/time/size  in f_stat(); Add f_utime(); Fix f_getcwd() on Windows; Fix f_mkdir() build error on Windows.

* [Lua] Add runtime support for script pre-compilation. Changes behavior with LUA_COMPILER (Re: https://github.com/opentx/opentx/issues/3318):
  All .lua scripts are now compiled and saved to binary "bytecode" file (.luac extension) upon first execution;
  Scripts are also automatically re-compiled if .lua source file is newer than existing .luac file;
  The pre-compiled .luac version is loaded if modification time is newer or equivalent to .lua source file;
  If a .luac version of a script exists, then the .lua version does not need to be present;
  Guards against bytecode compatibility issues (binaries from 64-bit sim will not run on 32-bit sim/radio);
  In SIMU and DEBUG builds, the source .lua file is always preferred in order to preserve full debug info (this is is controlled with new LUA_SCRIPT_LOAD_MODE macro, see lua_api.h);
  GC is now run after each script is loaded.

* [Lua] Add loadScript() API function as alternative to loadfile() from Lua base. This can take advantage of the new OTx script pre-compilation features to reduce memory footprint when loading functions dynamically. This is an interface to luaLoadScriptFileToState(). Fully documented.

* [SD][Lua] Flexible file extensions support:
  Allow for variable length file extensions throughout system (no longer hard-coded in LEN_FILE_EXTENSION);
  Fixes issues with renaming files in SD manager which have file extensions longer than ".ext";
  Expand general support for multiple file extensions per file type in sdListFiles() (eg. .lua and .luac for scripts);
  Lua scripts with .luac extensions can now be selected in custom/telemetry/function menus even if no .lua version exists (duplicates are not shown);
  .luac files can now also be executed from SD file manager UI.

* [Build] Added CMake options for LUA_COMPILER and LUA_SCRIPT_LOAD_MODE.

* Cosmetics.

* [SD][gui] Improve efficiency of some file name handling routines in sdmanager GUI and sdListFiles() by extending getFileExtension() function. Use shared isExtensionMatching() in place of isImageFileExtension(). Only allow executing .luac files when LUA_COMPILER defined (as per request).

* [simpgmspace] Fix f_mkdir() for MSVC build and misc. cleanup.

* [Lua] Use getFileExtension() in script loader to determine file type and check for buffer overflow.

radio/src/CMakeLists.txt

@@ -172,6 +172,12 @@ endif()
 
 if(NOT LUA STREQUAL NO)
   add_definitions(-DLUA)
+  if(LUA_COMPILER)
+    add_definitions(-DLUA_COMPILER)
+  endif()
+  if(NOT "${LUA_SCRIPT_LOAD_MODE}" STREQUAL "")
+    add_definitions(-DLUA_SCRIPT_LOAD_MODE="${LUA_SCRIPT_LOAD_MODE}")
+  endif()
   include_directories(${LUA_DIR})
   set(FIRMWARE_DEPENDENCIES ${FIRMWARE_DEPENDENCIES} ${LUA_EXPORT})
   if(LUA STREQUAL YES)

radio/src/gui/480x272/radio_sdmanager.cpp

@@ -29,11 +29,6 @@
 int currentBitmapIndex = 0;
 BitmapBuffer * currentBitmap = NULL;
 
-bool isImageFileExtension(const char * ext)
-{
-  return (ext && (!strcasecmp(ext, BMP_EXT) || !strcasecmp(ext, JPG_EXT) || !strcasecmp(ext, PNG_EXT)));
-}
-
 bool menuRadioSdManagerInfo(event_t event)
 {
   SIMPLE_SUBMENU(STR_SD_INFO_TITLE, ICON_RADIO_SD_BROWSER, 1);
@@ -109,17 +104,11 @@ void onSdManagerMenu(const char * result)
   }
   else if (result == STR_RENAME_FILE) {
     memcpy(reusableBuffer.sdmanager.originalName, line, sizeof(reusableBuffer.sdmanager.originalName));
-    char * ext = getFileExtension(line, SD_SCREEN_FILE_LENGTH+1);
+    uint8_t fnlen = 0, extlen = 0;
-    if (ext) {
+    getFileExtension(line, 0, LEN_FILE_EXTENSION_MAX, &fnlen, &extlen);
-      // write spaces to allow a longer filename
+    // write spaces to allow extending the length of a filename
-      memset(ext, ' ', SD_SCREEN_FILE_LENGTH-LEN_FILE_EXTENSION-(ext-line));
+    memset(line + fnlen - extlen, ' ', SD_SCREEN_FILE_LENGTH - fnlen + extlen);
-      line[SD_SCREEN_FILE_LENGTH-LEN_FILE_EXTENSION] = '\0';
+    line[SD_SCREEN_FILE_LENGTH-extlen] = '\0';
-    }
-    else {
-      int len = strlen(line);
-      memset(line + strlen(line), ' ', SD_SCREEN_FILE_LENGTH - len);
-      line[SD_SCREEN_FILE_LENGTH] = '\0';
-    }
     s_editMode = EDIT_MODIFY_STRING;
     editNameCursorPos = 0;
   }
@@ -224,18 +213,18 @@ bool menuRadioSdManager(event_t _event)
       if (s_editMode == 0) {
         killEvents(_event);
         char * line = reusableBuffer.sdmanager.lines[index];
-        char * ext = getFileExtension(line, SD_SCREEN_FILE_LENGTH+1);
         if (!strcmp(line, "..")) {
           break; // no menu for parent dir
         }
+        char * ext = getFileExtension(line);
         if (ext) {
           if (!strcasecmp(ext, SOUNDS_EXT)) {
             POPUP_MENU_ADD_ITEM(STR_PLAY_FILE);
           }
-          else if (isImageFileExtension(ext)) {
+          else if (isExtensionMatching(ext, BITMAPS_EXT)) {
             TCHAR lfn[_MAX_LFN+1];
             f_getcwd(lfn, _MAX_LFN);
-            if (!READ_ONLY() && unsigned(LEN_FILE_EXTENSION+ext-line) <= sizeof(g_model.header.bitmap) && !strcmp(lfn, BITMAPS_PATH)) {
+            if (!READ_ONLY() && unsigned(strlen(ext)+ext-line) <= sizeof(g_model.header.bitmap) && !strcmp(lfn, BITMAPS_PATH)) {
               POPUP_MENU_ADD_ITEM(STR_ASSIGN_BITMAP);
             }
           }
@@ -243,7 +232,7 @@ bool menuRadioSdManager(event_t _event)
             POPUP_MENU_ADD_ITEM(STR_VIEW_TEXT);
           }
 #if defined(LUA)
-          else if (!strcasecmp(ext, SCRIPTS_EXT)) {
+          else if (isExtensionMatching(ext, SCRIPTS_EXT)) {
             POPUP_MENU_ADD_ITEM(STR_EXECUTE_FILE);
           }
 #endif
@@ -347,15 +336,16 @@ bool menuRadioSdManager(event_t _event)
     LcdFlags attr = (index == i ? INVERS : 0);
     if (reusableBuffer.sdmanager.lines[i][0]) {
       if (s_editMode == EDIT_MODIFY_STRING && attr) {
-        editName(MENUS_MARGIN_LEFT, y, reusableBuffer.sdmanager.lines[i], SD_SCREEN_FILE_LENGTH-4, _event, attr, 0);
+        uint8_t extlen, efflen;
+        char * ext = getFileExtension(reusableBuffer.sdmanager.originalName, 0, 0, NULL, &extlen);
+        editName(MENUS_MARGIN_LEFT, y, reusableBuffer.sdmanager.lines[i], SD_SCREEN_FILE_LENGTH - extlen, _event, attr, 0);
+        efflen = effectiveLen(reusableBuffer.sdmanager.lines[i], SD_SCREEN_FILE_LENGTH - extlen);
         if (s_editMode == 0) {
-          unsigned int len = effectiveLen(reusableBuffer.sdmanager.lines[i], SD_SCREEN_FILE_LENGTH-LEN_FILE_EXTENSION);
-          char * ext = getFileExtension(reusableBuffer.sdmanager.originalName, sizeof(reusableBuffer.sdmanager.originalName));
           if (ext) {
-            strAppend(&reusableBuffer.sdmanager.lines[i][len], ext);
+            strAppend(&reusableBuffer.sdmanager.lines[i][efflen], ext);
           }
           else {
-            reusableBuffer.sdmanager.lines[i][len] = 0;
+            reusableBuffer.sdmanager.lines[i][efflen] = 0;
           }
           f_rename(reusableBuffer.sdmanager.originalName, reusableBuffer.sdmanager.lines[i]);
           REFRESH_FILES();
@@ -376,8 +366,8 @@ bool menuRadioSdManager(event_t _event)
     }
   }
 
-  char * ext = getFileExtension(reusableBuffer.sdmanager.lines[index], SD_SCREEN_FILE_LENGTH+1);
+  char * ext = getFileExtension(reusableBuffer.sdmanager.lines[index]);
-  if (isImageFileExtension(ext)) {
+  if (ext && isExtensionMatching(ext, BITMAPS_EXT)) {
     if (currentBitmapIndex != menuVerticalPosition) {
       currentBitmapIndex = menuVerticalPosition;
       delete currentBitmap;

radio/src/gui/common/stdlcd/radio_sdmanager.cpp

@@ -103,17 +103,11 @@ void onSdManagerMenu(const char * result)
   }
   else if (result == STR_RENAME_FILE) {
     memcpy(reusableBuffer.sdmanager.originalName, line, sizeof(reusableBuffer.sdmanager.originalName));
-    char * ext = getFileExtension(line, SD_SCREEN_FILE_LENGTH+1);
+    uint8_t fnlen = 0, extlen = 0;
-    if (ext) {
+    getFileExtension(line, 0, LEN_FILE_EXTENSION_MAX, &fnlen, &extlen);
-      // write spaces to allow a longer filename
+    // write spaces to allow extending the length of a filename
-      memset(ext, ' ', SD_SCREEN_FILE_LENGTH-LEN_FILE_EXTENSION-(ext-line));
+    memset(line + fnlen - extlen, ' ', SD_SCREEN_FILE_LENGTH - fnlen + extlen);
-      line[SD_SCREEN_FILE_LENGTH-LEN_FILE_EXTENSION] = '\0';
+    line[SD_SCREEN_FILE_LENGTH-extlen] = '\0';
-    }
-    else {
-      int len = strlen(line);
-      memset(line + strlen(line), ' ', SD_SCREEN_FILE_LENGTH - len);
-      line[SD_SCREEN_FILE_LENGTH] = '\0';
-    }
     s_editMode = EDIT_MODIFY_STRING;
     editNameCursorPos = 0;
   }
@@ -261,13 +255,13 @@ void menuRadioSdManager(event_t _event)
           break; // no menu for parent dir
         }
 #if defined(CPUARM)
-        char * ext = getFileExtension(line, SD_SCREEN_FILE_LENGTH+1);
+        char * ext = getFileExtension(line);
         if (ext) {
           if (!strcasecmp(ext, SOUNDS_EXT)) {
             POPUP_MENU_ADD_ITEM(STR_PLAY_FILE);
           }
 #if LCD_DEPTH > 1
-          else if (!strcasecmp(ext, BITMAPS_EXT)) {
+          else if (isExtensionMatching(ext, BITMAPS_EXT)) {
             if (!READ_ONLY() && (ext-line) <= (int)sizeof(g_model.header.bitmap)) {
               POPUP_MENU_ADD_ITEM(STR_ASSIGN_BITMAP);
             }
@@ -277,7 +271,7 @@ void menuRadioSdManager(event_t _event)
             POPUP_MENU_ADD_ITEM(STR_VIEW_TEXT);
           }
 #if defined(LUA)
-          else if (!strcasecmp(ext, SCRIPTS_EXT)) {
+          else if (isExtensionMatching(ext, SCRIPTS_EXT)) {
             POPUP_MENU_ADD_ITEM(STR_EXECUTE_FILE);
           }
 #endif
@@ -402,15 +396,16 @@ void menuRadioSdManager(event_t _event)
       }
 #if defined(CPUARM)
       if (s_editMode == EDIT_MODIFY_STRING && attr) {
-        editName(lcdNextPos, y, reusableBuffer.sdmanager.lines[i], SD_SCREEN_FILE_LENGTH-4, _event, attr, 0);
+        uint8_t extlen, efflen;
+        char * ext = getFileExtension(reusableBuffer.sdmanager.originalName, 0, 0, NULL, &extlen);
+        editName(lcdNextPos, y, reusableBuffer.sdmanager.lines[i], SD_SCREEN_FILE_LENGTH - extlen, _event, attr, 0);
+        efflen = effectiveLen(reusableBuffer.sdmanager.lines[i], SD_SCREEN_FILE_LENGTH - extlen);
         if (s_editMode == 0) {
-          unsigned int len = effectiveLen(reusableBuffer.sdmanager.lines[i], SD_SCREEN_FILE_LENGTH-LEN_FILE_EXTENSION);
-          char * ext = getFileExtension(reusableBuffer.sdmanager.originalName, sizeof(reusableBuffer.sdmanager.originalName));
           if (ext) {
-            strAppend(&reusableBuffer.sdmanager.lines[i][len], ext);
+            strAppend(&reusableBuffer.sdmanager.lines[i][efflen], ext);
           }
           else {
-            reusableBuffer.sdmanager.lines[i][len] = 0;
+            reusableBuffer.sdmanager.lines[i][efflen] = 0;
           }
           f_rename(reusableBuffer.sdmanager.originalName, reusableBuffer.sdmanager.lines[i]);
           REFRESH_FILES();
@@ -429,8 +424,8 @@ void menuRadioSdManager(event_t _event)
   }
 
 #if LCD_DEPTH > 1
-  char * ext = getFileExtension(reusableBuffer.sdmanager.lines[index], SD_SCREEN_FILE_LENGTH+1);
+  char * ext = getFileExtension(reusableBuffer.sdmanager.lines[index]);
-  if (ext && !strcasecmp(ext, BITMAPS_EXT)) {
+  if (ext && isExtensionMatching(ext, BITMAPS_EXT)) {
     if (lastPos != menuVerticalPosition) {
       if (!lcdLoadBitmap(modelBitmap, reusableBuffer.sdmanager.lines[index], MODEL_BITMAP_WIDTH, MODEL_BITMAP_HEIGHT)) {
         memcpy(modelBitmap, logo_taranis, MODEL_BITMAP_SIZE);

radio/src/lua/api_general.cpp

@@ -1131,6 +1131,78 @@ static int luaGetRSSI(lua_State * L)
   return 3;
 }
 
+/*luadoc
+@function loadScript(file [, mode], [,env])
+
+Load a Lua script file. This is similar to Lua's own loadfile() API method,  but it uses
+OpenTx's optional pre-compilation feature to (possibly) save memory and time during load.
+
+@param file (string) Full path and file name of script. The file extension is optional and ignored (see "mode" param to control
+  which extension will be used). However, if an extension is specified, it should be ".lua" (or ".luac"), otherwise it is treated
+  as part of the file name and the .lua/.luac will be appended to that.
+
+@param mode (string) (optional) Controls whether to force loading the text (.lua) or binary (.luac, that is, a pre-compiled file)
+  version of the script. By default OTx will load the newest version and compile a new binary if necessary (overwriting any
+  existing .luac version of the same script, and stripping some debug info like line numbers). You can use this to bypass the automatic
+  script version check and set specific compilation options in OpenTx.
+    Possible values are:
+      "b" only binary.
+      "t" only text.
+      "T" (default on simulator) prefer text but load binary if that is the only version available.
+      "bt" (default on radio) either binary or text, whichever is newer (binary preferred when timestamps are equal).
+      Add "x" to avoid automatic compilation of source file to .luac version.
+        Eg: "tx", "bx", or "btx".
+      Add "c" to force compilation of source file to .luac version (even if existing version is newer than source file).
+        Eg: "tc" or "btc" (forces "t", overrides "x").
+      Add "d" to keep extra debug info in the compiled binary.
+        Eg: "td", "btd", or "tcd" (no effect with just "b" or with "x").
+
+  Note that you will get an error if you specify "b" or "t" and that specific version of the file does not exist (eg. no .luac file when "b" is used).
+  Also note that mode is NOT passed on to Lua's loader function, so unlike with loadfile() the actual file content is not checked (as if no mode or "bt" were passed to loadfile()).
+
+@param env (integer) See documentation for Lua function loadfile().
+
+@returns Same as from Lua API loadfile() method.
+  If the script was loaded w/out errors then the loaded script (or "chunk") is returned as a function;
+  Otherwise, returns nil plus the error message
+
+@usage
+  fun, err = loadScript("/SCRIPTS/FUNCTIONS/print.lua")
+  if (fun ~= nil) then
+     fun("Hello from loadScript()")
+  else
+     print(err)
+  end
+
+@status current Introduced in 2.2.0
+*/
+static int luaLoadScript(lua_State * L)
+{
+  // this function is replicated pretty much verbatim from luaB_loadfile() and load_aux() in lbaselib.c
+  const char *fname = luaL_optstring(L, 1, NULL);
+  const char *mode = luaL_optstring(L, 2, NULL);
+  int env = (!lua_isnone(L, 3) ? 3 : 0);  // 'env' index or 0 if no 'env'
+  lua_settop(L, 0);
+  if (fname != NULL && luaLoadScriptFileToState(L, fname , mode) == SCRIPT_OK) {
+    if (env != 0) {  // 'env' parameter?
+      lua_pushvalue(L, env);  // environment for loaded function
+      if (!lua_setupvalue(L, -2, 1))  // set it as 1st upvalue
+        lua_pop(L, 1);  // remove 'env' if not used by previous call
+    }
+    return 1;
+  }
+  else {
+    // error (message should be on top of the stack)
+    if (!lua_isstring(L, -1)) {
+      // probably didn't find a file or had some other error before luaL_loadfile() was run
+      lua_pushfstring(L, "loadScript(\"%s\", \"%s\") error: File not found", (fname != NULL ? fname : "nul"), (mode != NULL ? mode : "bt"));
+    }
+    lua_pushnil(L);
+    lua_insert(L, -2);  // move nil before error message
+    return 2;  // return nil plus error message
+  }
+}
+
 const luaL_Reg opentxLib[] = {
   { "getTime", luaGetTime },
   { "getDateTime", luaGetDateTime },
@@ -1152,6 +1224,7 @@ const luaL_Reg opentxLib[] = {
   { "defaultChannel", luaDefaultChannel },
   { "getRSSI", luaGetRSSI },
   { "killEvents", luaKillEvents },
+  { "loadScript", luaLoadScript },
 #if LCD_DEPTH > 1 && !defined(COLORLCD)
   { "GREY", luaGrey },
 #endif

radio/src/lua/interface.cpp

@@ -18,17 +18,18 @@
  * GNU General Public License for more details.
  */
 
+/** @file Main interface layer handler for Lua API. */
+
 #include <ctype.h>
 #include <stdio.h>
 #include "opentx.h"
 #include "bin_allocator.h"
 #include "lua/lua_api.h"
+#include "sdcard.h"
 
-#if defined(LUA_COMPILER) && defined(SIMU)
+extern "C" {
-  extern "C" {
+  #include <lundump.h>
-    #include <lundump.h>
+}
-  }
-#endif
 
 #define PERMANENT_SCRIPTS_MAX_INSTRUCTIONS (10000/100)
 #define MANUAL_SCRIPTS_MAX_INSTRUCTIONS    (20000/100)
@@ -176,6 +177,28 @@ void luaRegisterLibraries(lua_State * L)
 #endif
 }
 
+void luaDoGc(lua_State * L)
+{
+  if (L) {
+    PROTECT_LUA() {
+      lua_gc(L, LUA_GCCOLLECT, 0);
+#if defined(SIMU) || defined(DEBUG)
+      static int lastgc = 0;
+      int gc = luaGetMemUsed(L);
+      if (gc != lastgc) {
+        lastgc = gc;
+        TRACE("GC Use: %dbytes", gc);
+      }
+#endif
+    }
+    else {
+      // we disable Lua for the rest of the session
+      luaDisable();
+    }
+    UNPROTECT_LUA();
+  }
+}
+
 void luaFree(lua_State * L, ScriptInternalData & sid)
 {
   PROTECT_LUA() {
@@ -187,15 +210,17 @@ void luaFree(lua_State * L, ScriptInternalData & sid)
       luaL_unref(L, LUA_REGISTRYINDEX, sid.background);
       sid.background = 0;
     }
-    lua_gc(L, LUA_GCCOLLECT, 0);
   }
   else {
     luaDisable();
   }
   UNPROTECT_LUA();
+
+  luaDoGc(L);
 }
 
-#if defined(LUA_COMPILER) && defined(SIMU)
+#if defined(LUA_COMPILER)
+/// callback for luaU_dump()
 static int luaDumpWriter(lua_State * L, const void* p, size_t size, void* u)
 {
   UNUSED(L);
@@ -204,64 +229,224 @@ static int luaDumpWriter(lua_State * L, const void* p, size_t size, void* u)
   return (result != FR_OK && !written);
 }
 
-void luaCompileAndSave(lua_State * L, const char *bytecodeName)
+/*
+  @fn luaDumpState(lua_State * L, const char * filename, const FILINFO * finfo, int stripDebug)
+
+  Save compiled bytecode from a given Lua stack to a file.
+
+  @param L The Lua stack to dump.
+  @param filename Full path and name of file to save to (typically with .luac extension).
+  @param finfo Can be NULL. If not NULL, sets timestamp of created file to match the one in finfo->fdate/ftime
+  @param stripDebug This is passed directly to luaU_dump()
+    1 = remove debug info from bytecode (smaller but errors are less informative)
+    0 = keep debug info
+*/
+static void luaDumpState(lua_State * L, const char * filename, const FILINFO * finfo, int stripDebug)
 {
   FIL D;
-  char srcName[1024];
+  if (f_open(&D, filename, FA_WRITE | FA_CREATE_ALWAYS) == FR_OK) {
-  strcpy(srcName, bytecodeName);
+    lua_lock(L);
-  strcat(srcName, ".src");
+    luaU_dump(L, getproto(L->top - 1), luaDumpWriter, &D, stripDebug);
+    lua_unlock(L);
+    if (f_close(&D) == FR_OK) {
+      if (finfo != NULL)
+        f_utime(filename, finfo);  // set the file mod time
+      TRACE("luaDumpState(%s): Saved bytecode to file.", filename);
+    }
+  } else
+    TRACE_ERROR("luaDumpState(%s): Error: Could not open output file.", filename);
+}
+#endif  // LUA_COMPILER
 
-  if (f_stat(srcName, 0) != FR_OK) {
+/**
-    return;   // no source to compile
+  @fn luaLoadScriptFileToState(lua_State * L, const char * filename, const char * mode)
+
+  Load a Lua script file into a given lua_State (stack).  May use OpenTx's optional pre-compilation
+   feature to save memory and time during load.
+
+  @param L (lua_State) the Lua stack to load into.
+
+  @param filename (string) full path and file name of script.
+
+  @param mode (string) controls whether the file can be text or binary (that is, a pre-compiled file).
+   Possible values are:
+    "b" only binary.
+    "t" only text.
+    "T" (default on simulator) prefer text but load binary if that is the only version available.
+    "bt" (default on radio) either binary or text, whichever is newer (binary preferred when timestamps are equal).
+    Add "x" to avoid automatic compilation of source file to .luac version.
+      Eg: "tx", "bx", or "btx".
+    Add "c" to force compilation of source file to .luac version (even if existing version is newer than source file).
+      Eg: "tc" or "btc" (forces "t", overrides "x").
+    Add "d" to keep extra debug info in the compiled binary.
+      Eg: "td", "btd", or "tcd" (no effect with just "b" or with "x").
+
+  @retval (int)
+  SCRIPT_OK on success (LUA_OK)
+  SCRIPT_NOFILE if file wasn't found for specified mode or Lua could not open file (LUA_ERRFILE)
+  SCRIPT_SYNTAX_ERROR if Lua returned a syntax error during pre/de-compilation (LUA_ERRSYNTAX)
+  SCRIPT_PANIC for Lua memory errors (LUA_ERRMEM or LUA_ERRGCMM)
+*/
+int luaLoadScriptFileToState(lua_State * L, const char * filename, const char * mode)
+{
+  if (luaState == INTERPRETER_PANIC) {
+    return SCRIPT_PANIC;
+  } else if (filename == NULL) {
+    return SCRIPT_NOFILE;
   }
 
-  if (f_open(&D, bytecodeName, FA_WRITE | FA_CREATE_ALWAYS) != FR_OK) {
+  int lstatus;
-    TRACE("Could not open Lua bytecode output file %s", bytecodeName);
+  char lmode[6] = "bt";
-    return;
+  uint8_t ret = SCRIPT_NOFILE;
+
+  if (mode != NULL) {
+    strncpy(lmode, mode, sizeof(lmode)-1);
+    lmode[sizeof(lmode)-1] = '\0';
   }
 
-  PROTECT_LUA() {
+#if defined(LUA_COMPILER)
-    if (luaL_loadfile(L, srcName) == 0) {
+  uint16_t fnamelen;
-      lua_lock(L);
+  uint8_t extlen;
-      luaU_dump(L, getproto(L->top - 1), luaDumpWriter, &D, 1);
+  char filenameFull[LEN_FILE_PATH_MAX + _MAX_LFN + 1] = "\0";
-      lua_unlock(L);
+  FILINFO fnoLuaS, fnoLuaC;
-      TRACE("Saved Lua bytecode to file %s", bytecodeName);
+  FRESULT frLuaS, frLuaC;
+
+  bool scriptNeedsCompile = false;
+  uint8_t loadFileType = 0;  // 1=text, 2=binary
+
+  memset(&fnoLuaS, 0, sizeof(FILINFO));
+  memset(&fnoLuaC, 0, sizeof(FILINFO));
+
+  fnamelen = strlen(filename);
+  // check if file extension is already in the file name and strip it
+  getFileExtension(filename, fnamelen, 0, NULL, &extlen);
+  fnamelen -= extlen;
+  if (fnamelen > sizeof(filenameFull) - sizeof(SCRIPT_BIN_EXT)) {
+    TRACE_ERROR("luaLoadScriptFileToState(%s, %s): Error loading script: filename buffer overflow.\n", filename, lmode);
+    return ret;
+  }
+  strncat(filenameFull, filename, fnamelen);
+
+  // check if binary version exists
+  strcpy(filenameFull + fnamelen, SCRIPT_BIN_EXT);
+  frLuaC = f_stat(filenameFull, &fnoLuaC);
+
+  // check if text version exists
+  strcpy(filenameFull + fnamelen, SCRIPT_EXT);
+  frLuaS = f_stat(filenameFull, &fnoLuaS);
+
+  // decide which version to load, text or binary
+  if (frLuaC != FR_OK && frLuaS == FR_OK) {
+    // only text version exists
+    loadFileType = 1;
+    scriptNeedsCompile = true;
+  }
+  else if (frLuaC == FR_OK && frLuaS != FR_OK) {
+    // only binary version exists
+    loadFileType = 2;
+  }
+  else if (frLuaS == FR_OK) {
+    // both versions exist, compare them
+    if (strchr(lmode, 'c') || (uint32_t)((fnoLuaC.fdate << 16) + fnoLuaC.ftime) < (uint32_t)((fnoLuaS.fdate << 16) + fnoLuaS.ftime)) {
+      // text version is newer than binary or forced by "c" mode flag, rebuild it
+      scriptNeedsCompile = true;
+    }
+    if (scriptNeedsCompile || !strchr(lmode, 'b')) {
+      // text version needs compilation or forced by mode
+      loadFileType = 1;
+    } else {
+      // use binary file
+      loadFileType = 2;
     }
   }
-  UNPROTECT_LUA();
+  // else both versions are missing
-  f_close(&D);
+
-}
+  // skip compilation based on mode flags? ("c" overrides "x")
+  if (scriptNeedsCompile && strchr(lmode, 'x') && !strchr(lmode, 'c')) {
+    scriptNeedsCompile = false;
+  }
+
+  if (loadFileType == 2) {
+    // change file extension to binary version
+    strcpy(filenameFull + fnamelen, SCRIPT_BIN_EXT);
+  }
+
+//  TRACE_DEBUG("luaLoadScriptFileToState(%s, %s):\n", filename, lmode);
+//  TRACE_DEBUG("\tldfile='%s'; ldtype=%u; compile=%u;\n", filenameFull, loadFileType, scriptNeedsCompile);
+//  TRACE_DEBUG("\t%-5s: %s; mtime: %04X%04X = %u/%02u/%02u %02u:%02u:%02u;\n", SCRIPT_EXT, (frLuaS == FR_OK ? "ok" : "nf"), fnoLuaS.fdate, fnoLuaS.ftime,
+//      (fnoLuaS.fdate >> 9) + 1980, (fnoLuaS.fdate >> 5) & 15, fnoLuaS.fdate & 31, fnoLuaS.ftime >> 11, (fnoLuaS.ftime >> 5) & 63, (fnoLuaS.ftime & 31) * 2);
+//  TRACE_DEBUG("\t%-5s: %s; mtime: %04X%04X = %u/%02u/%02u %02u:%02u:%02u;\n", SCRIPT_BIN_EXT, (frLuaC == FR_OK ? "ok" : "nf"), fnoLuaC.fdate, fnoLuaC.ftime,
+//      (fnoLuaC.fdate >> 9) + 1980, (fnoLuaC.fdate >> 5) & 15, fnoLuaC.fdate & 31, fnoLuaC.ftime >> 11, (fnoLuaC.ftime >> 5) & 63, (fnoLuaC.ftime & 31) * 2);
+
+  // final check that file exists and is allowed by mode flags
+  if (!loadFileType || (loadFileType == 1 && !strpbrk(lmode, "tTc")) || (loadFileType == 2 && !strpbrk(lmode, "bT"))) {
+    TRACE_ERROR("luaLoadScriptFileToState(%s, %s): Error loading script: file not found.\n", filename, lmode);
+    return SCRIPT_NOFILE;
+  }
+
+#else  // !defined(LUA_COMPILER)
+
+  // use passed file name as-is
+  const char *filenameFull = filename;
+
 #endif
 
-int luaLoad(lua_State * L, const char * filename, ScriptInternalData & sid, ScriptInputsOutputs * sio=NULL)
+  TRACE("luaLoadScriptFileToState(%s, %s): loading %s", filename, lmode, filenameFull);
+
+  // we don't pass <mode> on to loadfilex() because we want lua to load whatever file we specify, regardless of content
+  lstatus = luaL_loadfilex(L, filenameFull, NULL);
+#if defined(LUA_COMPILER)
+  // Check for bytecode encoding problem, eg. compiled for x64. Unfortunately Lua doesn't provide a unique error code for this. See Lua/src/lundump.c.
+  if (lstatus == LUA_ERRSYNTAX && loadFileType == 2 && frLuaS == FR_OK && strstr(lua_tostring(L, -1), "precompiled")) {
+    loadFileType = 1;
+    scriptNeedsCompile = true;
+    strcpy(filenameFull + fnamelen, SCRIPT_EXT);
+    TRACE_ERROR("luaLoadScriptFileToState(%s, %s): Error loading script: %s\n\tRetrying with %s\n", filename, lmode, lua_tostring(L, -1), filenameFull);
+    lstatus = luaL_loadfilex(L, filenameFull, NULL);
+  }
+  if (lstatus == LUA_OK) {
+    if (scriptNeedsCompile && loadFileType == 1) {
+      strcpy(filenameFull + fnamelen, SCRIPT_BIN_EXT);
+      luaDumpState(L, filenameFull, &fnoLuaS, (strchr(lmode, 'd') ? 0 : 1));
+    }
+    ret = SCRIPT_OK;
+  }
+#else
+  if (lstatus == LUA_OK) {
+    ret = SCRIPT_OK;
+  }
+#endif
+  else {
+    TRACE_ERROR("luaLoadScriptFileToState(%s, %s): Error loading script: %s\n", filename, lmode, lua_tostring(L, -1));
+    if (lstatus == LUA_ERRFILE) {
+      ret = SCRIPT_NOFILE;
+    } else if (lstatus == LUA_ERRSYNTAX) {
+      ret = SCRIPT_SYNTAX_ERROR;
+    } else {  //  LUA_ERRMEM or LUA_ERRGCMM
+      ret = SCRIPT_PANIC;
+    }
+  }
+
+  return ret;
+}
+
+static int luaLoad(lua_State * L, const char * filename, ScriptInternalData & sid, ScriptInputsOutputs * sio=NULL)
 {
   int init = 0;
+  int lstatus = 0;
 
   sid.instructions = 0;
   sid.state = SCRIPT_OK;
 
-#if 0
-  // not needed, we just called luaInit
-  luaFree(sid);
-#endif
-
   if (luaState == INTERPRETER_PANIC) {
     return SCRIPT_PANIC;
   }
 
-#if defined(LUA_COMPILER) && defined(SIMU)
-  luaCompileAndSave(L, filename);
-#endif
-
   luaSetInstructionsLimit(L, MANUAL_SCRIPTS_MAX_INSTRUCTIONS);
 
   PROTECT_LUA() {
-    if (luaL_loadfile(L, filename) == 0 &&
+    sid.state = luaLoadScriptFileToState(L, filename, LUA_SCRIPT_LOAD_MODE);
-        lua_pcall(L, 0, 1, 0) == 0 &&
+    if (sid.state == SCRIPT_OK && (lstatus = lua_pcall(L, 0, 1, 0)) == LUA_OK && lua_istable(L, -1)) {
-        lua_istable(L, -1)) {
-
-      luaL_checktype(L, -1, LUA_TTABLE);
-
       for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
         const char * key = lua_tostring(L, -2);
         if (!strcmp(key, "init")) {
@@ -287,15 +472,15 @@ int luaLoad(lua_State * L, const char * filename, ScriptInternalData & sid, Scri
       if (init) {
         lua_rawgeti(L, LUA_REGISTRYINDEX, init);
         if (lua_pcall(L, 0, 0, 0) != 0) {
-          TRACE("Error in script %s init: %s", filename, lua_tostring(L, -1));
+          TRACE_ERROR("luaLoad(%s): Error in script init(): %s\n", filename, lua_tostring(L, -1));
           sid.state = SCRIPT_SYNTAX_ERROR;
         }
         luaL_unref(L, LUA_REGISTRYINDEX, init);
         lua_gc(L, LUA_GCCOLLECT, 0);
       }
     }
-    else {
+    else if (sid.state == SCRIPT_OK) {
-      TRACE("Error in script %s: %s", filename, lua_tostring(L, -1));
+      TRACE_ERROR("luaLoad(%s): Error parsing script (%d): %s\n", filename, lstatus, lua_tostring(L, -1));
       sid.state = SCRIPT_SYNTAX_ERROR;
     }
   }
@@ -309,6 +494,8 @@ int luaLoad(lua_State * L, const char * filename, ScriptInternalData & sid, Scri
     luaFree(L, sid);
   }
 
+  luaDoGc(L);
+
   return sid.state;
 }
 
@@ -321,10 +508,10 @@ bool luaLoadMixScript(uint8_t index)
     ScriptInputsOutputs * sio = &scriptInputsOutputs[index];
     sid.reference = SCRIPT_MIX_FIRST+index;
     sid.state = SCRIPT_NOFILE;
-    char filename[sizeof(SCRIPTS_MIXES_PATH)+sizeof(sd.file)+sizeof(SCRIPTS_EXT)] = SCRIPTS_MIXES_PATH "/";
+    char filename[sizeof(SCRIPTS_MIXES_PATH)+sizeof(sd.file)+sizeof(SCRIPT_EXT)] = SCRIPTS_MIXES_PATH "/";
     strncpy(filename+sizeof(SCRIPTS_MIXES_PATH), sd.file, sizeof(sd.file));
     filename[sizeof(SCRIPTS_MIXES_PATH)+sizeof(sd.file)] = '\0';
-    strcat(filename+sizeof(SCRIPTS_MIXES_PATH), SCRIPTS_EXT);
+    strcat(filename+sizeof(SCRIPTS_MIXES_PATH), SCRIPT_EXT);
     if (luaLoad(lsScripts, filename, sid, sio) == SCRIPT_PANIC) {
       return false;
     }
@@ -341,10 +528,10 @@ bool luaLoadFunctionScript(uint8_t index)
       ScriptInternalData & sid = scriptInternalData[luaScriptsCount++];
       sid.reference = SCRIPT_FUNC_FIRST+index;
       sid.state = SCRIPT_NOFILE;
-      char filename[sizeof(SCRIPTS_FUNCS_PATH)+sizeof(fn.play.name)+sizeof(SCRIPTS_EXT)] = SCRIPTS_FUNCS_PATH "/";
+      char filename[sizeof(SCRIPTS_FUNCS_PATH)+sizeof(fn.play.name)+sizeof(SCRIPT_EXT)] = SCRIPTS_FUNCS_PATH "/";
       strncpy(filename+sizeof(SCRIPTS_FUNCS_PATH), fn.play.name, sizeof(fn.play.name));
       filename[sizeof(SCRIPTS_FUNCS_PATH)+sizeof(fn.play.name)] = '\0';
-      strcat(filename+sizeof(SCRIPTS_FUNCS_PATH), SCRIPTS_EXT);
+      strcat(filename+sizeof(SCRIPTS_FUNCS_PATH), SCRIPT_EXT);
       if (luaLoad(lsScripts, filename, sid) == SCRIPT_PANIC) {
         return false;
       }
@@ -369,10 +556,10 @@ bool luaLoadTelemetryScript(uint8_t index)
         ScriptInternalData & sid = scriptInternalData[luaScriptsCount++];
         sid.reference = SCRIPT_TELEMETRY_FIRST+index;
         sid.state = SCRIPT_NOFILE;
-        char filename[sizeof(SCRIPTS_TELEM_PATH)+sizeof(script.file)+sizeof(SCRIPTS_EXT)] = SCRIPTS_TELEM_PATH "/";
+        char filename[sizeof(SCRIPTS_TELEM_PATH)+sizeof(script.file)+sizeof(SCRIPT_EXT)] = SCRIPTS_TELEM_PATH "/";
         strncpy(filename+sizeof(SCRIPTS_TELEM_PATH), script.file, sizeof(script.file));
         filename[sizeof(SCRIPTS_TELEM_PATH)+sizeof(script.file)] = '\0';
-        strcat(filename+sizeof(SCRIPTS_TELEM_PATH), SCRIPTS_EXT);
+        strcat(filename+sizeof(SCRIPTS_TELEM_PATH), SCRIPT_EXT);
         if (luaLoad(lsScripts, filename, sid) == SCRIPT_PANIC) {
           return false;
         }
@@ -693,28 +880,6 @@ bool luaDoOneRunPermanentScript(uint8_t evt, int i, uint32_t scriptType)
   return true;
 }
 
-void luaDoGc(lua_State * L)
-{
-  if (L) {
-    PROTECT_LUA() {
-      lua_gc(L, LUA_GCCOLLECT, 0);
-#if defined(SIMU) || defined(DEBUG)
-      static int lastgc = 0;
-      int gc = luaGetMemUsed(L);
-      if (gc != lastgc) {
-        lastgc = gc;
-        TRACE("GC Use: %dbytes", gc);
-      }
-#endif
-    }
-    else {
-      // we disable Lua for the rest of the session
-      luaDisable();
-    }
-    UNPROTECT_LUA();
-  }
-}
-
 bool luaTask(event_t evt, uint8_t scriptType, bool allowLcdUsage)
 {
   if (luaState == INTERPRETER_PANIC) return false;

radio/src/lua/lua_api.h

@@ -31,6 +31,16 @@ extern "C" {
   #include <lgc.h>
 }
 
+#ifndef LUA_SCRIPT_LOAD_MODE
+  // Can force loading of binary (.luac) or plain-text (.lua) versions of scripts specifically, and control
+  //  compilation options. See interface.cpp:luaLoadScriptFileToState() <mode> parameter description for details.
+  #if !defined(LUA_COMPILER) || defined(SIMU) || defined(DEBUG)
+    #define LUA_SCRIPT_LOAD_MODE    "T"   // prefer loading .lua source file for full debug info
+  #else
+    #define LUA_SCRIPT_LOAD_MODE    "bt"  // binary or text, whichever is newer
+  #endif
+#endif
+
 extern lua_State * lsScripts;
 extern lua_State * lsWidgets;
 extern bool luaLcdAllowed;
@@ -144,7 +154,7 @@ void luaLoadThemes();
 void luaRegisterLibraries(lua_State * L);
 void registerBitmapClass(lua_State * L);
 void luaSetInstructionsLimit(lua_State* L, int count);
-void luaCompileAndSave(lua_State * L, const char *bytecodeName);
+int luaLoadScriptFileToState(lua_State * L, const char * filename, const char * mode);
 #else  // defined(LUA)
 #define luaInit()
 #define LUA_INIT_THEMES_AND_WIDGETS()

radio/src/lua/widgets.cpp

@@ -24,11 +24,6 @@
 #include "bin_allocator.h"
 #include "lua/lua_api.h"
 
-#if defined(LUA_COMPILER) && defined(SIMU)
-  #include <lundump.h>
-  #include <lstate.h>
-#endif
-
 #define WIDGET_SCRIPTS_MAX_INSTRUCTIONS    (10000/100)
 #define MANUAL_SCRIPTS_MAX_INSTRUCTIONS    (20000/100)
 #define LUA_WARNING_INFO_LEN               64
@@ -36,7 +31,7 @@
 lua_State *lsWidgets = NULL;
 extern int custom_lua_atpanic(lua_State *L);
 
-#define LUA_FULLPATH_MAXLEN                (42) // max length (example: /SCRIPTS/THEMES/mytheme.lua)
+#define LUA_FULLPATH_MAXLEN                (LEN_FILE_PATH_MAX + LEN_SCRIPT_FILENAME + LEN_FILE_EXTENSION_MAX)  // max length (example: /SCRIPTS/THEMES/mytheme.lua)
 
 void exec(int function, int nresults=0)
 {
@@ -369,24 +364,21 @@ void luaLoadWidgetCallback()
 
 void luaLoadFile(const char * filename, void (*callback)())
 {
-  if (lsWidgets == 0) return;
+  if (lsWidgets == NULL || callback == NULL)
+    return;
 
-  TRACE("luaLoadFile() %s", filename);
+  TRACE("luaLoadFile(%s)", filename);
-
-#if defined(LUA_COMPILER) && defined(SIMU)
-  luaCompileAndSave(lsWidgets, filename);
-#endif
 
   luaSetInstructionsLimit(lsWidgets, MANUAL_SCRIPTS_MAX_INSTRUCTIONS);
 
   PROTECT_LUA() {
-    if (luaL_loadfile(lsWidgets, filename) == 0 &&
+    if (luaLoadScriptFileToState(lsWidgets, filename, LUA_SCRIPT_LOAD_MODE) == SCRIPT_OK) {
-        lua_pcall(lsWidgets, 0, 1, 0) == 0 &&
+      if (lua_pcall(lsWidgets, 0, 1, 0) == LUA_OK && lua_istable(lsWidgets, -1)) {
-        lua_istable(lsWidgets, -1)) {
+        (*callback)();
-      (*callback)();
+      }
-    }
+      else {
-    else {
+        TRACE("luaLoadFile(%s): Error parsing script: %s", filename, lua_tostring(lsWidgets, -1));
-      TRACE("Error in script %s: %s", filename, lua_tostring(lsWidgets, -1));
+      }
     }
   }
   else {

radio/src/sdcard.cpp

@@ -40,18 +40,75 @@ const char * sdCheckAndCreateDirectory(const char * path)
   return NULL;
 }
 
-bool isFileAvailable(const char * path)
+bool isFileAvailable(const char * path, bool exclDir)
 {
+  if (exclDir) {
+    FILINFO fno;
+    return (f_stat(path, &fno) == FR_OK && !(fno.fattrib & AM_DIR));
+  }
   return f_stat(path, 0) == FR_OK;
 }
 
-bool isFileAvailable(const char * filename, const char * directory)
+/**
+  Search file system path for a file. Can optionally take a list of file extensions to search through.
+  Eg. find "splash.bmp", or the first occurrence of one of "splash.[bmp|jpeg|jpg|gif]".
+
+  @param path String with path name, no trailing slash. eg; "/BITMAPS"
+  @param file String containing file name to search for, with or w/out an extension.
+    eg; "splash.bmp" or "splash"
+  @param pattern Optional list of one or more file extensions concatenated together, including the period(s).
+    The list is searched backwards and the first match, if any, is returned.  If null, then only the actual filename
+    passed will be searched for.
+    eg: ".gif.jpg.jpeg.bmp"
+  @param exclDir true/false whether to allow directory matches (default true, excludes directories)
+  @param match Optional container to hold the matched file extension (wide enough to hold LEN_FILE_EXTENSION_MAX + 1).
+  @retval true if a file was found, false otherwise.
+*/
+bool isFilePatternAvailable(const char * path, const char * file, const char * pattern = NULL, bool exclDir = true, char * match = NULL)
 {
-  char path[256];
+  uint8_t fplen;
-  char * pos = strAppend(path, directory);
+  char fqfp[LEN_FILE_PATH_MAX + _MAX_LFN + 1] = "\0";
-  *pos = '/';
+
-  strAppend(pos+1, filename);
+  fplen = strlen(path);
-  return isFileAvailable(path);
+  if (fplen > LEN_FILE_PATH_MAX) {
+    TRACE_ERROR("isFilePatternAvailable(%s) = error: file path too long.\n", path, file);
+    return false;
+  }
+
+  strcpy(fqfp, path);
+  strcpy(fqfp + fplen, "/");
+  strncat(fqfp + (++fplen), file, _MAX_LFN);
+
+  if (pattern == NULL) {
+    // no extensions list, just check the filename as-is
+    return isFileAvailable(fqfp, exclDir);
+  }
+  else {
+    // extensions list search
+    const char *ext;
+    uint16_t len;
+    uint8_t extlen, fnlen;
+    int plen;
+
+    getFileExtension(file, 0, 0, &fnlen, &extlen);
+    len = fplen + fnlen - extlen;
+    fqfp[len] = '\0';
+    ext = getFileExtension(pattern, 0, 0, &fnlen, &extlen);
+    plen = (int)fnlen;
+    while (plen > 0 && ext) {
+      strncat(fqfp + len, ext, extlen);
+      if (isFileAvailable(fqfp, exclDir)) {
+        if (match != NULL) strncat(&(match[0]='\0'), ext, extlen);
+        return true;
+      }
+      plen -= extlen;
+      if (plen > 0) {
+        fqfp[len] = '\0';
+        ext = getFileExtension(pattern, plen, 0, NULL, &extlen);
+      }
+    }
+  }
+  return false;
 }
 
 char * getFileIndex(char * filename, unsigned int & value)
@@ -88,63 +145,73 @@ uint8_t getDigitsCount(unsigned int value)
 int findNextFileIndex(char * filename, uint8_t size, const char * directory)
 {
   unsigned int index;
+  uint8_t extlen;
   char * indexPos = getFileIndex(filename, index);
-  char extension[LEN_FILE_EXTENSION+1];
+  char extension[LEN_FILE_EXTENSION_MAX+1] = "\0";
-  strncpy(extension, getFileExtension(filename), sizeof(extension));
+  char *p = getFileExtension(filename, 0, 0, NULL, &extlen);
+  if (p) strncat(extension, p, sizeof(extension)-1);
   while (1) {
     index++;
-    if ((indexPos - filename) + getDigitsCount(index) + LEN_FILE_EXTENSION > size) {
+    if ((indexPos - filename) + getDigitsCount(index) + extlen > size) {
       return 0;
     }
     char * pos = strAppendUnsigned(indexPos, index);
     strAppend(pos, extension);
-    if (!isFileAvailable(filename, directory)) {
+    if (!isFilePatternAvailable(directory, filename, NULL, false)) {
       return index;
     }
   }
 }
 
-bool isExtensionMatching(const char * extension, const char * pattern, uint8_t flags)
+/**
+  Check if given extension exists in a list of extensions.
+  @param extension The extension to search for, including leading period.
+  @param pattern One or more file extensions concatenated together, including the periods.
+    The list is searched backwards and the first match, if any, is returned.
+    eg: ".gif.jpg.jpeg.png"
+  @param match Optional container to hold the matched file extension (wide enough to hold LEN_FILE_EXTENSION_MAX + 1).
+  @retval true if a extension was found in the lost, false otherwise.
+*/
+bool isExtensionMatching(const char * extension, const char * pattern, char * match)
 {
-  if (flags & LIST_SD_FILE_EXT) {
+  const char *ext;
-    for (int i=0; pattern[i]!='\0'; i+=LEN_FILE_EXTENSION) {
+  uint8_t extlen, fnlen;
-      if (strncasecmp(extension, &pattern[i], LEN_FILE_EXTENSION) == 0) {
+  int plen;
-        return true;
+
-      }
+  ext = getFileExtension(pattern, 0, 0, &fnlen, &extlen);
+  plen = (int)fnlen;
+  while (plen > 0 && ext) {
+    if (!strncasecmp(extension, ext, extlen)) {
+      if (match != NULL) strncat(&(match[0]='\0'), ext, extlen);
+      return true;
+    }
+    plen -= extlen;
+    if (plen > 0) {
+      ext = getFileExtension(pattern, plen, 0, NULL, &extlen);
     }
-    return false;
-  }
-  else {
-    return strcasecmp(extension, pattern) == 0;
   }
+  return false;
 }
 
 bool sdListFiles(const char * path, const char * extension, const uint8_t maxlen, const char * selection, uint8_t flags)
 {
+  static uint16_t lastpopupMenuOffset = 0;
   FILINFO fno;
   DIR dir;
+  char *fnExt;
+  uint8_t fnLen, extLen;
+  char tmpExt[LEN_FILE_EXTENSION_MAX+1] = "\0";
 
 #if defined(CPUARM)
   popupMenuOffsetType = MENU_OFFSET_EXTERNAL;
 #endif
   
-  static uint16_t lastpopupMenuOffset = 0;
-
 #if defined(CPUARM)
   static uint8_t s_last_flags;
 
   if (selection) {
     s_last_flags = flags;
-    memset(reusableBuffer.modelsel.menu_bss, 0, sizeof(reusableBuffer.modelsel.menu_bss));
+    if (!isFilePatternAvailable(path, selection, ((flags & LIST_SD_FILE_EXT) ? NULL : extension))) selection = NULL;
-    strcpy(reusableBuffer.modelsel.menu_bss[0], path);
-    strcat(reusableBuffer.modelsel.menu_bss[0], "/");
-    strncat(reusableBuffer.modelsel.menu_bss[0], selection, maxlen);
-    if (!(flags & LIST_SD_FILE_EXT)) {
-      strcat(reusableBuffer.modelsel.menu_bss[0], extension);
-    }
-    if (f_stat(reusableBuffer.modelsel.menu_bss[0], &fno) != FR_OK || (fno.fattrib & AM_DIR)) {
-      selection = NULL;
-    }
   }
   else {
     flags = s_last_flags;
@@ -194,15 +261,33 @@ bool sdListFiles(const char * path, const char * extension, const uint8_t maxlen
     for (;;) {
       res = f_readdir(&dir, &fno);                   /* Read a directory item */
       if (res != FR_OK || fno.fname[0] == 0) break;  /* Break on error or end of dir */
+      if (fno.fattrib & AM_DIR) continue;            /* Skip subfolders */
 
-      uint8_t len = strlen(fno.fname);
+      fnExt = getFileExtension(fno.fname, 0, 0, &fnLen, &extLen);
-      uint8_t maxlen_with_extension = (flags & LIST_SD_FILE_EXT) ? maxlen : maxlen+LEN_FILE_EXTENSION;
+      fnLen -= extLen;
-      if (len < LEN_FILE_EXTENSION+1 || len > maxlen_with_extension || !isExtensionMatching(fno.fname+len-LEN_FILE_EXTENSION, extension, flags) || (fno.fattrib & AM_DIR)) continue;
+
+//      TRACE_DEBUG("listSdFiles(%s, %s, %u, %s, %u): fn='%s'; fnExt='%s'; match=%d\n",
+//           path, extension, maxlen, (selection ? selection : "nul"), flags, fno.fname, (fnExt ? fnExt : "nul"), (fnExt && isExtensionMatching(fnExt, extension)));
+
+      // file validation checks
+      if (!fnLen || fnLen > maxlen || (                                              // wrong size
+            fnExt && extension && (                                                  // extension-based checks follow...
+              !isExtensionMatching(fnExt, extension) || (                            // wrong extension
+                !(flags & LIST_SD_FILE_EXT) &&                                       // only if we want unique file names...
+                strcmp(fnExt, getFileExtension(extension)) &&                        // possible duplicate file name...
+                isFilePatternAvailable(path, fno.fname, extension, true, tmpExt) &&  // find the first file from extensions list...
+                strncmp(fnExt, tmpExt, LEN_FILE_EXTENSION_MAX)                       // found file doesn't match, this is a duplicate
+              )
+            )
+          ))
+      {
+        continue;
+      }
 
       popupMenuNoItems++;
 
       if (!(flags & LIST_SD_FILE_EXT)) {
-        fno.fname[len-LEN_FILE_EXTENSION] = '\0';
+        fno.fname[fnLen] = '\0';  // strip extension
       }
 
       if (popupMenuOffset == 0) {

radio/src/sdcard.h

@@ -45,6 +45,8 @@
 #define SCRIPTS_FUNCS_PATH  SCRIPTS_PATH "/FUNCTIONS"
 #define SCRIPTS_TELEM_PATH  SCRIPTS_PATH "/TELEMETRY"
 
+#define LEN_FILE_PATH_MAX   (sizeof(SCRIPTS_TELEM_PATH)+1)  // longest + "/"
+
 #if defined(COLORLCD)
 const char RADIO_MODELSLIST_PATH[] = RADIO_PATH "/models.txt";
 const char RADIO_SETTINGS_PATH[] = RADIO_PATH "/radio.bin";
@@ -56,18 +58,27 @@ const char RADIO_SETTINGS_PATH[] = RADIO_PATH "/radio.bin";
 #define BMP_EXT             ".bmp"
 #define PNG_EXT             ".png"
 #define JPG_EXT             ".jpg"
-#define SCRIPTS_EXT         ".lua"
+#define SCRIPT_EXT          ".lua"
+#define SCRIPT_BIN_EXT      ".luac"
 #define TEXT_EXT            ".txt"
 #define FIRMWARE_EXT        ".bin"
 #define EEPROM_EXT          ".bin"
 #define SPORT_FIRMWARE_EXT  ".frk"
 
+#define LEN_FILE_EXTENSION_MAX  5  // longest used, including the dot, excluding null term.
+
 #if defined(PCBHORUS)
 #define BITMAPS_EXT         BMP_EXT JPG_EXT PNG_EXT
 #else
 #define BITMAPS_EXT         BMP_EXT
 #endif
 
+#ifdef LUA_COMPILER
+  #define SCRIPTS_EXT         SCRIPT_BIN_EXT SCRIPT_EXT
+#else
+  #define SCRIPTS_EXT         SCRIPT_EXT
+#endif
+
 #define GET_FILENAME(filename, path, var, ext) \
   char filename[sizeof(path) + sizeof(var) + sizeof(ext)]; \
   memcpy(filename, path, sizeof(path) - 1); \
@@ -99,19 +110,31 @@ inline const pm_char * SDCARD_ERROR(FRESULT result)
 }
 #endif
 
-#define LEN_FILE_EXTENSION             4
+// NOTE: 'size' must = 0 or be a valid character position within 'filename' array -- it is NOT validated
 template<class T>
-T * getFileExtension(T * filename, int size=0)
+T * getFileExtension(T * filename, uint8_t size=0, uint8_t extMaxLen=0, uint8_t *fnlen=NULL, uint8_t *extlen=NULL)
 {
-  int len = strlen(filename);
+  int len = size;
-  if (size != 0 && size < len) {
+  if (!size) {
-    len = size;
+    len = strlen(filename);
   }
-  for (int i=len; i>=len-LEN_FILE_EXTENSION; --i) {
+  if (!extMaxLen) {
+    extMaxLen = LEN_FILE_EXTENSION_MAX;
+  }
+  if (fnlen != NULL) {
+    *fnlen = (uint8_t)len;
+  }
+  for (int i=len-1; i >= 0 && len-i <= extMaxLen; --i) {
     if (filename[i] == '.') {
+      if (extlen) {
+        *extlen = len-i;
+      }
       return &filename[i];
     }
   }
+  if (extlen != NULL) {
+    *extlen = 0;
+  }
   return NULL;
 }
 
@@ -123,8 +146,9 @@ T * getFileExtension(T * filename, int size=0)
   #define O9X_FOURCC 0x3178396F // o9x for gruvin9x/MEGA2560
 #endif
 
-bool isFileAvailable(const char * filename);
+bool isFileAvailable(const char * filename, bool exclDir = false);
 int findNextFileIndex(char * filename, uint8_t size, const char * directory);
+bool isExtensionMatching(const char * extension, const char * pattern, char * match = NULL);
 
 const char * sdCopyFile(const char * src, const char * dest);
 const char * sdCopyFile(const char * srcFilename, const char * srcDir, const char * destFilename, const char * destDir);

radio/src/targets/common/arm/stm32/CMakeLists.txt

@@ -1,4 +1,6 @@
 set(LUA "NO" CACHE STRING "Lua scripts (YES/NO/NO_MODEL_SCRIPTS)")
+option(LUA_COMPILER "Pre-compile and save Lua scripts" OFF)
+set(LUA_SCRIPT_LOAD_MODE "" CACHE STRING "Script loading mode and compilation flags [btTxcd] (see loadScript() API docs). Blank for default ('bt' on radio, 'T' on SIMU/DEBUG builds)")
 set(USB "JOYSTICK" CACHE STRING "USB option (JOYSTICK/MASSSTORAGE/SERIAL)")
 set(ARCH ARM)
 set(STM32USB_DIR ${THIRDPARTY_DIR}/STM32_USB-Host-Device_Lib_V2.2.0/Libraries)

radio/src/targets/simu/simpgmspace.cpp

@@ -24,10 +24,7 @@
 #include <stdarg.h>
 #include <sys/stat.h>
 
-#if defined(_MSC_VER)
+#if !defined _MSC_VER
-  #include <direct.h>
-  #define mkdir(s, f) _mkdir(s)
-#else
   #include <sys/time.h>
 #endif
 
@@ -522,21 +519,24 @@ uint16_t stackAvailable()
 }
 
 #if defined(SDCARD) && !defined(SKIP_FATFS_DECLARATION) && !defined(SIMU_DISKIO)
+#if defined(_MSC_VER) || !defined(__GNUC__)
+  #include <direct.h>
+  #include <stdlib.h>
+  #include <sys/utime.h>
+  #define mkdir(s, f) _mkdir(s)
+#else
+  #include <utime.h>
+#endif
+#include "ff.h"
+#include <map>
+#include <string>
+
 namespace simu {
 #include <dirent.h>
-#if !defined WIN32
+#if !defined(_MSC_VER)
   #include <libgen.h>
 #endif
 }
-#include "ff.h"
-
-#if defined WIN32 || !defined __GNUC__
-#include <direct.h>
-#include <stdlib.h>
-#endif
-
-#include <map>
-#include <string>
 
 #if defined(CPUARM)
 FATFS g_FATFS_Obj;
@@ -647,6 +647,11 @@ FRESULT f_stat (const TCHAR * name, FILINFO *fno)
     TRACE_SIMPGMSPACE("f_stat(%s) = OK", path);
     if (fno) {
       fno->fattrib = (tmp.st_mode & S_IFDIR) ? AM_DIR : 0;
+      // convert to FatFs fdate/ftime
+      struct tm *ltime = localtime(&tmp.st_mtime);
+      fno->fdate = ((ltime->tm_year - 80) << 9) | ((ltime->tm_mon + 1) << 5) | ltime->tm_mday;
+      fno->ftime = (ltime->tm_hour << 11) | (ltime->tm_min << 5) | (ltime->tm_sec / 2);
+      fno->fsize = (DWORD)tmp.st_size;
     }
     return FR_OK;
   }
@@ -808,7 +813,11 @@ FRESULT f_mkfs (const TCHAR* path, BYTE opt, DWORD au, void* work, UINT len)
 FRESULT f_mkdir (const TCHAR * name)
 {
   char * path = convertSimuPath(name);
+#if defined(WIN32) && defined(__GNUC__)
+  if (mkdir(path)) {
+#else
   if (mkdir(path, 0777)) {
+#endif
     TRACE_SIMPGMSPACE("mkdir(%s) = error %d (%s)", path, errno, strerror(errno));
     return FR_INVALID_NAME;
   }
@@ -846,6 +855,38 @@ FRESULT f_rename(const TCHAR *oldname, const TCHAR *newname)
   return FR_OK;
 }
 
+FRESULT f_utime(const TCHAR* path, const FILINFO* fno)
+{
+  if (fno == NULL)
+    return FR_INVALID_PARAMETER;
+
+  char *simpath = convertSimuPath(path);
+  char *realPath = findTrueFileName(simpath);
+  struct utimbuf newTimes;
+  struct tm ltime;
+
+  // convert from FatFs fdate/ftime
+  ltime.tm_year = ((fno->fdate >> 9) & 0x7F) + 80;
+  ltime.tm_mon = ((fno->fdate >> 5) & 0xF) - 1;
+  ltime.tm_mday = (fno->fdate & 0x1F);
+  ltime.tm_hour = ((fno->ftime >> 11) & 0x1F);
+  ltime.tm_min = ((fno->ftime >> 5) & 0x3F);
+  ltime.tm_sec = (fno->ftime & 0x1F) * 2;
+  ltime.tm_isdst = -1;  // force mktime() to check dst
+
+  newTimes.modtime = mktime(&ltime);
+  newTimes.actime = newTimes.modtime;
+
+  if (utime(realPath, &newTimes)) {
+    TRACE_SIMPGMSPACE("f_utime(%s) = error %d (%s)", simpath, errno, strerror(errno));
+    return FR_DENIED;
+  }
+  else {
+    TRACE_SIMPGMSPACE("f_utime(%s) set mtime = %s", simpath, ctime(&newTimes.modtime));
+    return FR_OK;
+  }
+}
+
 int f_putc (TCHAR c, FIL * fil)
 {
   if (fil && fil->obj.fs) fwrite(&c, 1, 1, (FILE*)fil->obj.fs);
@@ -873,27 +914,33 @@ int f_printf (FIL *fil, const TCHAR * format, ...)
 FRESULT f_getcwd (TCHAR *path, UINT sz_path)
 {
   char cwd[1024];
+  size_t sdlen = strlen(simuSdDirectory);
   if (!getcwd(cwd, 1024)) {
     TRACE_SIMPGMSPACE("f_getcwd() = getcwd() error %d (%s)", errno, strerror(errno));
     strcpy(path, ".");
     return FR_NO_PATH;
   }
+  size_t cwdlen = strlen(cwd);
 
-  if (strlen(cwd) < strlen(simuSdDirectory)) {
+  if (cwdlen < sdlen) {
     TRACE_SIMPGMSPACE("f_getcwd() = logic error strlen(cwd) < strlen(simuSdDirectory):  cwd: \"%s\",  simuSdDirectory: \"%s\"", cwd, simuSdDirectory);
     strcpy(path, ".");
     return FR_NO_PATH;
   }
 
-  if (sz_path < (strlen(cwd) - strlen(simuSdDirectory))) {
+  if (sz_path < (cwdlen - sdlen)) {
-    TRACE_SIMPGMSPACE("f_getcwd(): buffer too short");
+    //TRACE_SIMPGMSPACE("f_getcwd(): buffer too short");
     return FR_NOT_ENOUGH_CORE;
   }
 
   // remove simuSdDirectory from the cwd
-  strcpy(path, cwd + strlen(simuSdDirectory));
+#ifdef _MSC_VER
+  strcpy(path, cwd + sdlen + 2);  // account for drive name
+#else
+  strcpy(path, cwd + sdlen);
+#endif
 
-  if (strlen(path) == 0) {
+  if (path[0] == '\0') {
     strcpy(path, "/");    // fix for the root directory
   }
 

radio/src/thirdparty/FatFs/ffconf.h

@@ -72,7 +72,7 @@ extern "C" {
 /* This option switches f_expand function. (0:Disable or 1:Enable) */
 
 
-#define _USE_CHMOD		0
+#define _USE_CHMOD		1
 /* This option switches attribute manipulation functions, f_chmod() and f_utime().
 /  (0:Disable or 1:Enable) Also _FS_READONLY needs to be 0 to enable this option. */