Merge pull request #3409 from opentx/projectkk2glider/issue_3374_disk_cache_horus

Fixes #3374: added SD card disk cache for Horus

radio/src/CMakeLists.txt

@@ -21,6 +21,7 @@ option(JITTER_MEASURE "Enable ADC jitter measurement" OFF)
 option(JITTER_FILTER "Enable ADC jitter filtering" ON)
 option(WATCHDOG_DISABLED "Disable hardware Watchdog on Horus" OFF)  # TODO remove it when it's OK
 option(SIMU_AUDIO "Enable simulator audio" OFF)
+option(SIMU_DISKIO "Enable disk IO simulation in simulator. Simulator will use FatFs module and simulated IO layer that  uses \"./sdcard.image\" file as image of SD card. This file must contain whole SD card from first to last sector" OFF)
 
 enable_language(ASM)
 set(OPT s)
@@ -77,6 +78,7 @@ if(PCB STREQUAL HORUS)
   set(FLAVOUR horus)
   set(VIRTUAL_INPUTS YES)
   set(RAMBACKUP YES)
+  option(DISK_CACHE "Enable SD card disk cache" YES)
   add_definitions(-DPCBHORUS -DCOLORLCD -DSTM32F429_439xx -DPPM_PIN_HW_SERIAL)
   add_definitions(-DEEPROM_VARIANT=0 -DAUDIO -DVOICE -DRTCLOCK)
   add_definitions(-DLUAINPUTS -DXCURVES -DVARIO)
@@ -105,6 +107,10 @@ if(PCB STREQUAL HORUS)
     ${WIDGETS_SRC}
     )
   set(SRC ${SRC} targets/taranis/rtc_driver.cpp)
+  if(DISK_CACHE)
+    set(SRC ${SRC} disk_cache.cpp)
+    add_definitions(-DDISK_CACHE)
+  endif()
   set(TARGET_SRC ${TARGET_SRC} board_horus.cpp)
   set(FIRMWARE_TARGET_SRC
     ${FIRMWARE_TARGET_SRC}

radio/src/cli.cpp

@@ -27,7 +27,7 @@
 #define CLI_COMMAND_MAX_LEN            256
 
 OS_TID cliTaskId;
-TaskStack<CLI_STACK_SIZE> cliStack;
+TaskStack<CLI_STACK_SIZE> _ALIGNED(8) cliStack; // stack must be aligned to 8 bytes otherwise printf for %f does not work!
 Fifo<uint8_t, 256> cliRxFifo;
 uint8_t cliTracesEnabled = true;
 // char cliLastLine[CLI_COMMAND_MAX_LEN+1];
@@ -631,6 +631,13 @@ int cliDisplay(const char ** argv)
   else if (!strcmp(argv[1], "audio")) {
     printAudioVars();
   }
+#if defined(DISK_CACHE)
+  else if (!strcmp(argv[1], "dc")) {
+    DiskCacheStats stats = diskCache.getStats();
+    uint32_t hitRate = diskCache.getHitRate();
+    serialPrint("Disk Cache stats: reads: %u, hits: %u, hit rate: %0.1f%%", (stats.noHits + stats.noMisses), stats.noHits, hitRate/10.0);
+  }
+#endif
   else if (toLongLongInt(argv, 1, &address) > 0) {
     int size = 256;
     if (toInt(argv, 2, &size) >= 0) {

radio/src/disk_cache.cpp

@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) OpenTX
+ *
+ * Based on code named
+ *   th9x - http://code.google.com/p/th9x 
+ *   er9x - http://code.google.com/p/er9x
+ *   gruvin9x - http://code.google.com/p/gruvin9x
+ *
+ * License GPLv2: http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <string.h>
+#include "opentx.h"
+
+#if defined(SIMU) && !defined(SIMU_DISKIO)
+  #define __disk_read(...)    (RES_OK)
+  #define __disk_write(...)   (RES_OK)
+#else
+  extern DRESULT __disk_read (BYTE pdrv, BYTE* buff, DWORD sector, UINT count);
+  extern DRESULT __disk_write (BYTE pdrv, const BYTE* buff, DWORD sector, UINT count);
+#endif
+
+#if 0     // set to 1 to enable traces
+  #define TRACE_DISK_CACHE(...)   TRACE(__VA_ARGS__)
+#else
+  #define TRACE_DISK_CACHE(...)
+#endif
+
+DiskCache diskCache;
+
+DiskCacheBlock::DiskCacheBlock() 
+  : startSector(0), 
+  endSector(0) 
+{
+}
+
+bool DiskCacheBlock::read(BYTE* buff, DWORD sector, UINT count)
+{
+  if (sector >= startSector && (sector+count) <= endSector) {
+    TRACE_DISK_CACHE("\tcache read(%u, %u) from %p", (uint32_t)sector, (uint32_t)count, this);
+    memcpy(buff, data + ((sector - startSector) * BLOCK_SIZE), count * BLOCK_SIZE);
+    return true;
+  }
+  return false;
+}
+
+DRESULT DiskCacheBlock::fill(BYTE drv, BYTE* buff, DWORD sector, UINT count) 
+{
+  // TODO: check if trying to read beyond the end of disk
+  DRESULT res = __disk_read(drv, data, sector, DISK_CACHE_BLOCK_SECTORS);
+  if (res != RES_OK) {
+    return res;
+  }
+  startSector = sector;
+  endSector = sector + DISK_CACHE_BLOCK_SECTORS;
+  memcpy(buff, data, count * BLOCK_SIZE);
+  TRACE_DISK_CACHE("\tcache %p FILLED from read(%u, %u)", this, (uint32_t)sector, (uint32_t)count);
+  return RES_OK;
+}
+
+void DiskCacheBlock::free(DWORD sector, UINT count) 
+{
+  if (sector < endSector && (sector+count) > startSector) {
+    TRACE_DISK_CACHE("\tINVALIDATING disk cache block %p (%u)", this, startSector);
+    endSector = 0;
+  }
+}
+
+bool DiskCacheBlock::empty() const
+{
+  return (endSector == 0);
+}
+
+DiskCache::DiskCache() 
+  : lastBlock(0) 
+{
+  stats.noHits = 0;
+  stats.noMisses = 0;
+  blocks = new DiskCacheBlock[DISK_CACHE_BLOCKS_NUM];
+}
+
+DRESULT DiskCache::read(BYTE drv, BYTE* buff, DWORD sector, UINT count)
+{
+  // TODO: check if not caching first sectors would improve anything
+  // if (sector < 1000) {
+  //   ++stats.noMisses;
+  //   return __disk_read(drv, buff, sector, count);  
+  // }
+
+  for(int n=0; n < DISK_CACHE_BLOCKS_NUM; ++n) {
+    if (blocks[n].read(buff, sector, count)) {
+      ++stats.noHits;
+      return RES_OK;
+    }
+  }
+
+  ++stats.noMisses;
+
+  // find free block
+  for(int n=0; n < DISK_CACHE_BLOCKS_NUM; ++n) {
+    if (blocks[n].empty()) {
+      TRACE_DISK_CACHE("\t\t using free block");
+      return blocks[n].fill(drv, buff, sector, count);
+    }
+  }
+
+  // use next block (round robin)
+  // TODO: use better strategy to select which used block gets used here
+  if (++lastBlock >= DISK_CACHE_BLOCKS_NUM) {
+    lastBlock = 0;
+  }
+  return blocks[lastBlock].fill(drv, buff, sector, count);
+}
+
+DRESULT DiskCache::write(BYTE drv, const BYTE* buff, DWORD sector, UINT count)
+{
+  for(int n=0; n < DISK_CACHE_BLOCKS_NUM; ++n) {
+    blocks[n].free(sector, count);
+  }
+  return __disk_write(drv, buff, sector, count);  
+}
+
+const DiskCacheStats & DiskCache::getStats() const 
+{ 
+  return stats; 
+}
+
+int DiskCache::getHitRate() const
+{
+  uint32_t all = stats.noHits + stats.noMisses;
+  if (all == 0) return 0;
+  return (stats.noHits * 1000) / all;
+}
+
+DRESULT disk_read (BYTE drv, BYTE *buff, DWORD sector, UINT count)
+{
+  return diskCache.read(drv, buff, sector, count);
+}
+
+
+DRESULT disk_write (BYTE drv, const BYTE *buff, DWORD sector, UINT count)
+{
+  return diskCache.write(drv, buff, sector, count);
+}

radio/src/disk_cache.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) OpenTX
+ *
+ * Based on code named
+ *   th9x - http://code.google.com/p/th9x 
+ *   er9x - http://code.google.com/p/er9x
+ *   gruvin9x - http://code.google.com/p/gruvin9x
+ *
+ * License GPLv2: http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DISK_CACHE_H_
+#define _DISK_CACHE_H_
+
+#include "diskio.h"
+
+// tunable parameters
+#define BLOCK_SIZE                512   // sector Size in Bytes
+#define DISK_CACHE_BLOCKS_NUM       8   // no cache blocks
+#define DISK_CACHE_BLOCK_SECTORS   16   // no sectors
+
+#define DISK_CACHE_BLOCK_SIZE   (DISK_CACHE_BLOCK_SECTORS * BLOCK_SIZE)
+
+class DiskCacheBlock
+{
+public:
+  DiskCacheBlock();
+  bool read(BYTE* buff, DWORD sector, UINT count);
+  DRESULT fill(BYTE drv, BYTE* buff, DWORD sector, UINT count);
+  void free(DWORD sector, UINT count);
+  bool empty() const;
+
+private:
+  uint8_t data[DISK_CACHE_BLOCK_SIZE];
+  DWORD startSector;
+  DWORD endSector;
+};
+
+struct DiskCacheStats
+{
+  uint32_t noHits;
+  uint32_t noMisses;
+};
+
+class DiskCache
+{
+public:
+  DiskCache();
+
+  DRESULT read(BYTE drv, BYTE* buff, DWORD sector, UINT count);
+  DRESULT write(BYTE drv, const BYTE* buff, DWORD sector, UINT count);
+  const DiskCacheStats & getStats() const; 
+  int getHitRate() const;
+private:
+  DiskCacheStats stats;
+  uint32_t lastBlock;
+  DiskCacheBlock * blocks;
+};
+
+extern DiskCache diskCache;
+
+#endif // _DISK_CACHE_H_

radio/src/gui/horus/view_statistics.cpp

@@ -127,12 +127,22 @@ bool menuStatsDebug(evt_t event)
   lcdDrawText(lcdNextPos+20, MENU_CONTENT_TOP+2*FH+1, "[Audio]", HEADER_COLOR|SMLSIZE);
   lcdDrawNumber(lcdNextPos+5, MENU_CONTENT_TOP+2*FH, audioStack.available(), LEFT);
 
-#if defined(LUA)
+  int line = 3;
-  lcdDrawText(MENUS_MARGIN_LEFT, MENU_CONTENT_TOP+3*FH, "Lua duration");
-  lcdDrawNumber(MENU_STATS_COLUMN1, MENU_CONTENT_TOP+3*FH, 10*maxLuaDuration, LEFT, 0, NULL, "ms");
 
-  lcdDrawText(MENUS_MARGIN_LEFT, MENU_CONTENT_TOP+4*FH, "Lua interval");
+#if defined(DISK_CACHE)
-  lcdDrawNumber(MENU_STATS_COLUMN1, MENU_CONTENT_TOP+4*FH, 10*maxLuaInterval, LEFT, 0, NULL, "ms");
+  lcdDrawText(MENUS_MARGIN_LEFT, MENU_CONTENT_TOP+line*FH, "SD Cache hits");
+  lcdDrawNumber(MENU_STATS_COLUMN1, MENU_CONTENT_TOP+line*FH, diskCache.getHitRate(), PREC1|LEFT, 0, NULL, "%");
+  ++line;
+#endif
+
+#if defined(LUA)
+  lcdDrawText(MENUS_MARGIN_LEFT, MENU_CONTENT_TOP+line*FH, "Lua duration");
+  lcdDrawNumber(MENU_STATS_COLUMN1, MENU_CONTENT_TOP+line*FH, 10*maxLuaDuration, LEFT, 0, NULL, "ms");
+  ++line;
+
+  lcdDrawText(MENUS_MARGIN_LEFT, MENU_CONTENT_TOP+line*FH, "Lua interval");
+  lcdDrawNumber(MENU_STATS_COLUMN1, MENU_CONTENT_TOP+line*FH, 10*maxLuaInterval, LEFT, 0, NULL, "ms");
+  ++line;
 #endif
 
   lcdDrawText(LCD_W/2, MENU_FOOTER_TOP+2, STR_MENUTORESET, CENTERED);

radio/src/opentx.h

@@ -270,6 +270,10 @@
   #include "targets/9x/board_stock.h"
 #endif
 
+#if defined(DISK_CACHE)
+  #include "disk_cache.h"
+#endif
+
 #if defined(SIMU)
   #include "targets/simu/simpgmspace.h"
 #elif defined(CPUARM)

radio/src/sdcard.cpp

@@ -20,10 +20,7 @@
 
 #include <stdint.h>
 #include "opentx.h"
-
-#if !defined(SIMU)
 #include "diskio.h"
-#endif
 
 const char * sdCheckAndCreateDirectory(const char * path)
 {

radio/src/targets/horus/board_horus.h

@@ -133,7 +133,7 @@ void getCPUUniqueID(char * s);
   #define SD_GET_SPEED()          (0)
 #endif
 
-#if defined(SIMU)
+#if defined(SIMU) && !defined(SIMU_DISKIO)
   #define sdInit()
   #define sdDone()
 #else

radio/src/targets/horus/diskio.cpp

@@ -112,7 +112,12 @@ uint32_t sdReadRetries = 0;
 /*-----------------------------------------------------------------------*/
 /* Read Sector(s)                                                        */
 
-DRESULT disk_read (
+#if !defined(DISK_CACHE)
+  #define __disk_read     disk_read
+  #define __disk_write    disk_write
+#endif
+
+DRESULT __disk_read (
   BYTE drv,               /* Physical drive nmuber (0..) */
   BYTE *buff,             /* Data buffer to store read data */
   DWORD sector,   				/* Sector address (LBA) */
@@ -193,7 +198,7 @@ DRESULT disk_read (
 /* Write Sector(s)                                                       */
 
 #if _READONLY == 0
-DRESULT disk_write (
+DRESULT __disk_write (
   BYTE drv,                       /* Physical drive nmuber (0..) */
   const BYTE *buff,       /* Data to be written */
   DWORD sector,           /* Sector address (LBA) */

radio/src/targets/simu/CMakeLists.txt

@@ -30,10 +30,17 @@ if(MSVC)
   include_directories(${PROJECT_SOURCE_DIR}/winbuild C:/Programs/pthreads/Pre-built.2/include C:/Programs/msinttypes C:/Programs/dirent/include)
   target_link_libraries(${SIMULATOR_TARGET} PRIVATE C:/Programs/pthreads/Pre-built.2/lib/pthreadVC2.lib)
 else()
-  add_definitions(-g)
+  set(CMAKE_C_FLAGS_DEBUG  "${COMMON_FLAGS} -g -O0")
+  set(CMAKE_CXX_FLAGS_DEBUG  "${COMMON_FLAGS} -g -O0")
 endif()
 
 if(NOT WIN32)
+
+  if(SIMU_DISKIO)
+    add_definitions(-DSIMU_DISKIO)
+    set(SIMU_SRC ${SIMU_SRC} ../../thirdparty/FatFs/ff.c ../../thirdparty/FatFs/option/ccsbcs.c)
+  endif()
+
   add_executable(simu WIN32 ${SIMU_SRC} ../../simu.cpp)
   add_dependencies(simu ${FIRMWARE_DEPENDENCIES})
   target_include_directories(simu PUBLIC /usr/local/include/fox-1.6 PUBLIC /usr/include/fox-1.6 /opt/local/include/fox-1.6)

radio/src/targets/simu/simpgmspace.cpp

@@ -1032,23 +1032,28 @@ uint32_t Card_CSD[4]; // TODO elsewhere
 FATFS g_FATFS_Obj = { 0};
 #endif
 
+pthread_mutex_t ioMutex;
+
 int ff_cre_syncobj (BYTE vol, _SYNC_t* sobj) /* Create a sync object */
 {
+  pthread_mutex_init(&ioMutex, 0);
   return 1;
 }
 
 int ff_req_grant (_SYNC_t sobj)        /* Lock sync object */
 {
+  pthread_mutex_lock(&ioMutex);
   return 1;
 }
 
 void ff_rel_grant (_SYNC_t sobj)        /* Unlock sync object */
 {
-
+  pthread_mutex_unlock(&ioMutex);
 }
 
 int ff_del_syncobj (_SYNC_t sobj)        /* Delete a sync object */
 {
+  pthread_mutex_destroy(&ioMutex);
   return 1;
 }
 
@@ -1091,7 +1096,12 @@ DSTATUS disk_status (BYTE pdrv)
   return (DSTATUS)0;
 }
 
-DRESULT disk_read (BYTE pdrv, BYTE* buff, DWORD sector, UINT count)
+#if !defined(DISK_CACHE)
+  #define __disk_read     disk_read
+  #define __disk_write    disk_write
+#endif
+
+DRESULT __disk_read (BYTE pdrv, BYTE* buff, DWORD sector, UINT count)
 {
   if (diskImage == 0) return RES_NOTRDY;
   traceDiskStatus();
@@ -1101,7 +1111,7 @@ DRESULT disk_read (BYTE pdrv, BYTE* buff, DWORD sector, UINT count)
   return RES_OK;
 }
 
-DRESULT disk_write (BYTE pdrv, const BYTE* buff, DWORD sector, UINT count)
+DRESULT __disk_write (BYTE pdrv, const BYTE* buff, DWORD sector, UINT count)
 {
   if (diskImage == 0) return RES_NOTRDY;
   traceDiskStatus();
@@ -1194,6 +1204,48 @@ DRESULT disk_ioctl (BYTE pdrv, BYTE cmd, void* buff)
   return RES_OK;
 }
 
+void sdInit(void)
+{
+  // ioMutex = CoCreateMutex();
+  // if (ioMutex >= CFG_MAX_MUTEX ) {
+  //   // sd error
+  //   return;
+  // }
+
+  if (f_mount(&g_FATFS_Obj, "", 1) == FR_OK) {
+    // call sdGetFreeSectors() now because f_getfree() takes a long time first time it's called
+    sdGetFreeSectors();
+
+    referenceSystemAudioFiles();
+
+#if defined(LOG_TELEMETRY)
+    f_open(&g_telemetryFile, LOGS_PATH "/telemetry.log", FA_OPEN_ALWAYS | FA_WRITE);
+    if (f_size(&g_telemetryFile) > 0) {
+      f_lseek(&g_telemetryFile, f_size(&g_telemetryFile)); // append
+    }
+#endif
+  }
+  else {
+    TRACE("f_mount() failed");
+  }
+}
+
+void sdDone()
+{
+  if (sdMounted()) {
+    audioQueue.stopSD();
+#if defined(LOG_TELEMETRY)
+    f_close(&g_telemetryFile);
+#endif
+    f_mount(NULL, "", 0); // unmount SD
+  }
+}
+
+uint32_t sdMounted()
+{
+  return g_FATFS_Obj.fs_type != 0;
+}
+
 uint32_t sdIsHC()
 {
   return sdGetSize() > 2000000;

radio/src/targets/simu/simpgmspace.h

@@ -474,7 +474,9 @@ extern char simuSdDirectory[1024];
 #define sdMountPoll()
 #define sdPoll10ms()
 #define sd_card_ready()  (true)
-#define sdMounted()      (true)
+#if !defined(SIMU_DISKIO)
+	#define sdMounted()      (true)
+#endif
 
 inline void ledOff() { }
 

radio/src/thirdparty/FatFs/diskio.h

@@ -5,7 +5,7 @@
 #ifndef _DISKIO_DEFINED
 #define _DISKIO_DEFINED
 
-#if defined(__cplusplus) && !defined(SIMU)
+#if defined(__cplusplus)
 extern "C" {
 #endif
 
@@ -73,7 +73,7 @@ DRESULT disk_ioctl (BYTE pdrv, BYTE cmd, void* buff);
 #define ATA_GET_MODEL		21	/* Get model name */
 #define ATA_GET_SN			22	/* Get serial number */
 
-#if defined(__cplusplus) && !defined(SIMU)
+#if defined(__cplusplus)
 }
 #endif
 

radio/src/thirdparty/FatFs/ff.h

@@ -238,7 +238,7 @@ TCHAR* f_gets (TCHAR* buff, int len, FIL* fp);						/* Get a string from the fil
 #define f_eof(fp) ((int)((fp)->fptr == (fp)->fsize))
 #define f_error(fp) ((fp)->err)
 #define f_tell(fp) ((fp)->fptr)
-#if !defined(SIMU)
+#if !defined(SIMU) || defined(SIMU_DISKIO)
 	#define f_size(fp) ((fp)->fsize)
 #else
 	UINT f_size(FIL* fil);