Merge pull request adafruit#7000 from MicroDev1/storage-extend

Add Storage Extension Support

Author: dhalbert

locale/circuitpython.pot

@@ -117,6 +117,10 @@ msgstr ""
 msgid "%q init failed"
 msgstr ""
 
+#: shared-bindings/dualbank/__init__.c
+msgid "%q is %q"
+msgstr ""
+
 #: py/argcheck.c
 msgid "%q length must be %d"
 msgstr ""
@@ -211,7 +215,7 @@ msgstr ""
 msgid "%q, %q, and %q must all be the same length"
 msgstr ""
 
-#: py/objint.c
+#: py/objint.c shared-bindings/storage/__init__.c
 msgid "%q=%q"
 msgstr ""
 

ports/espressif/common-hal/dualbank/__init__.c

@@ -32,6 +32,8 @@
 #include "esp_log.h"
 #include "esp_ota_ops.h"
 
+#include "supervisor/flash.h"
+
 static const esp_partition_t *update_partition = NULL;
 static esp_ota_handle_t update_handle = 0;
 

ports/espressif/supervisor/internal_flash.c

@@ -24,6 +24,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #include "supervisor/internal_flash.h"
 
 #include <stdint.h>
@@ -32,47 +33,114 @@
 
 #include "extmod/vfs.h"
 #include "extmod/vfs_fat.h"
+
 #include "py/mphal.h"
 #include "py/obj.h"
 #include "py/runtime.h"
-#include "lib/oofatfs/ff.h"
 
-#include "components/spi_flash/include/esp_partition.h"
+#include "esp_ota_ops.h"
+#include "esp_partition.h"
 
+#include "supervisor/filesystem.h"
 #include "supervisor/flash.h"
 #include "supervisor/usb.h"
 
-STATIC const esp_partition_t *_partition;
+#define OP_READ     0
+#define OP_WRITE    1
 
 // TODO: Split the caching out of supervisor/shared/external_flash so we can use it.
 #define SECTOR_SIZE 4096
 STATIC uint8_t _cache[SECTOR_SIZE];
 STATIC uint32_t _cache_lba = 0xffffffff;
 
+#if CIRCUITPY_STORAGE_EXTEND
+#if FF_MAX_SS == FF_MIN_SS
+#define SECSIZE(fs) (FF_MIN_SS)
+#else
+#define SECSIZE(fs) ((fs)->ssize)
+#endif // FF_MAX_SS == FF_MIN_SS
+STATIC DWORD fatfs_bytes(void) {
+    FATFS *fatfs = filesystem_circuitpy();
+    return (fatfs->csize * SECSIZE(fatfs)) * (fatfs->n_fatent - 2);
+}
+STATIC bool storage_extended = true;
+STATIC const esp_partition_t *_partition[2];
+#else
+STATIC const esp_partition_t *_partition[1];
+#endif // CIRCUITPY_STORAGE_EXTEND
+
 void supervisor_flash_init(void) {
-    _partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA,
+    if (_partition[0] != NULL) {
+        return;
+    }
+    _partition[0] = esp_partition_find_first(ESP_PARTITION_TYPE_DATA,
         ESP_PARTITION_SUBTYPE_DATA_FAT,
         NULL);
+    #if CIRCUITPY_STORAGE_EXTEND
+    _partition[1] = esp_ota_get_next_update_partition(NULL);
+    #endif
 }
 
 uint32_t supervisor_flash_get_block_size(void) {
     return FILESYSTEM_BLOCK_SIZE;
 }
 
 uint32_t supervisor_flash_get_block_count(void) {
-    return _partition->size / FILESYSTEM_BLOCK_SIZE;
+    #if CIRCUITPY_STORAGE_EXTEND
+    return ((storage_extended) ? (_partition[0]->size + _partition[1]->size) : _partition[0]->size) / FILESYSTEM_BLOCK_SIZE;
+    #else
+    return _partition[0]->size / FILESYSTEM_BLOCK_SIZE;
+    #endif
 }
 
 void port_internal_flash_flush(void) {
+}
 
+STATIC void single_partition_rw(const esp_partition_t *partition, uint8_t *data,
+    const uint32_t offset, const uint32_t size_total, const bool op) {
+    if (op == OP_READ) {
+        esp_partition_read(partition, offset, data, size_total);
+    } else {
+        esp_partition_erase_range(partition, offset, size_total);
+        esp_partition_write(partition, offset, _cache, size_total);
+    }
+}
+
+#if CIRCUITPY_STORAGE_EXTEND
+STATIC void multi_partition_rw(uint8_t *data,
+    const uint32_t offset, const uint32_t size_total, const bool op) {
+    if (offset > _partition[0]->size) {
+        // only r/w partition 1
+        single_partition_rw(_partition[1], data, (offset - _partition[0]->size), size_total, op);
+    } else if ((offset + size_total) > _partition[0]->size) {
+        // first r/w partition 0, then partition 1
+        uint32_t size_0 = _partition[0]->size - offset;
+        uint32_t size_1 = size_total - size_0;
+        if (op == OP_READ) {
+            esp_partition_read(_partition[0], offset, data, size_0);
+            esp_partition_read(_partition[1], 0, (data + size_0), size_1);
+        } else {
+            esp_partition_erase_range(_partition[0], offset, size_0);
+            esp_partition_write(_partition[0], offset, _cache, size_0);
+            esp_partition_erase_range(_partition[1], 0, size_1);
+            esp_partition_write(_partition[1], 0, (_cache + size_0), size_1);
+        }
+    } else {
+        // only r/w partition 0
+        single_partition_rw(_partition[0], data, offset, size_total, op);
+    }
 }
+#endif
 
 mp_uint_t supervisor_flash_read_blocks(uint8_t *dest, uint32_t block, uint32_t num_blocks) {
-    esp_partition_read(_partition,
-        block * FILESYSTEM_BLOCK_SIZE,
-        dest,
-        num_blocks * FILESYSTEM_BLOCK_SIZE);
-    return 0;
+    const uint32_t offset = block * FILESYSTEM_BLOCK_SIZE;
+    const uint32_t read_total = num_blocks * FILESYSTEM_BLOCK_SIZE;
+    #if CIRCUITPY_STORAGE_EXTEND
+    multi_partition_rw(dest, offset, read_total, OP_READ);
+    #else
+    single_partition_rw(_partition[0], dest, offset, read_total, OP_READ);
+    #endif
+    return 0; // success
 }
 
 mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32_t num_blocks) {
@@ -82,12 +150,8 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32
         uint32_t block_address = lba + block;
         uint32_t sector_offset = block_address / blocks_per_sector * SECTOR_SIZE;
         uint8_t block_offset = block_address % blocks_per_sector;
-
         if (_cache_lba != block_address) {
-            esp_partition_read(_partition,
-                sector_offset,
-                _cache,
-                SECTOR_SIZE);
+            supervisor_flash_read_blocks(_cache, sector_offset / FILESYSTEM_BLOCK_SIZE, blocks_per_sector);
             _cache_lba = sector_offset;
         }
         for (uint8_t b = block_offset; b < blocks_per_sector; b++) {
@@ -100,15 +164,34 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32
                 FILESYSTEM_BLOCK_SIZE);
             block++;
         }
-        esp_partition_erase_range(_partition, sector_offset, SECTOR_SIZE);
-        esp_partition_write(_partition,
-            sector_offset,
-            _cache,
-            SECTOR_SIZE);
+        #if CIRCUITPY_STORAGE_EXTEND
+        multi_partition_rw(_cache, sector_offset, SECTOR_SIZE, OP_WRITE);
+        #else
+        single_partition_rw(_partition[0], _cache, sector_offset, SECTOR_SIZE, OP_READ);
+        #endif
     }
-
     return 0; // success
 }
 
 void supervisor_flash_release_cache(void) {
 }
+
+void supervisor_flash_set_extended(bool extended) {
+    #if CIRCUITPY_STORAGE_EXTEND
+    storage_extended = extended;
+    #endif
+}
+
+bool supervisor_flash_get_extended(void) {
+    #if CIRCUITPY_STORAGE_EXTEND
+    return storage_extended;
+    #else
+    return false;
+    #endif
+}
+
+void supervisor_flash_update_extended(void) {
+    #if CIRCUITPY_STORAGE_EXTEND
+    storage_extended = (_partition[0]->size < fatfs_bytes());
+    #endif
+}

py/circuitpy_mpconfig.mk

@@ -401,6 +401,9 @@ CFLAGS += -DCIRCUITPY_STATUS_BAR=$(CIRCUITPY_STATUS_BAR)
 CIRCUITPY_STORAGE ?= 1
 CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE)
 
+CIRCUITPY_STORAGE_EXTEND ?= $(CIRCUITPY_DUALBANK)
+CFLAGS += -DCIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND)
+
 CIRCUITPY_STRUCT ?= 1
 CFLAGS += -DCIRCUITPY_STRUCT=$(CIRCUITPY_STRUCT)
 

shared-bindings/dualbank/__init__.c

@@ -26,6 +26,10 @@
 
 #include "shared-bindings/dualbank/__init__.h"
 
+#if CIRCUITPY_STORAGE_EXTEND
+#include "supervisor/flash.h"
+#endif
+
 //| """DUALBANK Module
 //|
 //| The `dualbank` module adds ability to update and switch
@@ -55,6 +59,14 @@
 //| """
 //| ...
 
+#if CIRCUITPY_STORAGE_EXTEND
+STATIC void raise_error_if_storage_extended(void) {
+    if (supervisor_flash_get_extended()) {
+        mp_raise_msg_varg(&mp_type_RuntimeError, translate("%q is %q"), MP_QSTR_storage, MP_QSTR_extended);
+    }
+}
+#endif
+
 //| def flash(buffer: ReadableBuffer, offset: int = 0) -> None:
 //|     """Writes one of two app partitions at the given offset.
 //|
@@ -70,6 +82,10 @@ STATIC mp_obj_t dualbank_flash(size_t n_args, const mp_obj_t *pos_args, mp_map_t
         { MP_QSTR_offset, MP_ARG_INT | MP_ARG_KW_ONLY, {.u_int = 0} },
     };
 
+    #if CIRCUITPY_STORAGE_EXTEND
+    raise_error_if_storage_extended();
+    #endif
+
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
 
@@ -94,6 +110,9 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_KW(dualbank_flash_obj, 0, dualbank_flash);
 //|     ...
 //|
 STATIC mp_obj_t dualbank_switch(void) {
+    #if CIRCUITPY_STORAGE_EXTEND
+    raise_error_if_storage_extended();
+    #endif
     common_hal_dualbank_switch();
     return mp_const_none;
 }

shared-bindings/storage/__init__.c

@@ -34,6 +34,7 @@
 #include "py/runtime.h"
 #include "shared-bindings/storage/__init__.h"
 #include "supervisor/shared/translate/translate.h"
+#include "supervisor/flash.h"
 
 //| """Storage management
 //|
@@ -150,7 +151,7 @@ STATIC mp_obj_t storage_getmount(const mp_obj_t mnt_in) {
 }
 MP_DEFINE_CONST_FUN_OBJ_1(storage_getmount_obj, storage_getmount);
 
-//| def erase_filesystem() -> None:
+//| def erase_filesystem(extended: Optional[bool] = None) -> None:
 //|     """Erase and re-create the ``CIRCUITPY`` filesystem.
 //|
 //|     On boards that present USB-visible ``CIRCUITPY`` drive (e.g., SAMD21 and SAMD51),
@@ -160,16 +161,38 @@ MP_DEFINE_CONST_FUN_OBJ_1(storage_getmount_obj, storage_getmount);
 //|     This function can be called from the REPL when ``CIRCUITPY``
 //|     has become corrupted.
 //|
+//|     :param bool extended: On boards that support ``dualbank`` module
+//|         and the ``extended`` parameter, the ``CIRCUITPY`` storage can be
+//|         extended by setting this to `True`. If this isn't provided or
+//|         set to `None` (default), the existing configuration will be used.
+//|
 //|     .. warning:: All the data on ``CIRCUITPY`` will be lost, and
 //|          CircuitPython will restart on certain boards."""
 //|     ...
 //|
 
-STATIC mp_obj_t storage_erase_filesystem(void) {
-    common_hal_storage_erase_filesystem();
+STATIC mp_obj_t storage_erase_filesystem(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    enum { ARG_extended };
+    static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_extended, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = mp_const_none} },
+    };
+
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+
+    #if CIRCUITPY_STORAGE_EXTEND
+    bool extended = (args[ARG_extended].u_obj == mp_const_none) ? supervisor_flash_get_extended() : mp_obj_is_true(args[ARG_extended].u_obj);
+    common_hal_storage_erase_filesystem(extended);
+    #else
+    if (mp_obj_is_true(args[ARG_extended].u_obj)) {
+        mp_raise_NotImplementedError_varg(translate("%q=%q"), MP_QSTR_extended, MP_QSTR_True);
+    }
+    common_hal_storage_erase_filesystem(false);
+    #endif
+
     return mp_const_none;
 }
-MP_DEFINE_CONST_FUN_OBJ_0(storage_erase_filesystem_obj, storage_erase_filesystem);
+MP_DEFINE_CONST_FUN_OBJ_KW(storage_erase_filesystem_obj, 0, storage_erase_filesystem);
 
 //| def disable_usb_drive() -> None:
 //|     """Disable presenting ``CIRCUITPY`` as a USB mass storage device.

shared-bindings/storage/__init__.h

@@ -37,7 +37,7 @@ void common_hal_storage_umount_path(const char *path);
 void common_hal_storage_umount_object(mp_obj_t vfs_obj);
 void common_hal_storage_remount(const char *path, bool readonly, bool disable_concurrent_write_protection);
 mp_obj_t common_hal_storage_getmount(const char *path);
-void common_hal_storage_erase_filesystem(void);
+void common_hal_storage_erase_filesystem(bool extended);
 
 bool common_hal_storage_disable_usb_drive(void);
 bool common_hal_storage_enable_usb_drive(void);

shared-module/storage/__init__.c

@@ -267,11 +267,14 @@ void common_hal_storage_remount(const char *mount_path, bool readonly, bool disa
     filesystem_set_internal_concurrent_write_protection(!disable_concurrent_write_protection);
 }
 
-void common_hal_storage_erase_filesystem(void) {
+void common_hal_storage_erase_filesystem(bool extended) {
     #if CIRCUITPY_USB
     usb_disconnect();
     #endif
     mp_hal_delay_ms(1000);
+    #if CIRCUITPY_STORAGE_EXTEND
+    supervisor_flash_set_extended(extended);
+    #endif
     (void)filesystem_init(false, true);  // Force a re-format. Ignore failure.
     common_hal_mcu_reset();
     // We won't actually get here, since we're resetting.

supervisor/flash.h

@@ -50,4 +50,8 @@ void supervisor_flash_init_vfs(struct _fs_user_mount_t *vfs);
 void supervisor_flash_flush(void);
 void supervisor_flash_release_cache(void);
 
+void supervisor_flash_set_extended(bool extended);
+bool supervisor_flash_get_extended(void);
+void supervisor_flash_update_extended(void);
+
 #endif  // MICROPY_INCLUDED_SUPERVISOR_FLASH_H

supervisor/shared/filesystem.c

@@ -143,16 +143,22 @@ bool filesystem_init(bool create_allowed, bool force_create) {
     } else if (res != FR_OK) {
         return false;
     }
+
     vfs->str = "/";
     vfs->len = 1;
     vfs->obj = MP_OBJ_FROM_PTR(vfs_fat);
     vfs->next = NULL;
+
     MP_STATE_VM(vfs_mount_table) = vfs;
 
     // The current directory is used as the boot up directory.
     // It is set to the internal flash filesystem by default.
     MP_STATE_PORT(vfs_cur) = vfs;
 
+    #if CIRCUITPY_STORAGE_EXTEND
+    supervisor_flash_update_extended();
+    #endif
+
     return true;
 }