Merge pull request #26 from tasmota/master

Updates from upstream master

.github/workflows/dev_release_esptool_pypi.yml

@@ -0,0 +1,42 @@
+# This workflow will upload an esptool Python package when a dev release tag (e.g. "v4.7.dev2") is pushed
+
+name: PyPI dev release
+
+on:
+  push:
+    tags:
+      - v*.*.dev*
+
+jobs:
+  build_and_upload:
+
+    runs-on: ubuntu-latest
+
+    if: startsWith(github.ref, 'refs/tags/') && contains(github.ref_name, 'dev')
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python 3.8
+      uses: actions/setup-python@master
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install twine setuptools
+
+    - name: Create development release ${{ github.ref_name }}
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        TWINE_NON_INTERACTIVE: true
+      run: |
+        DEV_VERSION=$(echo "${{ github.ref_name }}" | grep -oE 'dev[0-9]+' | sed 's/dev//')
+        python ci/patch_dev_release.py --dev-no ${DEV_VERSION} esptool/__init__.py
+        git diff
+        python -m pip download esptool==$(python setup.py -V) && echo "Version ${{ github.ref_name }} already published, skipping..." && exit 1
+
+        echo "Packaging and publishing new esptool development release: ${{ github.ref_name }}"
+        python setup.py sdist
+        tar -ztvf dist/*
+        twine upload dist/*

.gitlab-ci.yml

@@ -43,8 +43,8 @@ cache:
 
 version_check:
   <<: *test_template
-  only:
-    - tags
+  rules:
+    - if: '$CI_COMMIT_TAG != null'
   script:
     - VERSION=$(esptool.py version | head -n 1)
     - |
@@ -65,6 +65,15 @@ version_check:
     reports:
       junit: test/report.xml
 
+check_uf2_ids:
+  <<: *host_tests_template
+  allow_failure: true
+  variables:
+    COVERAGE_PROCESS_START: "${CI_PROJECT_DIR}/test/.covconf"
+    PYTEST_ADDOPTS: "-sv --junitxml=test/report.xml --color=yes"
+  script:
+    - coverage run -m pytest ${CI_PROJECT_DIR}/test/test_uf2_ids.py
+
 host_tests:
   <<: *host_tests_template
   variables:
@@ -291,10 +300,19 @@ target_esp32s3:
   script:
     - coverage run --parallel-mode -m pytest ${CI_PROJECT_DIR}/test/test_esptool.py --port /dev/serial_ports/ESP32S3 --chip esp32s3 --baud 115200
 
-target_esp32s3_32MB:
+target_esp32s3_32MB_octal:
+  extends: .target_esptool_test
+  tags:
+    - esptool_esp32s3_32MB_octal_target
+  variables:
+    ESPTOOL_TEST_FLASH_SIZE: "32"
+  script:
+    - coverage run --parallel-mode -m pytest ${CI_PROJECT_DIR}/test/test_esptool.py --port /dev/serial_ports/ESP32S3_32MB --chip esp32s3 --baud 115200
+
+target_esp32s3_32MB_quad:
   extends: .target_esptool_test
   tags:
-    - esptool_esp32s3_32MB_target
+    - esptool_esp32s3_32MB_quad_target
   variables:
     ESPTOOL_TEST_FLASH_SIZE: "32"
   script:
@@ -429,8 +447,8 @@ build_docs:
   image: python:3.7-bullseye
   tags:
     - build_docs
-  only:
-    changes:
+  rules:
+    - changes:
       - "docs/**/*"
       - "CONTRIBUTING.rst"
   needs: []
@@ -452,10 +470,6 @@ build_docs:
     - deploy
   needs:
     - build_docs
-  only:
-    changes:
-      - "docs/**/*"
-      - "CONTRIBUTING.rst"
   script:
     - source ${CI_PROJECT_DIR}/docs/utils.sh
     - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
@@ -466,9 +480,12 @@ build_docs:
 deploy_docs_preview:
   extends:
     - .deploy_docs_template
-  except:
-    refs:
-      - master
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "master"'
+      when: never
+    - changes:
+      - "docs/**/*"
+      - "CONTRIBUTING.rst"
   variables:
     TYPE: "preview"
     DOCS_BUILD_DIR: "${CI_PROJECT_DIR}/docs/_build/"
@@ -481,9 +498,8 @@ deploy_docs_preview:
 deploy_docs_production:
   extends:
     - .deploy_docs_template
-  only:
-    refs:
-      - master
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "master"'
   variables:
     TYPE: "production"
     DOCS_BUILD_DIR: "${CI_PROJECT_DIR}/docs/_build/"
@@ -492,25 +508,3 @@ deploy_docs_production:
     DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
     DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
     DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esptool"
-
-deploy_dev_package:
-  <<: *test_template
-  rules:
-    - if: $CI_DEV_PUBLISH != null && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-  variables:
-    TWINE_NON_INTERACTIVE: "true"
-    TWINE_USERNAME: __token__
-    TWINE_PASSWORD: ${PYPI_ESPTOOL_TOKEN}
-  stage: deploy_development_package
-  dependencies: []
-  script:
-    - python -m pip install --upgrade pip
-    - python -m pip install twine setuptools
-    - python ci/patch_dev_release.py --dev-no ${CI_DEV_PUBLISH} esptool/__init__.py
-    # check what has been changed with git
-    - git diff
-    - python setup.py sdist
-    # skip release if it has already been released (with failure)
-    - python -m pip download esptool==$(python setup.py -V) && exit 1
-    - tar -ztvf dist/*
-    - python -m twine upload dist/*

docs/en/esptool/basic-commands.rst

@@ -236,8 +236,7 @@ This information corresponds to the headers described in :ref:`image-format`.
 
 Merge Binaries for Flashing: merge_bin
 --------------------------------------
-
-The ``merge_bin`` command will merge multiple binary files (of any kind) into a single file that can be flashed to a device later. Any gaps between the input files are padded with 0xFF bytes (same as unwritten flash contents).
+The ``merge_bin`` command will merge multiple binary files (of any kind) into a single file that can be flashed to a device later. Any gaps between the input files are padded based on selected output format.
 
 For example:
 
@@ -247,23 +246,54 @@ For example:
 
 Will create a file ``merged-flash.bin`` with the contents of the other 3 files. This file can be later be written to flash with ``esptool.py write_flash 0x0 merged-flash.bin``.
 
-.. note:
-
-    Because gaps between the input files are padded with 0xFF bytes, when the merged binary is written then any flash sectors between the individual files will be erased. To avoid this, write the files individually.
 
-**Options:**
+**Common options:**
 
 *  The ``merge_bin`` command supports the same ``--flash_mode``, ``--flash_size`` and ``--flash_freq`` options as the ``write_flash`` command to override the bootloader flash header (see above for details).
    These options are applied to the output file contents in the same way as when writing to flash. Make sure to pass the ``--chip`` parameter if using these options, as the supported values and the bootloader offset both depend on the chip.
-*  The ``--target-offset 0xNNN`` option will create a merged binary that should be flashed at the specified offset, instead of at offset 0x0.
-*  The ``--fill-flash-size SIZE`` option will pad the merged binary with 0xFF bytes to the full flash specified size, for example ``--fill-flash-size 4MB`` will create a 4MB binary file.
+*  The ``--format`` option will change the format of the output file. For more information about formats see formats description below.
 *  It is possible to append options from a text file with ``@filename``. As an example, this can be conveniently used with the ESP-IDF build system, which produces a ``flash_args`` file in the build directory of a project:
 
 .. code:: sh
 
     cd build    # The build directory of an ESP-IDF project
     esptool.py --chip {IDF_TARGET_NAME} merge_bin -o merged-flash.bin @flash_args
 
+
+RAW Output Format
+^^^^^^^^^^^^^^^^^
+
+The output of the command will be in ``raw`` format and gaps between individual files will be filled with `0xFF` bytes (same as unwritten flash contents).
+
+.. note::
+
+    Because gaps between the input files are padded with `0xFF` bytes, when the merged binary is written then any flash sectors between the individual files will be erased. To avoid this, write the files individually.
+
+
+**RAW options:**
+
+*  The ``--fill-flash-size SIZE`` option will pad the merged binary with `0xFF` bytes to the full flash specified size, for example ``--fill-flash-size 4MB`` will create a 4MB binary file.
+*  The ``--target-offset 0xNNN`` option will create a merged binary that should be flashed at the specified offset, instead of at offset 0x0.
+
+
+UF2 Output Format
+^^^^^^^^^^^^^^^^^
+
+This command will generate a UF2 (`USB Flashing Format <https://github.com/microsoft/uf2>`_) binary. 
+This UF2 file can be copied to a USB mass storage device exposed by another ESP running the `ESP USB Bridge <https://github.com/espressif/esp-usb-bridge>`_ project. The bridge MCU will use it to flash the target MCU. This is as simple copying (or "drag-and-dropping") the file to the exposed disk accessed by a file explorer in your machine.
+
+Gaps between the files will be filled with `0x00` bytes.
+
+**UF2 options:**
+
+*  The ``--chunk-size`` option will set what portion of 512 byte block will be used for data. Common value is 256 bytes. By default the largest possible value will be used.
+*  The ``--md5-disable`` option will disable MD5 checksums at the end of each block. This can be useful for integration with e.g. `tinyuf2 <https://github.com/adafruit/tinyuf2>`__.
+
+.. code:: sh
+
+    esptool.py --chip {IDF_TARGET_NAME} merge_bin --format uf2 -o merged-flash.uf2 --flash_mode dio --flash_size 4MB 0x1000 bootloader.bin 0x8000 partition-table.bin 0x10000 app.bin
+
+
 Advanced Commands
 -----------------
 

esptool/__init__.py

@@ -587,18 +587,36 @@ def add_spi_flash_subparsers(parent, allow_keep, auto_detect):
         "--output", "-o", help="Output filename", type=str, required=True
     )
     parser_merge_bin.add_argument(
-        "--format", "-f", help="Format of the output file", choices="raw", default="raw"
-    )  # for future expansion
+        "--format",
+        "-f",
+        help="Format of the output file",
+        choices=["raw", "uf2"],
+        default="raw",
+    )
+    uf2_group = parser_merge_bin.add_argument_group("UF2 format")
+    uf2_group.add_argument(
+        "--chunk-size",
+        help="Specify the used data part of the 512 byte UF2 block. "
+        "A common value is 256. By default the largest possible value will be used.",
+        default=None,
+        type=arg_auto_chunk_size,
+    )
+    uf2_group.add_argument(
+        "--md5-disable",
+        help="Disable MD5 checksum in UF2 output",
+        action="store_true",
+    )
     add_spi_flash_subparsers(parser_merge_bin, allow_keep=True, auto_detect=False)
 
-    parser_merge_bin.add_argument(
+    raw_group = parser_merge_bin.add_argument_group("RAW format")
+    raw_group.add_argument(
         "--target-offset",
         "-t",
         help="Target offset where the output file will be flashed",
         type=arg_auto_int,
         default=0,
     )
-    parser_merge_bin.add_argument(
+    raw_group.add_argument(
         "--fill-flash-size",
         help="If set, the final binary file will be padded with FF "
         "bytes up to this flash size.",
@@ -834,7 +852,11 @@ def flash_xmc_startup():
             if flash_size is not None:  # Secure download mode
                 esp.flash_set_parameters(flash_size_bytes(flash_size))
                 # Check if stub supports chosen flash size
-                if esp.IS_STUB and flash_size in ("32MB", "64MB", "128MB"):
+                if (
+                    esp.IS_STUB
+                    and esp.CHIP_NAME != "ESP32-S3"
+                    and flash_size_bytes(flash_size) > 16 * 1024 * 1024
+                ):
                     print(
                         "WARNING: Flasher stub doesn't fully support flash size larger "
                         "than 16MB, in case of failure use --no-stub."
@@ -858,7 +880,7 @@ def flash_xmc_startup():
             args.size = flash_size_bytes(size_str)
 
         if esp.IS_STUB and hasattr(args, "address") and hasattr(args, "size"):
-            if args.address + args.size > 0x1000000:
+            if esp.CHIP_NAME != "ESP32-S3" and args.address + args.size > 0x1000000:
                 print(
                     "WARNING: Flasher stub doesn't fully support flash size larger "
                     "than 16MB, in case of failure use --no-stub."
@@ -906,6 +928,13 @@ def arg_auto_size(x):
     return x if x == "all" else arg_auto_int(x)
 
 
+def arg_auto_chunk_size(string: str) -> int:
+    num = int(string, 0)
+    if num & 3 != 0:
+        raise argparse.ArgumentTypeError("Chunk size should be a 4-byte aligned number")
+    return num
+
+
 def get_port_list():
     if list_ports is None:
         raise FatalError(