Merge pull request #844 from bernd-herzog/reduce_image_size_lz4

Reduce image size lz4

dockerfile

@@ -14,7 +14,7 @@ COPY ./ /havocsrc
 
 #Fetch dependencies from APT
 RUN apt-get update && \
-        apt-get install -y tar wget dfu-util cmake python bzip2 curl python3 python3-yaml && \
+        apt-get install -y tar wget dfu-util cmake python bzip2 lz4 curl python3 python3-yaml && \
         apt-get -qy autoremove
 
 #Install current pip from PyPa

dockerfile-alpine

@@ -13,7 +13,7 @@ COPY ./ /havocsrc
 # Fetch dependencies from APK
 RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
 RUN apk update -U
-RUN apk add --no-cache git tar wget cmake curl bzip2 make
+RUN apk add --no-cache git tar wget cmake curl bzip2 lz4 make
 RUN apk add --no-cache dfu-util ccache icu-data-full
 RUN apk add --no-cache python3 py3-pip py3-yaml
 RUN apk add --no-cache py3-pyyaml-env-tag

dockerfile-nogit

@@ -11,7 +11,7 @@ WORKDIR /havoc/firmware
 
 # Fetch dependencies from APT
 RUN apt-get update && \
-	apt-get install -y git tar wget dfu-util cmake python3 ccache bzip2 curl && \
+	apt-get install -y git tar wget dfu-util cmake python3 ccache bzip2 liblz4-tool curl && \
 	apt-get -qy autoremove
 
 #Install current pip from PyPa

dockerfile-nogit-alpine

@@ -10,7 +10,7 @@ WORKDIR /havoc/firmware
 # Fetch dependencies from APK
 RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
 RUN apk update -U
-RUN apk add --no-cache git tar wget cmake curl bzip2 make
+RUN apk add --no-cache git tar wget cmake curl bzip2 lz4 make
 RUN apk add --no-cache dfu-util ccache icu-data-full
 RUN apk add --no-cache python3 py3-pip py3-yaml
 RUN apk add --no-cache py3-pyyaml-env-tag

firmware/CMakeLists.txt

@@ -28,6 +28,7 @@ set(CHIBIOS_PORTAPACK ${PROJECT_SOURCE_DIR}/chibios-portapack)
 set(EXTRACT_CPLD_DATA ${PROJECT_SOURCE_DIR}/tools/extract_cpld_data.py)
 set(MAKE_SPI_IMAGE ${PROJECT_SOURCE_DIR}/tools/make_spi_image.py)
 set(MAKE_IMAGE_CHUNK ${PROJECT_SOURCE_DIR}/tools/make_image_chunk.py)
+set(LZ4 lz4)
 
 set(FIRMWARE_NAME portapack-h1_h2-mayhem)
 set(FIRMWARE_FILENAME ${FIRMWARE_NAME}.bin)

firmware/application/CMakeLists.txt

@@ -317,7 +317,10 @@ set(TCSRC)
 set(TCPPSRC)
 
 # List ASM source files here
-set(ASMSRC ${PORTASM})
+set(ASMSRC
+	${PORTASM}
+	lz4.S
+)
 
 set(INCDIR ${CMAKE_CURRENT_BINARY_DIR} ${COMMON} ${PORTINC} ${KERNINC} ${TESTINC}
 	${HALINC} ${PLATFORMINC} ${BOARDINC}

firmware/application/core_control.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (C) 2015 Jared Boone, ShareBrained Technology, Inc.
+ * Copyright (C) 2023 Bernd Herzog
  *
  * This file is part of PortaPack.
  *
@@ -28,22 +29,20 @@ using namespace lpc43xx;
 
 #include "message.hpp"
 #include "baseband_api.hpp"
+#include "lz4.h"
 
 #include <cstring>
 
-/* TODO: OK, this is cool, but how do I put the M4 to sleep so I can switch to
- * a different image? Other than asking the old image to sleep while the M0
- * makes changes?
- *
- * I suppose I could force M4MEMMAP to an invalid memory reason which would
- * cause an exception and effectively halt the M4. But that feels gross.
- */
 void m4_init(const portapack::spi_flash::image_tag_t image_tag, const portapack::memory::region_t to, const bool full_reset) {
 	const portapack::spi_flash::chunk_t* chunk = reinterpret_cast<const portapack::spi_flash::chunk_t*>(portapack::spi_flash::images.base());
 	while(chunk->tag) {
 		if(chunk->tag == image_tag) {
-			/* Initialize M4 code RAM */
+
-			std::memcpy(reinterpret_cast<void*>(to.base()), &chunk->data[0], chunk->length);
+			const void *src = &chunk->data[0];
+			void *dst = reinterpret_cast<void*>(to.base());
+
+			/* extract and initialize M4 code RAM */
+			unlz4_len(src, dst, chunk->compressed_data_size);
 
 			/* M4 core is assumed to be sleeping with interrupts off, so we can mess
 			 * with its address space and RAM without concern.

firmware/application/lz4.S

@@ -0,0 +1,70 @@
+/* source: https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/lz4-decompression-routine-for-cortex-m0-and-later */
+
+                    .syntax             unified
+                    .cpu                cortex-m0
+                    .thumb
+
+/* License: Public Domain - I cannot be held responsible for what it does or does not do if you use it, whether it's modified or not. */
+/* Entry point = unlz4. On entry: r0 = source, r1 = destination. The first two bytes of the source must contain the length of the compressed data. */
+
+                    .func               unlz4
+                    .global             unlz4,unlz4_len
+                    .type               unlz4,%function
+                    .type               unlz4_len,%function
+
+                    .thumb_func
+unlz4:              ldrh                r2,[r0]             /* get length of compressed data */
+                    adds                r0,r0,#2            /* advance source pointer */
+
+                    .thumb_func
+unlz4_len:          push                {r4-r6,lr}          /* save r4, r5, r6 and return-address */
+                    adds                r5,r2,r0            /* point r5 to end of compressed data */
+
+getToken:           ldrb                r6,[r0]             /* get token */
+                    adds                r0,r0,#1            /* advance source pointer */
+                    lsrs                r4,r6,#4            /* get literal length, keep token in r6 */
+                    beq                 getOffset           /* jump forward if there are no literals */
+                    bl                  getLength           /* get length of literals */
+                    movs                r2,r0               /* point r2 to literals */
+                    bl                  copyData            /* copy literals (r2=src, r1=dst, r4=len) */
+                    movs                r0,r2               /* update source pointer */
+
+getOffset:          ldrb                r3,[r0,#0]          /* get match offset's low byte */
+                    subs                r2,r1,r3            /* subtract from destination; this will become the match position */
+                    ldrb                r3,[r0,#1]          /* get match offset's high byte */
+                    lsls                r3,r3,#8            /* shift to high byte */
+                    subs                r2,r2,r3            /* subtract from match position */
+                    adds                r0,r0,#2            /* advance source pointer */
+                    lsls                r4,r6,#28           /* get rid of token's high 28 bits */
+                    lsrs                r4,r4,#28           /* move the 4 low bits back where they were */
+                    bl                  getLength           /* get length of match data */
+                    adds                r4,r4,#4            /* minimum match length is 4 bytes */
+                    bl                  copyData            /* copy match data (r2=src, r1=dst, r4=len) */
+                    cmp                 r0,r5               /* check if we've reached the end of the compressed data */
+                    blt                 getToken            /* if not, go get the next token */
+                    pop                 {r4-r6,pc}          /* restore r4, r5 and r6, then return */
+
+                    .thumb_func
+getLength:          cmp                 r4,#0x0f            /* if length is 15, then more length info follows */
+                    bne                 gotLength           /* jump forward if we have the complete length */
+
+getLengthLoop:      ldrb                r3,[r0]             /* read another byte */
+                    adds                r0,r0,#1            /* advance source pointer */
+                    adds                r4,r4,r3            /* add byte to length */
+                    cmp                 r3,#0xff            /* check if end reached */
+                    beq                 getLengthLoop       /* if not, go round loop */
+gotLength:          bx                  lr                  /* return */
+
+                    .thumb_func
+copyData:           rsbs                r4,r4,#0            /* index = -length */
+                    subs                r2,r2,r4            /* point to end of source */
+                    subs                r1,r1,r4            /* point to end of destination */
+                    
+copyDataLoop:       ldrb                r3,[r2,r4]          /* read byte from source_end[-index] */
+                    strb                r3,[r1,r4]          /* store byte in destination_end[-index] */
+                    adds                r4,r4,#1            /* increment index */
+                    bne                 copyDataLoop        /* keep going until index wraps to 0 */
+                    bx                  lr                  /* return */
+                    .size               unlz4,.-unlz4
+                    .endfunc
+/* 42 narrow instructions = 84 bytes */

firmware/application/lz4.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2015 Jared Boone, ShareBrained Technology, Inc.
+ * Copyright (C) 2023 Bernd Herzog
+ *
+ * This file is part of PortaPack.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, Inc., 51 Franklin Street,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __LZ4_H__
+#define __LZ4_H__
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+    extern void unlz4_len(const void *aSource, void *aDestination, uint32_t aLength);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*__LZ4_H__*/

firmware/baseband/CMakeLists.txt

@@ -273,7 +273,8 @@ macro(DeclareTargets chunk_tag name)
 	add_custom_command(
 		OUTPUT ${PROJECT_NAME}.bin ${PROJECT_NAME}.img
 		COMMAND ${CMAKE_OBJCOPY} -O binary ${PROJECT_NAME}.elf ${PROJECT_NAME}.bin
-		COMMAND ${MAKE_IMAGE_CHUNK} ${PROJECT_NAME}.bin ${chunk_tag} ${PROJECT_NAME}.img
+		COMMAND ${LZ4} -f -9 ${PROJECT_NAME}.bin ${PROJECT_NAME}.lz4
+		COMMAND ${MAKE_IMAGE_CHUNK} ${PROJECT_NAME}.lz4 ${chunk_tag} ${PROJECT_NAME}.img
 		DEPENDS ${PROJECT_NAME}.elf ${MAKE_IMAGE_CHUNK}
 		VERBATIM
 	)
@@ -508,7 +509,8 @@ DeclareTargets(PWFM wfm_audio)
 
 add_custom_command(
 	OUTPUT hackrf.img
-	COMMAND ${MAKE_IMAGE_CHUNK} ${HACKRF_FIRMWARE_BIN_IMAGE} HRF1 hackrf.img 98304
+	COMMAND ${LZ4} -f -9 ${HACKRF_FIRMWARE_BIN_IMAGE} hackrf.lz4
+	COMMAND ${MAKE_IMAGE_CHUNK} hackrf.lz4 HRF1 hackrf.img
 	DEPENDS ${HACKRF_FIRMWARE_BIN_FILENAME} ${MAKE_IMAGE_CHUNK}
 	VERBATIM
 )

firmware/common/spi_image.hpp

@@ -114,6 +114,7 @@ constexpr image_tag_t image_tag_hackrf				{ 'H', 'R', 'F', '1' };
 struct chunk_t {
 	const image_tag_t tag;
 	const uint32_t length;
+	const uint32_t compressed_data_size;
 	const uint8_t data[];
 
 	const chunk_t* next() const {

firmware/tools/make_image_chunk.py

@@ -27,7 +27,7 @@ import struct
 usage_message = """
 PortaPack image chunk writer
 
-Usage: <command> <input_binary> <four-characer tag> <output_tagged_binary> [<chunk max size>]
+Usage: <command> <input_binary> <four-characer tag> <output_tagged_binary>
 """
 
 def read_image(path):
@@ -41,17 +41,33 @@ def write_image(data, path):
 	f.write(data)
 	f.close()
 
-input_image_max_length = 32768
+if len(sys.argv) == 4:
-if len(sys.argv) in (4, 5):
 	input_image = read_image(sys.argv[1])
+	input_image = bytearray(input_image)
 	tag = tuple(map(ord, sys.argv[2]))
 	output_path = sys.argv[3]
-	if len(sys.argv) == 5:
+
-		input_image_max_length = int(sys.argv[4])
+	if input_image[4] & 8 == 8:
+		input_image = input_image[15:]
+	else:
+		input_image = input_image[7:]
+
+	if (len(input_image) & 3) != 0:
+		for i in range(4 - (len(input_image) & 3)):
+			input_image.append(0)
+
+	output_image = bytearray()
+	output_image += struct.pack('<4BI', tag[0], tag[1], tag[2], tag[3], len(input_image) - 4)
+	output_image += input_image
+	write_image(output_image, output_path)
+
 elif len(sys.argv) == 2:
-	input_image = bytearray()
+	null_image = bytearray()
 	tag = (0, 0, 0, 0)
 	output_path = sys.argv[1]
+	null_image += struct.pack('<4BI', tag[0], tag[1], tag[2], tag[3], 0)
+	write_image(null_image, output_path)
+
 else:
 	print(usage_message)
 	sys.exit(-1)
@@ -59,14 +75,3 @@ else:
 if len(tag) != 4:
 	print(usage_message)
 	sys.exit(-2)
-
-if len(input_image) > input_image_max_length:
-	raise RuntimeError('image size of %d exceeds device size of %d bytes' % (len(input_image), input_image_max_length))
-if (len(input_image) & 3) != 0:
-	raise RuntimeError('image size of %d is not multiple of four' % (len(input_image,)))
-
-output_image = bytearray()
-output_image += struct.pack('<4BI', tag[0], tag[1], tag[2], tag[3], len(input_image))
-output_image += input_image
-
-write_image(output_image, output_path)