diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..bee8a64b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/build.sh b/build.sh index 5bb08a04..579c9300 100755 --- a/build.sh +++ b/build.sh @@ -17,16 +17,10 @@ ORIG_ENV="$(env)" ROOTFS_OVERLAYS="" source ./phoenix-rtos-build/build.subr -source ./build.project +# set all static output dirs before sourcing target/project build script PREFIX_PROJECT="$(pwd)" -# Some makefiles add "$PROJECT_PATH/" to their include path so it has to be set -if [ -z "$PROJECT_PATH" ]; then - echo "PROJECT_PATH is not set (or is empty)" - exit 1; -fi - _TARGET_FOR_HOST_BUILD="host-generic-pc" PREFIX_BUILD="$PREFIX_PROJECT/_build/$TARGET" @@ -44,6 +38,17 @@ PREFIX_SYSROOT="" # empty by default (use toolchain sysroot) PLO_SCRIPT_DIR="$PREFIX_BUILD/plo-scripts" PREFIX_ROOTFS="$PREFIX_FS/root/" + +export TARGET TARGET_FAMILY TARGET_SUBFAMILY TARGET_PROJECT PROJECT_PATH PREFIX_PROJECT PREFIX_BUILD\ + PREFIX_BUILD_HOST PREFIX_FS PREFIX_BOOT PREFIX_PROG PREFIX_PROG_STRIPPED PREFIX_A\ + PREFIX_H PREFIX_ROOTFS CROSS CFLAGS CXXFLAGS LDFLAGS CC LD AR AS MAKEFLAGS DEVICE_FLAGS PLO_SCRIPT_DIR\ + PREFIX_SYSROOT LIBPHOENIX_DEVEL_MODE + +source ./build.project + +# Some makefiles add "$PROJECT_PATH/" to their include path so it has to be set +[ -z "$PROJECT_PATH" ] && b_die "PROJECT_PATH is not set (or is empty)" + : "${PREFIX_ROOTSKEL:="$PREFIX_PROJECT/_fs/root-skel/"}" @@ -67,11 +72,6 @@ AR=${CROSS}ar MAKEFLAGS="--no-print-directory -j 9" -export TARGET TARGET_FAMILY TARGET_SUBFAMILY TARGET_PROJECT PROJECT_PATH PREFIX_PROJECT PREFIX_BUILD\ - PREFIX_BUILD_HOST PREFIX_FS PREFIX_BOOT PREFIX_PROG PREFIX_PROG_STRIPPED PREFIX_A\ - PREFIX_H PREFIX_ROOTFS CROSS CFLAGS CXXFLAGS LDFLAGS CC LD AR AS MAKEFLAGS DEVICE_FLAGS PLO_SCRIPT_DIR\ - PREFIX_SYSROOT LIBPHOENIX_DEVEL_MODE - # export flags for ports - call make only after all necessary env variables are already set EXPORT_CFLAGS="$(make -f phoenix-rtos-build/Makefile.common export-cflags)" EXPORT_LDFLAGS="$(make -f phoenix-rtos-build/Makefile.common export-ldflags)" @@ -200,7 +200,7 @@ fi if [ "${B_HOST}" = "y" ]; then if [ "$TARGET" != "$_TARGET_FOR_HOST_BUILD" ]; then # if not already building for host - re-exec with clean env - (env "$ORIG_ENV" TARGET=$_TARGET_FOR_HOST_BUILD ./phoenix-rtos-build/build.sh host) + (env "$ORIG_ENV" NOSAN=1 TARGET=$_TARGET_FOR_HOST_BUILD ./phoenix-rtos-build/build.sh host) else source ./phoenix-rtos-build/build-host-tools.sh fi diff --git a/scripts/gr716-bch.py b/scripts/gr716-bch.py index 425bef9f..0b42fc41 100755 --- a/scripts/gr716-bch.py +++ b/scripts/gr716-bch.py @@ -21,7 +21,8 @@ def parse_args(): ) parser.add_argument("input", help="file to convert") parser.add_argument("output", help="output file") - parser.add_argument("-s", "--size", help="size of the flash memory (default 16 MiB)") + parser.add_argument("-s", "--size", type=int, default=(16*1024*1024), + help="size of the flash memory in bytes (default %(default)s)") return parser.parse_args() @@ -60,11 +61,6 @@ def generate_bch(data: bytearray): def main(): args = parse_args() - try: - size = int(args.size) if args.size else 16 * 1024 * 1024 - except: - print(f'Error: invalid size "{args.size}"') - exit(1) validate_file(args.input) @@ -83,7 +79,7 @@ def main(): f.write(bch) print(f"Generated BCH of {args.input} to {args.output}") - print(f"{BOLD}{GREEN}Please load the BCH file to the SPI flash at offset {hex(size - len(bch))}{NORMAL}") + print(f"{BOLD}{GREEN}Please load the BCH file to the SPI flash at offset {args.size - len(bch):#x}{NORMAL}") if __name__ == "__main__": diff --git a/scripts/image_builder.py b/scripts/image_builder.py new file mode 100755 index 00000000..20adde98 --- /dev/null +++ b/scripts/image_builder.py @@ -0,0 +1,806 @@ +#!/usr/bin/env python3 +# +# Generic image builder for Phoenix-RTOS based projects. +# Parses plo scripts in YAML format to produce target plo scripts partition/disk images. +# +# Copyright 2024 Phoenix Systems +# Author: Marek Bialowas +# + +import argparse +import logging +import os +import sys +import subprocess +from collections import defaultdict +from enum import Enum, IntEnum +from pathlib import Path +from dataclasses import InitVar, asdict, dataclass, field, KW_ONLY, fields +from typing import IO, Any, BinaryIO, ClassVar, Dict, List, Optional, TextIO, Tuple, Union +import yaml +import jinja2 + +from nvm_config import FlashMemory, read_nvm, find_target_part +from strip import ElfParser, PhFlags, PhType + +VERSION = (1, 0, 0) + + +# global consts (taken from env/commandline) +TARGET: str +SIZE_PAGE: int +PREFIX_BOOT: Path +PREFIX_ROOTFS: Path +PREFIX_PROG_STRIPPED: Path +PLO_SCRIPT_DIR: Path + + +@dataclass +class ProgInfo: + path: Path + offs: int + size: int + + def __str__(self) -> str: + return f"{str(self.path.name):30s} (offs={self.offs:#10x}, size={self.size:#8x})" + + +def round_up(size: int, size_page: int) -> int: + return (size + size_page - 1) & ~(size_page - 1) + + +def get_elf_sizes(path : Path) -> Tuple[int, int, int, int]: + """Returns vaddr + ceil(mem_size, SIZE_PAGE) of TEXT and BSS sections""" + + with open(path, "rb") as f: + elf = ElfParser(f) + logging.debug(elf.get_program_headers()) + + text_ph = [] + bss_ph = [] + + for ph, _ in elf.get_program_headers(): + if ph.p_type == PhType.PT_LOAD: + if ph.p_flags == PhFlags.PF_R | PhFlags.PF_X: + text_ph.append(ph) + + if ph.p_flags == PhFlags.PF_R | PhFlags.PF_W: + bss_ph.append(ph) + + assert len(text_ph) == 1 + assert len(bss_ph) == 1 + + return (text_ph[0].p_vaddr, round_up(text_ph[0].p_memsz, SIZE_PAGE), + bss_ph[0].p_vaddr, round_up(bss_ph[0].p_memsz, SIZE_PAGE)) + + +class PloScriptEncoding(Enum): + """All supported types of PLO script encoding""" + DEBUG_ASDICT = -1 # debug-only output + STRING_MAGIC_V1 = 0 # human-readable string, beginning with 8-char magic string + # BINARY_V1 = 10 # packed binary representation of the script NOTE: not yet implemented + + +class PloCmdFactory: + """Plo command factory from different input args/kwargs""" + + @staticmethod + def _extract_extra_flags(cmd_args: List[str]) -> Optional[str]: + """Deletes from cmd_args and returns first param beginning with '-'""" + for arg in cmd_args: + if arg.startswith('-'): + cmd_args.remove(arg) + return arg + + return None + + @classmethod + def build(cls, cmd: str = "", **kwargs): + cmd_args = [] + if "name" in kwargs: + cmd_name = kwargs.pop("name") + elif "action" in kwargs: + cmd_name = kwargs.pop("action") + else: # command as a string + cmd_name, *cmd_args = cmd.split() + + if not cmd_name: + raise ValueError(f"unknown CMD format: str='{cmd}', kwargs={kwargs}") + + kwargs["name"] = cmd_name + extra_flags = cls._extract_extra_flags(cmd_args) + # set extra_flags only if not explicitly provided + if extra_flags is not None: + kwargs["extra_flags"] = kwargs.get("extra_flags", extra_flags) + + # TODO: use more generic approach with cls.NAME? + if cmd_name in ("kernel", "kernelimg"): + return PloCmdKernel(*cmd_args, **kwargs) + if cmd_name in ("app", "blob"): + return PloCmdApp(*cmd_args, **kwargs) + if cmd_name in ("call"): + return PloCmdCall(*cmd_args, **kwargs) + + # TODO: add compile-time checks for scripts validity (eg. memory regions cross-check)? + + # generic PLO command - treated for now as string + return PloCmdGeneric(cmd) + + +@dataclass +class PloCmdBase: + """Base class for all PLO commands""" + NAME: ClassVar[str] = "unknown" + name: str = field(default=NAME, kw_only=True) + # command extra flags (first param from string beginning with '-') + extra_flags: InitVar[str] = field(default='', kw_only=True) + + def emit(self, file: TextIO, enc: PloScriptEncoding, payload_offs: int, is_relative: bool) -> Tuple[int, Optional[ProgInfo]]: + """ + Emit plo command into `file` using `enc` encoding. + + Args: + payload_offs: Current offset in partition file + is_relative: True if the current script should use relative aliases (alias -b needs to be called in one of the previous scripts) + + Returns: + Tuple (new payload_offs and optional ProgInfo if data needs to be added to the partition file) + """ + + raise NotImplementedError(f"{self.__class__.__name__}: emit not implemented!") + + +@dataclass +class PloCmdGeneric(PloCmdBase): + """Generic PLO command - treated only as a string - fallback for unknown specific PLO cmd type""" + cmd: str + + def emit(self, file: TextIO, enc: PloScriptEncoding, payload_offs: int, is_relative: bool) -> Tuple[int, Optional[ProgInfo]]: + if enc == PloScriptEncoding.DEBUG_ASDICT: + file.write(str(asdict(self)) + "\n") + elif enc == PloScriptEncoding.STRING_MAGIC_V1: + # basic plo cmd - just emit as a string + file.write(self.cmd + "\n") + else: + raise NotImplementedError(f"PloScriptEncoding {enc.value} not implemented") + + return payload_offs, None + + +@dataclass +class PloCmdAlias(PloCmdBase): + """Alias command - used for memory aliases - emitted usually by app/kernel/blob/call commands""" + NAME: ClassVar = "alias" + name: str = field(default=NAME, kw_only=True) + + filename: str # target alias filename + size: int # target alias size + set_base: bool = False # should we set new base before aliasing? + + def emit(self, file: TextIO, enc: PloScriptEncoding, payload_offs: int, is_relative: bool) -> Tuple[int, Optional[ProgInfo]]: + if enc == PloScriptEncoding.DEBUG_ASDICT: + file.write(str(asdict(self)) + "\n") + elif enc == PloScriptEncoding.STRING_MAGIC_V1: + if self.set_base: + if is_relative: + file.write(f"alias -rb {payload_offs:#x}\n") + + # TODO: add real support for virtual base change (payload_offs in partition, virtual_offs as relative alias) + # for now we'll just reset payload_offs - any relative `app` after base change in the same script would break + payload_offs = 0 + else: + file.write(f"alias -b {payload_offs:#x}\n") + + aliasCmd = "alias -r" if is_relative else "alias" + file.write(f"{aliasCmd} {self.filename} {payload_offs:#x} {self.size:#x}\n") + else: + raise NotImplementedError(f"PloScriptEncoding {enc.value} not implemented") + + return payload_offs + round_up(self.size, SIZE_PAGE), ProgInfo(PREFIX_PROG_STRIPPED / self.filename, payload_offs, self.size) + + +@dataclass +class PloCmdKernel(PloCmdBase): + """Kernel is a special type of 'app' due to kernelimg command subtype: + kernel[img] + kernel flash0 + """ + NAME: ClassVar = "kernel" + device: str + + # internal fields + suffix: str = field(default="elf", init=False) + size: int = field(init=False) + filename: str = field(init=False) + abspath: Path = field(init=False) + + def __post_init__(self, extra_flags: str = ''): + if self.name == "kernelimg": + self.suffix = "bin" + + self.filename = f"phoenix-{'-'.join(TARGET.split('-')[:2])}.{self.suffix}" + self.abspath = PREFIX_PROG_STRIPPED / self.filename + self.size = os.path.getsize(self.abspath) + + def emit(self, file: TextIO, enc: PloScriptEncoding, payload_offs: int, is_relative: bool) -> Tuple[int, Optional[ProgInfo]]: + alias = PloCmdAlias(self.filename, self.size) + new_offs, prog_info = alias.emit(file, enc, payload_offs, is_relative) + + if enc == PloScriptEncoding.DEBUG_ASDICT: + file.write(str(asdict(self)) + "\n") + elif enc == PloScriptEncoding.STRING_MAGIC_V1: + if self.name == "kernel": + file.write(f"{self.name} {self.device}\n") + else: # kernelimg + tbeg, tsz, dbeg, dsz = get_elf_sizes(self.abspath.with_suffix(".elf")) + file.write(f"{self.name} {self.device} {self.filename} {tbeg:#x} {tsz:#x} {dbeg:#x} {dsz:#x}\n") + else: + raise NotImplementedError(f"PloScriptEncoding {enc.value} not implemented") + + return new_offs, prog_info + + +class CmdAppFlags(IntEnum): + """extra flags for `app` command""" + # NOTE: they are not flags actually as you can't use `-n` only + NONE = 0 + EXEC = 1 + EXEC_NO_COPY = 2 + + def emit_as_string(self) -> str: + if self.value == CmdAppFlags.EXEC: + return " -x" + if self.value == CmdAppFlags.EXEC_NO_COPY: + return " -xn" + return "" + + @classmethod + def _missing_(cls, value): + """Makes it possible to initialize class from name-as-a-string""" + if isinstance(value, str): + value = value.upper() + if value in dir(cls): + return cls[value] + + raise ValueError(f"{value} is not a valid {cls.__name__}") + + +@dataclass +class PloCmdApp(PloCmdBase): + """Support for app/blob command: + app [-x|-xn] + blob + app flash0 -x psh;-i;/etc/rc.psh ddr ddr + """ + NAME: ClassVar = "app" + name: str = field(default=NAME, kw_only=True) + + device: str # PLO device name + filename_args: InitVar[str] = "" # program/blob name/full path with optional args separated by `;` + text_map: str = "" # target memory map for program .text + data_maps: str = "" # target memory map for program data + extra maps the process should have access to + _ = KW_ONLY + filename: str = "" + args: List[str] = field(default_factory=list) + flags: CmdAppFlags | str | int = CmdAppFlags.NONE + + # internal fields + size: int = field(init=False) + abspath: Path = field(init=False) + + @staticmethod + def _resolve_filename(filename: str) -> Tuple[str, Path]: + if filename.startswith("/"): + return os.path.basename(filename), PREFIX_ROOTFS / filename.lstrip("/") + + return filename, PREFIX_PROG_STRIPPED / filename + + def _parse_flags(self, extra_flags: str): + # flags attr takes precedence + if self.flags and isinstance(self.flags, str): + self.flags = CmdAppFlags(self.flags) + return + + if extra_flags == "-x": + self.flags = CmdAppFlags.EXEC + elif extra_flags == "-xn": + self.flags = CmdAppFlags.EXEC_NO_COPY + + def __post_init__(self, extra_flags: str = '', filename_args: str = ''): + self._parse_flags(extra_flags) + + if filename_args: + self.filename, *self.args = filename_args.split(";", maxsplit=1) + + if isinstance(self.args, str): + self.args = self.args.split(";") + + # filename can be either relative to PROG_STRIPPED or absolute (in ROOTFS) + self.filename, self.abspath = self._resolve_filename(self.filename) + + self.size = os.path.getsize(self.abspath) + + # TODO: add parsing maps? + for req_val_name in ("filename", "text_map", "data_maps"): + if not asdict(self).get(req_val_name): + raise TypeError(f"Required attribute '{req_val_name}' not present/empty") + + def emit(self, file: TextIO, enc: PloScriptEncoding, payload_offs: int, is_relative: bool) -> Tuple[int, Optional[ProgInfo]]: + alias = PloCmdAlias(self.filename, self.size) + new_offs, prog_info = alias.emit(file, enc, payload_offs, is_relative) + + if enc == PloScriptEncoding.DEBUG_ASDICT: + file.write(str(asdict(self)) + "\n") + elif enc == PloScriptEncoding.STRING_MAGIC_V1: + file.write(f"{self.name} {self.device}{self.flags.emit_as_string()} " + f"{';'.join([self.filename, *self.args])} {self.text_map} {self.data_maps}\n") + else: + raise NotImplementedError(f"PloScriptEncoding {enc.value} not implemented") + + return new_offs, prog_info + + +@dataclass +class PloCmdCall(PloCmdBase): + """Support for call command: + call [-setbase|-absolute] + call flash0 nlr0.plo 0x400000 0xdabaabad + """ + NAME: ClassVar = "call" + device: str # PLO device name + filename: str # target script (alias) filename + offset: int # target script offset (absolute or relative) + target_magic: str | None # target script magic if used + set_base: bool = False # should we set new base before calling the script? + absolute: bool = False # should we force absolute call even if the current script is relative? + name: str = field(default=NAME, kw_only=True) + + # internal fields + size: int = field(init=False) + + def __post_init__(self, extra_flags: str = ''): + + if extra_flags == "-setbase": + self.set_base = True + if extra_flags == "-absolute": + self.absolute = True + + # change str->desired type TODO: use decorators? + self.offset = int(self.offset, 0) + + self.size = 0x1000 # FIXME: get real defined script size + + def emit(self, file: TextIO, enc: PloScriptEncoding, payload_offs: int, is_relative: bool) -> Tuple[int, Optional[ProgInfo]]: + alias = PloCmdAlias(self.filename, self.size, set_base=self.set_base) + if self.absolute: # force absolute call even if the current script is relative + is_relative = False + + alias.emit(file, enc, self.offset, is_relative) + + if enc == PloScriptEncoding.DEBUG_ASDICT: + file.write(str(asdict(self)) + "\n") + elif enc == PloScriptEncoding.STRING_MAGIC_V1: + file.write(f"call {self.device} {self.filename} {self.target_magic}\n") + else: + raise NotImplementedError(f"PloScriptEncoding {enc.value} not implemented") + + # call doesn't change the payload offset nor write anything to the target partition + return payload_offs, None + + +@dataclass +class PloScript: + """Full PLO script definition""" + size: int # reserved size for the plo script + offs: int = 0 # script offset from the beginning of flash (if not relative) or 0 (if relative) + is_relative: bool = False # if relative, all aliases start from offset `0`, otherwise they start from offset `offs` + magic: str | None = None # PLO script magic + contents: List[PloCmdBase] = field(default_factory=list, init=False) + + def __post_init__(self): + # fix types + if isinstance(self.size, str): + self.size = int(self.size) + if isinstance(self.offs, str): + self.offs = int(self.offs) + + def write(self, file: TextIO, enc: PloScriptEncoding = PloScriptEncoding.STRING_MAGIC_V1) -> List[ProgInfo]: + prog_offs = self.offs + self.size # init with "just after the script" + progs = [] + + if self.magic is not None: + if len(self.magic) != 8: + raise ValueError(f"PLO magic string '{self.magic}' with invalid len ({len(self.magic)} != 8)") + file.write(f"{self.magic}\n") + for cmd in self.contents: + prog_offs, prog_spec = cmd.emit(file, enc, prog_offs, self.is_relative) + if prog_spec: + progs.append(prog_spec) + + file.write("\0") + + if file.tell() > self.size: + raise ValueError(f"Generated user script is too large (allocated size: {self.size} < actual size {file.tell()})") + + return progs + + +def render_val(tpl: Any, **kwargs) -> Any: # mostly str | List[str] | Dict[str, str] + """Uses jinja2 to render possible template variable - kwargs will be defined global variables""" + + # use recurrence for collections + if isinstance(tpl, list): + return [render_val(item, **kwargs) for item in tpl] + if isinstance(tpl, dict): + return {k: render_val(v, **kwargs) for k, v in tpl.items()} + + if isinstance(tpl, str): + rendered = jinja2.Template(tpl, undefined=jinja2.StrictUndefined).render(env=os.environ, **kwargs) + if tpl!= rendered: + logging.debug("render_val: '%s' -> '%s'", tpl, rendered) + return rendered + + # shortcut for lazy callers - return value with original type + return tpl + + +def str2bool(v: str | bool) -> bool: + """False is denoted by empty string or any literal sensible false values""" + if isinstance(v, bool): + return v + return v.lower() not in ("", "no", "false", "n", "0") + + +def nvm_to_dict(nvm: List[FlashMemory]) -> Dict[str, Dict[str, Any]]: + """Convert NVM to basic data types for jinja2 templates resolution""" + nvm_dict = {f.name: {p.name: p for p in f.parts} for f in nvm} + for flash in nvm: + nvm_dict[flash.name]['_meta'] = {f.name: getattr(flash, f.name) for f in fields(flash)} + + return nvm_dict + + +def parse_plo_script(nvm: List[FlashMemory], script_name: str) -> PloScript: + """Parse YAML/jinja2 plo script and return it as PloScript object""" + + nvm_dict = nvm_to_dict(nvm) + + with open(script_name, "r", encoding="utf-8") as f: + script_dict = yaml.safe_load(f) + + # render templates in basic plo script attributes + plo_param_names = [f.name for f in fields(PloScript) if f.init] + plo_kwargs = {k: render_val(script_dict.get(k), nvm=nvm_dict) for k in plo_param_names if k in script_dict} + script = PloScript(**plo_kwargs) + + tpl_context = {'nvm': nvm_dict, 'script': script} + for cmd in script_dict['contents']: + if isinstance(cmd, str): + cmd_rendered = render_val(cmd, **tpl_context) + cmddef = PloCmdFactory.build(cmd_rendered) + else: + # render all values + args = {k: render_val(v, **tpl_context) for k, v in cmd.items()} + enabled = args.pop('if', True) + if not str2bool(enabled): + logging.debug("PLO command disabled (if: '%s'): %s", enabled, str(args)) + continue + + if 'str' in args: + # command still as string, just conditional + cmddef = PloCmdFactory.build(args["str"]) + else: + # treat all dict elements as keyword arguments + cmddef = PloCmdFactory.build(**args) + + script.contents.append(cmddef) + + return script + + +def write_plo_script(nvm: List[FlashMemory], + script_name: + str, out_name: str | None = None, + enc: PloScriptEncoding = PloScriptEncoding.STRING_MAGIC_V1) -> List[ProgInfo]: + """Write desired PLO script and return contents of the target partition (including plo script itself)""" + os.makedirs(PLO_SCRIPT_DIR, exist_ok=True) + plo_script = parse_plo_script(nvm, script_name) + + if out_name is not None: + if out_name.startswith("/"): + path = Path(out_name) + else: + path = PLO_SCRIPT_DIR / out_name + else: + path = PLO_SCRIPT_DIR / os.path.basename(script_name).removesuffix(".yaml") + + with open(path, "w", encoding="ascii") as f: + progs = plo_script.write(f, enc) + + # TODO: what about plo_script.size? + progs = [ProgInfo(path, plo_script.offs, os.path.getsize(path))] + progs + return progs + + + +def set_offset(file: IO[bytes], target_offset: int, padding_byte: int): + """Sets the file position to `target_offset` - write `padding_byte` to extend the file if necessary""" + assert 0 <= padding_byte <= 255 + CHUNK_SIZE = 512 + + # always move to the end of the file (previous write might have been in the middle) + file.seek(0, os.SEEK_END) + curr_offset = file.tell() + + if (diff := target_offset - curr_offset) > 0: + + full, part = (diff // CHUNK_SIZE, diff % CHUNK_SIZE) + pad_chunk = bytes([padding_byte]) * CHUNK_SIZE + file.write(pad_chunk[:part]) + for _ in range(full): + file.write(pad_chunk) + elif target_offset != curr_offset: # overwriting parts of the existing image - just set the position directly + file.seek(target_offset) + + assert file.tell() == target_offset, f"{file.tell()} != {target_offset}" + + +def add_to_image(fout: BinaryIO, offset: int, fpath: Union[str, Path], padding_byte: int): + """Add file data (`fpath`) to the target image file (`fout`) at a given `offset`""" + set_offset(fout, offset, padding_byte) + + CHUNK_SIZE = 512 + written = 0 + with open(fpath, "rb") as fIn: + while True: + data = fIn.read(CHUNK_SIZE) + written += fout.write(data) + if len(data) != CHUNK_SIZE: + break + + return written + + +def remove_if_exists(path: Path): + try: + os.remove(path) + except FileNotFoundError: + pass + + +def create_ptable(flash: FlashMemory) -> Path: + tool = PREFIX_BOOT / "psdisk" + out_fn = PREFIX_BOOT / flash.ptable_filename + remove_if_exists(out_fn) + + cmd: List[str] = [str(tool), str(out_fn), "-m", f"0x{flash.size:x},0x{flash.block_size:x}"] + for part in flash.parts: + if part.virtual: + continue + cmd.extend(("-p", f"{part.name},0x{part.offs:x},0x{part.size:x},0x{part.type.value:x}")) + + logging.debug("command: %s", " ".join(cmd)) + proc = subprocess.run(cmd, capture_output=True, check=True) + logging.info(proc.stdout.decode()) + + return out_fn + + +def write_image(contents: List[ProgInfo], img_out_name: Path, img_max_size: int, padding_byte) -> int: + """Creates partition/disk image with desired `contents`""" + remove_if_exists(img_out_name) + + logging.verbose("program images:\n%s", "\n".join([str(prog) for prog in contents])) + + if not contents: + raise ValueError("Empty partition definition") + + with open(img_out_name, "wb") as fout: + for prog in contents: + written = add_to_image(fout, prog.offs, prog.path, padding_byte) + assert written == prog.size, f"{prog.path}: write failed: written={written}, size={prog.size}" + + img_size = os.path.getsize(img_out_name) + logging.info("Created %-20s with size %8u / %8u (%4u kB %3u %%)", os.path.basename(img_out_name), + img_size, img_max_size, img_size / 1024, 100 * img_size // img_max_size) + + assert img_size <= img_max_size, f"Partition image exceeds total size {img_size} > {img_max_size}" + return 0 + + +def parse_args() ->argparse.Namespace: + def env_or_required(key): + """required as a commandline param or ENV var""" + return {'default': os.environ.get(key)} if key in os.environ else {'required': True} + + parser = argparse.ArgumentParser(description="Create PLO scripts, partitions and disk images") + parser.add_argument("-v", "--verbose", action='count', default=0, help="Increase verbosity (can be used multiple times)") + parser.add_argument("--version", action="version", version=f"{parser.prog} {VERSION[0]}.{VERSION[1]}.{VERSION[2]}") + + # common paths - usually taken from build via env + parser.add_argument("--target", **env_or_required("TARGET"), help="TARGET identification string") + parser.add_argument("--size-page", **env_or_required("SIZE_PAGE"), type=int, help="Target page size") + parser.add_argument("--prefix-boot", **env_or_required("PREFIX_BOOT"), help="boot directory path") + parser.add_argument("--prefix-rootfs", **env_or_required("PREFIX_ROOTFS"), help="boot directory path") + parser.add_argument("--prefix-prog-stripped", **env_or_required("PREFIX_PROG_STRIPPED"), help="prog.stripped directory path") + parser.add_argument("--plo-script-dir", **env_or_required("PLO_SCRIPT_DIR"), help="output PLO scripts directory path") + + subparsers = parser.add_subparsers(help="subcommands", dest="cmd") + ptable = subparsers.add_parser("ptable", help="prepare partition tables") + ptable.add_argument("--nvm", type=str, default="nvm.yaml", help="Path to NVM config (default: %(default)s)") + + query = subparsers.add_parser("query", help="Render jinja2 template") + query.add_argument("--nvm", type=str, default="nvm.yaml", help="Path to NVM config (default: %(default)s)") + query.add_argument("query", type=str, help="Template to render") + + + partition = subparsers.add_parser("partition", aliases=["part"], help="prepare partition image") + + # opt 1 - use NVM config + part name + partition.add_argument("--nvm", type=str, default="nvm.yaml", help="Path to NVM config (default: %(default)s)") + partition.add_argument("--name", type=str, dest="part_name", help="Partition name from NVM in format [flash_name:]part_name") + # TODO: opt 2 - provide partition details by hand? + + part_exclusive_group = partition.add_mutually_exclusive_group(required=True) + # opt 1 - use PLO script for partition definition + part_exclusive_group.add_argument("--script", type=str, dest="script_name", help="YAML PLO script definition") + # opt 2 - define partition contents by hand + part_exclusive_group.add_argument("--contents", type=str, action="append", help="filename to append to the partition image in format `path[:offset]`") + + + script = subparsers.add_parser("script", help="prepare PLO script from yaml/template") + script.add_argument("--nvm", type=str, default="nvm.yaml", help="Path to NVM config (default: %(default)s)") + script.add_argument("--script", type=str, required=True, dest="script_name", help="YAML PLO script definition") + script.add_argument("--out", type=str, dest="out_name", help="Output script name (or full path) - default is the script name without .yaml suffix") + + + disk = subparsers.add_parser("disk", help="prepare disk image") + disk.add_argument("--nvm", type=str, default="nvm.yaml", help="Path to NVM config (default: %(default)s)\nBy default all partition image files will be used.") + disk.add_argument("--name", type=str, dest="flash_name", help="Flash name from NVM") + disk.add_argument("--part", type=str, action="append", help="Custom partition mapping in format [flash_name:]part_name=img_path") + disk.add_argument("--out", type=str, dest="out_name", help="Output disk file name (or full path) - default is the name from nvm config") + + args = parser.parse_args() + + # set common paths/vars as globals + global TARGET, SIZE_PAGE, PREFIX_BOOT, PREFIX_ROOTFS, PREFIX_PROG_STRIPPED, PLO_SCRIPT_DIR + TARGET = args.target + SIZE_PAGE = args.size_page + PREFIX_BOOT = Path(args.prefix_boot) + PREFIX_ROOTFS = Path(args.prefix_rootfs) + PREFIX_PROG_STRIPPED = Path(args.prefix_prog_stripped) + PLO_SCRIPT_DIR = Path(args.plo_script_dir) + + return args + + +VERBOSE = 15 + + +def main() -> int: + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) + logging.addLevelName(VERBOSE, 'VERBOSE') + # HACKISH: add function to the module for verbose logging + logging.verbose = lambda msg, *args, **kwargs: logging.log(VERBOSE, msg, *args, **kwargs) + + args = parse_args() + + if args.verbose > 0: + logging.getLogger().setLevel(logging.DEBUG if args.verbose == 2 else VERBOSE) + + + nvm: List[FlashMemory] = [] + if args.nvm: + nvm = read_nvm(args.nvm) + + if args.cmd == "ptable": + for flash in nvm: + ptable_path = create_ptable(flash) + + # prepare ptable partition image + ptable_part = find_target_part(nvm, f"{flash.name}:ptable") + if not ptable_part: + raise ValueError(f"No ptable partition defined for flash {flash.name}") + + ptable_size = os.path.getsize(ptable_path) + progs = [] + for offs in range(0, ptable_part.size, flash.block_size): + progs.append(ProgInfo(ptable_path, offs, ptable_size)) + + write_image(progs, PREFIX_BOOT / ptable_part.filename, ptable_part.size, ptable_part.flash.padding_byte) + + # errors during ptable creation will raise an exception with problem description + return 0 + + if args.cmd == "query": + nvm_dict = nvm_to_dict(nvm) + print(render_val(args.query, nvm=nvm_dict)) + return 0 + + if args.cmd == "script": + #TODO: update the result file only if different (avoid unnecessary re-linking plo with every build) + progs = write_plo_script(nvm, args.script_name, args.out_name) + out_script = progs[0] + logging.info("PLO script written to %s (size=%u)", out_script.path, os.path.getsize(out_script.path)) + logging.debug("program images:\n%s", "\n".join([str(prog) for prog in progs])) + return 0 + + if args.cmd in ("part", "partition"): + target_part = find_target_part(nvm, args.part_name) + if not target_part: + raise ValueError("Can't find target partition with given params") + + contents: List[ProgInfo] = [] + if args.script_name: + contents = write_plo_script(nvm, args.script_name) + + # if we're making non-relative plo script - change offsets by partition beginning + if contents and (contents[0].offs - target_part.offs) >= 0: + for prog in contents: + prog.offs -= target_part.offs + + elif args.contents: + curr_offs = 0 + for name in args.contents: + if ":" in name: + name, offs = name.split(":") + assert int(offs) >= curr_offs, f"offset {offs} larger than current offset ({curr_offs})" + curr_offs = int(offs) + + contents.append(ProgInfo(Path(name), curr_offs, os.path.getsize(name))) + + return write_image(contents, PREFIX_BOOT / target_part.filename, target_part.size, target_part.flash.padding_byte) + + if args.cmd == "disk": + # support `--part` overrides in format: `[flash_name:]part_name=img_path` + overrides = defaultdict(dict) + if args.part: + for pdef in args.part: + part_name, img_path = pdef.split("=") + part = find_target_part(nvm, part_name) + if part is None: + raise KeyError(f"Unknown partition definition: {part_name}") + overrides[part.flash.name][part.name] = img_path + + at_least_one_image_created = False + for flash in nvm: + if args.flash_name and flash.name != args.flash_name: + continue + + progs: List[ProgInfo] = [] + for part in flash.parts: + if part.empty: # never try to write the file + continue + if part.name in overrides[flash.name]: + part_img = overrides[flash.name][part.name] + if part_img.lower() in ("none", "null"): + continue + if not part_img.startswith("/"): + part_img = PREFIX_BOOT / part_img + else: + part_img = Path(part_img) + else: + part_img = PREFIX_BOOT / part.filename + + # for virtual ranges: allow explicit writing, missing partition is not an error + if part.virtual and not os.path.exists(part_img): + continue + + progs.append(ProgInfo(part_img, part.offs, os.path.getsize(part_img))) + + disk_path = PREFIX_BOOT / f"{flash.name}.disk" + if args.out_name: + disk_path = Path(args.out_name) if args.out_name.startswith('/') else PREFIX_BOOT / args.out_name + write_image(progs, disk_path, flash.size, flash.padding_byte) + at_least_one_image_created = True + + if not at_least_one_image_created: + raise ValueError(f"No disk image created - check `name` param ('{args.flash_name}') and NVM config!") + + return 0 + + return 1 # unknown command + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/nvm_config.py b/scripts/nvm_config.py new file mode 100644 index 00000000..55a57a80 --- /dev/null +++ b/scripts/nvm_config.py @@ -0,0 +1,178 @@ +# +# Non-Volatile Memory configuration file handler (YAML format). +# +# Copyright 2024 Phoenix Systems +# Author: Marek Bialowas +# +import logging +from enum import Enum +from dataclasses import dataclass, field +from typing import Any, List, Optional + +import yaml + + +class PartitionType(Enum): + """ptable partition type definition - keep in sync with libptable/ptable.h""" + RAW = 0x51 + JFFS2 = 0x72 + METERFS = 0x75 + + @classmethod + def _missing_(cls, value): + """Makes it possible to initialize class from name-as-a-string""" + if isinstance(value, str): + value = value.upper() + if value in dir(cls): + return cls[value] + + raise ValueError(f"{value} is not a valid {cls.__name__}") + + +@dataclass +class Partition: + """Single partition specification""" + offs: int + size: int + name: str + type: PartitionType + + flash: Any = field(kw_only=True) # access to the parent flash metadata + virtual: bool = field(default=False, kw_only=True) # don't put this partition into ptable + empty: bool = field(default=False, kw_only=True) # don't search for partition image when creating disk file + + def __str__(self): + def flags(self) -> str: + if self.virtual: + return 'E' + return ' ' + + return f"{flags(self)} {self.offs:#08x} {self.size:#08x} [{int(self.size / 1024):5d} kB] {self.name:12s} {self.type.name.lower()}" + + @property + def filename(self): + if self.virtual: + return f"part_{self.flash.name}_{self.name}.img" + + return f"part_{self.name}.img" + + +@dataclass +class FlashMemory: + """Single flash memory specification""" + name: str + size: int + block_size: int + padding_byte: int = 0x0 + + parts: List[Partition] = field(default_factory=list, kw_only=True) + + @property + def ptable_filename(self): + return f"{self.name}.ptable" + + def validate(self): + """Sanity checks for memory layout""" + prev_part = None + part_names = set() + for part in self.parts: + if part.name in part_names: + raise ValueError(f"{self.name}: duplicate partition name '{part.name}'") + + part_names.add(part.name) + + if part.virtual: # virtual partitions can overlap existing ones and not be aligned to block + continue + + if part.offs % self.block_size != 0: + raise ValueError(f"{self.name}: partition '{part.name}' start 0x{part.offs:x} is not aligned to block size 0x{self.block_size:x}") + if part.size % self.block_size != 0: + raise ValueError(f"{self.name}: partition '{part.name}' size 0x{part.size:x} is not aligned to block size 0x{self.block_size:x}") + + if prev_part and part.offs < prev_part.offs: + raise ValueError(f"{self.name}: partition offsets are not monotonic (error at {part.name})") + if prev_part and part.offs < (prev_part.offs + prev_part.size): + raise ValueError(f"{self.name}: partitions '{part.name}' and '{prev_part.name}' are overlapping") + + if part.offs + part.size > self.size: + raise ValueError(f"{self.name}: partition '{part.name}' size extends over the end of the flash") + + prev_part = part + + if prev_part and (free_size := self.size - (prev_part.offs + prev_part.size)) > 0: + logging.debug(f"{self.name}: free space at the end: 0x{free_size:x} [{int(free_size / 1024)} kB]") + + def __str__(self): + return f"{self.__class__.__name__}({self.name}) size={self.size:#x} [{int(self.size / 1024 / 1024)} MB] block_size={self.block_size:#x}\n" \ + + "\n".join(["\t" + str(p) for p in self.parts]) + + +def round_up(size: int, size_page: int) -> int: + return (size + size_page - 1) & ~(size_page - 1) + + +def read_nvm(fname: str) -> List[FlashMemory]: + """reads full Non-Volatile Memory layout from a file {fname}""" + nvm = [] + with open(fname, "r", encoding="utf-8") as f: + nvm_dict = yaml.safe_load(f) + # TODO: validate against JSON template? + + for (name, attrs) in nvm_dict.items(): + f = FlashMemory(name, attrs['size'], attrs['block_size'], attrs.get('padding_byte', 0)) + curr_offs = 0 + prev_p: Partition | None = None + for part_attrs in attrs.get('partitions', []): + p = Partition(part_attrs.get('offs', curr_offs), part_attrs.get('size', 0), part_attrs['name'], + PartitionType(part_attrs.get('type', 'RAW')), flash=f, empty=part_attrs.get('empty', False)) + + if part_attrs.get('virtual'): + p.virtual = True + else: + curr_offs = round_up(p.offs + p.size, f.block_size) + + # set previous partition size from the absolute offset of the current one + if not p.virtual and 'offs' in part_attrs and prev_p is not None and prev_p.size == 0: + prev_p.size = p.offs - prev_p.offs + + f.parts.append(p) + prev_p = p + + # add "virtual" ptable partition + ptable_blocks = attrs.get('ptable_blocks', 0) + if ptable_blocks > 0: + ptable_size = ptable_blocks * f.block_size + p = Partition(f.size - ptable_size, ptable_size, "ptable", PartitionType.RAW, flash=f, virtual=True) + f.parts.append(p) + + # set last non-virtual partition size (if 0) to the end of flash + for part in reversed(f.parts): + if not part.virtual: + if part.size == 0: + part.size = f.size - part.offs + break + + logging.debug(f) + f.validate() + logging.debug(f) + nvm.append(f) + + return nvm + + +def find_target_part(nvm: List[FlashMemory], name: str) -> Optional[Partition]: + """Finds matching partition from name in format `[flash_name:]part_name`""" + part_name = name + flash_name = None + if ":" in part_name: + (flash_name, part_name) = part_name.split(":") + + for flash in nvm: + if flash_name is not None and flash.name != flash_name: + continue + + for part in flash.parts: + if part.name == part_name: + return part + + return None diff --git a/scripts/strip.py b/scripts/strip.py index ec274fba..a348aa9f 100755 --- a/scripts/strip.py +++ b/scripts/strip.py @@ -2,8 +2,8 @@ # # Script to remove references to .symtab from relocation section and then run strip # -# Copyright 2022 Phoenix Systems -# Author: Andrzej Glowinski +# Copyright 2022, 2024 Phoenix Systems +# Author: Andrzej Glowinski, Marek Bialowas # import shutil @@ -12,7 +12,7 @@ import tempfile from io import BytesIO from dataclasses import dataclass -from enum import IntEnum +from enum import Flag, IntEnum import struct from typing import ClassVar, Type, List, Tuple @@ -42,6 +42,63 @@ class ShType(IntEnum): SHT_REL = 9 +class PhType(IntEnum): + PT_NULL = 0 # Unused segment. + PT_LOAD = 1 # Loadable segment. + PT_DYNAMIC = 2 # Dynamic linking information. + PT_INTERP = 3 # Interpreter pathname. + PT_NOTE = 4 # Auxiliary information. + PT_SHLIB = 5 # Reserved. + PT_PHDR = 6 # The program header table itself. + PT_TLS = 7 # The thread-local storage template. + PT_LOOS = 0x60000000 # Lowest operating system-specific pt entry type. + PT_HIOS = 0x6fffffff # Highest operating system-specific pt entry type. + PT_LOPROC = 0x70000000 # Lowest processor-specific program hdr entry type. + PT_HIPROC = 0x7fffffff # Highest processor-specific program hdr entry type. + + # x86-64 program header types. + # These all contain stack unwind tables. + PT_GNU_EH_FRAME = 0x6474e50 + PT_SUNW_EH_FRAME = 0x6474e50 + PT_SUNW_UNWIND = 0x6464e50 + + PT_GNU_STACK = 0x6474e551 # Indicates stack executability. + PT_GNU_RELRO = 0x6474e552 # Read-only after relocation. + PT_GNU_PROPERTY = 0x6474e553 # .note.gnu.property notes sections. + + PT_OPENBSD_MUTABLE = 0x65a3dbe5 # Like bss, but not immutable. + PT_OPENBSD_RANDOMIZE = 0x65a3dbe6 # Fill with random data. + PT_OPENBSD_WXNEEDED = 0x65a3dbe7 # Program does W^X violations. + PT_OPENBSD_NOBTCFI = 0x65a3dbe8 # Do not enforce branch target CFI. + PT_OPENBSD_SYSCALLS = 0x65a3dbe9 # System call sites. + PT_OPENBSD_BOOTDATA = 0x65a41be6 # Section for boot arguments. + + # ARM program header types. + PT_ARM_ARCHEXT = 0x70000000 # Platform architecture compatibility info + # These all contain stack unwind tables. + PT_ARM_EXIDX = 0x70000001 + PT_ARM_UNWIND = 0x70000001 + # MTE memory tag segment type + PT_AARCH64_MEMTAG_MTE = 0x70000002 + + # MIPS program header types. + PT_MIPS_REGINFO = 0x70000000 # Register usage information. + PT_MIPS_RTPROC = 0x70000001 # Runtime procedure table. + PT_MIPS_OPTIONS = 0x70000002 # Options segment. + PT_MIPS_ABIFLAGS = 0x70000003 # Abiflags segment. + + # RISCV program header types. + PT_RISCV_ATTRIBUTES = 0x70000003 + + +# Segment flag bits. +class PhFlags(Flag): + PF_X = 1 # Execute + PF_W = 2 # Write + PF_R = 4 # Read + PF_MASKOS = 0x0ff00000 # Bits for operating system-specific semantics. + PF_MASKPROC = 0xf0000000 # Bits for processor-specific semantics. + class ElfStruct: """Abstract for every ELF struct""" FORMAT: ClassVar[List[Tuple[str, str]]] @@ -163,6 +220,38 @@ class Elf32Shdr(ElfShdr): ] +@dataclass +class ElfPhdr(ElfStruct): + """"Abstraction for structs Elf32_Phdr and Elf64_Phdr""" + p_type: PhType + p_offset: int + p_vaddr: int + p_paddr: int + p_filesz: int + p_memsz: int + p_flags: PhFlags + p_align: int + + def __post_init__(self): + self.p_type = PhType(self.p_type) + self.p_flags = PhFlags(self.p_flags) + + +@dataclass +class Elf32Phdr(ElfPhdr): + """"Struct Elf32_Phdr""" + FORMAT = [ + ("I", "p_type"), + ("I", "p_offset"), + ("I", "p_vaddr"), + ("I", "p_paddr"), + ("I", "p_filesz"), + ("I", "p_memsz"), + ("I", "p_flags"), + ("I", "p_align") + ] + + @dataclass class ElfRelx(ElfStruct): """Abstraction for structs Elf32_Rel, Elf64_Rel, Elf32_Rela, Elf64_Rela""" @@ -194,6 +283,9 @@ def __iter__(self): assert self.entrySize == self.header.get_size() yield self.parser.read_struct(self.header, off), off + def __str__(self) -> str: + return "\n".join([str(s) for s, _ in self]) + class ElfSectionTable(ElfFixedSizeTable): header: Type[ElfShdr] @@ -206,6 +298,17 @@ def __init__(self, e: ElfEhdr, p: "ElfParser"): self.entrySize = e.e_shentsize +class ElfPhdrTable(ElfFixedSizeTable): + header: Type[ElfPhdr] + + def __init__(self, e: ElfEhdr, p: "ElfParser"): + super().__init__(p) + self.header = {EiClass.ELFCLASS32: Elf32Phdr}[p.ident.get_class()] + self.offset = e.e_phoff + self.size = e.e_phnum * e.e_phentsize + self.entrySize = e.e_phentsize + + class ElfRelocationTable(ElfFixedSizeTable): header: Type[ElfRelx] @@ -245,6 +348,9 @@ def get_sections(self) -> ElfSectionTable: def get_relocations(self, s: ElfShdr) -> ElfRelocationTable: return ElfRelocationTable(s, self) + def get_program_headers(self) -> ElfPhdrTable: + return ElfPhdrTable(self.header, self) + def remove_symtab_references(in_file, out_file): with open(in_file, "rb") as file: