Build optimization: pre-built PM3 binaries, ARM64 CI, base image caching

Replace PM3 compile-from-source in pi-gen with pre-built tarball extraction (saves 43-58 min). Merge stagePM3 into stageDangerousPi as 02-pm3-install substage, renumber all subsequent substages. Switch CI PM3 build to native ARM64 runner (ubuntu-24.04-arm64) eliminating QEMU overhead. Add weekly base-image workflow for pre-baking stages 0-2. Support PM3_TARBALL, BASE_IMAGE, and APT_PROXY env vars in build-image.sh. Also includes prior Phase 5 work: theme system, design system integration, component update system, OS updates, CI build pipeline, and test results. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-04 12:01:01 -08:00
parent 2ec89041ef
commit a9acdb85ce
163 changed files with 8124 additions and 921 deletions
--- a/ci/build-backend.sh
+++ b/ci/build-backend.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Package the backend Python source for distribution.
+#
+# Environment variables:
+#   OUTPUT_DIR  - where to write the tarball (default: ./dist)
+#   VERSION     - version string (default: read from VERSION file)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WORKSPACE="${SCRIPT_DIR}/.."
+OUTPUT_DIR="${OUTPUT_DIR:-${WORKSPACE}/dist}"
+VERSION="${VERSION:-$(cat "${WORKSPACE}/VERSION" 2>/dev/null || echo "0.1.0")}"
+
+echo "=== Dangerous Pi Backend Package ==="
+echo "Version: ${VERSION}"
+
+STAGING="/tmp/backend-staging/backend"
+rm -rf /tmp/backend-staging
+mkdir -p "$STAGING"
+
+# Copy backend source
+cp -r "${WORKSPACE}/app/backend" "${STAGING}/app/backend"
+
+# Copy requirements
+cp "${WORKSPACE}/requirements.txt" "${STAGING}/"
+
+cat > "${STAGING}/COMPONENT_VERSION" << EOF
+{
+  "component_id": "backend",
+  "version": "${VERSION}",
+  "build_date": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+}
+EOF
+
+mkdir -p "$OUTPUT_DIR"
+TARBALL="backend-${VERSION}.tar.gz"
+cd /tmp/backend-staging
+tar -czf "${OUTPUT_DIR}/${TARBALL}" backend/
+echo "✅ Built: ${OUTPUT_DIR}/${TARBALL}"
+
+rm -rf /tmp/backend-staging
--- a/ci/build-frontend.sh
+++ b/ci/build-frontend.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Build the Remix frontend and package it for distribution.
+#
+# Environment variables:
+#   OUTPUT_DIR  - where to write the tarball (default: ./dist)
+#   VERSION     - version string (default: read from VERSION file)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WORKSPACE="${SCRIPT_DIR}/.."
+OUTPUT_DIR="${OUTPUT_DIR:-${WORKSPACE}/dist}"
+VERSION="${VERSION:-$(cat "${WORKSPACE}/VERSION" 2>/dev/null || echo "0.1.0")}"
+FRONTEND_DIR="${WORKSPACE}/app/frontend"
+
+echo "=== Dangerous Pi Frontend Build ==="
+echo "Version: ${VERSION}"
+
+cd "$FRONTEND_DIR"
+
+# Install dependencies (including dev for build)
+npm ci --production=false
+
+# Build Remix app
+npm run build
+
+# Remove dev dependencies after build
+npm prune --production
+
+# Package
+STAGING="/tmp/frontend-staging/frontend"
+rm -rf /tmp/frontend-staging
+mkdir -p "$STAGING"
+
+cp -r build/ "$STAGING/"
+
+cat > "${STAGING}/COMPONENT_VERSION" << EOF
+{
+  "component_id": "frontend",
+  "version": "${VERSION}",
+  "build_date": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+}
+EOF
+
+mkdir -p "$OUTPUT_DIR"
+TARBALL="frontend-${VERSION}.tar.gz"
+cd /tmp/frontend-staging
+tar -czf "${OUTPUT_DIR}/${TARBALL}" frontend/
+echo "✅ Built: ${OUTPUT_DIR}/${TARBALL}"
+
+rm -rf /tmp/frontend-staging
--- a/ci/build-pm3.sh
+++ b/ci/build-pm3.sh
@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+# Build Proxmark3 client + firmware + Python bindings for aarch64.
+# Designed to run inside an aarch64 Docker container (via QEMU on CI).
+#
+# Environment variables:
+#   PYTHON_VERSION  - e.g. "3.12" (default: auto-detect)
+#   PM3_REPO        - upstream repo (default: RfidResearchGroup/proxmark3)
+#   OUTPUT_DIR      - where to write the tarball (default: ./dist)
+#
+# Reads patches from pi-gen/stageDangerousPi/02-pm3-install/ in the workspace.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WORKSPACE="${SCRIPT_DIR}/.."
+PYTHON_VERSION="${PYTHON_VERSION:-$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')}"
+PYTHON_ABI="cp$(echo "$PYTHON_VERSION" | tr -d '.')"
+PM3_REPO="${PM3_REPO:-https://github.com/RfidResearchGroup/proxmark3}"
+OUTPUT_DIR="${OUTPUT_DIR:-${WORKSPACE}/dist}"
+ARCH="$(uname -m)"  # aarch64 when in container
+BUILD_DIR="/tmp/pm3-build"
+
+echo "=== Dangerous Pi PM3 Build ==="
+echo "Python: ${PYTHON_VERSION} (${PYTHON_ABI})"
+echo "Arch:   ${ARCH}"
+echo "Repo:   ${PM3_REPO}"
+
+# -----------------------------------------------------------------------
+# 1. Install build dependencies
+# -----------------------------------------------------------------------
+echo "--- Installing build dependencies ---"
+apt-get update -qq
+apt-get install -y -qq \
+    cmake python3-dev swig build-essential \
+    libreadline-dev libusb-1.0-0-dev liblz4-dev libbz2-dev \
+    libjansson-dev libc6-dev pkg-config git \
+    gcc-arm-none-eabi libnewlib-dev \
+    ca-certificates libssl-dev libbluetooth-dev libgd-dev
+
+# -----------------------------------------------------------------------
+# 2. Clone and patch
+# -----------------------------------------------------------------------
+echo "--- Cloning Proxmark3 ---"
+rm -rf "$BUILD_DIR"
+git clone --depth 1 "$PM3_REPO" "$BUILD_DIR"
+cd "$BUILD_DIR"
+
+PATCH_DIR="${WORKSPACE}/pi-gen/stageDangerousPi/02-pm3-install"
+
+# Apply LED PWM control patch
+if [ -f "${PATCH_DIR}/led-pwm-control-fixed.patch" ]; then
+    echo "Applying LED PWM control patch (fixed)..."
+    patch -p1 < "${PATCH_DIR}/led-pwm-control-fixed.patch" || \
+        echo "Warning: LED PWM patch failed, continuing"
+elif [ -f "${PATCH_DIR}/led-pwm-control.patch" ]; then
+    echo "Applying LED PWM control patch..."
+    patch -p1 < "${PATCH_DIR}/led-pwm-control.patch" || \
+        echo "Warning: LED PWM patch failed, continuing"
+fi
+
+# Apply HF booster detection patch
+if [ -f "${PATCH_DIR}/hf-booster-detection.patch" ]; then
+    echo "Applying HF booster detection patch..."
+    patch -p1 < "${PATCH_DIR}/hf-booster-detection.patch" || \
+        echo "Warning: HF booster patch failed, continuing"
+fi
+
+# DangerousPi version branding
+sed -i 's/^fullgitinfo="Iceman"$/fullgitinfo="Iceman\/DangerousPi"/' tools/mkversion.sh 2>/dev/null || true
+
+# -----------------------------------------------------------------------
+# 3. Configure platform
+# -----------------------------------------------------------------------
+cat > Makefile.platform << EOF
+PLATFORM=PM3GENERIC
+LED_ORDER=PM3EASY
+EOF
+
+# -----------------------------------------------------------------------
+# 4. Build firmware (bootrom + fullimage)
+# -----------------------------------------------------------------------
+echo "--- Building firmware ---"
+make -j"$(nproc)" bootrom fullimage
+
+# -----------------------------------------------------------------------
+# 5. Build client
+# -----------------------------------------------------------------------
+echo "--- Building client ---"
+make -j"$(nproc)" client
+
+# -----------------------------------------------------------------------
+# 6. Build Python bindings (SWIG)
+# -----------------------------------------------------------------------
+echo "--- Building Python bindings ---"
+
+# Fix CMakeLists.txt for experimental_lib (add missing sources)
+if [ -f client/experimental_lib/CMakeLists.txt ]; then
+    sed -i '/TARGET_SOURCES.*pm3rrg_rdv4_experimental/,/)/s|\(${PM3_ROOT}/client/src/wiegand_formatutils.c\)|\1\n    ${PM3_ROOT}/client/src/qrcode/qrcode.c|' \
+        client/experimental_lib/CMakeLists.txt 2>/dev/null || true
+    sed -i 's|\(${PM3_ROOT}/client/src/pm3\.c\)|${PM3_ROOT}/client/src/pla.c\n        \1|' \
+        client/experimental_lib/CMakeLists.txt 2>/dev/null || true
+fi
+
+mkdir -p client/build && cd client/build
+cmake .. -DBUILD_PYTHON_LIB=ON
+make -j"$(nproc)"
+cd ../..
+
+# Build SWIG library
+if [ -f client/experimental_lib/00make_swig.sh ]; then
+    cd client/experimental_lib
+    ./00make_swig.sh
+    ./01make_lib.sh
+    cd ../..
+fi
+
+# -----------------------------------------------------------------------
+# 7. Package artifacts
+# -----------------------------------------------------------------------
+echo "--- Packaging ---"
+PM3_VERSION="$(git describe --always --tags 2>/dev/null || echo unknown)"
+STAGING="/tmp/pm3-staging/pm3"
+rm -rf /tmp/pm3-staging
+mkdir -p "${STAGING}/client/pyscripts" \
+         "${STAGING}/firmware" \
+         "${STAGING}/patches" \
+         "${STAGING}/scripts"
+
+# Client binary
+cp client/build/proxmark3 "${STAGING}/client/"
+
+# Python bindings
+cp client/experimental_lib/build/libpm3rrg_rdv4.so "${STAGING}/client/pyscripts/" 2>/dev/null || true
+cp client/pyscripts/*.py "${STAGING}/client/pyscripts/" 2>/dev/null || true
+cd "${STAGING}/client/pyscripts" && ln -sf libpm3rrg_rdv4.so _pm3.so && cd "$BUILD_DIR"
+
+# Firmware
+cp bootrom/obj/bootrom.elf "${STAGING}/firmware/"
+cp armsrc/obj/fullimage.elf "${STAGING}/firmware/"
+
+# Flash utilities
+cp pm3-flash-all pm3-flash-bootrom pm3-flash-fullimage "${STAGING}/" 2>/dev/null || true
+chmod +x "${STAGING}"/pm3-flash-* 2>/dev/null || true
+
+# Patches (for rebuild-from-source fallback)
+cp "${PATCH_DIR}"/*.patch "${STAGING}/patches/" 2>/dev/null || true
+
+# Version metadata
+cat > "${STAGING}/COMPONENT_VERSION" << VEOF
+{
+  "component_id": "pm3",
+  "version": "${PM3_VERSION}",
+  "platform": "${ARCH}-linux",
+  "python_version": "${PYTHON_VERSION}",
+  "python_abi": "${PYTHON_ABI}",
+  "build_date": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
+  "upstream_commit": "$(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
+}
+VEOF
+
+# Create tarball
+mkdir -p "$OUTPUT_DIR"
+TARBALL="pm3-${PM3_VERSION}-${ARCH}-linux-${PYTHON_ABI}.tar.gz"
+cd /tmp/pm3-staging
+tar -czf "${OUTPUT_DIR}/${TARBALL}" pm3/
+echo "✅ Built: ${OUTPUT_DIR}/${TARBALL}"
+
+# Cleanup
+rm -rf "$BUILD_DIR" /tmp/pm3-staging
--- a/ci/build-theme.sh
+++ b/ci/build-theme.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# Build the theme component from the @dangerousthings/tokens package.
+# Generates token CSS files and packages them for distribution.
+#
+# Environment variables:
+#   OUTPUT_DIR         - where to write the tarball (default: ./dist)
+#   VERSION            - version string (default: read from VERSION file)
+#   DESIGN_SYSTEM_DIR  - path to dt-design-system monorepo
+#                        (default: ../dt-design-system relative to workspace)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WORKSPACE="${SCRIPT_DIR}/.."
+OUTPUT_DIR="${OUTPUT_DIR:-${WORKSPACE}/dist}"
+VERSION="${VERSION:-$(cat "${WORKSPACE}/VERSION" 2>/dev/null || echo "0.1.0")}"
+DESIGN_SYSTEM_DIR="${DESIGN_SYSTEM_DIR:-$(realpath "${WORKSPACE}/../dt-design-system")}"
+
+echo "=== Dangerous Pi Theme Build ==="
+echo "Version:       ${VERSION}"
+echo "Design system: ${DESIGN_SYSTEM_DIR}"
+
+if [ ! -d "$DESIGN_SYSTEM_DIR" ]; then
+    echo "Error: dt-design-system not found at ${DESIGN_SYSTEM_DIR}"
+    echo "Set DESIGN_SYSTEM_DIR to the monorepo path"
+    exit 1
+fi
+
+# Build tokens to generate CSS
+cd "$DESIGN_SYSTEM_DIR"
+npm ci --production=false
+npm run build:tokens
+
+# Package theme CSS files
+STAGING="/tmp/theme-staging/theme"
+rm -rf /tmp/theme-staging
+
+BRANDS=("dt" "classic" "supra")
+for brand in "${BRANDS[@]}"; do
+    CSS_FILE="${DESIGN_SYSTEM_DIR}/packages/tokens/dist/css/${brand}.css"
+    if [ -f "$CSS_FILE" ]; then
+        mkdir -p "${STAGING}/${brand}"
+        cp "$CSS_FILE" "${STAGING}/${brand}/tokens.css"
+
+        # Read brand metadata from the generated CSS or tokens
+        cat > "${STAGING}/${brand}/theme.json" << TJEOF
+{
+  "id": "${brand}",
+  "supportsModes": ["dark", "light", "auto"],
+  "defaultMode": "dark"
+}
+TJEOF
+    else
+        echo "Warning: ${CSS_FILE} not found, skipping ${brand}"
+    fi
+done
+
+# Add brand metadata from tokens package if available
+TOKENS_INDEX="${DESIGN_SYSTEM_DIR}/packages/tokens/dist/index.js"
+if [ -f "$TOKENS_INDEX" ]; then
+    # Extract brand names/descriptions from the built tokens
+    node -e "
+const tokens = require('${TOKENS_INDEX}');
+const brands = Object.values(tokens.brands || {}).map(b => ({
+  id: b.id, name: b.name, description: b.description,
+  supportsModes: ['dark', 'light', 'auto'], defaultMode: 'dark'
+}));
+require('fs').writeFileSync('/tmp/theme-staging/theme/themes.json',
+  JSON.stringify({ schema_version: 1, themes: brands }, null, 2));
+" 2>/dev/null || echo '{"schema_version": 1, "themes": []}' > "${STAGING}/themes.json"
+else
+    # Fallback: generate registry from what we packaged
+    cat > "${STAGING}/themes.json" << REOF
+{
+  "schema_version": 1,
+  "themes": [
+    {"id": "dt", "name": "Dangerous Things", "description": "Official DT brand", "supportsModes": ["dark", "light", "auto"], "defaultMode": "dark"},
+    {"id": "classic", "name": "Classic Cyberpunk", "description": "Original dark navy aesthetic", "supportsModes": ["dark", "light", "auto"], "defaultMode": "dark"},
+    {"id": "supra", "name": "VivoKey Supra", "description": "Material Design 3 with VivoKey blue", "supportsModes": ["dark", "light", "auto"], "defaultMode": "dark"}
+  ]
+}
+REOF
+fi
+
+cat > "${STAGING}/COMPONENT_VERSION" << EOF
+{
+  "component_id": "theme",
+  "version": "${VERSION}",
+  "build_date": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")",
+  "tokens_package_version": "$(node -p "require('${DESIGN_SYSTEM_DIR}/packages/tokens/package.json').version" 2>/dev/null || echo "unknown")"
+}
+EOF
+
+mkdir -p "$OUTPUT_DIR"
+TARBALL="theme-${VERSION}.tar.gz"
+cd /tmp/theme-staging
+tar -czf "${OUTPUT_DIR}/${TARBALL}" theme/
+echo "✅ Built: ${OUTPUT_DIR}/${TARBALL}"
+
+rm -rf /tmp/theme-staging
--- a/ci/generate-manifest.py
+++ b/ci/generate-manifest.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Generate release-manifest.json from built component tarballs.
+
+Scans the dist/ directory for component tarballs, reads their
+COMPONENT_VERSION metadata, computes SHA256 checksums, and writes
+a release-manifest.json suitable for upload as a GitHub Release asset.
+
+Usage:
+    python3 ci/generate-manifest.py [dist_dir] [--version VERSION]
+"""
+import hashlib
+import json
+import os
+import sys
+import tarfile
+from pathlib import Path
+
+
+def sha256_file(path: Path) -> str:
+    """Compute SHA256 checksum of a file."""
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while chunk := f.read(65536):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def read_component_version(tarball_path: Path) -> dict:
+    """Read COMPONENT_VERSION JSON from inside a tarball."""
+    with tarfile.open(tarball_path, "r:gz") as tf:
+        for member in tf.getmembers():
+            if member.name.endswith("COMPONENT_VERSION"):
+                f = tf.extractfile(member)
+                if f:
+                    return json.loads(f.read().decode())
+    return {}
+
+
+def main():
+    dist_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("dist")
+    version = None
+    if "--version" in sys.argv:
+        idx = sys.argv.index("--version")
+        version = sys.argv[idx + 1]
+
+    if not dist_dir.exists():
+        print(f"Error: {dist_dir} does not exist")
+        sys.exit(1)
+
+    # Read VERSION file if version not specified
+    if not version:
+        version_file = Path("VERSION")
+        version = version_file.read_text().strip() if version_file.exists() else "0.1.0"
+
+    manifest = {
+        "schema_version": 1,
+        "release_version": version,
+        "components": {},
+    }
+
+    tarballs = sorted(dist_dir.glob("*.tar.gz"))
+    if not tarballs:
+        print(f"Warning: no tarballs found in {dist_dir}")
+
+    for tarball in tarballs:
+        filename = tarball.name
+        checksum = sha256_file(tarball)
+        size = tarball.stat().st_size
+        comp_version = read_component_version(tarball)
+        comp_id = comp_version.get("component_id", "")
+
+        if not comp_id:
+            # Try to infer from filename
+            for known in ("pm3", "frontend", "backend", "theme"):
+                if filename.startswith(known):
+                    comp_id = known
+                    break
+
+        if not comp_id:
+            print(f"  Skipping {filename} (unknown component)")
+            continue
+
+        comp_ver = comp_version.get("version", version)
+
+        # Build asset key (platform-specific for pm3)
+        platform_str = comp_version.get("platform")
+        python_abi = comp_version.get("python_abi")
+        if platform_str and python_abi:
+            asset_key = f"{platform_str}-{python_abi}"
+        else:
+            asset_key = "default"
+
+        asset_info = {
+            "filename": filename,
+            "checksum_sha256": checksum,
+            "size": size,
+        }
+        if comp_version.get("python_version"):
+            asset_info["python_version"] = comp_version["python_version"]
+        if platform_str:
+            asset_info["platform"] = platform_str
+
+        # Initialize component entry if needed
+        if comp_id not in manifest["components"]:
+            manifest["components"][comp_id] = {
+                "version": comp_ver,
+                "assets": {},
+                "changelog": "",
+            }
+
+        manifest["components"][comp_id]["assets"][asset_key] = asset_info
+        print(f"  {comp_id}/{asset_key}: {filename} ({size} bytes)")
+
+    # Write manifest
+    manifest_path = dist_dir / "release-manifest.json"
+    with open(manifest_path, "w") as f:
+        json.dump(manifest, f, indent=2)
+
+    print(f"\n✅ Wrote {manifest_path}")
+    print(f"   Components: {list(manifest['components'].keys())}")
+
+
+if __name__ == "__main__":
+    main()