From 7c2d7313997557edb40ed735d088d7f8c574de38 Mon Sep 17 00:00:00 2001 From: Slim B Date: Thu, 12 Feb 2026 09:20:55 +0300 Subject: [PATCH 01/37] "#" --- .shellcheckrc | 7 + CHANGELOG.md | 25 ++ LICENSE | 21 + Makefile | 42 ++ README.md | 84 ++++ VERSION | 1 + action.yml | 64 +++ bin/josh-sync | 672 ++++++++++++++++++++++++++++++++ examples/devenv.nix | 36 ++ examples/forward.yml | 74 ++++ examples/josh-sync.yml | 43 ++ examples/reverse.yml | 52 +++ flake.nix | 83 ++++ lib/auth.sh | 61 +++ lib/config.sh | 122 ++++++ lib/core.sh | 32 ++ lib/state.sh | 75 ++++ lib/sync.sh | 303 ++++++++++++++ schema/config-schema.json | 96 +++++ tests/fixtures/minimal.yml | 16 + tests/fixtures/multi-target.yml | 29 ++ tests/unit/auth.bats | 39 ++ tests/unit/config.bats | 106 +++++ tests/unit/state.bats | 40 ++ 24 files changed, 2123 insertions(+) create mode 100644 .shellcheckrc create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 VERSION create mode 100644 action.yml create mode 100755 bin/josh-sync create mode 100644 examples/devenv.nix create mode 100644 examples/forward.yml create mode 100644 examples/josh-sync.yml create mode 100644 examples/reverse.yml create mode 100644 flake.nix create mode 100644 lib/auth.sh create mode 100644 lib/config.sh create mode 100644 lib/core.sh create mode 100644 lib/state.sh create mode 100644 lib/sync.sh create mode 100644 schema/config-schema.json create mode 100644 tests/fixtures/minimal.yml create mode 100644 tests/fixtures/multi-target.yml create mode 100644 tests/unit/auth.bats create mode 100644 tests/unit/config.bats create mode 100644 tests/unit/state.bats diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..94d00ee --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,7 @@ +# josh-sync shellcheck configuration +# SC2155: Declare and assign separately to avoid masking return values +# We accept this pattern for jq/yq pipeline assignments where failure is handled by set -e +disable=SC2155 + +# Bash 4+ features +shell=bash diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f53e45c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,25 @@ +# Changelog + +## 1.0.0 + +Initial release. Extracted from [private-monorepo-example](https://code.itkan.io/pe/private-monorepo-example) into a standalone reusable library. + +### Features + +- Bidirectional sync: forward (mono → subrepo) and reverse (subrepo → mono) +- Multi-target support via `.josh-sync.yml` config +- Per-target credential overrides (SSH keys, HTTPS tokens) +- Force-with-lease safety for forward sync +- Loop prevention via git trailers +- State tracking on orphan branch (`josh-sync-state`) +- Initial import and subrepo reset commands +- Composite action for Gitea/GitHub CI +- Nix flake with devenv module +- Preflight validation checks +- Config schema (JSON Schema) + +### Breaking Changes (vs. inline scripts) + +- Python + pyyaml replaced by yq-go (single static binary) +- 7 separate scripts replaced by single `josh-sync` CLI +- Consumer workflows use composite action (`uses: org/josh-sync@v1`) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5019ec7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 itkan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..073f855 --- /dev/null +++ b/Makefile @@ -0,0 +1,42 @@ +.PHONY: lint test build clean + +# Lint all shell scripts with shellcheck +lint: + @echo "Running shellcheck..." + shellcheck -x lib/*.sh bin/josh-sync + @echo "All checks passed." + +# Run bats-core unit tests +test: + @echo "Running tests..." + bats tests/unit/ + @echo "All tests passed." + +# Bundle into a single self-contained script +build: dist/josh-sync + +dist/josh-sync: bin/josh-sync lib/*.sh VERSION + @mkdir -p dist + @echo "Bundling josh-sync..." + @echo '#!/usr/bin/env bash' > dist/josh-sync + @echo "# josh-sync $(cat VERSION) — bundled distribution" >> dist/josh-sync + @echo '# Generated by: make build' >> dist/josh-sync + @echo '' >> dist/josh-sync + @# Inline all library modules (strip shebangs and source directives) + @for f in lib/core.sh lib/config.sh lib/auth.sh lib/state.sh lib/sync.sh; do \ + echo "# --- $$f ---" >> dist/josh-sync; \ + grep -v '^#!/' "$$f" | grep -v '^# shellcheck source=' >> dist/josh-sync; \ + echo '' >> dist/josh-sync; \ + done + @# Append CLI (strip shebangs, source directives, and source commands) + @echo '# --- bin/josh-sync ---' >> dist/josh-sync + @grep -v '^#!/' bin/josh-sync \ + | grep -v '^# shellcheck source=' \ + | grep -v '^source "\$${JOSH_LIB_DIR}/' \ + | sed '/^JOSH_LIB_DIR=/,/^source/d' \ + >> dist/josh-sync + @chmod +x dist/josh-sync + @echo "Built: dist/josh-sync ($(wc -l < dist/josh-sync) lines)" + +clean: + rm -rf dist/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..94e16f9 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# josh-sync + +Bidirectional monorepo ↔ subrepo sync via [josh-proxy](https://josh-project.github.io/josh/). Supports multiple sync targets from a single config. + +## Quick Start + +### 1. Add config + +Create `.josh-sync.yml` in your monorepo root: + +```yaml +josh: + proxy_url: "https://josh.example.com" + monorepo_path: "org/monorepo" + +targets: + - name: "billing" + subfolder: "services/billing" + josh_filter: ":/services/billing" + subrepo_url: "git@gitea.example.com:ext/billing.git" + subrepo_auth: "ssh" + branches: + main: main + forward_only: [] + +bot: + name: "josh-sync-bot" + email: "josh-sync-bot@example.com" + trailer: "Josh-Sync-Origin" +``` + +### 2. Add CI workflows + +Copy from [examples/](examples/) and customize paths/branches: + +```yaml +# .gitea/workflows/josh-sync-forward.yml +- uses: org/josh-sync@v1 + with: + direction: forward + env: + SYNC_BOT_USER: ${{ secrets.SYNC_BOT_USER }} + SYNC_BOT_TOKEN: ${{ secrets.SYNC_BOT_TOKEN }} + SUBREPO_SSH_KEY: ${{ secrets.SUBREPO_SSH_KEY }} +``` + +### 3. Local dev (Nix) + +Add josh-sync as a flake input, then: + +```nix +{ inputs, ... }: { + imports = [ inputs.josh-sync.devenvModules.default ]; +} +``` + +Run `josh-sync preflight` to validate your setup. + +## CLI + +``` +josh-sync sync [--forward|--reverse] [--target NAME] [--branch BRANCH] +josh-sync preflight +josh-sync import +josh-sync reset +josh-sync status +josh-sync state show [branch] +josh-sync state reset [branch] +``` + +## How It Works + +- **Forward sync** (mono → subrepo): pushes directly if clean, creates conflict PR if not. Uses `--force-with-lease` for safety. +- **Reverse sync** (subrepo → mono): always creates a PR, never pushes directly. +- **Loop prevention**: `Josh-Sync-Origin:` git trailer filters out bot commits. +- **State tracking**: orphan branch `josh-sync-state` stores JSON per target/branch. + +## Dependencies + +`bash >=4`, `git`, `curl`, `jq`, `yq` ([mikefarah/yq](https://github.com/mikefarah/yq) v4+), `openssh` + +## License + +MIT diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..5faf00f --- /dev/null +++ b/action.yml @@ -0,0 +1,64 @@ +name: josh-sync +description: Bidirectional monorepo ↔ subrepo sync via josh-proxy + +inputs: + config: + description: "Path to .josh-sync.yml" + default: ".josh-sync.yml" + direction: + description: "forward, reverse, or both" + default: "both" + target: + description: "Sync only this target (default: all)" + required: false + branch: + description: "Sync only this branch (default: all)" + required: false + debug: + description: "Enable debug output" + default: "false" + +runs: + using: composite + steps: + - name: Setup josh-sync + shell: bash + run: | + JOSH_DIR="$(mktemp -d)" + cp -r "${{ github.action_path }}/bin" "${{ github.action_path }}/lib" "${JOSH_DIR}/" + chmod +x "${JOSH_DIR}/bin/josh-sync" + echo "${JOSH_DIR}/bin" >> "$GITHUB_PATH" + echo "JOSH_SYNC_ROOT=${JOSH_DIR}" >> "$GITHUB_ENV" + + - name: Check dependencies + shell: bash + run: | + for cmd in git curl jq yq; do + command -v "$cmd" &>/dev/null || { echo "::error::Missing required tool: $cmd"; exit 1; } + done + + - name: Loop guard (forward) + id: guard + if: inputs.direction == 'forward' || inputs.direction == 'both' + shell: bash + run: | + TRAILER=$(yq '.bot.trailer' "${{ inputs.config }}" 2>/dev/null || echo "Josh-Sync-Origin") + if git log -1 --format=%B | grep -q "^${TRAILER}:"; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "::notice::Skipping sync — HEAD commit has sync trailer (loop prevention)" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Sync + if: steps.guard.outputs.skip != 'true' + shell: bash + env: + JOSH_SYNC_DEBUG: ${{ inputs.debug == 'true' && '1' || '0' }} + run: | + ARGS="--config ${{ inputs.config }}" + [[ "${{ inputs.direction }}" == "forward" ]] && ARGS+=" --forward" + [[ "${{ inputs.direction }}" == "reverse" ]] && ARGS+=" --reverse" + [[ -n "${{ inputs.target }}" ]] && ARGS+=" --target ${{ inputs.target }}" + [[ -n "${{ inputs.branch }}" ]] && ARGS+=" --branch ${{ inputs.branch }}" + josh-sync sync $ARGS diff --git a/bin/josh-sync b/bin/josh-sync new file mode 100755 index 0000000..38f8a63 --- /dev/null +++ b/bin/josh-sync @@ -0,0 +1,672 @@ +#!/usr/bin/env bash +# bin/josh-sync — CLI entrypoint for josh-sync +# +# Usage: josh-sync [flags] +# +# Commands: +# sync Run forward and/or reverse sync +# preflight Validate config, connectivity, auth +# import Initial import: pull subrepo into monorepo +# reset Reset subrepo to josh-filtered view +# status Show target config and sync state +# state show|reset Manage sync state directly +# +# Global flags: +# --config FILE Config path (default: .josh-sync.yml) +# --debug Verbose logging +# --version Show version +# --help Show usage + +set -euo pipefail + +# ─── Resolve library root ────────────────────────────────────────── + +if [ -n "${JOSH_SYNC_ROOT:-}" ]; then + JOSH_LIB_DIR="${JOSH_SYNC_ROOT}/lib" +else + JOSH_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)" +fi + +# Source library modules +# shellcheck source=../lib/core.sh +source "${JOSH_LIB_DIR}/core.sh" +# shellcheck source=../lib/config.sh +source "${JOSH_LIB_DIR}/config.sh" +# shellcheck source=../lib/auth.sh +source "${JOSH_LIB_DIR}/auth.sh" +# shellcheck source=../lib/state.sh +source "${JOSH_LIB_DIR}/state.sh" +# shellcheck source=../lib/sync.sh +source "${JOSH_LIB_DIR}/sync.sh" + +# ─── Version ──────────────────────────────────────────────────────── + +josh_sync_version() { + local version_file + if [ -n "${JOSH_SYNC_ROOT:-}" ]; then + version_file="${JOSH_SYNC_ROOT}/VERSION" + else + version_file="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/VERSION" + fi + if [ -f "$version_file" ]; then + cat "$version_file" + else + echo "dev" + fi +} + +# ─── Usage ────────────────────────────────────────────────────────── + +usage() { + cat >&2 < [flags] + +Commands: + sync Run forward and/or reverse sync + preflight Validate config, connectivity, auth, workflow coverage + import Initial import: pull existing subrepo into monorepo (creates PR) + reset Reset subrepo to josh-filtered view (after merging import PR) + status Show target config and sync state + state show [branch] Show sync state JSON + state reset [branch] Reset sync state to {} + +Global flags: + --config FILE Config path (default: .josh-sync.yml) + --debug Verbose logging + --version Show version and exit + --help Show this help + +Sync flags: + --forward Forward only (mono → subrepo) + --reverse Reverse only (subrepo → mono) + --target NAME Filter to one target (env: JOSH_SYNC_TARGET) + --branch BRANCH Filter to one branch + +Environment: + JOSH_SYNC_TARGET Restrict to a single target name + JOSH_SYNC_STATE_BRANCH State branch name (default: josh-sync-state) + SYNC_BOT_USER Git username for auth + SYNC_BOT_TOKEN Token with repo scope + SUBREPO_TOKEN Subrepo-specific token (optional) + SUBREPO_SSH_KEY SSH key for SSH targets (optional) +EOF +} + +# ─── Sync Command ─────────────────────────────────────────────────── + +cmd_sync() { + local direction="both" + local filter_target="${JOSH_SYNC_TARGET:-}" + local filter_branch="" + local config_file=".josh-sync.yml" + + while [ $# -gt 0 ]; do + case "$1" in + --forward) direction="forward"; shift ;; + --reverse) direction="reverse"; shift ;; + --target) filter_target="$2"; shift 2 ;; + --branch) filter_branch="$2"; shift 2 ;; + --config) config_file="$2"; shift 2 ;; + --debug) export JOSH_SYNC_DEBUG=1; shift ;; + *) die "Unknown flag: $1" ;; + esac + done + + parse_config "$config_file" + + # Forward sync + if [ "$direction" = "forward" ] || [ "$direction" = "both" ]; then + _sync_direction "forward" "$filter_target" "$filter_branch" + fi + + # Reverse sync + if [ "$direction" = "reverse" ] || [ "$direction" = "both" ]; then + _sync_direction "reverse" "$filter_target" "$filter_branch" + fi +} + +_sync_direction() { + local direction="$1" + local filter_target="$2" + local filter_branch="$3" + + while read -r TARGET_JSON; do + local target_name + target_name=$(echo "$TARGET_JSON" | jq -r '.name') + + # Filter to specific target if requested + if [ -n "$filter_target" ] && [ "$filter_target" != "$target_name" ]; then + continue + fi + + log "INFO" "══════ Target: ${target_name} (${direction}) ══════" + load_target "$TARGET_JSON" + + # Resolve branches + local branches + if [ -n "$filter_branch" ]; then + branches="$filter_branch" + elif [ "$direction" = "reverse" ]; then + # Exclude forward_only branches for reverse sync + branches=$(echo "$TARGET_JSON" | jq -r ' + (.forward_only // []) as $fwd | + .branches | keys[] | select(. as $b | $fwd | index($b) | not) + ') + else + branches=$(echo "$TARGET_JSON" | jq -r '.branches | keys[]') + fi + + # Sync each branch + for branch in $branches; do + echo "" >&2 + log "INFO" "━━━ Processing branch: ${branch} (${direction}) ━━━" + + # Resolve branch mapping + local mapped + mapped=$(echo "$TARGET_JSON" | jq -r --arg b "$branch" '.branches[$b] // empty') + if [ -z "$mapped" ]; then + log "WARN" "No mapping for branch ${branch} — skipping" + continue + fi + + export SYNC_BRANCH_MONO="$branch" + export SYNC_BRANCH_SUBREPO="$mapped" + + # Check state BEFORE cloning (skip if unchanged) + local state prev_sha current_sha + state=$(read_state "$branch") + + if [ "$direction" = "forward" ]; then + prev_sha=$(echo "$state" | jq -r '.last_forward.mono_sha // empty') + current_sha=$(git rev-parse "origin/${branch}" 2>/dev/null || echo "unknown") + + if [ "$prev_sha" = "$current_sha" ] && [ -n "$prev_sha" ]; then + log "INFO" "Branch ${branch} unchanged since last sync — skipping" + continue + fi + else + prev_sha=$(echo "$state" | jq -r '.last_reverse.subrepo_sha // empty') + current_sha=$(subrepo_ls_remote "${mapped}") + + if [ -z "$current_sha" ]; then + log "WARN" "Subrepo branch ${mapped} doesn't exist — skipping" + continue + fi + + if [ "$prev_sha" = "$current_sha" ] && [ -n "$prev_sha" ]; then + log "INFO" "Subrepo branch ${mapped} unchanged — skipping" + continue + fi + fi + + # Run sync + local result + if [ "$direction" = "forward" ]; then + result=$(forward_sync) + else + result=$(reverse_sync) + fi + log "INFO" "Result: ${result}" + + # Handle warnings + if [ "$result" = "lease-rejected" ]; then + echo "::warning::Target ${target_name}, branch ${branch}: subrepo changed during sync — will retry" + continue + fi + if [ "$result" = "conflict" ]; then + echo "::warning::Target ${target_name}, branch ${branch}: merge conflict — PR created on subrepo" + fi + if [ "$result" = "josh-rejected" ]; then + echo "::error::Target ${target_name}, branch ${branch}: josh rejected push — check proxy logs" + continue + fi + + # Update state (only on success) + local new_state + if [ "$direction" = "forward" ]; then + local subrepo_sha_now + subrepo_sha_now=$(subrepo_ls_remote "${mapped}") + new_state=$(jq -n \ + --arg m_sha "$current_sha" \ + --arg s_sha "${subrepo_sha_now:-}" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg status "$result" \ + --argjson prev "$state" \ + '$prev + {last_forward: {mono_sha:$m_sha, subrepo_sha:$s_sha, timestamp:$ts, status:$status}}') + else + local mono_sha_now + mono_sha_now=$(git rev-parse "origin/${branch}" 2>/dev/null || echo "") + new_state=$(jq -n \ + --arg s_sha "$current_sha" \ + --arg m_sha "${mono_sha_now:-}" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg status "$result" \ + --argjson prev "$state" \ + '$prev + {last_reverse: {subrepo_sha:$s_sha, mono_sha:$m_sha, timestamp:$ts, status:$status}}') + fi + + write_state "$branch" "$new_state" + done + done < <(echo "$JOSH_SYNC_TARGETS" | jq -c '.[]') +} + +# ─── Preflight Command ───────────────────────────────────────────── + +cmd_preflight() { + local config_file=".josh-sync.yml" + + while [ $# -gt 0 ]; do + case "$1" in + --config) config_file="$2"; shift 2 ;; + --debug) export JOSH_SYNC_DEBUG=1; shift ;; + *) die "Unknown flag: $1" ;; + esac + done + + local RED='\033[0;31m' + local GREEN='\033[0;32m' + local YELLOW='\033[1;33m' + local NC='\033[0m' + + pass() { echo -e " ${GREEN}✓${NC} $1"; } + fail() { echo -e " ${RED}✗${NC} $1"; ERRORS=$((ERRORS + 1)); } + warn() { echo -e " ${YELLOW}!${NC} $1"; } + + local ERRORS=0 + + echo "Josh Sync — Pre-flight Check" + echo "=============================" + echo "" + + # 1. Config + echo "1. Configuration" + + if [ ! -f "$config_file" ]; then + fail "${config_file} not found (run from monorepo root)" + exit 1 + fi + pass "${config_file} exists" + + parse_config "$config_file" + + [ -n "$JOSH_PROXY_URL" ] && pass "proxy_url: ${JOSH_PROXY_URL}" || fail "proxy_url missing" + [ -n "$MONOREPO_PATH" ] && pass "monorepo_path: ${MONOREPO_PATH}" || fail "monorepo_path missing" + [ -n "$BOT_TRAILER" ] && pass "trailer: ${BOT_TRAILER}" || fail "trailer missing" + [ -n "${GITEA_TOKEN:-}" ] && pass "SYNC_BOT_TOKEN is set" || warn "SYNC_BOT_TOKEN missing in .env" + [ -n "${BOT_USER:-}" ] && pass "BOT_USER: ${BOT_USER}" || warn "BOT_USER missing in .env" + + local target_count + target_count=$(echo "$JOSH_SYNC_TARGETS" | jq 'length') + pass "Targets configured: ${target_count}" + + # 2. Per-target checks + echo "" + echo "2. Targets" + + while read -r TARGET_JSON; do + local target_name subfolder auth + target_name=$(echo "$TARGET_JSON" | jq -r '.name') + subfolder=$(echo "$TARGET_JSON" | jq -r '.subfolder') + auth=$(echo "$TARGET_JSON" | jq -r '.subrepo_auth // "https"') + + echo "" + echo " ── Target: ${target_name} ──" + + load_target "$TARGET_JSON" + + [ -n "$JOSH_FILTER" ] && pass "josh_filter: ${JOSH_FILTER}" || fail "josh_filter missing" + [ -n "$SUBREPO_URL" ] && pass "subrepo_url: ${SUBREPO_URL}" || fail "subrepo_url missing" + + # Subfolder exists + if [ -d "$subfolder" ]; then + local file_count + file_count=$(find "$subfolder" -type f | wc -l) + pass "Subfolder ${subfolder}/ exists (${file_count} files)" + else + fail "Subfolder ${subfolder}/ not found" + if [ -z "${CI:-}" ]; then + echo " If the subrepo already has content, run: josh-sync import ${target_name}" + fi + fi + + # SSH key check + if [ "$auth" = "ssh" ]; then + local local_ssh_key_var + local_ssh_key_var=$(echo "$TARGET_JSON" | jq -r '.subrepo_ssh_key_var // "SUBREPO_SSH_KEY"') + if [ -n "${!local_ssh_key_var:-}" ]; then + pass "SSH key set (${local_ssh_key_var})" + else + warn "SSH key missing (${local_ssh_key_var})" + fi + fi + + # Josh proxy connectivity + if timeout 60 git ls-remote "$(josh_auth_url)" HEAD >/dev/null 2>&1; then + pass "Josh filter works for ${target_name}" + else + fail "Cannot ls-remote through josh for ${target_name}" + echo " Try: git ls-remote ${JOSH_PROXY_URL}/${MONOREPO_PATH}.git${JOSH_FILTER}.git" + fi + + # Subrepo connectivity + if [ "$auth" = "ssh" ]; then + local local_ssh_key_var2 + local_ssh_key_var2=$(echo "$TARGET_JSON" | jq -r '.subrepo_ssh_key_var // "SUBREPO_SSH_KEY"') + if [ -n "${!local_ssh_key_var2:-}" ]; then + if timeout 10 git ls-remote "$(subrepo_auth_url)" HEAD >/dev/null 2>&1; then + pass "Subrepo reachable via SSH" + else + fail "Cannot ls-remote subrepo via SSH" + fi + else + warn "Skipping subrepo check — no SSH key" + fi + else + if timeout 10 git ls-remote "$(subrepo_auth_url)" HEAD >/dev/null 2>&1; then + pass "Subrepo reachable via HTTPS" + else + fail "Cannot ls-remote subrepo via HTTPS" + fi + fi + + # Branch mapping + echo "$TARGET_JSON" | jq -r ' + (.forward_only // []) as $fwd | + .branches | to_entries[] | + .key as $k | .value as $v | + " mono/\($k) \(if ($fwd | index($k)) then "→ only" else "↔" end) subrepo/\($v)" + ' + done < <(echo "$JOSH_SYNC_TARGETS" | jq -c '.[]') + + # 3. Workflow path coverage + echo "" + echo "3. Workflow path coverage" + + local forward_workflow=".gitea/workflows/josh-sync-forward.yml" + if [ -f "$forward_workflow" ]; then + pass "${forward_workflow} exists" + + local workflow_paths + workflow_paths=$(yq -r '.on.push.paths[]?' "$forward_workflow" 2>/dev/null \ + | sed 's|/\*\*$||; s|/\*$||' || echo "") + + while read -r subfolder; do + if echo "$workflow_paths" | grep -q "^${subfolder}$"; then + pass "Workflow path covers ${subfolder}" + else + warn "Workflow paths missing ${subfolder}/** — add it to ${forward_workflow}" + fi + done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[].subfolder') + else + warn "${forward_workflow} not found (optional for local-only use)" + fi + + if [ -f ".gitea/workflows/josh-sync-reverse.yml" ]; then + pass ".gitea/workflows/josh-sync-reverse.yml exists" + else + warn ".gitea/workflows/josh-sync-reverse.yml not found (optional)" + fi + + # Summary + echo "" + echo "=============================" + if [ "$ERRORS" -eq 0 ]; then + echo -e "${GREEN}All checks passed.${NC} Ready to deploy." + else + echo -e "${RED}${ERRORS} check(s) failed.${NC} Fix issues above before deploying." + exit 1 + fi +} + +# ─── Import Command ──────────────────────────────────────────────── + +cmd_import() { + local config_file=".josh-sync.yml" + local target_name="" + + while [ $# -gt 0 ]; do + case "$1" in + --config) config_file="$2"; shift 2 ;; + --debug) export JOSH_SYNC_DEBUG=1; shift ;; + -*) die "Unknown flag: $1" ;; + *) target_name="$1"; shift ;; + esac + done + + if [ -z "$target_name" ]; then + echo "Usage: josh-sync import " >&2 + parse_config "$config_file" + echo "Available targets:" >&2 + echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2 + exit 1 + fi + + parse_config "$config_file" + + local target_json + target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)') + [ -n "$target_json" ] || die "Target '${target_name}' not found in config" + + log "INFO" "══════ Import target: ${target_name} ══════" + load_target "$target_json" + + local branches + branches=$(echo "$target_json" | jq -r '.branches | keys[]') + + for branch in $branches; do + local mapped + mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') + [ -z "$mapped" ] && continue + + export SYNC_BRANCH_MONO="$branch" + export SYNC_BRANCH_SUBREPO="$mapped" + + local result + result=$(initial_import) + log "INFO" "Result: ${result}" + done +} + +# ─── Reset Command ───────────────────────────────────────────────── + +cmd_reset() { + local config_file=".josh-sync.yml" + local target_name="" + + while [ $# -gt 0 ]; do + case "$1" in + --config) config_file="$2"; shift 2 ;; + --debug) export JOSH_SYNC_DEBUG=1; shift ;; + -*) die "Unknown flag: $1" ;; + *) target_name="$1"; shift ;; + esac + done + + if [ -z "$target_name" ]; then + echo "Usage: josh-sync reset " >&2 + parse_config "$config_file" + echo "Available targets:" >&2 + echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2 + exit 1 + fi + + parse_config "$config_file" + + local target_json + target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)') + [ -n "$target_json" ] || die "Target '${target_name}' not found in config" + + log "INFO" "══════ Reset target: ${target_name} ══════" + load_target "$target_json" + + local branches + branches=$(echo "$target_json" | jq -r '.branches | keys[]') + + for branch in $branches; do + local mapped + mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') + [ -z "$mapped" ] && continue + + export SYNC_BRANCH_MONO="$branch" + export SYNC_BRANCH_SUBREPO="$mapped" + + local result + result=$(subrepo_reset) + log "INFO" "Result: ${result}" + done +} + +# ─── Status Command ──────────────────────────────────────────────── + +cmd_status() { + local config_file=".josh-sync.yml" + + while [ $# -gt 0 ]; do + case "$1" in + --config) config_file="$2"; shift 2 ;; + --debug) export JOSH_SYNC_DEBUG=1; shift ;; + *) die "Unknown flag: $1" ;; + esac + done + + parse_config "$config_file" + + echo "Josh Sync Status" + echo "================" + echo "" + echo "Josh proxy: ${JOSH_PROXY_URL}" + echo "Monorepo: ${MONOREPO_PATH}" + echo "Bot: ${BOT_NAME} <${BOT_EMAIL}>" + echo "" + + while read -r TARGET_JSON; do + local target_name subfolder auth + target_name=$(echo "$TARGET_JSON" | jq -r '.name') + subfolder=$(echo "$TARGET_JSON" | jq -r '.subfolder') + auth=$(echo "$TARGET_JSON" | jq -r '.subrepo_auth // "https"') + + echo "Target: ${target_name}" + echo " Subfolder: ${subfolder}" + echo " Subrepo: $(echo "$TARGET_JSON" | jq -r '.subrepo_url')" + echo " Auth: ${auth}" + echo " Filter: $(echo "$TARGET_JSON" | jq -r '.josh_filter')" + + load_target "$TARGET_JSON" + + echo " Branches:" + echo "$TARGET_JSON" | jq -r '.branches | to_entries[] | " \(.key) → \(.value)"' + + local fwd_only + fwd_only=$(echo "$TARGET_JSON" | jq -r '(.forward_only // []) | join(", ")') + [ -n "$fwd_only" ] && echo " Forward only: ${fwd_only}" + + # Show state for each branch + echo " State:" + for branch in $(echo "$TARGET_JSON" | jq -r '.branches | keys[]'); do + local state + state=$(read_state "$branch") + if [ "$state" = "{}" ]; then + echo " ${branch}: (no state)" + else + local fwd_status fwd_ts rev_status rev_ts + fwd_status=$(echo "$state" | jq -r '.last_forward.status // "-"') + fwd_ts=$(echo "$state" | jq -r '.last_forward.timestamp // "-"') + rev_status=$(echo "$state" | jq -r '.last_reverse.status // "-"') + rev_ts=$(echo "$state" | jq -r '.last_reverse.timestamp // "-"') + echo " ${branch}: fwd=${fwd_status} (${fwd_ts}), rev=${rev_status} (${rev_ts})" + fi + done + echo "" + done < <(echo "$JOSH_SYNC_TARGETS" | jq -c '.[]') +} + +# ─── State Command ───────────────────────────────────────────────── + +cmd_state() { + local subcmd="${1:-}" + shift || true + + local config_file=".josh-sync.yml" + local target_name="" + local branch="main" + + # Parse remaining args + local args=() + while [ $# -gt 0 ]; do + case "$1" in + --config) config_file="$2"; shift 2 ;; + -*) die "Unknown flag: $1" ;; + *) args+=("$1"); shift ;; + esac + done + + [ "${#args[@]}" -ge 1 ] && target_name="${args[0]}" + [ "${#args[@]}" -ge 2 ] && branch="${args[1]}" + + case "$subcmd" in + show) + [ -n "$target_name" ] || { echo "Usage: josh-sync state show [branch]" >&2; exit 1; } + parse_config "$config_file" + + local target_json + target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)') + [ -n "$target_json" ] || die "Target '${target_name}' not found" + + load_target "$target_json" + read_state "$branch" | jq '.' + ;; + reset) + [ -n "$target_name" ] || { echo "Usage: josh-sync state reset [branch]" >&2; exit 1; } + parse_config "$config_file" + + local target_json + target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)') + [ -n "$target_json" ] || die "Target '${target_name}' not found" + + load_target "$target_json" + write_state "$branch" "{}" + log "INFO" "State reset for target '${target_name}', branch '${branch}'" + ;; + *) + echo "Usage: josh-sync state [branch]" >&2 + exit 1 + ;; + esac +} + +# ─── Main ─────────────────────────────────────────────────────────── + +main() { + local command="${1:-}" + shift || true + + # Handle global flags that can appear before command + case "$command" in + --version|-v) + echo "josh-sync $(josh_sync_version)" + exit 0 + ;; + --help|-h|"") + usage + exit 0 + ;; + esac + + case "$command" in + sync) cmd_sync "$@" ;; + preflight) cmd_preflight "$@" ;; + import) cmd_import "$@" ;; + reset) cmd_reset "$@" ;; + status) cmd_status "$@" ;; + state) cmd_state "$@" ;; + *) + echo "Unknown command: ${command}" >&2 + usage + exit 1 + ;; + esac +} + +main "$@" diff --git a/examples/devenv.nix b/examples/devenv.nix new file mode 100644 index 0000000..e7b0c17 --- /dev/null +++ b/examples/devenv.nix @@ -0,0 +1,36 @@ +# Consumer devenv.nix example +# Add josh-sync as a flake input in your devenv.yaml or flake.nix, +# then import the module here. +# +# In devenv.yaml: +# inputs: +# josh-sync: +# url: github:org/josh-sync/v1.0.0 +# flake: true +# +# Or in flake.nix: +# inputs.josh-sync = { +# url = "github:org/josh-sync/v1.0.0"; +# inputs.nixpkgs.follows = "nixpkgs"; +# }; + +{ inputs, pkgs, ... }: + +{ + imports = [ inputs.josh-sync.devenvModules.default ]; + + # josh-sync CLI is now available in the shell. + # Commands: + # josh-sync sync --forward Forward sync (mono → subrepo) + # josh-sync sync --reverse Reverse sync (subrepo → mono) + # josh-sync preflight Validate config and connectivity + # josh-sync import Initial import from subrepo + # josh-sync reset Reset subrepo to josh-filtered view + # josh-sync status Show target config and sync state + # josh-sync state show [b] Show state JSON + # josh-sync state reset [b] Reset state + + enterShell = '' + echo "Josh Sync available — run 'josh-sync --help' for commands" + ''; +} diff --git a/examples/forward.yml b/examples/forward.yml new file mode 100644 index 0000000..5c2fad7 --- /dev/null +++ b/examples/forward.yml @@ -0,0 +1,74 @@ +# .gitea/workflows/josh-sync-forward.yml — Consumer workflow template +# Syncs monorepo subfolder(s) → external subrepo(s) +# +# Customize: +# - paths: list all target subfolders +# - branches: list all monorepo branches to trigger on +# - org/josh-sync@v1: pin to your library repo and version + +name: "Josh Sync → Subrepo" + +on: + push: + branches: [main] + paths: + # List all target subfolders here (must match .josh-sync.yml targets[].subfolder) + - "services/billing/**" + - "services/auth/**" + - "libs/shared/**" + schedule: + - cron: "0 */6 * * *" + workflow_dispatch: + inputs: + target: + description: "Target to sync (empty = detect from push or all on schedule)" + required: false + default: "" + branch: + description: "Branch to sync (empty = triggered branch or all)" + required: false + default: "" + +concurrency: + group: josh-sync-fwd-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + sync: + runs-on: docker + container: node:20-bookworm + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Install tools + run: | + apt-get update -qq && apt-get install -y -qq jq curl git openssh-client >/dev/null 2>&1 + YQ_VERSION=v4.44.6 + curl -sL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ + -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + + - name: Detect changed target + if: github.event_name == 'push' + id: detect + run: | + CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") + TARGETS=$(yq -o json '.targets' .josh-sync.yml \ + | jq -r '.[] | "\(.name):\(.subfolder)"' \ + | while IFS=: read -r name prefix; do + echo "$CHANGED" | grep -q "^${prefix}/" && echo "$name" + done | sort -u | tr '\n' ' ') + echo "targets=${TARGETS}" >> "$GITHUB_OUTPUT" + + - uses: org/josh-sync@v1 + with: + direction: forward + target: ${{ github.event.inputs.target || steps.detect.outputs.targets }} + branch: ${{ github.event.inputs.branch || github.ref_name }} + env: + SYNC_BOT_USER: ${{ secrets.SYNC_BOT_USER }} + SYNC_BOT_TOKEN: ${{ secrets.SYNC_BOT_TOKEN }} + SUBREPO_TOKEN: ${{ secrets.SUBREPO_TOKEN || secrets.SYNC_BOT_TOKEN }} + SUBREPO_SSH_KEY: ${{ secrets.SUBREPO_SSH_KEY }} diff --git a/examples/josh-sync.yml b/examples/josh-sync.yml new file mode 100644 index 0000000..5db4223 --- /dev/null +++ b/examples/josh-sync.yml @@ -0,0 +1,43 @@ +# .josh-sync.yml — Multi-target configuration example +# Place this at the root of your monorepo. + +josh: + # Your josh-proxy instance URL (no trailing slash) + proxy_url: "https://josh.example.com" + # Repo path as josh sees it (org/repo on your Gitea/GitHub) + monorepo_path: "org/monorepo" + +targets: + - name: "billing" + subfolder: "services/billing" + josh_filter: ":/services/billing" + subrepo_url: "https://gitea.example.com/ext/billing.git" + subrepo_auth: "https" + branches: + main: main + develop: develop + forward_only: [] + + - name: "auth" + subfolder: "services/auth" + josh_filter: ":/services/auth" + subrepo_url: "git@gitea.example.com:ext/auth.git" + subrepo_auth: "ssh" + # Per-target credential override (reads from $AUTH_SSH_KEY instead of $SUBREPO_SSH_KEY) + subrepo_ssh_key_var: "AUTH_SSH_KEY" + branches: + main: main + forward_only: [] + + - name: "shared-lib" + subfolder: "libs/shared" + josh_filter: ":/libs/shared" + subrepo_url: "https://gitea.example.com/ext/shared-lib.git" + branches: + main: main + forward_only: [main] # one-way: mono → subrepo only + +bot: + name: "josh-sync-bot" + email: "josh-sync-bot@example.com" + trailer: "Josh-Sync-Origin" diff --git a/examples/reverse.yml b/examples/reverse.yml new file mode 100644 index 0000000..e532b01 --- /dev/null +++ b/examples/reverse.yml @@ -0,0 +1,52 @@ +# .gitea/workflows/josh-sync-reverse.yml — Consumer workflow template +# Checks external subrepo(s) for new commits and creates PRs on monorepo. +# Always creates PRs, never pushes directly. +# +# Customize: +# - cron schedule +# - org/josh-sync@v1: pin to your library repo and version + +name: "Josh Sync ← Subrepo" + +on: + schedule: + - cron: "0 1,7,13,19 * * *" # Every 6h, offset from forward + workflow_dispatch: + inputs: + target: + description: "Target to reverse-sync (empty = all)" + required: false + default: "" + branch: + description: "Branch to reverse-sync (empty = all eligible)" + required: false + default: "" + +concurrency: + group: josh-sync-rev-${{ github.event.inputs.target || 'all' }} + cancel-in-progress: false + +jobs: + sync: + runs-on: docker + container: node:20-bookworm + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Install tools + run: | + apt-get update -qq && apt-get install -y -qq jq curl git openssh-client >/dev/null 2>&1 + curl -sL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" \ + -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + + - uses: org/josh-sync@v1 + with: + direction: reverse + target: ${{ github.event.inputs.target || '' }} + branch: ${{ github.event.inputs.branch || '' }} + env: + SYNC_BOT_USER: ${{ secrets.SYNC_BOT_USER }} + SYNC_BOT_TOKEN: ${{ secrets.SYNC_BOT_TOKEN }} + SUBREPO_TOKEN: ${{ secrets.SUBREPO_TOKEN || secrets.SYNC_BOT_TOKEN }} + SUBREPO_SSH_KEY: ${{ secrets.SUBREPO_SSH_KEY }} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..cfee63e --- /dev/null +++ b/flake.nix @@ -0,0 +1,83 @@ +{ + description = "josh-sync: bidirectional monorepo ↔ subrepo sync via josh-proxy"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + outputs = { self, nixpkgs }: + let + forAllSystems = nixpkgs.lib.genAttrs [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + in + { + packages = forAllSystems (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + version = builtins.replaceStrings [ "\n" ] [ "" ] (builtins.readFile ./VERSION); + in + { + default = pkgs.stdenv.mkDerivation { + pname = "josh-sync"; + inherit version; + src = ./.; + + nativeBuildInputs = [ pkgs.makeWrapper ]; + + installPhase = '' + mkdir -p $out/{bin,lib} + cp lib/*.sh $out/lib/ + cp bin/josh-sync $out/bin/ + chmod +x $out/bin/josh-sync + + wrapProgram $out/bin/josh-sync \ + --set JOSH_SYNC_ROOT "$out" \ + --prefix PATH : ${pkgs.lib.makeBinPath [ + pkgs.bash + pkgs.git + pkgs.curl + pkgs.jq + pkgs.yq-go + pkgs.openssh + pkgs.coreutils + pkgs.findutils + pkgs.rsync + ]} + ''; + + meta = { + description = "Bidirectional monorepo ↔ subrepo sync via josh-proxy"; + license = pkgs.lib.licenses.mit; + mainProgram = "josh-sync"; + }; + }; + }); + + # devenv module for consumers + devenvModules.default = { pkgs, ... }: { + packages = [ self.packages.${pkgs.system}.default ]; + dotenv.disableHint = true; + }; + + # Dev shell for library contributors + devShells = forAllSystems (system: + let pkgs = nixpkgs.legacyPackages.${system}; + in { + default = pkgs.mkShell { + packages = [ + pkgs.bash + pkgs.git + pkgs.curl + pkgs.jq + pkgs.yq-go + pkgs.openssh + pkgs.shellcheck + pkgs.bats + pkgs.rsync + ]; + }; + }); + }; +} diff --git a/lib/auth.sh b/lib/auth.sh new file mode 100644 index 0000000..58e404d --- /dev/null +++ b/lib/auth.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# lib/auth.sh — Authenticated URLs, remote queries, and PR creation +# +# Requires: lib/core.sh and lib/config.sh sourced first +# Expects: JOSH_PROXY_URL, MONOREPO_PATH, BOT_USER, GITEA_TOKEN, JOSH_FILTER, +# SUBREPO_URL, SUBREPO_AUTH, SUBREPO_TOKEN (set by parse_config + load_target) + +# ─── Josh-Proxy Auth URL ─────────────────────────────────────────── +# Josh always uses HTTPS. Filter is embedded in the URL path. +# Result: https://user:token@proxy/org/repo.git:/services/app.git + +josh_auth_url() { + local base="${JOSH_PROXY_URL}/${MONOREPO_PATH}.git${JOSH_FILTER}.git" + echo "$base" | sed "s|https://|https://${BOT_USER}:${GITEA_TOKEN}@|" +} + +# ─── Subrepo Auth URL ────────────────────────────────────────────── +# HTTPS: injects user:token into URL +# SSH: returns bare URL (auth via GIT_SSH_COMMAND set by load_target) + +subrepo_auth_url() { + if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then + echo "$SUBREPO_URL" + else + echo "$SUBREPO_URL" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|" + fi +} + +# ─── Remote Queries ───────────────────────────────────────────────── + +subrepo_ls_remote() { + local ref="${1:-HEAD}" + local output + output=$(git ls-remote "$(subrepo_auth_url)" "refs/heads/${ref}") \ + || die "Failed to reach subrepo (check SSH key / auth)" + echo "$output" | awk '{print $1}' +} + +# ─── PR Creation ──────────────────────────────────────────────────── +# Shared helper for creating PRs on Gitea/GitHub API. +# Usage: create_pr <body> + +create_pr() { + local api_url="$1" + local token="$2" + local base="$3" + local head="$4" + local title="$5" + local body="$6" + + curl -sf -X POST \ + -H "Authorization: token ${token}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n \ + --arg base "$base" \ + --arg head "$head" \ + --arg title "$title" \ + --arg body "$body" \ + '{base:$base, head:$head, title:$title, body:$body}')" \ + "${api_url}/pulls" >/dev/null +} diff --git a/lib/config.sh b/lib/config.sh new file mode 100644 index 0000000..e11214c --- /dev/null +++ b/lib/config.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# lib/config.sh — Config loading and target resolution +# +# Two-phase loading: +# 1. parse_config() — reads .josh-sync.yml via yq+jq, exports globals + JOSH_SYNC_TARGETS +# 2. load_target() — called per-target during iteration, sets target-specific env vars +# +# Requires: lib/core.sh sourced first, yq and jq on PATH + +# ─── Phase 1: Parse Config ───────────────────────────────────────── + +parse_config() { + local config_file="${1:-.josh-sync.yml}" + [ -f "$config_file" ] || die "Config not found: ${config_file} (run from monorepo root)" + + local config_json + config_json=$(yq -o json "$config_file") || die "Failed to parse ${config_file}" + + # Export global values + export JOSH_PROXY_URL + JOSH_PROXY_URL=$(echo "$config_json" | jq -r '.josh.proxy_url') + export MONOREPO_PATH + MONOREPO_PATH=$(echo "$config_json" | jq -r '.josh.monorepo_path') + export BOT_NAME + BOT_NAME=$(echo "$config_json" | jq -r '.bot.name') + export BOT_EMAIL + BOT_EMAIL=$(echo "$config_json" | jq -r '.bot.email') + export BOT_TRAILER + BOT_TRAILER=$(echo "$config_json" | jq -r '.bot.trailer') + + [ -n "$JOSH_PROXY_URL" ] && [ "$JOSH_PROXY_URL" != "null" ] || die "josh.proxy_url missing in config" + [ -n "$MONOREPO_PATH" ] && [ "$MONOREPO_PATH" != "null" ] || die "josh.monorepo_path missing in config" + [ -n "$BOT_TRAILER" ] && [ "$BOT_TRAILER" != "null" ] || die "bot.trailer missing in config" + + # Enrich targets with derived fields (gitea_host, subrepo_repo_path, auto josh_filter) + export JOSH_SYNC_TARGETS + JOSH_SYNC_TARGETS=$(echo "$config_json" | jq '[.targets[] | . + + # Auto-derive josh_filter from subfolder if not set + (if (.josh_filter // "") == "" then + {josh_filter: (":/" + .subfolder)} + else {} end) + + # Derive gitea_host and subrepo_repo_path from subrepo_url + (.subrepo_url as $url | + if ($url | test("^ssh://")) then + ($url | capture("ssh://[^@]*@(?<h>[^/]+)/(?<p>.+?)(\\.git)?$") // + {h: "", p: ""} | {gitea_host: .h, subrepo_repo_path: .p}) + elif ($url | test("^git@")) then + ($url | capture("git@(?<h>[^:/]+)[:/](?<p>.+?)(\\.git)?$") // + {h: "", p: ""} | {gitea_host: .h, subrepo_repo_path: .p}) + elif ($url | test("^https?://")) then + ($url | capture("https?://(?<h>[^/]+)/(?<p>.+?)(\\.git)?$") // + {h: "", p: ""} | {gitea_host: .h, subrepo_repo_path: .p}) + else + {gitea_host: "", subrepo_repo_path: ""} + end + ) + ]') + + # Load .env credentials (if present, not required — CI sets these via secrets) + if [ -f .env ]; then + # shellcheck source=/dev/null + source .env + fi + + export GITEA_TOKEN="${GITEA_TOKEN:-${SYNC_BOT_TOKEN:-}}" + export BOT_USER="${BOT_USER:-${SYNC_BOT_USER:-}}" + + # Monorepo API URL (derived from first target's host, overridable via env) + local gitea_host + gitea_host=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[0].gitea_host') + export MONOREPO_API="${MONOREPO_API:-https://${gitea_host}/api/v1/repos/${MONOREPO_PATH}}" + + log "INFO" "Config loaded: $(echo "$JOSH_SYNC_TARGETS" | jq 'length') target(s)" +} + +# ─── Phase 2: Load Target ────────────────────────────────────────── +# Called per-target during iteration. Takes a JSON object (one element +# of JOSH_SYNC_TARGETS) and sets all target-specific env vars. + +load_target() { + local tj="$1" + + export JOSH_SYNC_TARGET_NAME + JOSH_SYNC_TARGET_NAME=$(echo "$tj" | jq -r '.name') + export JOSH_FILTER + JOSH_FILTER=$(echo "$tj" | jq -r '.josh_filter') + export SUBREPO_URL + SUBREPO_URL=$(echo "$tj" | jq -r '.subrepo_url') + export SUBREPO_AUTH + SUBREPO_AUTH=$(echo "$tj" | jq -r '.subrepo_auth // "https"') + + # API URL from pre-derived fields + local gitea_host subrepo_repo_path + gitea_host=$(echo "$tj" | jq -r '.gitea_host') + subrepo_repo_path=$(echo "$tj" | jq -r '.subrepo_repo_path') + export SUBREPO_API="https://${gitea_host}/api/v1/repos/${subrepo_repo_path}" + + # Per-target credential resolution (indirect variable reference) + local token_var ssh_key_var + token_var=$(echo "$tj" | jq -r '.subrepo_token_var // "SUBREPO_TOKEN"') + ssh_key_var=$(echo "$tj" | jq -r '.subrepo_ssh_key_var // "SUBREPO_SSH_KEY"') + + # Resolve: per-target var → default var → SYNC_BOT_TOKEN fallback + export SUBREPO_TOKEN="${!token_var:-${SUBREPO_TOKEN:-${SYNC_BOT_TOKEN:-}}}" + local ssh_key_value="${!ssh_key_var:-${SUBREPO_SSH_KEY:-}}" + + # Clean up previous SSH state and set up new if needed + if [ -n "${JOSH_SSH_DIR:-}" ]; then + rm -rf "$JOSH_SSH_DIR" + unset JOSH_SSH_DIR GIT_SSH_COMMAND + fi + + if [ "$SUBREPO_AUTH" = "ssh" ] && [ -n "$ssh_key_value" ]; then + JOSH_SSH_DIR=$(mktemp -d) + echo "$ssh_key_value" > "${JOSH_SSH_DIR}/subrepo_key" + chmod 600 "${JOSH_SSH_DIR}/subrepo_key" + export GIT_SSH_COMMAND="ssh -i ${JOSH_SSH_DIR}/subrepo_key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + log "INFO" "SSH auth configured for target ${JOSH_SYNC_TARGET_NAME}" + fi + + log "INFO" "Loaded target: ${JOSH_SYNC_TARGET_NAME} (${SUBREPO_AUTH})" +} diff --git a/lib/core.sh b/lib/core.sh new file mode 100644 index 0000000..64ab9b7 --- /dev/null +++ b/lib/core.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# lib/core.sh — Foundation: logging, exit codes, git environment isolation +# +# Source this first. All other modules depend on it. + +set -euo pipefail + +# ─── Exit Codes ───────────────────────────────────────────────────── + +readonly E_OK=0 +readonly E_GENERAL=1 +readonly E_CONFIG=10 +readonly E_AUTH=11 + +# ─── Logging ──────────────────────────────────────────────────────── +# All log output goes to stderr. Sync functions use stdout for return values. + +log() { + local level="$1"; shift + echo "$(date -u +%H:%M:%S) [${level}] $*" >&2 +} + +die() { log "FATAL" "$@"; exit "$E_GENERAL"; } + +# ─── Git Environment Isolation ────────────────────────────────────── +# Prevent user/system config from interfering with sync operations. +# Safe because josh-sync always runs as a subprocess, never sourced into +# an interactive shell. + +export GIT_CONFIG_GLOBAL=/dev/null +export GIT_CONFIG_SYSTEM=/dev/null +export GIT_TERMINAL_PROMPT=0 diff --git a/lib/state.sh b/lib/state.sh new file mode 100644 index 0000000..90a4c41 --- /dev/null +++ b/lib/state.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# lib/state.sh — Sync state management on orphan branch +# +# State persists on an orphan git branch (default: josh-sync-state), +# committed and pushed to origin. Survives CI runner teardown. +# +# Storage layout: +# origin/josh-sync-state/ +# <target>/<branch>.json (e.g., billing/main.json) +# +# JSON per state file: +# { +# "last_forward": { "mono_sha": "...", "subrepo_sha": "...", "timestamp": "...", "status": "..." }, +# "last_reverse": { "subrepo_sha": "...", "mono_sha": "...", "timestamp": "...", "status": "..." } +# } +# +# Forward and reverse state are independent — updated with jq merge. +# +# Requires: lib/core.sh sourced first +# Expects: JOSH_SYNC_TARGET_NAME, BOT_NAME, BOT_EMAIL (set by load_target) + +STATE_BRANCH="${JOSH_SYNC_STATE_BRANCH:-josh-sync-state}" + +# ─── State Key ────────────────────────────────────────────────────── +# Namespace with target: "billing" + "main" → "billing/main" +# Slashes in branch names converted to hyphens. + +state_key() { + local branch_key="${1//\//-}" + echo "${JOSH_SYNC_TARGET_NAME}/${branch_key}" +} + +# ─── Read State ───────────────────────────────────────────────────── + +read_state() { + local key + key=$(state_key "$1") + git fetch origin "$STATE_BRANCH" 2>/dev/null || true + git show "origin/${STATE_BRANCH}:${key}.json" 2>/dev/null || echo '{}' +} + +# ─── Write State ──────────────────────────────────────────────────── +# Uses git worktree to avoid touching the working tree. + +write_state() { + local key + key=$(state_key "$1") + local state_json="$2" + local tmp_dir + tmp_dir=$(mktemp -d) + + # Try to check out existing state branch, or create orphan + if git rev-parse "origin/${STATE_BRANCH}" >/dev/null 2>&1; then + git worktree add "$tmp_dir" "origin/${STATE_BRANCH}" 2>/dev/null + else + git worktree add --detach "$tmp_dir" 2>/dev/null + (cd "$tmp_dir" && git checkout --orphan "$STATE_BRANCH" && git rm -rf . 2>/dev/null || true) + fi + + # Create target subdirectory and write state + mkdir -p "$(dirname "${tmp_dir}/${key}.json")" + echo "$state_json" | jq '.' > "${tmp_dir}/${key}.json" + + ( + cd "$tmp_dir" + git add -A + if ! git diff --cached --quiet 2>/dev/null; then + git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ + commit -m "state: update ${key}" + git push origin "HEAD:${STATE_BRANCH}" || log "WARN" "Failed to push state" + fi + ) + + git worktree remove "$tmp_dir" 2>/dev/null || rm -rf "$tmp_dir" +} diff --git a/lib/sync.sh b/lib/sync.sh new file mode 100644 index 0000000..f18a472 --- /dev/null +++ b/lib/sync.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# lib/sync.sh — Sync algorithms: forward, reverse, import, reset +# +# All four operations: +# 1. Create a temp work dir with cleanup trap +# 2. Perform git operations +# 3. Return a status string via stdout +# +# Requires: lib/core.sh, lib/config.sh, lib/auth.sh, lib/state.sh sourced +# Expects: SYNC_BRANCH_MONO, SYNC_BRANCH_SUBREPO, JOSH_FILTER, etc. + +# ─── Forward Sync: Monorepo → Subrepo ────────────────────────────── +# +# Returns: fresh | skip | clean | lease-rejected | conflict + +forward_sync() { + local mono_branch="$SYNC_BRANCH_MONO" + local subrepo_branch="$SYNC_BRANCH_SUBREPO" + local work_dir + work_dir=$(mktemp -d) + trap "rm -rf '$work_dir'" EXIT + + log "INFO" "=== Forward sync: mono/${mono_branch} → subrepo/${subrepo_branch} ===" + + # 1. Clone the monorepo subfolder through josh (filtered view) + log "INFO" "Cloning filtered monorepo via josh-proxy..." + git clone "$(josh_auth_url)" \ + --branch "$mono_branch" --single-branch \ + "${work_dir}/filtered" || die "Failed to clone through josh-proxy" + + cd "${work_dir}/filtered" + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + + local mono_head + mono_head=$(git rev-parse HEAD) + log "INFO" "Mono filtered HEAD: ${mono_head:0:12}" + + # 2. Record subrepo HEAD before any operations (the "lease") + local subrepo_sha + subrepo_sha=$(subrepo_ls_remote "$subrepo_branch") + log "INFO" "Subrepo HEAD (lease): ${subrepo_sha:-(empty)}" + + # 3. Handle fresh push (subrepo branch doesn't exist) + if [ -z "$subrepo_sha" ]; then + log "INFO" "Subrepo branch doesn't exist — doing fresh push" + git push "$(subrepo_auth_url)" "HEAD:refs/heads/${subrepo_branch}" \ + || die "Failed to push to subrepo" + echo "fresh" + return + fi + + # 4. Fetch subrepo for comparison + git remote add subrepo "$(subrepo_auth_url)" + git fetch subrepo "$subrepo_branch" + + # 5. Compare trees — skip if identical + local mono_tree subrepo_tree + mono_tree=$(git rev-parse HEAD^{tree}) + subrepo_tree=$(git rev-parse "subrepo/${subrepo_branch}^{tree}" 2>/dev/null || echo "none") + + if [ "$mono_tree" = "$subrepo_tree" ]; then + log "INFO" "Trees identical — nothing to sync" + echo "skip" + return + fi + + # 6. Attempt merge: start from subrepo state, merge mono changes + git checkout -B sync-attempt "subrepo/${subrepo_branch}" >&2 + + if git merge --no-commit --no-ff "$mono_head" >&2 2>&1; then + # Clean merge + if git diff --cached --quiet; then + log "INFO" "Merge empty — skip" + git merge --abort 2>/dev/null || true + echo "skip" + return + fi + + git commit -m "Sync from monorepo $(date -u +%Y-%m-%dT%H:%M:%SZ) + +${BOT_TRAILER}: forward/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 + + # 7. Push with force-with-lease (explicit SHA) + if git push \ + --force-with-lease="refs/heads/${subrepo_branch}:${subrepo_sha}" \ + "$(subrepo_auth_url)" \ + "HEAD:refs/heads/${subrepo_branch}"; then + + log "INFO" "Forward sync complete" + echo "clean" + else + log "WARN" "Force-with-lease rejected — subrepo changed during sync" + echo "lease-rejected" + fi + + else + # Conflict! + local conflicted + conflicted=$(git diff --name-only --diff-filter=U 2>/dev/null || echo "(unknown)") + git merge --abort + + log "WARN" "Merge conflict on: ${conflicted}" + + # Push mono state as a conflict branch for PR + local ts + ts=$(date +%Y%m%d-%H%M%S) + local conflict_branch="auto-sync/mono-${mono_branch}-${ts}" + git checkout -B "$conflict_branch" "$mono_head" >&2 + git push "$(subrepo_auth_url)" "${conflict_branch}" + + # Create PR on subrepo + local pr_body + pr_body="## Sync Conflict\n\nMonorepo \`${mono_branch}\` has changes that conflict with \`${subrepo_branch}\`.\n\n**Conflicted files:**\n$(echo "$conflicted" | sed 's/^/- /')\n\nPlease resolve and merge this PR to complete the sync." + + create_pr "${SUBREPO_API}" "${SUBREPO_TOKEN}" \ + "$subrepo_branch" "$conflict_branch" \ + "[Sync Conflict] mono/${mono_branch} → ${subrepo_branch}" \ + "$pr_body" \ + || die "Failed to create conflict PR on subrepo (check SUBREPO_TOKEN)" + + log "INFO" "Conflict PR created on subrepo" + echo "conflict" + fi +} + +# ─── Reverse Sync: Subrepo → Monorepo ────────────────────────────── +# +# Always creates a PR on the monorepo — never pushes directly. +# Returns: skip | pr-created | josh-rejected + +reverse_sync() { + local mono_branch="$SYNC_BRANCH_MONO" + local subrepo_branch="$SYNC_BRANCH_SUBREPO" + local work_dir + work_dir=$(mktemp -d) + trap "rm -rf '$work_dir'" EXIT + + log "INFO" "=== Reverse sync: subrepo/${subrepo_branch} → mono/${mono_branch} ===" + + # 1. Clone subrepo + git clone "$(subrepo_auth_url)" \ + --branch "$subrepo_branch" --single-branch \ + "${work_dir}/subrepo" || die "Failed to clone subrepo" + + cd "${work_dir}/subrepo" + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + + # 2. Fetch monorepo's filtered view for comparison + git remote add mono-filtered "$(josh_auth_url)" + git fetch mono-filtered "$mono_branch" || die "Failed to fetch from josh-proxy" + + # 3. Find new human commits (excludes bot commits from forward sync) + local human_commits + human_commits=$(git log "mono-filtered/${mono_branch}..HEAD" \ + --oneline --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") + + if [ -z "$human_commits" ]; then + log "INFO" "No new human commits in subrepo — nothing to sync" + echo "skip" + return + fi + + log "INFO" "New human commits to sync:" + echo "$human_commits" >&2 + + # 4. Push through josh to a staging branch + local ts + ts=$(date +%Y%m%d-%H%M%S) + local staging_branch="auto-sync/subrepo-${subrepo_branch}-${ts}" + + if git push -o "base=${mono_branch}" "$(josh_auth_url)" "HEAD:refs/heads/${staging_branch}"; then + log "INFO" "Pushed to staging branch via josh: ${staging_branch}" + + # 5. Create PR on monorepo (NEVER direct push) + local pr_body + pr_body="## Subrepo changes\n\nNew commits from subrepo \`${subrepo_branch}\`:\n\n\`\`\`\n${human_commits}\n\`\`\`\n\n**Review checklist:**\n- [ ] Changes scoped to synced subfolder\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" + + create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ + "$mono_branch" "$staging_branch" \ + "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ + "$pr_body" \ + || die "Failed to create PR on monorepo (check GITEA_TOKEN)" + + log "INFO" "Reverse sync PR created on monorepo" + echo "pr-created" + else + log "ERROR" "Josh rejected push — check josh-proxy logs" + echo "josh-rejected" + fi +} + +# ─── Initial Import: Subrepo → Monorepo (first time) ─────────────── +# +# Used when a subrepo already has content and you're adding it to the +# monorepo for the first time. Creates a PR. +# Returns: skip | pr-created + +initial_import() { + local mono_branch="$SYNC_BRANCH_MONO" + local subrepo_branch="$SYNC_BRANCH_SUBREPO" + local subfolder + subfolder=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$JOSH_SYNC_TARGET_NAME" \ + '.[] | select(.name == $n) | .subfolder') + local work_dir + work_dir=$(mktemp -d) + trap "rm -rf '$work_dir'" EXIT + + log "INFO" "=== Initial import: subrepo/${subrepo_branch} → mono/${mono_branch}:${subfolder}/ ===" + + # 1. Clone monorepo (directly, not through josh — we need the real repo) + local mono_auth_url + mono_auth_url=$(echo "https://${BOT_USER}:${GITEA_TOKEN}@$(echo "$MONOREPO_API" | sed 's|https://||; s|/api/v1/repos/|/|').git") + + git clone "$mono_auth_url" \ + --branch "$mono_branch" --single-branch \ + "${work_dir}/monorepo" || die "Failed to clone monorepo" + + # 2. Clone subrepo + git clone "$(subrepo_auth_url)" \ + --branch "$subrepo_branch" --single-branch \ + "${work_dir}/subrepo" || die "Failed to clone subrepo" + + local file_count + file_count=$(find "${work_dir}/subrepo" -not -path '*/.git/*' -not -path '*/.git' -type f | wc -l | tr -d ' ') + log "INFO" "Subrepo has ${file_count} files" + + # 3. Copy subrepo content into monorepo subfolder + cd "${work_dir}/monorepo" + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + + local ts + ts=$(date +%Y%m%d-%H%M%S) + local staging_branch="auto-sync/import-${JOSH_SYNC_TARGET_NAME}-${ts}" + git checkout -B "$staging_branch" >&2 + + mkdir -p "$subfolder" + rsync -a --exclude='.git' "${work_dir}/subrepo/" "${subfolder}/" + git add "$subfolder" + + if git diff --cached --quiet; then + log "INFO" "No changes — subfolder already matches subrepo" + echo "skip" + return + fi + + git commit -m "Import ${JOSH_SYNC_TARGET_NAME} from subrepo/${subrepo_branch} + +${BOT_TRAILER}: import/${JOSH_SYNC_TARGET_NAME}/${ts}" >&2 + + # 4. Push branch + git push origin "$staging_branch" || die "Failed to push import branch" + log "INFO" "Pushed import branch: ${staging_branch}" + + # 5. Create PR on monorepo + local pr_body + pr_body="## Initial import\n\nImporting existing subrepo \`${subrepo_branch}\` (${file_count} files) into \`${subfolder}/\`.\n\n**Review checklist:**\n- [ ] Content looks correct\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" + + create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ + "$mono_branch" "$staging_branch" \ + "[Import] ${JOSH_SYNC_TARGET_NAME}: ${subrepo_branch}" \ + "$pr_body" \ + || die "Failed to create PR on monorepo (check GITEA_TOKEN)" + + log "INFO" "Import PR created on monorepo" + echo "pr-created" +} + +# ─── Subrepo Reset: Force-push josh-filtered view to subrepo ─────── +# Run this AFTER merging an import PR to establish shared josh history. +# Returns: reset + +subrepo_reset() { + local mono_branch="$SYNC_BRANCH_MONO" + local subrepo_branch="$SYNC_BRANCH_SUBREPO" + local work_dir + work_dir=$(mktemp -d) + trap "rm -rf '$work_dir'" EXIT + + log "INFO" "=== Subrepo reset: josh-filtered mono/${mono_branch} → subrepo/${subrepo_branch} ===" + + # 1. Clone monorepo through josh (filtered view — this IS the shared history) + log "INFO" "Cloning filtered monorepo via josh-proxy..." + git clone "$(josh_auth_url)" \ + --branch "$mono_branch" --single-branch \ + "${work_dir}/filtered" || die "Failed to clone through josh-proxy" + + cd "${work_dir}/filtered" + + local head_sha + head_sha=$(git rev-parse HEAD) + log "INFO" "Josh-filtered HEAD: ${head_sha:0:12}" + + # 2. Force-push to subrepo (replaces subrepo history with josh-filtered history) + log "WARN" "Force-pushing to subrepo/${subrepo_branch} — this replaces subrepo history" + git push --force "$(subrepo_auth_url)" "HEAD:refs/heads/${subrepo_branch}" \ + || die "Failed to force-push to subrepo" + + log "INFO" "Subrepo reset complete — histories are now linked through josh" + echo "reset" +} diff --git a/schema/config-schema.json b/schema/config-schema.json new file mode 100644 index 0000000..bb222e9 --- /dev/null +++ b/schema/config-schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "josh-sync configuration", + "description": "Configuration for bidirectional monorepo ↔ subrepo sync via josh-proxy", + "type": "object", + "required": ["josh", "targets", "bot"], + "properties": { + "josh": { + "type": "object", + "required": ["proxy_url", "monorepo_path"], + "properties": { + "proxy_url": { + "type": "string", + "description": "Josh-proxy URL (no trailing slash)", + "pattern": "^https?://" + }, + "monorepo_path": { + "type": "string", + "description": "Repo path as josh sees it (org/repo)" + } + } + }, + "targets": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["name", "subfolder", "subrepo_url", "branches"], + "properties": { + "name": { + "type": "string", + "description": "Unique target identifier" + }, + "subfolder": { + "type": "string", + "description": "Monorepo subfolder path" + }, + "josh_filter": { + "type": "string", + "description": "Josh filter expression (auto-derived from subfolder if omitted)", + "pattern": "^:" + }, + "subrepo_url": { + "type": "string", + "description": "External subrepo git URL (HTTPS or SSH)" + }, + "subrepo_auth": { + "type": "string", + "enum": ["https", "ssh"], + "default": "https", + "description": "Auth method for subrepo" + }, + "subrepo_token_var": { + "type": "string", + "description": "Env var name for per-target HTTPS token (default: SUBREPO_TOKEN)" + }, + "subrepo_ssh_key_var": { + "type": "string", + "description": "Env var name for per-target SSH key (default: SUBREPO_SSH_KEY)" + }, + "branches": { + "type": "object", + "description": "Branch mapping: mono_branch → subrepo_branch", + "additionalProperties": { + "type": "string" + } + }, + "forward_only": { + "type": "array", + "items": { "type": "string" }, + "default": [], + "description": "Branches that only sync mono → subrepo (never reverse)" + } + } + } + }, + "bot": { + "type": "object", + "required": ["name", "email", "trailer"], + "properties": { + "name": { + "type": "string", + "description": "Git commit author name for sync commits" + }, + "email": { + "type": "string", + "description": "Git commit author email" + }, + "trailer": { + "type": "string", + "description": "Git trailer key for loop prevention" + } + } + } + } +} diff --git a/tests/fixtures/minimal.yml b/tests/fixtures/minimal.yml new file mode 100644 index 0000000..d767d4e --- /dev/null +++ b/tests/fixtures/minimal.yml @@ -0,0 +1,16 @@ +# Test fixture: minimal single-target config +josh: + proxy_url: "https://josh.test.local" + monorepo_path: "org/repo" + +targets: + - name: "example" + subfolder: "services/example" + subrepo_url: "https://gitea.test.local/ext/example.git" + branches: + main: main + +bot: + name: "test-bot" + email: "test@test.local" + trailer: "Josh-Sync-Origin" diff --git a/tests/fixtures/multi-target.yml b/tests/fixtures/multi-target.yml new file mode 100644 index 0000000..d0ced19 --- /dev/null +++ b/tests/fixtures/multi-target.yml @@ -0,0 +1,29 @@ +# Test fixture: multi-target config +josh: + proxy_url: "https://josh.test.local" + monorepo_path: "org/monorepo" + +targets: + - name: "app-a" + subfolder: "services/app-a" + josh_filter: ":/services/app-a" + subrepo_url: "https://gitea.test.local/ext/app-a.git" + subrepo_auth: "https" + branches: + main: main + forward_only: [] + + - name: "app-b" + subfolder: "services/app-b" + subrepo_url: "git@gitea.test.local:ext/app-b.git" + subrepo_auth: "ssh" + subrepo_ssh_key_var: "APP_B_SSH_KEY" + branches: + main: main + develop: dev + forward_only: [develop] + +bot: + name: "test-bot" + email: "test-bot@test.local" + trailer: "Josh-Sync-Origin" diff --git a/tests/unit/auth.bats b/tests/unit/auth.bats new file mode 100644 index 0000000..e94b5cc --- /dev/null +++ b/tests/unit/auth.bats @@ -0,0 +1,39 @@ +#!/usr/bin/env bats +# tests/unit/auth.bats — Auth URL construction tests + +setup() { + export JOSH_SYNC_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + source "$JOSH_SYNC_ROOT/lib/core.sh" + source "$JOSH_SYNC_ROOT/lib/auth.sh" + + # Set up env vars as parse_config + load_target would + export JOSH_PROXY_URL="https://josh.test.local" + export MONOREPO_PATH="org/monorepo" + export BOT_USER="sync-bot" + export GITEA_TOKEN="test-token-123" + export JOSH_FILTER=":/services/app-a" + export SUBREPO_URL="https://gitea.test.local/ext/app-a.git" + export SUBREPO_AUTH="https" + export SUBREPO_TOKEN="subrepo-token-456" +} + +@test "josh_auth_url builds correct HTTPS URL with credentials and filter" { + local url + url=$(josh_auth_url) + [ "$url" = "https://sync-bot:test-token-123@josh.test.local/org/monorepo.git:/services/app-a.git" ] +} + +@test "subrepo_auth_url injects credentials for HTTPS" { + local url + url=$(subrepo_auth_url) + [ "$url" = "https://sync-bot:subrepo-token-456@gitea.test.local/ext/app-a.git" ] +} + +@test "subrepo_auth_url returns bare URL for SSH" { + export SUBREPO_AUTH="ssh" + export SUBREPO_URL="git@gitea.test.local:ext/app-a.git" + + local url + url=$(subrepo_auth_url) + [ "$url" = "git@gitea.test.local:ext/app-a.git" ] +} diff --git a/tests/unit/config.bats b/tests/unit/config.bats new file mode 100644 index 0000000..7c306d0 --- /dev/null +++ b/tests/unit/config.bats @@ -0,0 +1,106 @@ +#!/usr/bin/env bats +# tests/unit/config.bats — Config parsing tests + +setup() { + export JOSH_SYNC_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + source "$JOSH_SYNC_ROOT/lib/core.sh" + source "$JOSH_SYNC_ROOT/lib/config.sh" + + FIXTURES="$JOSH_SYNC_ROOT/tests/fixtures" +} + +@test "parse_config loads multi-target config" { + cd "$(mktemp -d)" + cp "$FIXTURES/multi-target.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + [ "$JOSH_PROXY_URL" = "https://josh.test.local" ] + [ "$MONOREPO_PATH" = "org/monorepo" ] + [ "$BOT_NAME" = "test-bot" ] + [ "$BOT_EMAIL" = "test-bot@test.local" ] + [ "$BOT_TRAILER" = "Josh-Sync-Origin" ] +} + +@test "parse_config derives gitea_host from HTTPS URL" { + cd "$(mktemp -d)" + cp "$FIXTURES/multi-target.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + local host + host=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[0].gitea_host') + [ "$host" = "gitea.test.local" ] +} + +@test "parse_config derives gitea_host from SSH URL" { + cd "$(mktemp -d)" + cp "$FIXTURES/multi-target.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + local host + host=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[1].gitea_host') + [ "$host" = "gitea.test.local" ] +} + +@test "parse_config derives subrepo_repo_path from HTTPS URL" { + cd "$(mktemp -d)" + cp "$FIXTURES/multi-target.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + local path + path=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[0].subrepo_repo_path') + [ "$path" = "ext/app-a" ] +} + +@test "parse_config derives subrepo_repo_path from SSH URL" { + cd "$(mktemp -d)" + cp "$FIXTURES/multi-target.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + local path + path=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[1].subrepo_repo_path') + [ "$path" = "ext/app-b" ] +} + +@test "parse_config auto-derives josh_filter from subfolder when not set" { + cd "$(mktemp -d)" + cp "$FIXTURES/minimal.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + local filter + filter=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[0].josh_filter') + [ "$filter" = ":/services/example" ] +} + +@test "parse_config preserves explicit josh_filter" { + cd "$(mktemp -d)" + cp "$FIXTURES/multi-target.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + local filter + filter=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[0].josh_filter') + [ "$filter" = ":/services/app-a" ] +} + +@test "parse_config reports correct target count" { + cd "$(mktemp -d)" + cp "$FIXTURES/multi-target.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + local count + count=$(echo "$JOSH_SYNC_TARGETS" | jq 'length') + [ "$count" -eq 2 ] +} + +@test "parse_config fails on missing config file" { + cd "$(mktemp -d)" + run bash -c 'source "$JOSH_SYNC_ROOT/lib/core.sh"; source "$JOSH_SYNC_ROOT/lib/config.sh"; parse_config "nonexistent.yml"' + [ "$status" -ne 0 ] +} diff --git a/tests/unit/state.bats b/tests/unit/state.bats new file mode 100644 index 0000000..9f41e43 --- /dev/null +++ b/tests/unit/state.bats @@ -0,0 +1,40 @@ +#!/usr/bin/env bats +# tests/unit/state.bats — State key generation tests + +setup() { + export JOSH_SYNC_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + source "$JOSH_SYNC_ROOT/lib/core.sh" + source "$JOSH_SYNC_ROOT/lib/state.sh" + + export JOSH_SYNC_TARGET_NAME="billing" +} + +@test "state_key generates target/branch format" { + local key + key=$(state_key "main") + [ "$key" = "billing/main" ] +} + +@test "state_key converts slashes to hyphens in branch name" { + local key + key=$(state_key "feature/my-branch") + [ "$key" = "billing/feature-my-branch" ] +} + +@test "state_key works with different target names" { + export JOSH_SYNC_TARGET_NAME="auth-service" + local key + key=$(state_key "develop") + [ "$key" = "auth-service/develop" ] +} + +@test "STATE_BRANCH defaults to josh-sync-state" { + [ "$STATE_BRANCH" = "josh-sync-state" ] +} + +@test "STATE_BRANCH can be overridden via env" { + export JOSH_SYNC_STATE_BRANCH="custom-state" + # Re-source to pick up the new value + source "$JOSH_SYNC_ROOT/lib/state.sh" + [ "$STATE_BRANCH" = "custom-state" ] +} From f2785241bfc95a04120d130aa3251a12e42c7a90 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Thu, 12 Feb 2026 11:22:51 +0300 Subject: [PATCH 02/37] "#" --- CHANGELOG.md | 2 +- README.md | 4 ++-- action.yml | 12 ++++++------ bin/josh-sync | 4 ++-- examples/forward.yml | 6 +++--- examples/reverse.yml | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f53e45c..8af14fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,4 +22,4 @@ Initial release. Extracted from [private-monorepo-example](https://code.itkan.io - Python + pyyaml replaced by yq-go (single static binary) - 7 separate scripts replaced by single `josh-sync` CLI -- Consumer workflows use composite action (`uses: org/josh-sync@v1`) +- Consumer workflows use composite action (`uses: https://your-gitea/org/josh-sync@v1`) diff --git a/README.md b/README.md index 94e16f9..c3ad370 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Copy from [examples/](examples/) and customize paths/branches: ```yaml # .gitea/workflows/josh-sync-forward.yml -- uses: org/josh-sync@v1 +- uses: https://your-gitea.example.com/org/josh-sync@v1 with: direction: forward env: @@ -59,7 +59,7 @@ Run `josh-sync preflight` to validate your setup. ## CLI ``` -josh-sync sync [--forward|--reverse] [--target NAME] [--branch BRANCH] +josh-sync sync [--forward|--reverse] [--target NAME[,NAME]] [--branch BRANCH] josh-sync preflight josh-sync import <target> josh-sync reset <target> diff --git a/action.yml b/action.yml index 5faf00f..629cf21 100644 --- a/action.yml +++ b/action.yml @@ -56,9 +56,9 @@ runs: env: JOSH_SYNC_DEBUG: ${{ inputs.debug == 'true' && '1' || '0' }} run: | - ARGS="--config ${{ inputs.config }}" - [[ "${{ inputs.direction }}" == "forward" ]] && ARGS+=" --forward" - [[ "${{ inputs.direction }}" == "reverse" ]] && ARGS+=" --reverse" - [[ -n "${{ inputs.target }}" ]] && ARGS+=" --target ${{ inputs.target }}" - [[ -n "${{ inputs.branch }}" ]] && ARGS+=" --branch ${{ inputs.branch }}" - josh-sync sync $ARGS + ARGS=(--config "${{ inputs.config }}") + [[ "${{ inputs.direction }}" == "forward" ]] && ARGS+=(--forward) + [[ "${{ inputs.direction }}" == "reverse" ]] && ARGS+=(--reverse) + [[ -n "${{ inputs.target }}" ]] && ARGS+=(--target "${{ inputs.target }}") + [[ -n "${{ inputs.branch }}" ]] && ARGS+=(--branch "${{ inputs.branch }}") + josh-sync sync "${ARGS[@]}" diff --git a/bin/josh-sync b/bin/josh-sync index 38f8a63..610460c 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -81,7 +81,7 @@ Global flags: Sync flags: --forward Forward only (mono → subrepo) --reverse Reverse only (subrepo → mono) - --target NAME Filter to one target (env: JOSH_SYNC_TARGET) + --target NAME Filter to target(s) — comma-separated for multiple (env: JOSH_SYNC_TARGET) --branch BRANCH Filter to one branch Environment: @@ -137,7 +137,7 @@ _sync_direction() { target_name=$(echo "$TARGET_JSON" | jq -r '.name') # Filter to specific target if requested - if [ -n "$filter_target" ] && [ "$filter_target" != "$target_name" ]; then + if [ -n "$filter_target" ] && ! echo ",$filter_target," | grep -q ",${target_name},"; then continue fi diff --git a/examples/forward.yml b/examples/forward.yml index 5c2fad7..97d499f 100644 --- a/examples/forward.yml +++ b/examples/forward.yml @@ -4,7 +4,7 @@ # Customize: # - paths: list all target subfolders # - branches: list all monorepo branches to trigger on -# - org/josh-sync@v1: pin to your library repo and version +# - uses: https://your-gitea.example.com/org/josh-sync@v1 — pin to your library repo and version name: "Josh Sync → Subrepo" @@ -59,10 +59,10 @@ jobs: | jq -r '.[] | "\(.name):\(.subfolder)"' \ | while IFS=: read -r name prefix; do echo "$CHANGED" | grep -q "^${prefix}/" && echo "$name" - done | sort -u | tr '\n' ' ') + done | sort -u | paste -sd ',' -) echo "targets=${TARGETS}" >> "$GITHUB_OUTPUT" - - uses: org/josh-sync@v1 + - uses: https://your-gitea.example.com/org/josh-sync@v1 with: direction: forward target: ${{ github.event.inputs.target || steps.detect.outputs.targets }} diff --git a/examples/reverse.yml b/examples/reverse.yml index e532b01..def838a 100644 --- a/examples/reverse.yml +++ b/examples/reverse.yml @@ -4,7 +4,7 @@ # # Customize: # - cron schedule -# - org/josh-sync@v1: pin to your library repo and version +# - uses: https://your-gitea.example.com/org/josh-sync@v1 — pin to your library repo and version name: "Josh Sync ← Subrepo" @@ -40,7 +40,7 @@ jobs: curl -sL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" \ -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq - - uses: org/josh-sync@v1 + - uses: https://your-gitea.example.com/org/josh-sync@v1 with: direction: reverse target: ${{ github.event.inputs.target || '' }} From ad925d822856085fd9019981cc83dd5eb9c70339 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Thu, 12 Feb 2026 13:21:37 +0300 Subject: [PATCH 03/37] Update flake.nix --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index cfee63e..063826e 100644 --- a/flake.nix +++ b/flake.nix @@ -24,7 +24,7 @@ inherit version; src = ./.; - nativeBuildInputs = [ pkgs.makeWrapper ]; + nativeBuildInputs = [ pkgs.makeWrapper pkgs.shellcheck ]; installPhase = '' mkdir -p $out/{bin,lib} From 0d2aea9664d83357caf26cd71f8b3adfa27ec879 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Thu, 12 Feb 2026 14:33:26 +0300 Subject: [PATCH 04/37] Fix all shellcheck warnings for nix build gate - SC2015: Wrap A && B || C patterns in brace groups for directive scope - SC2064: Suppress for intentional early trap expansion (local vars) - SC2164: Add || exit to cd commands in subshells - SC2001: Suppress for sed URL injection (clearer than parameter expansion) - SC1083: Handle {tree} git syntax (quote or suppress) - SC1091: Suppress for runtime-resolved source paths - SC2034: Remove unused exit codes (E_OK, E_CONFIG, E_AUTH) - SC2116: Eliminate useless echo in mono_auth_url construction Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- bin/josh-sync | 7 +++++++ flake.lock | 27 +++++++++++++++++++++++++++ lib/auth.sh | 2 ++ lib/core.sh | 3 --- lib/state.sh | 2 +- lib/sync.sh | 25 +++++++++++++++++-------- 6 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 flake.lock diff --git a/bin/josh-sync b/bin/josh-sync index 610460c..e3c244d 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC1091 # Source paths resolved at runtime # bin/josh-sync — CLI entrypoint for josh-sync # # Usage: josh-sync <command> [flags] @@ -291,11 +292,14 @@ cmd_preflight() { parse_config "$config_file" + # shellcheck disable=SC2015 # A && B || C is intentional — pass/warn/fail never fail + { [ -n "$JOSH_PROXY_URL" ] && pass "proxy_url: ${JOSH_PROXY_URL}" || fail "proxy_url missing" [ -n "$MONOREPO_PATH" ] && pass "monorepo_path: ${MONOREPO_PATH}" || fail "monorepo_path missing" [ -n "$BOT_TRAILER" ] && pass "trailer: ${BOT_TRAILER}" || fail "trailer missing" [ -n "${GITEA_TOKEN:-}" ] && pass "SYNC_BOT_TOKEN is set" || warn "SYNC_BOT_TOKEN missing in .env" [ -n "${BOT_USER:-}" ] && pass "BOT_USER: ${BOT_USER}" || warn "BOT_USER missing in .env" + } local target_count target_count=$(echo "$JOSH_SYNC_TARGETS" | jq 'length') @@ -316,8 +320,11 @@ cmd_preflight() { load_target "$TARGET_JSON" + # shellcheck disable=SC2015 + { [ -n "$JOSH_FILTER" ] && pass "josh_filter: ${JOSH_FILTER}" || fail "josh_filter missing" [ -n "$SUBREPO_URL" ] && pass "subrepo_url: ${SUBREPO_URL}" || fail "subrepo_url missing" + } # Subfolder exists if [ -d "$subfolder" ]; then diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..e5578c1 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1770562336, + "narHash": "sha256-ub1gpAONMFsT/GU2hV6ZWJjur8rJ6kKxdm9IlCT0j84=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d6c71932130818840fc8fe9509cf50be8c64634f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/lib/auth.sh b/lib/auth.sh index 58e404d..229ff88 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -11,6 +11,7 @@ josh_auth_url() { local base="${JOSH_PROXY_URL}/${MONOREPO_PATH}.git${JOSH_FILTER}.git" + # shellcheck disable=SC2001 # sed is clearer than ${var//} for URL injection echo "$base" | sed "s|https://|https://${BOT_USER}:${GITEA_TOKEN}@|" } @@ -22,6 +23,7 @@ subrepo_auth_url() { if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then echo "$SUBREPO_URL" else + # shellcheck disable=SC2001 echo "$SUBREPO_URL" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|" fi } diff --git a/lib/core.sh b/lib/core.sh index 64ab9b7..e9dabb5 100644 --- a/lib/core.sh +++ b/lib/core.sh @@ -7,10 +7,7 @@ set -euo pipefail # ─── Exit Codes ───────────────────────────────────────────────────── -readonly E_OK=0 readonly E_GENERAL=1 -readonly E_CONFIG=10 -readonly E_AUTH=11 # ─── Logging ──────────────────────────────────────────────────────── # All log output goes to stderr. Sync functions use stdout for return values. diff --git a/lib/state.sh b/lib/state.sh index 90a4c41..2410fac 100644 --- a/lib/state.sh +++ b/lib/state.sh @@ -62,7 +62,7 @@ write_state() { echo "$state_json" | jq '.' > "${tmp_dir}/${key}.json" ( - cd "$tmp_dir" + cd "$tmp_dir" || exit git add -A if ! git diff --cached --quiet 2>/dev/null; then git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ diff --git a/lib/sync.sh b/lib/sync.sh index f18a472..91d2903 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -18,6 +18,7 @@ forward_sync() { local subrepo_branch="$SYNC_BRANCH_SUBREPO" local work_dir work_dir=$(mktemp -d) + # shellcheck disable=SC2064 # Intentional early expansion — work_dir is local trap "rm -rf '$work_dir'" EXIT log "INFO" "=== Forward sync: mono/${mono_branch} → subrepo/${subrepo_branch} ===" @@ -28,7 +29,7 @@ forward_sync() { --branch "$mono_branch" --single-branch \ "${work_dir}/filtered" || die "Failed to clone through josh-proxy" - cd "${work_dir}/filtered" + cd "${work_dir}/filtered" || exit git config user.name "$BOT_NAME" git config user.email "$BOT_EMAIL" @@ -56,7 +57,8 @@ forward_sync() { # 5. Compare trees — skip if identical local mono_tree subrepo_tree - mono_tree=$(git rev-parse HEAD^{tree}) + mono_tree=$(git rev-parse 'HEAD^{tree}') + # shellcheck disable=SC1083 # {tree} is git syntax, not shell brace expansion subrepo_tree=$(git rev-parse "subrepo/${subrepo_branch}^{tree}" 2>/dev/null || echo "none") if [ "$mono_tree" = "$subrepo_tree" ]; then @@ -110,8 +112,10 @@ ${BOT_TRAILER}: forward/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 git push "$(subrepo_auth_url)" "${conflict_branch}" # Create PR on subrepo - local pr_body - pr_body="## Sync Conflict\n\nMonorepo \`${mono_branch}\` has changes that conflict with \`${subrepo_branch}\`.\n\n**Conflicted files:**\n$(echo "$conflicted" | sed 's/^/- /')\n\nPlease resolve and merge this PR to complete the sync." + local pr_body conflicted_list + # shellcheck disable=SC2001 + conflicted_list=$(echo "$conflicted" | sed 's/^/- /') + pr_body="## Sync Conflict\n\nMonorepo \`${mono_branch}\` has changes that conflict with \`${subrepo_branch}\`.\n\n**Conflicted files:**\n${conflicted_list}\n\nPlease resolve and merge this PR to complete the sync." create_pr "${SUBREPO_API}" "${SUBREPO_TOKEN}" \ "$subrepo_branch" "$conflict_branch" \ @@ -134,6 +138,7 @@ reverse_sync() { local subrepo_branch="$SYNC_BRANCH_SUBREPO" local work_dir work_dir=$(mktemp -d) + # shellcheck disable=SC2064 # Intentional early expansion — work_dir is local trap "rm -rf '$work_dir'" EXIT log "INFO" "=== Reverse sync: subrepo/${subrepo_branch} → mono/${mono_branch} ===" @@ -143,7 +148,7 @@ reverse_sync() { --branch "$subrepo_branch" --single-branch \ "${work_dir}/subrepo" || die "Failed to clone subrepo" - cd "${work_dir}/subrepo" + cd "${work_dir}/subrepo" || exit git config user.name "$BOT_NAME" git config user.email "$BOT_EMAIL" @@ -205,13 +210,16 @@ initial_import() { '.[] | select(.name == $n) | .subfolder') local work_dir work_dir=$(mktemp -d) + # shellcheck disable=SC2064 # Intentional early expansion — work_dir is local trap "rm -rf '$work_dir'" EXIT log "INFO" "=== Initial import: subrepo/${subrepo_branch} → mono/${mono_branch}:${subfolder}/ ===" # 1. Clone monorepo (directly, not through josh — we need the real repo) local mono_auth_url - mono_auth_url=$(echo "https://${BOT_USER}:${GITEA_TOKEN}@$(echo "$MONOREPO_API" | sed 's|https://||; s|/api/v1/repos/|/|').git") + local api_host_path + api_host_path=$(echo "$MONOREPO_API" | sed 's|https://||; s|/api/v1/repos/|/|') + mono_auth_url="https://${BOT_USER}:${GITEA_TOKEN}@${api_host_path}.git" git clone "$mono_auth_url" \ --branch "$mono_branch" --single-branch \ @@ -227,7 +235,7 @@ initial_import() { log "INFO" "Subrepo has ${file_count} files" # 3. Copy subrepo content into monorepo subfolder - cd "${work_dir}/monorepo" + cd "${work_dir}/monorepo" || exit git config user.name "$BOT_NAME" git config user.email "$BOT_EMAIL" @@ -277,6 +285,7 @@ subrepo_reset() { local subrepo_branch="$SYNC_BRANCH_SUBREPO" local work_dir work_dir=$(mktemp -d) + # shellcheck disable=SC2064 # Intentional early expansion — work_dir is local trap "rm -rf '$work_dir'" EXIT log "INFO" "=== Subrepo reset: josh-filtered mono/${mono_branch} → subrepo/${subrepo_branch} ===" @@ -287,7 +296,7 @@ subrepo_reset() { --branch "$mono_branch" --single-branch \ "${work_dir}/filtered" || die "Failed to clone through josh-proxy" - cd "${work_dir}/filtered" + cd "${work_dir}/filtered" || exit local head_sha head_sha=$(git rev-parse HEAD) From a19b795f9b9b703e6dd524c06f19f3b6bffa19cf Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Thu, 12 Feb 2026 14:37:11 +0300 Subject: [PATCH 05/37] Create .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5c9e080 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +result From 0edbdae558b9fd0c44289a1493963f4429409ccd Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Thu, 12 Feb 2026 18:00:08 +0300 Subject: [PATCH 06/37] "#" --- .gitignore | 1 + README.md | 5 + docs/config-reference.md | 79 +++++++ docs/guide.md | 482 +++++++++++++++++++++++++++++++++++++++ lib/sync.sh | 17 +- 5 files changed, 582 insertions(+), 2 deletions(-) create mode 100644 docs/config-reference.md create mode 100644 docs/guide.md diff --git a/.gitignore b/.gitignore index 5c9e080..58c99d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +.claude/*local* result diff --git a/README.md b/README.md index c3ad370..3b52a36 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,11 @@ Add josh-sync as a flake input, then: Run `josh-sync preflight` to validate your setup. +## Documentation + +- **[Setup Guide](docs/guide.md)** — Step-by-step: prerequisites, importing existing subrepos, CI workflows, and troubleshooting +- **[Configuration Reference](docs/config-reference.md)** — Full `.josh-sync.yml` field documentation + ## CLI ``` diff --git a/docs/config-reference.md b/docs/config-reference.md new file mode 100644 index 0000000..01b3af8 --- /dev/null +++ b/docs/config-reference.md @@ -0,0 +1,79 @@ +# Configuration Reference + +Full reference for `.josh-sync.yml` fields and environment variables. + +## `.josh-sync.yml` Structure + +```yaml +josh: # josh-proxy settings (required) +targets: # sync targets (required, at least 1) +bot: # bot identity for sync commits (required) +``` + +## `josh` Section + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `proxy_url` | string | Yes | Josh-proxy URL, no trailing slash. Must start with `http://` or `https://`. | +| `monorepo_path` | string | Yes | Repository path as josh-proxy sees it (e.g., `org/monorepo`). | + +## `targets[]` Section + +Each target maps a monorepo subfolder to an external subrepo. + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `name` | string | Yes | — | Unique target identifier. Used in CLI commands and state tracking. | +| `subfolder` | string | Yes | — | Monorepo subfolder path (e.g., `services/billing`). | +| `josh_filter` | string | No | `:/<subfolder>` | Josh filter expression. Auto-derived from `subfolder` if omitted. Must start with `:`. | +| `subrepo_url` | string | Yes | — | External subrepo Git URL. Supports HTTPS (`https://...`), SSH (`git@host:path`), and `ssh://` formats. | +| `subrepo_auth` | string | No | `"https"` | Auth method: `"https"` or `"ssh"`. | +| `subrepo_token_var` | string | No | `"SUBREPO_TOKEN"` | Name of the env var holding the HTTPS token for this target. | +| `subrepo_ssh_key_var` | string | No | `"SUBREPO_SSH_KEY"` | Name of the env var holding the SSH private key for this target. | +| `branches` | object | Yes | — | Branch mapping: `mono_branch: subrepo_branch`. Each key-value pair syncs those branches bidirectionally. | +| `forward_only` | string[] | No | `[]` | Branches that only sync mono → subrepo, never reverse. | + +## `bot` Section + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Git author name for sync commits. | +| `email` | string | Yes | Git author email for sync commits. | +| `trailer` | string | Yes | Git trailer key for loop prevention (e.g., `Josh-Sync-Origin`). | + +## Environment Variables + +### Credentials + +| Variable | Purpose | Default | +|----------|---------|---------| +| `SYNC_BOT_USER` | Bot's Git username | — | +| `SYNC_BOT_TOKEN` | API token for monorepo access and josh-proxy HTTPS auth | — | +| `SUBREPO_TOKEN` | HTTPS token for subrepo access | Falls back to `SYNC_BOT_TOKEN` | +| `SUBREPO_SSH_KEY` | SSH private key content (not a file path) for subrepo access | — | + +### Per-target credential overrides + +Set `subrepo_token_var` or `subrepo_ssh_key_var` in a target's config to read credentials from a different env var: + +```yaml +targets: + - name: "auth" + subrepo_token_var: "AUTH_REPO_TOKEN" # reads from $AUTH_REPO_TOKEN + subrepo_ssh_key_var: "AUTH_SSH_KEY" # reads from $AUTH_SSH_KEY +``` + +**Resolution order:** per-target env var → default env var (`SUBREPO_TOKEN` / `SUBREPO_SSH_KEY`) → `SYNC_BOT_TOKEN` fallback. + +### Runtime + +| Variable | Purpose | Default | +|----------|---------|---------| +| `JOSH_SYNC_TARGET` | Restrict sync to a single target | All targets | +| `JOSH_SYNC_STATE_BRANCH` | Name of the orphan branch for state storage | `josh-sync-state` | +| `JOSH_SYNC_DEBUG` | Enable verbose logging (`1` to enable) | `0` | +| `MONOREPO_API` | Override monorepo API URL | Auto-derived from first target's host | + +## JSON Schema + +The config file can be validated against the JSON Schema at [`schema/config-schema.json`](../schema/config-schema.json). diff --git a/docs/guide.md b/docs/guide.md new file mode 100644 index 0000000..816af1d --- /dev/null +++ b/docs/guide.md @@ -0,0 +1,482 @@ +# Setup Guide + +Step-by-step guide to setting up josh-sync for a new monorepo with existing subrepos. + +## Overview + +josh-sync provides bidirectional sync between a monorepo and N external subrepos via [josh-proxy](https://josh-project.github.io/josh/): + +``` +MONOREPO SUBREPOS +├── services/billing/ ──── forward ────► billing-repo/ +├── services/auth/ (push or cron) auth-repo/ +└── libs/shared/ ◄──── reverse ───── shared-lib-repo/ + (cron → always PR) + via josh-proxy (filtered git views) +``` + +**Key safety properties:** +- Forward sync (mono → subrepo) uses `--force-with-lease` — never overwrites concurrent changes +- Reverse sync (subrepo → mono) always creates a PR — never pushes directly +- Git trailers (`Josh-Sync-Origin:`) prevent infinite sync loops +- State tracked on an orphan branch (`josh-sync-state`) — survives CI runner teardown + +## Prerequisites + +Before you begin, you need: + +### josh-proxy instance + +A running [josh-proxy](https://josh-project.github.io/josh/) that can access your monorepo's Git server. Verify connectivity: + +```bash +git ls-remote https://josh.example.com/org/monorepo.git HEAD +``` + +### Bot account + +A dedicated Git user (e.g., `josh-sync-bot`) with: +- Write access to the monorepo +- Write access to all subrepos +- Ability to create PRs on both monorepo and subrepo platforms + +### Credentials + +| Variable | Purpose | Required | +|----------|---------|----------| +| `SYNC_BOT_USER` | Bot's Git username | Yes | +| `SYNC_BOT_TOKEN` | API token with repo scope (monorepo + josh-proxy auth) | Yes | +| `SUBREPO_SSH_KEY` | SSH private key for subrepo access (if using SSH auth) | If SSH | +| `SUBREPO_TOKEN` | HTTPS token for subrepo access (defaults to `SYNC_BOT_TOKEN`) | No | + +Per-target credential overrides are supported — see [Configuration Reference](config-reference.md). + +### Tool dependencies + +`bash >=4`, `git`, `curl`, `jq`, `yq` ([mikefarah/yq](https://github.com/mikefarah/yq) v4+), `openssh`, `rsync` + +> The Nix flake bundles all dependencies automatically. + +## Step 1: Create the Monorepo + +Create a new repository on your Git server (e.g., `org/monorepo`). Create subdirectories for each subrepo you want to sync: + +```bash +mkdir -p services/billing services/auth libs/shared +``` + +These directories will be populated during the import step. They can be empty or contain `.gitkeep` files for now. + +Verify josh-proxy can see the monorepo: + +```bash +git ls-remote https://josh.example.com/org/monorepo.git HEAD +``` + +## Step 2: Configure `.josh-sync.yml` + +Create `.josh-sync.yml` at the monorepo root. Each target maps a monorepo subfolder to an external subrepo: + +```yaml +josh: + proxy_url: "https://josh.example.com" # josh-proxy URL (no trailing slash) + monorepo_path: "org/monorepo" # repo path as josh sees it + +targets: + - name: "billing" # unique identifier + subfolder: "services/billing" # monorepo subfolder + # josh_filter auto-derived as ":/services/billing" if omitted + subrepo_url: "git@gitea.example.com:ext/billing.git" + subrepo_auth: "ssh" # "https" (default) or "ssh" + branches: + main: main # mono_branch: subrepo_branch + forward_only: [] + + - name: "auth" + subfolder: "services/auth" + subrepo_url: "https://gitea.example.com/ext/auth.git" + subrepo_auth: "https" + subrepo_token_var: "AUTH_REPO_TOKEN" # per-target credential override + branches: + main: main + develop: develop # multiple branches supported + forward_only: [] + + - name: "shared-lib" + subfolder: "libs/shared" + subrepo_url: "https://gitea.example.com/ext/shared-lib.git" + branches: + main: main + forward_only: [main] # one-way: mono → subrepo only + +bot: + name: "josh-sync-bot" + email: "josh-sync-bot@example.com" + trailer: "Josh-Sync-Origin" # git trailer for loop prevention +``` + +For the full field reference, see [Configuration Reference](config-reference.md). + +## Step 3: Set Up Local Dev Environment + +### Option A: Nix + devenv (recommended) + +**`devenv.yaml`** — declare josh-sync as a flake input: + +```yaml +inputs: + nixpkgs: + url: github:cachix/devenv-nixpkgs/rolling + josh-sync: + url: git+https://your-gitea.example.com/org/josh-sync?ref=main + flake: true +``` + +**`devenv.nix`** — import the josh-sync module: + +```nix +{ inputs, ... }: +{ + imports = [ inputs.josh-sync.devenvModules.default ]; + + name = "my-monorepo"; + + # .env contains secrets, not devenv config + dotenv.disableHint = true; +} +``` + +**`.envrc`** — activate devenv automatically: + +```bash +DEVENV_WARN_TIMEOUT=20 +use devenv +``` + +**`.env`** — local credentials (add to `.gitignore`): + +```bash +SYNC_BOT_USER=sync-bot +SYNC_BOT_TOKEN=<your-api-token> +SUBREPO_SSH_KEY="-----BEGIN OPENSSH PRIVATE KEY----- +... +-----END OPENSSH PRIVATE KEY-----" +# Per-target overrides: +# AUTH_REPO_TOKEN=<auth-specific-token> +``` + +### Option B: Manual installation + +Install the required tools, then either: + +- Clone the josh-sync repo and add `bin/` to your `PATH` +- Run `make build` to create a single bundled script at `dist/josh-sync` + +## Step 4: Validate with Preflight + +```bash +josh-sync preflight +``` + +This validates: +- Config syntax and required fields +- josh-proxy connectivity (via `git ls-remote` through josh) +- Subrepo connectivity and authentication +- Branch mappings +- CI workflow path coverage (checks if `.gitea/workflows/josh-sync-forward.yml` paths match target subfolders) + +For a new monorepo before import, preflight may warn that subfolders don't exist yet — that's expected. + +## Step 5: Import Existing Subrepos + +This is the critical onboarding step. For each existing subrepo, you run a three-step cycle: **import → merge → reset**. + +> Do this **one target at a time** to keep PRs reviewable. + +### 5a. Import + +```bash +josh-sync import billing +``` + +This: +1. Clones the monorepo directly (not through josh) +2. Clones the subrepo +3. Copies subrepo content into the monorepo subfolder via `rsync` +4. Creates a branch `auto-sync/import-billing-<timestamp>` +5. Pushes it and creates a PR on the monorepo + +Review the import PR — check for leaked credentials, environment-specific config, or files that shouldn't be in the monorepo. + +### 5b. Merge the import PR + +Merge the PR using your Git platform's UI. This lands the subrepo content into the monorepo's main branch. + +> At this point, the monorepo has the content but the histories are disconnected. Sync will **not** work until you complete the reset step. + +### 5c. Reset + +```bash +josh-sync reset billing +``` + +> **You do NOT need to `git pull` locally before running reset.** The reset command clones fresh from josh-proxy — it never uses your local working copy. + +This: +1. Clones the monorepo through josh-proxy with the josh filter (the "filtered view") +2. Force-pushes that filtered view to the subrepo, replacing its history + +This establishes **shared commit ancestry** between josh's filtered view and the subrepo. Without this, josh-proxy can't compute diffs between the two. + +> **Warning:** This is a destructive force-push that replaces the subrepo's history. Back up any important branches or tags in the subrepo beforehand. + +### 5d. Repeat for each target + +``` +For each target: + 1. josh-sync import <target> + 2. Review and merge the import PR on the monorepo + 3. josh-sync reset <target> +``` + +### 5e. Verify + +After all targets are imported and reset: + +```bash +# Check all targets show state +josh-sync status + +# Test forward sync — should return "skip" (trees are identical after reset) +josh-sync sync --forward --target billing + +# Test reverse sync — should return "skip" (no new human commits) +josh-sync sync --reverse --target billing +``` + +## Step 6: Set Up CI Workflows + +### Forward sync (mono → subrepo) + +Create `.gitea/workflows/josh-sync-forward.yml`: + +```yaml +name: "Josh Sync → Subrepo" + +on: + push: + branches: [main] + paths: + # List ALL target subfolders: + - "services/billing/**" + - "services/auth/**" + - "libs/shared/**" + schedule: + - cron: "0 */6 * * *" # every 6 hours as fallback + workflow_dispatch: + inputs: + target: + description: "Target to sync (empty = detect from push or all)" + required: false + default: "" + branch: + description: "Branch to sync (empty = triggered branch or all)" + required: false + default: "" + +concurrency: + group: josh-sync-fwd-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + sync: + runs-on: docker + container: node:20-bookworm + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 # needed for target detection + + - name: Install tools + run: | + apt-get update -qq && apt-get install -y -qq jq curl git openssh-client >/dev/null 2>&1 + curl -sL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" \ + -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + + - name: Detect changed target + if: github.event_name == 'push' + id: detect + run: | + CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") + TARGETS=$(yq -o json '.targets' .josh-sync.yml \ + | jq -r '.[] | "\(.name):\(.subfolder)"' \ + | while IFS=: read -r name prefix; do + echo "$CHANGED" | grep -q "^${prefix}/" && echo "$name" + done | sort -u | paste -sd ',' -) + echo "targets=${TARGETS}" >> "$GITHUB_OUTPUT" + + - uses: https://your-gitea.example.com/org/josh-sync@v1 + with: + direction: forward + target: ${{ github.event.inputs.target || steps.detect.outputs.targets }} + branch: ${{ github.event.inputs.branch || github.ref_name }} + env: + SYNC_BOT_USER: ${{ secrets.SYNC_BOT_USER }} + SYNC_BOT_TOKEN: ${{ secrets.SYNC_BOT_TOKEN }} + SUBREPO_TOKEN: ${{ secrets.SUBREPO_TOKEN || secrets.SYNC_BOT_TOKEN }} + SUBREPO_SSH_KEY: ${{ secrets.SUBREPO_SSH_KEY }} +``` + +### Reverse sync (subrepo → mono) + +Create `.gitea/workflows/josh-sync-reverse.yml`: + +```yaml +name: "Josh Sync ← Subrepo" + +on: + schedule: + - cron: "0 1,7,13,19 * * *" # every 6h, offset from forward + workflow_dispatch: + inputs: + target: + description: "Target to sync (empty = all)" + required: false + default: "" + branch: + description: "Branch to sync (empty = all eligible)" + required: false + default: "" + +concurrency: + group: josh-sync-rev-${{ github.event.inputs.target || 'all' }} + cancel-in-progress: false + +jobs: + sync: + runs-on: docker + container: node:20-bookworm + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Install tools + run: | + apt-get update -qq && apt-get install -y -qq jq curl git openssh-client >/dev/null 2>&1 + curl -sL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" \ + -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + + - uses: https://your-gitea.example.com/org/josh-sync@v1 + with: + direction: reverse + target: ${{ github.event.inputs.target || '' }} + branch: ${{ github.event.inputs.branch || '' }} + env: + SYNC_BOT_USER: ${{ secrets.SYNC_BOT_USER }} + SYNC_BOT_TOKEN: ${{ secrets.SYNC_BOT_TOKEN }} + SUBREPO_TOKEN: ${{ secrets.SUBREPO_TOKEN || secrets.SYNC_BOT_TOKEN }} + SUBREPO_SSH_KEY: ${{ secrets.SUBREPO_SSH_KEY }} +``` + +### Required CI secrets + +| Secret | Purpose | +|--------|---------| +| `SYNC_BOT_USER` | Bot username | +| `SYNC_BOT_TOKEN` | Bot API token (monorepo access + josh-proxy auth) | +| `SUBREPO_SSH_KEY` | SSH private key for subrepo push (if using SSH auth) | +| `SUBREPO_TOKEN` | Optional separate subrepo token (defaults to `SYNC_BOT_TOKEN`) | + +> **GitHub Actions note:** These examples target Gitea Actions. For GitHub Actions, change the `uses:` reference to a GitHub repo (e.g., `org/josh-sync@v1`) and `runs-on:` to a GitHub runner (e.g., `ubuntu-latest`). + +## How Ongoing Sync Works + +Once set up, sync runs automatically: + +### Forward sync (mono → subrepo) + +Triggered by pushes to target subfolders or on a cron schedule: + +1. Clones the monorepo through josh-proxy (filtered view of the subfolder) +2. Fetches the subrepo branch for comparison +3. If trees are identical → skip +4. If subrepo branch doesn't exist → fresh push +5. Merges mono changes on top of subrepo state +6. If clean merge → pushes with `--force-with-lease` (protects against concurrent changes) +7. If lease rejected → retries on next run (subrepo changed during sync) +8. If merge conflict → creates a conflict PR on the subrepo + +### Reverse sync (subrepo → mono) + +Runs on a cron schedule (never triggered by subrepo pushes): + +1. Clones the subrepo +2. Fetches the monorepo's josh-filtered view for comparison +3. Finds new human commits (filters out bot commits by checking for the `Josh-Sync-Origin:` trailer) +4. If no new human commits → skip +5. Pushes through josh-proxy to a staging branch +6. Creates a PR on the monorepo — **never pushes directly** + +### Loop prevention + +Bot commits include a git trailer like `Josh-Sync-Origin: forward/main/2024-02-12T10:30:00Z`. Each sync direction filters out commits with this trailer, preventing changes from bouncing back and forth. The CI action also has a loop guard that skips entirely if the HEAD commit has the trailer. + +### State tracking + +Sync state is stored as JSON files on an orphan branch (`josh-sync-state`), one file per target/branch. This tracks the last-synced commit SHAs and timestamps to avoid re-syncing the same changes. + +## Adding a New Target + +To add a new subrepo after initial setup: + +1. Add the target to `.josh-sync.yml` +2. Update the forward workflow's `paths:` list to include the new subfolder +3. Commit and push +4. Run the import-merge-reset cycle for the new target: + ```bash + josh-sync import new-target + # merge the PR + josh-sync reset new-target + ``` +5. Verify with `josh-sync status` + +## Troubleshooting + +### "Failed to clone through josh-proxy" + +- Check josh-proxy is running and accessible +- Verify `monorepo_path` matches what josh-proxy expects +- Test manually: `git ls-remote https://<user>:<token>@josh.example.com/org/repo.git:/services/app.git` + +### SSH authentication failures + +- `SUBREPO_SSH_KEY` must contain the actual key content, not a file path +- For per-target keys, ensure `subrepo_ssh_key_var` in config matches the env var name +- Check the key has write access to the subrepo + +### "Force-with-lease rejected" + +Normal: the subrepo changed while sync was running. The next sync run will pick it up. If persistent, check for another process pushing to the subrepo simultaneously. + +### "Josh rejected push" (reverse sync) + +Josh-proxy couldn't map the push back to the monorepo. Check josh-proxy logs, verify the josh filter is correct. May indicate a history divergence — consider running `josh-sync reset <target>`. + +### Import PR shows "No changes" + +The subfolder already contains the same content as the subrepo. This is fine — the import is a no-op. + +### Duplicate/looping commits + +Verify `bot.trailer` in config matches what's in commit messages. Check the loop guard in the CI workflow is active. + +### State issues + +```bash +# View current state +josh-sync state show <target> [branch] + +# Reset state (forces next sync to run regardless of SHA comparison) +josh-sync state reset <target> [branch] +``` diff --git a/lib/sync.sh b/lib/sync.sh index 91d2903..5476807 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -170,7 +170,20 @@ reverse_sync() { log "INFO" "New human commits to sync:" echo "$human_commits" >&2 - # 4. Push through josh to a staging branch + # 4. Merge subrepo changes onto the latest josh-filtered monorepo view + # This ensures the staging branch is based on the latest monorepo main, + # not on the common ancestor between subrepo and monorepo histories. + local subrepo_head + subrepo_head=$(git rev-parse HEAD) + + git checkout -B sync-push "mono-filtered/${mono_branch}" >&2 + git merge --no-ff "$subrepo_head" \ + -m "Sync from subrepo $(date -u +%Y-%m-%dT%H:%M:%SZ) + +${BOT_TRAILER}: reverse/${subrepo_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 \ + || die "Merge conflict during reverse sync — manual intervention needed" + + # 5. Push merged branch through josh to a staging branch local ts ts=$(date +%Y%m%d-%H%M%S) local staging_branch="auto-sync/subrepo-${subrepo_branch}-${ts}" @@ -178,7 +191,7 @@ reverse_sync() { if git push -o "base=${mono_branch}" "$(josh_auth_url)" "HEAD:refs/heads/${staging_branch}"; then log "INFO" "Pushed to staging branch via josh: ${staging_branch}" - # 5. Create PR on monorepo (NEVER direct push) + # 6. Create PR on monorepo (NEVER direct push) local pr_body pr_body="## Subrepo changes\n\nNew commits from subrepo \`${subrepo_branch}\`:\n\n\`\`\`\n${human_commits}\n\`\`\`\n\n**Review checklist:**\n- [ ] Changes scoped to synced subfolder\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" From 57589879425b19c5d8b3763693ea86f116835e6c Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Thu, 12 Feb 2026 19:32:21 +0300 Subject: [PATCH 07/37] Update sync.sh --- lib/sync.sh | 71 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/lib/sync.sh b/lib/sync.sh index 5476807..0328280 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -131,7 +131,7 @@ ${BOT_TRAILER}: forward/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 # ─── Reverse Sync: Subrepo → Monorepo ────────────────────────────── # # Always creates a PR on the monorepo — never pushes directly. -# Returns: skip | pr-created | josh-rejected +# Returns: skip | pr-created reverse_sync() { local mono_branch="$SYNC_BRANCH_MONO" @@ -170,43 +170,56 @@ reverse_sync() { log "INFO" "New human commits to sync:" echo "$human_commits" >&2 - # 4. Merge subrepo changes onto the latest josh-filtered monorepo view - # This ensures the staging branch is based on the latest monorepo main, - # not on the common ancestor between subrepo and monorepo histories. - local subrepo_head - subrepo_head=$(git rev-parse HEAD) + # 4. Clone monorepo directly (not through josh — we need a real branch from main) + local mono_auth_url api_host_path subfolder + api_host_path=$(echo "$MONOREPO_API" | sed 's|https://||; s|/api/v1/repos/|/|') + mono_auth_url="https://${BOT_USER}:${GITEA_TOKEN}@${api_host_path}.git" + subfolder=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$JOSH_SYNC_TARGET_NAME" \ + '.[] | select(.name == $n) | .subfolder') - git checkout -B sync-push "mono-filtered/${mono_branch}" >&2 - git merge --no-ff "$subrepo_head" \ - -m "Sync from subrepo $(date -u +%Y-%m-%dT%H:%M:%SZ) + git clone "$mono_auth_url" \ + --branch "$mono_branch" --single-branch \ + "${work_dir}/monorepo" || die "Failed to clone monorepo" -${BOT_TRAILER}: reverse/${subrepo_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 \ - || die "Merge conflict during reverse sync — manual intervention needed" + cd "${work_dir}/monorepo" || exit + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" - # 5. Push merged branch through josh to a staging branch local ts ts=$(date +%Y%m%d-%H%M%S) local staging_branch="auto-sync/subrepo-${subrepo_branch}-${ts}" + git checkout -B "$staging_branch" >&2 - if git push -o "base=${mono_branch}" "$(josh_auth_url)" "HEAD:refs/heads/${staging_branch}"; then - log "INFO" "Pushed to staging branch via josh: ${staging_branch}" + # 5. Rsync subrepo content into subfolder (--delete handles removals) + mkdir -p "$subfolder" + rsync -a --delete --exclude='.git' "${work_dir}/subrepo/" "${subfolder}/" + git add "$subfolder" - # 6. Create PR on monorepo (NEVER direct push) - local pr_body - pr_body="## Subrepo changes\n\nNew commits from subrepo \`${subrepo_branch}\`:\n\n\`\`\`\n${human_commits}\n\`\`\`\n\n**Review checklist:**\n- [ ] Changes scoped to synced subfolder\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" - - create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ - "$mono_branch" "$staging_branch" \ - "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ - "$pr_body" \ - || die "Failed to create PR on monorepo (check GITEA_TOKEN)" - - log "INFO" "Reverse sync PR created on monorepo" - echo "pr-created" - else - log "ERROR" "Josh rejected push — check josh-proxy logs" - echo "josh-rejected" + if git diff --cached --quiet; then + log "INFO" "No tree changes after rsync — skip" + echo "skip" + return fi + + git commit -m "Sync from subrepo/${subrepo_branch} $(date -u +%Y-%m-%dT%H:%M:%SZ) + +${BOT_TRAILER}: reverse/${subrepo_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 + + # 6. Push branch and create PR on monorepo (NEVER direct push to main) + git push origin "$staging_branch" || die "Failed to push staging branch" + log "INFO" "Pushed staging branch: ${staging_branch}" + + local pr_body + pr_body="## Subrepo changes\n\nNew commits from subrepo \`${subrepo_branch}\`:\n\n\`\`\`\n${human_commits}\n\`\`\`\n\n**Review checklist:**\n- [ ] Changes scoped to synced subfolder\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" + + create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ + "$mono_branch" "$staging_branch" \ + "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ + "$pr_body" \ + || die "Failed to create PR on monorepo (check GITEA_TOKEN)" + + log "INFO" "Reverse sync PR created on monorepo" + echo "pr-created" } # ─── Initial Import: Subrepo → Monorepo (first time) ─────────────── From cc08c530d12698a924e5109cbe36c6547ddd7a4f Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Thu, 12 Feb 2026 20:02:34 +0300 Subject: [PATCH 08/37] Update sync.sh --- lib/sync.sh | 64 ++++++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 45 deletions(-) diff --git a/lib/sync.sh b/lib/sync.sh index 0328280..91d2903 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -131,7 +131,7 @@ ${BOT_TRAILER}: forward/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 # ─── Reverse Sync: Subrepo → Monorepo ────────────────────────────── # # Always creates a PR on the monorepo — never pushes directly. -# Returns: skip | pr-created +# Returns: skip | pr-created | josh-rejected reverse_sync() { local mono_branch="$SYNC_BRANCH_MONO" @@ -170,56 +170,30 @@ reverse_sync() { log "INFO" "New human commits to sync:" echo "$human_commits" >&2 - # 4. Clone monorepo directly (not through josh — we need a real branch from main) - local mono_auth_url api_host_path subfolder - api_host_path=$(echo "$MONOREPO_API" | sed 's|https://||; s|/api/v1/repos/|/|') - mono_auth_url="https://${BOT_USER}:${GITEA_TOKEN}@${api_host_path}.git" - subfolder=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$JOSH_SYNC_TARGET_NAME" \ - '.[] | select(.name == $n) | .subfolder') - - git clone "$mono_auth_url" \ - --branch "$mono_branch" --single-branch \ - "${work_dir}/monorepo" || die "Failed to clone monorepo" - - cd "${work_dir}/monorepo" || exit - git config user.name "$BOT_NAME" - git config user.email "$BOT_EMAIL" - + # 4. Push through josh to a staging branch local ts ts=$(date +%Y%m%d-%H%M%S) local staging_branch="auto-sync/subrepo-${subrepo_branch}-${ts}" - git checkout -B "$staging_branch" >&2 - # 5. Rsync subrepo content into subfolder (--delete handles removals) - mkdir -p "$subfolder" - rsync -a --delete --exclude='.git' "${work_dir}/subrepo/" "${subfolder}/" - git add "$subfolder" + if git push -o "base=${mono_branch}" "$(josh_auth_url)" "HEAD:refs/heads/${staging_branch}"; then + log "INFO" "Pushed to staging branch via josh: ${staging_branch}" - if git diff --cached --quiet; then - log "INFO" "No tree changes after rsync — skip" - echo "skip" - return + # 5. Create PR on monorepo (NEVER direct push) + local pr_body + pr_body="## Subrepo changes\n\nNew commits from subrepo \`${subrepo_branch}\`:\n\n\`\`\`\n${human_commits}\n\`\`\`\n\n**Review checklist:**\n- [ ] Changes scoped to synced subfolder\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" + + create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ + "$mono_branch" "$staging_branch" \ + "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ + "$pr_body" \ + || die "Failed to create PR on monorepo (check GITEA_TOKEN)" + + log "INFO" "Reverse sync PR created on monorepo" + echo "pr-created" + else + log "ERROR" "Josh rejected push — check josh-proxy logs" + echo "josh-rejected" fi - - git commit -m "Sync from subrepo/${subrepo_branch} $(date -u +%Y-%m-%dT%H:%M:%SZ) - -${BOT_TRAILER}: reverse/${subrepo_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 - - # 6. Push branch and create PR on monorepo (NEVER direct push to main) - git push origin "$staging_branch" || die "Failed to push staging branch" - log "INFO" "Pushed staging branch: ${staging_branch}" - - local pr_body - pr_body="## Subrepo changes\n\nNew commits from subrepo \`${subrepo_branch}\`:\n\n\`\`\`\n${human_commits}\n\`\`\`\n\n**Review checklist:**\n- [ ] Changes scoped to synced subfolder\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" - - create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ - "$mono_branch" "$staging_branch" \ - "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ - "$pr_body" \ - || die "Failed to create PR on monorepo (check GITEA_TOKEN)" - - log "INFO" "Reverse sync PR created on monorepo" - echo "pr-created" } # ─── Initial Import: Subrepo → Monorepo (first time) ─────────────── From 405e5f453580ef0c13b8882e8ca1727f0fd8b807 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 09:31:41 +0300 Subject: [PATCH 09/37] Update guide.md --- docs/guide.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/guide.md b/docs/guide.md index 816af1d..8ba7143 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -228,7 +228,18 @@ This: This establishes **shared commit ancestry** between josh's filtered view and the subrepo. Without this, josh-proxy can't compute diffs between the two. -> **Warning:** This is a destructive force-push that replaces the subrepo's history. Back up any important branches or tags in the subrepo beforehand. +> **Warning:** This is a destructive force-push that replaces the subrepo's history. Back up any important branches or tags in the subrepo beforehand. Merge or close all open pull requests on the subrepo first — they will be invalidated. + +After reset, **every developer with a local clone of the subrepo** must update their local copy to match the new history: + +```bash +cd /path/to/local-subrepo +git fetch origin +git checkout main && git reset --hard origin/main +git checkout stage && git reset --hard origin/stage # repeat for each branch +``` + +Or simply delete and re-clone the subrepo. Local-only branches (not pushed to the remote) will be lost either way. ### 5d. Repeat for each target @@ -471,6 +482,24 @@ The subfolder already contains the same content as the subrepo. This is fine — Verify `bot.trailer` in config matches what's in commit messages. Check the loop guard in the CI workflow is active. +### "cannot lock ref" or "expected X but got Y" + +**After reset (subrepo):** The subrepo's history was replaced by force-push. Local clones still have the old history: + +```bash +cd /path/to/subrepo +git fetch origin +git checkout main && git reset --hard origin/main +``` + +Or simply delete and re-clone. + +**After import/reset cycle (monorepo):** The import and reset steps create and update branches rapidly (`auto-sync/import-*`, `josh-sync-state`). If your local clone fetched partway through, tracking refs go stale: + +```bash +git remote prune origin && git pull +``` + ### State issues ```bash From 105216a27eb532ce5ca77def09faf8a7a374700d Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 12:41:44 +0300 Subject: [PATCH 10/37] Add onboard and migrate-pr commands (v1.1.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New commands for safely onboarding existing subrepos into the monorepo without losing open PRs: - josh-sync onboard <target>: interactive, resumable 5-step flow (import → wait for merge → reset to new repo) - josh-sync migrate-pr <target> [PR#...] [--all]: migrate PRs from archived repo to new repo via patch application Also refactors create_pr() to wrap create_pr_number(), eliminating duplicated curl/jq logic. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- VERSION | 2 +- bin/josh-sync | 160 ++++++++++++++++++- lib/auth.sh | 48 ++++-- lib/onboard.sh | 425 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 619 insertions(+), 16 deletions(-) create mode 100644 lib/onboard.sh diff --git a/VERSION b/VERSION index 3eefcb9..9084fa2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/bin/josh-sync b/bin/josh-sync index e3c244d..c3a99a1 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -9,6 +9,8 @@ # preflight Validate config, connectivity, auth # import <target> Initial import: pull subrepo into monorepo # reset <target> Reset subrepo to josh-filtered view +# onboard <target> Import existing subrepo into monorepo (interactive) +# migrate-pr <target> [PR#...] [--all] Move PRs from archived to new subrepo # status Show target config and sync state # state show|reset Manage sync state directly # @@ -39,6 +41,8 @@ source "${JOSH_LIB_DIR}/auth.sh" source "${JOSH_LIB_DIR}/state.sh" # shellcheck source=../lib/sync.sh source "${JOSH_LIB_DIR}/sync.sh" +# shellcheck source=../lib/onboard.sh +source "${JOSH_LIB_DIR}/onboard.sh" # ─── Version ──────────────────────────────────────────────────────── @@ -69,6 +73,8 @@ Commands: preflight Validate config, connectivity, auth, workflow coverage import <target> Initial import: pull existing subrepo into monorepo (creates PR) reset <target> Reset subrepo to josh-filtered view (after merging import PR) + onboard <target> Import existing subrepo into monorepo (interactive, resumable) + migrate-pr <target> [PR#...] [--all] Move PRs from archived to new subrepo status Show target config and sync state state show <target> [branch] Show sync state JSON state reset <target> [branch] Reset sync state to {} @@ -643,6 +649,146 @@ cmd_state() { esac } +# ─── Onboard Command ────────────────────────────────────────────── + +cmd_onboard() { + local config_file=".josh-sync.yml" + local target_name="" + local restart=false + + while [ $# -gt 0 ]; do + case "$1" in + --config) config_file="$2"; shift 2 ;; + --debug) export JOSH_SYNC_DEBUG=1; shift ;; + --restart) restart=true; shift ;; + -*) die "Unknown flag: $1" ;; + *) target_name="$1"; shift ;; + esac + done + + if [ -z "$target_name" ]; then + echo "Usage: josh-sync onboard <target> [--restart]" >&2 + parse_config "$config_file" + echo "Available targets:" >&2 + echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2 + exit 1 + fi + + parse_config "$config_file" + + local target_json + target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)') + [ -n "$target_json" ] || die "Target '${target_name}' not found in config" + + log "INFO" "══════ Onboard target: ${target_name} ══════" + load_target "$target_json" + onboard_flow "$target_json" "$restart" +} + +# ─── Migrate PR Command ────────────────────────────────────────── + +cmd_migrate_pr() { + local config_file=".josh-sync.yml" + local target_name="" + local all=false + local pr_numbers=() + + while [ $# -gt 0 ]; do + case "$1" in + --config) config_file="$2"; shift 2 ;; + --debug) export JOSH_SYNC_DEBUG=1; shift ;; + --all) all=true; shift ;; + -*) die "Unknown flag: $1" ;; + *) + if [ -z "$target_name" ]; then + target_name="$1" + else + pr_numbers+=("$1") + fi + shift ;; + esac + done + + if [ -z "$target_name" ]; then + echo "Usage: josh-sync migrate-pr <target> [PR#...] [--all]" >&2 + parse_config "$config_file" + echo "Available targets:" >&2 + echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2 + exit 1 + fi + + parse_config "$config_file" + + local target_json + target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)') + [ -n "$target_json" ] || die "Target '${target_name}' not found in config" + + load_target "$target_json" + + # Load archived repo info from onboard state + local onboard_state archived_api + onboard_state=$(read_onboard_state "$target_name") + archived_api=$(echo "$onboard_state" | jq -r '.archived_api') + if [ -z "$archived_api" ] || [ "$archived_api" = "null" ]; then + die "No archived repo info found. Run 'josh-sync onboard ${target_name}' first." + fi + + log "INFO" "Archived repo: ${archived_api}" + + if [ "$all" = true ]; then + # Migrate all open PRs from archived repo + local prs + prs=$(list_open_prs "$archived_api" "$SUBREPO_TOKEN") \ + || die "Failed to list PRs on archived repo" + local count + count=$(echo "$prs" | jq 'length') + log "INFO" "Found ${count} open PR(s) on archived repo" + + echo "$prs" | jq -r '.[] | .number' | while read -r num; do + migrate_one_pr "$num" || true + done + + elif [ ${#pr_numbers[@]} -gt 0 ]; then + # Migrate specific PR numbers + for num in "${pr_numbers[@]}"; do + migrate_one_pr "$num" || true + done + + else + # Interactive: list open PRs, let user pick + local prs + prs=$(list_open_prs "$archived_api" "$SUBREPO_TOKEN") \ + || die "Failed to list PRs on archived repo" + local count + count=$(echo "$prs" | jq 'length') + + if [ "$count" -eq 0 ]; then + log "INFO" "No open PRs on archived repo" + return + fi + + echo "" >&2 + echo "Open PRs on archived repo:" >&2 + echo "$prs" | jq -r '.[] | " #\(.number): \(.title) (\(.base.ref) <- \(.head.ref))"' >&2 + echo "" >&2 + echo "Enter PR numbers to migrate (space-separated), or 'all':" >&2 + local selection + read -r selection + + if [ "$selection" = "all" ]; then + echo "$prs" | jq -r '.[] | .number' | while read -r num; do + migrate_one_pr "$num" || true + done + else + for num in $selection; do + migrate_one_pr "$num" || true + done + fi + fi + + log "INFO" "PR migration complete" +} + # ─── Main ─────────────────────────────────────────────────────────── main() { @@ -662,12 +808,14 @@ main() { esac case "$command" in - sync) cmd_sync "$@" ;; - preflight) cmd_preflight "$@" ;; - import) cmd_import "$@" ;; - reset) cmd_reset "$@" ;; - status) cmd_status "$@" ;; - state) cmd_state "$@" ;; + sync) cmd_sync "$@" ;; + preflight) cmd_preflight "$@" ;; + import) cmd_import "$@" ;; + reset) cmd_reset "$@" ;; + onboard) cmd_onboard "$@" ;; + migrate-pr) cmd_migrate_pr "$@" ;; + status) cmd_status "$@" ;; + state) cmd_state "$@" ;; *) echo "Unknown command: ${command}" >&2 usage diff --git a/lib/auth.sh b/lib/auth.sh index 229ff88..851f940 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -39,16 +39,15 @@ subrepo_ls_remote() { } # ─── PR Creation ──────────────────────────────────────────────────── -# Shared helper for creating PRs on Gitea/GitHub API. +# Shared helpers for creating PRs on Gitea/GitHub API. # Usage: create_pr <api_url> <token> <base> <head> <title> <body> +# number=$(create_pr_number <api_url> <token> <base> <head> <title> <body>) +# +# create_pr — fire-and-forget (stdout suppressed, safe inside sync functions) +# create_pr_number — returns the new PR number via stdout -create_pr() { - local api_url="$1" - local token="$2" - local base="$3" - local head="$4" - local title="$5" - local body="$6" +create_pr_number() { + local api_url="$1" token="$2" base="$3" head="$4" title="$5" body="$6" curl -sf -X POST \ -H "Authorization: token ${token}" \ @@ -59,5 +58,36 @@ create_pr() { --arg title "$title" \ --arg body "$body" \ '{base:$base, head:$head, title:$title, body:$body}')" \ - "${api_url}/pulls" >/dev/null + "${api_url}/pulls" | jq -r '.number' +} + +create_pr() { + create_pr_number "$@" >/dev/null +} + +# ─── PR API Helpers ────────────────────────────────────────────── +# Used by onboard and migrate-pr commands. + +# List open PRs on a repo. Returns JSON array. +# Usage: list_open_prs <api_url> <token> +list_open_prs() { + local api_url="$1" token="$2" + curl -sf -H "Authorization: token ${token}" \ + "${api_url}/pulls?state=open&limit=50" +} + +# Get PR diff as plain text. +# Usage: get_pr_diff <api_url> <token> <pr_number> +get_pr_diff() { + local api_url="$1" token="$2" pr_number="$3" + curl -sf -H "Authorization: token ${token}" \ + "${api_url}/pulls/${pr_number}.diff" +} + +# Get single PR as JSON (for checking merge status, metadata, etc.). +# Usage: get_pr <api_url> <token> <pr_number> +get_pr() { + local api_url="$1" token="$2" pr_number="$3" + curl -sf -H "Authorization: token ${token}" \ + "${api_url}/pulls/${pr_number}" } diff --git a/lib/onboard.sh b/lib/onboard.sh new file mode 100644 index 0000000..01a229a --- /dev/null +++ b/lib/onboard.sh @@ -0,0 +1,425 @@ +#!/usr/bin/env bash +# lib/onboard.sh — Onboard orchestration and PR migration +# +# Provides: +# onboard_flow() — Interactive: import → wait for merge → reset to new repo +# migrate_one_pr() — Migrate a single PR from archived repo to new repo +# +# Onboard state is stored on the josh-sync-state branch at <target>/onboard.json. +# Steps: start → importing → waiting-for-merge → resetting → complete +# +# Requires: lib/core.sh, lib/config.sh, lib/auth.sh, lib/state.sh, lib/sync.sh sourced +# Expects: JOSH_SYNC_TARGET_NAME, BOT_NAME, BOT_EMAIL, SUBREPO_API, SUBREPO_TOKEN, etc. + +# ─── Onboard State Helpers ──────────────────────────────────────── +# Follow the same pattern as read_state()/write_state() in lib/state.sh. + +read_onboard_state() { + local target_name="${1:-$JOSH_SYNC_TARGET_NAME}" + git fetch origin "$STATE_BRANCH" 2>/dev/null || true + git show "origin/${STATE_BRANCH}:${target_name}/onboard.json" 2>/dev/null || echo '{}' +} + +write_onboard_state() { + local target_name="${1:-$JOSH_SYNC_TARGET_NAME}" + local state_json="$2" + local key="${target_name}/onboard" + local tmp_dir + tmp_dir=$(mktemp -d) + + if git rev-parse "origin/${STATE_BRANCH}" >/dev/null 2>&1; then + git worktree add "$tmp_dir" "origin/${STATE_BRANCH}" 2>/dev/null + else + git worktree add --detach "$tmp_dir" 2>/dev/null + (cd "$tmp_dir" && git checkout --orphan "$STATE_BRANCH" && { git rm -rf . 2>/dev/null || true; }) + fi + + mkdir -p "$(dirname "${tmp_dir}/${key}.json")" + echo "$state_json" | jq '.' > "${tmp_dir}/${key}.json" + + ( + cd "$tmp_dir" || exit + git add -A + if ! git diff --cached --quiet 2>/dev/null; then + git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ + commit -m "onboard: update ${target_name}" + git push origin "HEAD:${STATE_BRANCH}" || log "WARN" "Failed to push onboard state" + fi + ) + + git worktree remove "$tmp_dir" 2>/dev/null || rm -rf "$tmp_dir" +} + +# ─── Derive Archived API URL ───────────────────────────────────── +# Given a URL like "git@host:org/repo-archived.git" or +# "https://host/org/repo-archived.git", derive the Gitea API URL. + +_archived_api_from_url() { + local url="$1" + # Strip .git suffix first — avoids non-greedy regex issues in POSIX ERE + url="${url%.git}" + local host repo_path + + if echo "$url" | grep -qE '^(ssh://|git@)'; then + # SSH URL + if echo "$url" | grep -q '^ssh://'; then + host=$(echo "$url" | sed -E 's|ssh://[^@]*@([^/]+)/.*|\1|') + repo_path=$(echo "$url" | sed -E 's|ssh://[^@]*@[^/]+/(.+)$|\1|') + else + host=$(echo "$url" | sed -E 's|git@([^:/]+)[:/].*|\1|') + repo_path=$(echo "$url" | sed -E 's|git@[^:/]+[:/](.+)$|\1|') + fi + else + # HTTPS URL + host=$(echo "$url" | sed -E 's|https?://([^/]+)/.*|\1|') + repo_path=$(echo "$url" | sed -E 's|https?://[^/]+/(.+)$|\1|') + fi + + echo "https://${host}/api/v1/repos/${repo_path}" +} + +# ─── Onboard Flow ──────────────────────────────────────────────── +# Interactive orchestrator with checkpoint/resume. +# Usage: onboard_flow <target_json> <restart> + +onboard_flow() { + local target_json="$1" + local restart="${2:-false}" + local target_name="$JOSH_SYNC_TARGET_NAME" + + # Load existing onboard state (or empty) + local onboard_state + onboard_state=$(read_onboard_state "$target_name") + local current_step + current_step=$(echo "$onboard_state" | jq -r '.step // "start"') + + if [ "$restart" = true ]; then + log "INFO" "Restarting onboard from scratch" + current_step="start" + onboard_state='{}' + fi + + log "INFO" "Onboard step: ${current_step}" + + # ── Step 1: Prerequisites + archived repo info ── + if [ "$current_step" = "start" ]; then + echo "" >&2 + echo "=== Onboarding ${target_name} ===" >&2 + echo "" >&2 + echo "Before proceeding, you should have:" >&2 + echo " 1. Renamed the existing subrepo (e.g., storefront → storefront-archived)" >&2 + echo " 2. Created a new EMPTY repo at the original URL" >&2 + echo "" >&2 + + # Verify the new (empty) subrepo is reachable (no HEAD ref — works on empty repos) + if git ls-remote "$(subrepo_auth_url)" >/dev/null 2>&1; then + # shellcheck disable=SC2001 # sed is clearer for URL pattern replacement + log "INFO" "New subrepo is reachable at $(echo "$SUBREPO_URL" | sed 's|://[^@]*@|://***@|')" + else + log "WARN" "New subrepo is not reachable — make sure you created the new empty repo" + fi + + echo "Enter the archived repo URL (e.g., git@host:org/repo-archived.git):" >&2 + local archived_url + read -r archived_url + [ -n "$archived_url" ] || die "Archived URL is required" + + # Determine auth type for archived repo (same as current subrepo) + local archived_auth="${SUBREPO_AUTH:-https}" + + # Derive API URL + local archived_api + archived_api=$(_archived_api_from_url "$archived_url") + + # Verify archived repo is reachable via API + if curl -sf -H "Authorization: token ${SUBREPO_TOKEN}" \ + "${archived_api}" >/dev/null 2>&1; then + log "INFO" "Archived repo reachable: ${archived_api}" + else + log "WARN" "Cannot reach archived repo API — check URL and token" + echo "Continue anyway? (y/N):" >&2 + local confirm + read -r confirm + [ "$confirm" = "y" ] || [ "$confirm" = "Y" ] || die "Aborted" + fi + + # Save state + onboard_state=$(jq -n \ + --arg step "importing" \ + --arg archived_api "$archived_api" \ + --arg archived_url "$archived_url" \ + --arg archived_auth "$archived_auth" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{step:$step, archived_api:$archived_api, archived_url:$archived_url, + archived_auth:$archived_auth, import_prs:{}, reset_branches:[], + migrated_prs:[], timestamp:$ts}') + write_onboard_state "$target_name" "$onboard_state" + current_step="importing" + fi + + # ── Step 2: Import (reuses initial_import()) ── + if [ "$current_step" = "importing" ]; then + echo "" >&2 + log "INFO" "Step 2: Importing subrepo content into monorepo..." + + local branches + branches=$(echo "$target_json" | jq -r '.branches | keys[]') + + # Load existing import_prs from state (resume support) + local import_prs + import_prs=$(echo "$onboard_state" | jq -r '.import_prs // {}') + + for branch in $branches; do + local mapped + mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') + [ -z "$mapped" ] && continue + + # Skip branches that already have an import PR recorded + if echo "$import_prs" | jq -e --arg b "$branch" 'has($b)' >/dev/null 2>&1; then + log "INFO" "Import PR already recorded for ${branch} — skipping" + continue + fi + + export SYNC_BRANCH_MONO="$branch" + export SYNC_BRANCH_SUBREPO="$mapped" + + log "INFO" "Importing branch: ${branch} (subrepo: ${mapped})" + local result + result=$(initial_import) + log "INFO" "Import result for ${branch}: ${result}" + + if [ "$result" = "pr-created" ]; then + # Find the import PR number via API + local prs pr_number + prs=$(list_open_prs "$MONOREPO_API" "$GITEA_TOKEN") + pr_number=$(echo "$prs" | jq -r --arg t "$target_name" --arg b "$branch" \ + '[.[] | select(.title | test("\\[Import\\] " + $t + ":")) | select(.base.ref == $b)] | .[0].number // empty') + + if [ -n "$pr_number" ]; then + import_prs=$(echo "$import_prs" | jq --arg b "$branch" --arg n "$pr_number" '. + {($b): ($n | tonumber)}') + log "INFO" "Import PR for ${branch}: #${pr_number}" + else + log "WARN" "Could not find import PR number for ${branch} — check monorepo PRs" + fi + fi + + # Save progress after each branch (resume support) + onboard_state=$(echo "$onboard_state" | jq --argjson prs "$import_prs" '.import_prs = $prs') + write_onboard_state "$target_name" "$onboard_state" + done + + # Update state + onboard_state=$(echo "$onboard_state" | jq \ + --arg step "waiting-for-merge" \ + --argjson prs "$import_prs" \ + '.step = $step | .import_prs = $prs') + write_onboard_state "$target_name" "$onboard_state" + current_step="waiting-for-merge" + fi + + # ── Step 3: Wait for merge ── + if [ "$current_step" = "waiting-for-merge" ]; then + echo "" >&2 + log "INFO" "Step 3: Waiting for import PR(s) to be merged..." + + local import_prs + import_prs=$(echo "$onboard_state" | jq -r '.import_prs') + local pr_count + pr_count=$(echo "$import_prs" | jq 'length') + + if [ "$pr_count" -eq 0 ]; then + log "WARN" "No import PRs recorded — skipping merge check" + else + echo "" >&2 + echo "Import PRs to merge:" >&2 + echo "$import_prs" | jq -r 'to_entries[] | " \(.key): PR #\(.value)"' >&2 + echo "" >&2 + echo "Merge the import PR(s) on the monorepo, then press Enter..." >&2 + read -r + + # Verify each PR is merged + local all_merged=true + for branch in $(echo "$import_prs" | jq -r 'keys[]'); do + local pr_number + pr_number=$(echo "$import_prs" | jq -r --arg b "$branch" '.[$b]') + local pr_json merged + pr_json=$(get_pr "$MONOREPO_API" "$GITEA_TOKEN" "$pr_number") + merged=$(echo "$pr_json" | jq -r '.merged // false') + + if [ "$merged" = "true" ]; then + log "INFO" "PR #${pr_number} (${branch}): merged" + else + log "ERROR" "PR #${pr_number} (${branch}): NOT merged — merge it first" + all_merged=false + fi + done + + if [ "$all_merged" = false ]; then + die "Not all import PRs are merged. Re-run 'josh-sync onboard ${target_name}' after merging." + fi + fi + + # Update state + onboard_state=$(echo "$onboard_state" | jq '.step = "resetting"') + write_onboard_state "$target_name" "$onboard_state" + current_step="resetting" + fi + + # ── Step 4: Reset (pushes josh-filtered history to new repo) ── + if [ "$current_step" = "resetting" ]; then + echo "" >&2 + log "INFO" "Step 4: Pushing josh-filtered history to new subrepo..." + + local branches + branches=$(echo "$target_json" | jq -r '.branches | keys[]') + local already_reset + already_reset=$(echo "$onboard_state" | jq -r '.reset_branches // []') + + for branch in $branches; do + # Skip branches already reset (resume support) + if echo "$already_reset" | jq -e --arg b "$branch" 'index($b) != null' >/dev/null 2>&1; then + log "INFO" "Branch ${branch} already reset — skipping" + continue + fi + + local mapped + mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') + [ -z "$mapped" ] && continue + + export SYNC_BRANCH_MONO="$branch" + export SYNC_BRANCH_SUBREPO="$mapped" + + local result + result=$(subrepo_reset) + log "INFO" "Reset result for ${branch}: ${result}" + + # Track progress + onboard_state=$(echo "$onboard_state" | jq --arg b "$branch" \ + '.reset_branches += [$b]') + write_onboard_state "$target_name" "$onboard_state" + done + + # Update state + onboard_state=$(echo "$onboard_state" | jq \ + --arg step "complete" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '.step = $step | .timestamp = $ts') + write_onboard_state "$target_name" "$onboard_state" + current_step="complete" + fi + + # ── Step 5: Done ── + if [ "$current_step" = "complete" ]; then + echo "" >&2 + echo "=== Onboarding complete! ===" >&2 + echo "" >&2 + echo "The new subrepo now has josh-filtered history." >&2 + echo "Developers should re-clone or reset their local copies:" >&2 + echo " git fetch origin && git reset --hard origin/main" >&2 + echo "" >&2 + echo "To migrate open PRs from the archived repo:" >&2 + echo " josh-sync migrate-pr ${target_name} # interactive picker" >&2 + echo " josh-sync migrate-pr ${target_name} --all # migrate all" >&2 + echo " josh-sync migrate-pr ${target_name} 5 8 12 # specific PRs" >&2 + fi +} + +# ─── Migrate One PR ────────────────────────────────────────────── +# Applies a PR's diff from the archived repo to the new subrepo. +# Usage: migrate_one_pr <pr_number> +# +# Expects: JOSH_SYNC_TARGET_NAME, SUBREPO_API, SUBREPO_TOKEN, BOT_NAME, BOT_EMAIL loaded + +migrate_one_pr() { + local pr_number="$1" + local target_name="$JOSH_SYNC_TARGET_NAME" + + # Read archived repo info from onboard state + local onboard_state archived_api + onboard_state=$(read_onboard_state "$target_name") + archived_api=$(echo "$onboard_state" | jq -r '.archived_api') + if [ -z "$archived_api" ] || [ "$archived_api" = "null" ]; then + die "No archived repo info found. Run 'josh-sync onboard ${target_name}' first." + fi + + # Check if this PR was already migrated + local already_migrated + already_migrated=$(echo "$onboard_state" | jq -r \ + --argjson num "$pr_number" '.migrated_prs // [] | map(select(.old_number == $num)) | length') + if [ "$already_migrated" -gt 0 ]; then + log "INFO" "PR #${pr_number} already migrated — skipping" + return 0 + fi + + # Same credentials — the repo was just renamed + local archived_token="$SUBREPO_TOKEN" + + # 1. Get PR metadata from archived repo + local pr_json title base head body + pr_json=$(get_pr "$archived_api" "$archived_token" "$pr_number") \ + || die "Failed to fetch PR #${pr_number} from archived repo" + title=$(echo "$pr_json" | jq -r '.title') + base=$(echo "$pr_json" | jq -r '.base.ref') + head=$(echo "$pr_json" | jq -r '.head.ref') + body=$(echo "$pr_json" | jq -r '.body // ""') + + log "INFO" "Migrating PR #${pr_number}: \"${title}\" (${base} <- ${head})" + + # 2. Get diff from archived repo + local diff + diff=$(get_pr_diff "$archived_api" "$archived_token" "$pr_number") + if [ -z "$diff" ]; then + log "WARN" "Empty diff for PR #${pr_number} — skipping" + return 1 + fi + + # 3. Clone new subrepo, apply patch + # Save cwd so we can restore it (function runs in caller's shell, not subshell) + local original_dir + original_dir=$(pwd) + + local work_dir + work_dir=$(mktemp -d) + # shellcheck disable=SC2064 # Intentional early expansion + trap "cd '$original_dir' 2>/dev/null; rm -rf '$work_dir'" RETURN + + git clone "$(subrepo_auth_url)" --branch "$base" --single-branch \ + "${work_dir}/subrepo" 2>&1 || die "Failed to clone new subrepo (branch: ${base})" + + cd "${work_dir}/subrepo" || exit + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + + git checkout -B "$head" >&2 + + if echo "$diff" | git apply --check 2>/dev/null; then + echo "$diff" | git apply + git add -A + git commit -m "${title} + +Migrated from archived repo PR #${pr_number}" >&2 + + git push "$(subrepo_auth_url)" "$head" >&2 \ + || die "Failed to push branch ${head}" + + # 4. Create PR on new repo + local new_number + new_number=$(create_pr_number "$SUBREPO_API" "$SUBREPO_TOKEN" \ + "$base" "$head" "$title" "$body") + log "INFO" "Migrated PR #${pr_number} -> #${new_number}: \"${title}\"" + + # 5. Record in onboard state + cd "$original_dir" || true + onboard_state=$(read_onboard_state "$target_name") + onboard_state=$(echo "$onboard_state" | jq \ + --argjson old "$pr_number" \ + --argjson new_num "${new_number}" \ + --arg title "$title" \ + '.migrated_prs += [{"old_number":$old, "new_number":$new_num, "title":$title}]') + write_onboard_state "$target_name" "$onboard_state" + else + log "ERROR" "Patch doesn't apply cleanly for PR #${pr_number} — skipping" + log "ERROR" "Manual migration needed: get diff from archived repo and resolve conflicts" + return 1 + fi +} From 72430714af8c4d725ca35e869fc552b8754b6a4e Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 18:33:53 +0300 Subject: [PATCH 11/37] Update docs for onboard and migrate-pr commands - README: add onboard and migrate-pr to CLI reference - Guide Step 5: add onboard as recommended Option A, move manual import/reset to Option B, document migrate-pr usage - Guide "Adding a New Target": mention onboard as preferred path Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- README.md | 2 ++ docs/guide.md | 70 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3b52a36..6979fee 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ josh-sync sync [--forward|--reverse] [--target NAME[,NAME]] [--branch BRANCH] josh-sync preflight josh-sync import <target> josh-sync reset <target> +josh-sync onboard <target> [--restart] +josh-sync migrate-pr <target> [PR#...] [--all] josh-sync status josh-sync state show <target> [branch] josh-sync state reset <target> [branch] diff --git a/docs/guide.md b/docs/guide.md index 8ba7143..f3c16c2 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -189,11 +189,61 @@ For a new monorepo before import, preflight may warn that subfolders don't exist ## Step 5: Import Existing Subrepos -This is the critical onboarding step. For each existing subrepo, you run a three-step cycle: **import → merge → reset**. +This is the critical onboarding step. There are two approaches: + +- **`josh-sync onboard`** (recommended) — interactive, resumable, preserves open PRs +- **Manual `import` → merge → `reset`** — lower-level, for automation or when there are no open PRs to preserve + +### Option A: Onboard (recommended) + +The `onboard` command walks you through the entire process interactively, with checkpoint/resume at every step. + +**Before you start:** + +1. **Rename** the existing subrepo on your Git server (e.g., `stores/storefront` → `stores/storefront-archived`) +2. **Create a new empty repo** at the original path (e.g., a new `stores/storefront` with no commits) + +The rename preserves the archived repo with all its history and open PRs. The new empty repo will receive josh-filtered history. + +**Run onboard:** + +```bash +josh-sync onboard billing +``` + +The command will: +1. **Verify prerequisites** — checks the new empty repo is reachable, asks for the archived repo URL +2. **Import** — copies subrepo content into monorepo and creates import PRs (one per branch) +3. **Wait for merge** — shows PR numbers and waits for you to merge them +4. **Reset** — pushes josh-filtered history to the new subrepo (per-branch, with resume) +5. **Done** — prints instructions for developers and PR migration + +If the process is interrupted at any point, re-run `josh-sync onboard billing` to resume from where it left off. Use `--restart` to start over. + +**Migrate open PRs:** + +After onboard completes, migrate PRs from the archived repo to the new one: + +```bash +# Interactive — lists open PRs and lets you pick +josh-sync migrate-pr billing + +# Migrate all open PRs at once +josh-sync migrate-pr billing --all + +# Migrate specific PRs by number +josh-sync migrate-pr billing 5 8 12 +``` + +PR migration works by fetching the diff from the archived repo's PR, applying it to the new repo, and creating a new PR. File content is identical after reset, so patches apply cleanly. + +### Option B: Manual import → merge → reset + +Use this when the subrepo has no open PRs to preserve, or for scripted automation. > Do this **one target at a time** to keep PRs reviewable. -### 5a. Import +#### 5b-1. Import ```bash josh-sync import billing @@ -208,13 +258,13 @@ This: Review the import PR — check for leaked credentials, environment-specific config, or files that shouldn't be in the monorepo. -### 5b. Merge the import PR +#### 5b-2. Merge the import PR Merge the PR using your Git platform's UI. This lands the subrepo content into the monorepo's main branch. > At this point, the monorepo has the content but the histories are disconnected. Sync will **not** work until you complete the reset step. -### 5c. Reset +#### 5b-3. Reset ```bash josh-sync reset billing @@ -241,7 +291,7 @@ git checkout stage && git reset --hard origin/stage # repeat for each branch Or simply delete and re-clone the subrepo. Local-only branches (not pushed to the remote) will be lost either way. -### 5d. Repeat for each target +#### 5b-4. Repeat for each target ``` For each target: @@ -250,9 +300,9 @@ For each target: 3. josh-sync reset <target> ``` -### 5e. Verify +### Verify -After all targets are imported and reset: +After all targets are imported and reset (whichever option you used): ```bash # Check all targets show state @@ -444,8 +494,12 @@ To add a new subrepo after initial setup: 1. Add the target to `.josh-sync.yml` 2. Update the forward workflow's `paths:` list to include the new subfolder 3. Commit and push -4. Run the import-merge-reset cycle for the new target: +4. Import the target: ```bash + # Recommended: interactive onboard (preserves open PRs) + josh-sync onboard new-target + + # Or manual: import → merge PR → reset josh-sync import new-target # merge the PR josh-sync reset new-target From 0363b0ee7713ae86c443a34a5202a0ccba3b460f Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 18:38:07 +0300 Subject: [PATCH 12/37] Fix VERSION not included in Nix package and Makefile bundle - flake.nix: copy VERSION file to $out/ so josh_sync_version() finds it - Makefile: add lib/onboard.sh to the bundle loop Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- Makefile | 2 +- flake.nix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 073f855..a748e3f 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ dist/josh-sync: bin/josh-sync lib/*.sh VERSION @echo '# Generated by: make build' >> dist/josh-sync @echo '' >> dist/josh-sync @# Inline all library modules (strip shebangs and source directives) - @for f in lib/core.sh lib/config.sh lib/auth.sh lib/state.sh lib/sync.sh; do \ + @for f in lib/core.sh lib/config.sh lib/auth.sh lib/state.sh lib/sync.sh lib/onboard.sh; do \ echo "# --- $$f ---" >> dist/josh-sync; \ grep -v '^#!/' "$$f" | grep -v '^# shellcheck source=' >> dist/josh-sync; \ echo '' >> dist/josh-sync; \ diff --git a/flake.nix b/flake.nix index 063826e..d6a1de9 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,7 @@ installPhase = '' mkdir -p $out/{bin,lib} + cp VERSION $out/ cp lib/*.sh $out/lib/ cp bin/josh-sync $out/bin/ chmod +x $out/bin/josh-sync From cb14cf9bd47d3b13a2ec07872ee85d98ebb4a9a9 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 18:38:44 +0300 Subject: [PATCH 13/37] Add docs for updating josh-sync version in Nix devenv Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- docs/guide.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/guide.md b/docs/guide.md index f3c16c2..2a7349c 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -165,6 +165,34 @@ SUBREPO_SSH_KEY="-----BEGIN OPENSSH PRIVATE KEY----- # AUTH_REPO_TOKEN=<auth-specific-token> ``` +### Updating josh-sync in devenv + +To update to the latest version: + +```bash +devenv update josh-sync +``` + +Or with plain Nix flakes: + +```bash +nix flake lock --update-input josh-sync +``` + +To pin to a specific version, use a tag ref in `devenv.yaml`: + +```yaml +josh-sync: + url: git+https://your-gitea.example.com/org/josh-sync?ref=v1.1 + flake: true +``` + +After updating, verify the version: + +```bash +josh-sync --version +``` + ### Option B: Manual installation Install the required tools, then either: From 553f0061741eba44592dd9c77df4ba4cd7dddb5f Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 19:48:46 +0300 Subject: [PATCH 14/37] Fix onboard import cloning from empty new repo instead of archived repo initial_import() now accepts an optional clone URL override parameter. onboard_flow() passes the archived repo URL so content is cloned from the right source. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- lib/onboard.sh | 13 ++++++++++++- lib/sync.sh | 8 ++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/onboard.sh b/lib/onboard.sh index 01a229a..69f8734 100644 --- a/lib/onboard.sh +++ b/lib/onboard.sh @@ -169,6 +169,17 @@ onboard_flow() { local import_prs import_prs=$(echo "$onboard_state" | jq -r '.import_prs // {}') + # Build the archived repo clone URL for initial_import(). + # The content lives in the archived repo — the new repo at SUBREPO_URL is empty. + local archived_url archived_clone_url + archived_url=$(echo "$onboard_state" | jq -r '.archived_url') + if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then + archived_clone_url="$archived_url" + else + # shellcheck disable=SC2001 + archived_clone_url=$(echo "$archived_url" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|") + fi + for branch in $branches; do local mapped mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') @@ -185,7 +196,7 @@ onboard_flow() { log "INFO" "Importing branch: ${branch} (subrepo: ${mapped})" local result - result=$(initial_import) + result=$(initial_import "$archived_clone_url") log "INFO" "Import result for ${branch}: ${result}" if [ "$result" = "pr-created" ]; then diff --git a/lib/sync.sh b/lib/sync.sh index 91d2903..ab1b7e8 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -200,9 +200,13 @@ reverse_sync() { # # Used when a subrepo already has content and you're adding it to the # monorepo for the first time. Creates a PR. +# Usage: initial_import [clone_url_override] +# clone_url_override — if set, clone from this URL instead of subrepo_auth_url() +# (used by onboard to clone from the archived repo) # Returns: skip | pr-created initial_import() { + local clone_url="${1:-$(subrepo_auth_url)}" local mono_branch="$SYNC_BRANCH_MONO" local subrepo_branch="$SYNC_BRANCH_SUBREPO" local subfolder @@ -225,8 +229,8 @@ initial_import() { --branch "$mono_branch" --single-branch \ "${work_dir}/monorepo" || die "Failed to clone monorepo" - # 2. Clone subrepo - git clone "$(subrepo_auth_url)" \ + # 2. Clone subrepo (or archived repo when clone_url is overridden) + git clone "$clone_url" \ --branch "$subrepo_branch" --single-branch \ "${work_dir}/subrepo" || die "Failed to clone subrepo" From fbacec7f6fb5273653653101b350ebd46061de12 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 20:51:22 +0300 Subject: [PATCH 15/37] Improve PR migration: fetch branches locally + 3-way merge Instead of fetching the API diff (which has context-sensitive patches that break after josh-filtered reset), fetch the archived repo's branches directly as a second remote and compute the diff locally. Apply with git apply --3way for resilience against context mismatches. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- lib/onboard.sh | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/lib/onboard.sh b/lib/onboard.sh index 69f8734..1cc6916 100644 --- a/lib/onboard.sh +++ b/lib/onboard.sh @@ -336,7 +336,8 @@ onboard_flow() { } # ─── Migrate One PR ────────────────────────────────────────────── -# Applies a PR's diff from the archived repo to the new subrepo. +# Fetches the PR's branch from the archived repo, computes a local diff, +# and applies it to the new subrepo with --3way for resilience. # Usage: migrate_one_pr <pr_number> # # Expects: JOSH_SYNC_TARGET_NAME, SUBREPO_API, SUBREPO_TOKEN, BOT_NAME, BOT_EMAIL loaded @@ -376,15 +377,7 @@ migrate_one_pr() { log "INFO" "Migrating PR #${pr_number}: \"${title}\" (${base} <- ${head})" - # 2. Get diff from archived repo - local diff - diff=$(get_pr_diff "$archived_api" "$archived_token" "$pr_number") - if [ -z "$diff" ]; then - log "WARN" "Empty diff for PR #${pr_number} — skipping" - return 1 - fi - - # 3. Clone new subrepo, apply patch + # 2. Clone new subrepo, add archived repo as second remote # Save cwd so we can restore it (function runs in caller's shell, not subshell) local original_dir original_dir=$(pwd) @@ -401,10 +394,32 @@ migrate_one_pr() { git config user.name "$BOT_NAME" git config user.email "$BOT_EMAIL" + # Build authenticated URL for the archived repo + local archived_url archived_clone_url + archived_url=$(echo "$onboard_state" | jq -r '.archived_url') + if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then + archived_clone_url="$archived_url" + else + # shellcheck disable=SC2001 + archived_clone_url=$(echo "$archived_url" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|") + fi + + # Fetch the PR's head and base branches from the archived repo + git remote add archived "$archived_clone_url" + git fetch archived "$head" "$base" 2>&1 \ + || die "Failed to fetch branches from archived repo" + + # 3. Compute diff locally and apply with --3way git checkout -B "$head" >&2 - if echo "$diff" | git apply --check 2>/dev/null; then - echo "$diff" | git apply + local diff + diff=$(git diff "archived/${base}..archived/${head}") + if [ -z "$diff" ]; then + log "WARN" "Empty diff for PR #${pr_number} — skipping" + return 1 + fi + + if echo "$diff" | git apply --3way 2>&1; then git add -A git commit -m "${title} @@ -429,8 +444,8 @@ Migrated from archived repo PR #${pr_number}" >&2 '.migrated_prs += [{"old_number":$old, "new_number":$new_num, "title":$title}]') write_onboard_state "$target_name" "$onboard_state" else - log "ERROR" "Patch doesn't apply cleanly for PR #${pr_number} — skipping" - log "ERROR" "Manual migration needed: get diff from archived repo and resolve conflicts" + log "ERROR" "Could not apply changes for PR #${pr_number} even with 3-way merge" + log "ERROR" "Manual migration needed: branch '${head}' from archived repo" return 1 fi } From 401d0e87a4a51c7d7e678dc5477339a8a3f8b372 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 21:08:23 +0300 Subject: [PATCH 16/37] Show [migrated] marker and summary in migrate-pr Interactive picker now marks already-migrated PRs. All modes (--all, explicit numbers, interactive) track and display success/fail/skip counts at the end. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- bin/josh-sync | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/bin/josh-sync b/bin/josh-sync index c3a99a1..f3dbd3b 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -735,6 +735,26 @@ cmd_migrate_pr() { log "INFO" "Archived repo: ${archived_api}" + # Load already-migrated PR numbers for skip detection and display + local migrated_numbers + migrated_numbers=$(echo "$onboard_state" | jq -r '[.migrated_prs // [] | .[].old_number] | map(tostring) | .[]') + + # Counters for summary + local migrated=0 failed=0 skipped=0 + + # Helper: attempt migration of one PR with counting + _try_migrate() { + local num="$1" + if echo "$migrated_numbers" | grep -qx "$num"; then + log "INFO" "PR #${num} already migrated — skipping" + skipped=$((skipped + 1)) + elif migrate_one_pr "$num"; then + migrated=$((migrated + 1)) + else + failed=$((failed + 1)) + fi + } + if [ "$all" = true ]; then # Migrate all open PRs from archived repo local prs @@ -744,14 +764,14 @@ cmd_migrate_pr() { count=$(echo "$prs" | jq 'length') log "INFO" "Found ${count} open PR(s) on archived repo" - echo "$prs" | jq -r '.[] | .number' | while read -r num; do - migrate_one_pr "$num" || true - done + while read -r num; do + _try_migrate "$num" + done < <(echo "$prs" | jq -r '.[] | .number') elif [ ${#pr_numbers[@]} -gt 0 ]; then # Migrate specific PR numbers for num in "${pr_numbers[@]}"; do - migrate_one_pr "$num" || true + _try_migrate "$num" done else @@ -767,26 +787,33 @@ cmd_migrate_pr() { return fi + # Display PRs with [migrated] marker for already-processed ones echo "" >&2 echo "Open PRs on archived repo:" >&2 - echo "$prs" | jq -r '.[] | " #\(.number): \(.title) (\(.base.ref) <- \(.head.ref))"' >&2 + while IFS=$'\t' read -r num title base_ref head_ref; do + if echo "$migrated_numbers" | grep -qx "$num"; then + echo " #${num}: ${title} (${base_ref} <- ${head_ref}) [migrated]" >&2 + else + echo " #${num}: ${title} (${base_ref} <- ${head_ref})" >&2 + fi + done < <(echo "$prs" | jq -r '.[] | "\(.number)\t\(.title)\t\(.base.ref)\t\(.head.ref)"') echo "" >&2 echo "Enter PR numbers to migrate (space-separated), or 'all':" >&2 local selection read -r selection if [ "$selection" = "all" ]; then - echo "$prs" | jq -r '.[] | .number' | while read -r num; do - migrate_one_pr "$num" || true - done + while read -r num; do + _try_migrate "$num" + done < <(echo "$prs" | jq -r '.[] | .number') else for num in $selection; do - migrate_one_pr "$num" || true + _try_migrate "$num" done fi fi - log "INFO" "PR migration complete" + log "INFO" "Migration complete: ${migrated} migrated, ${failed} failed, ${skipped} skipped" } # ─── Main ─────────────────────────────────────────────────────────── From 187a9ead14304bbcb91f864e2863124695501931 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Fri, 13 Feb 2026 22:45:13 +0300 Subject: [PATCH 17/37] Add file exclusion via josh stored filters (v1.2.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New `exclude` config field per target generates .josh-filters/<name>.josh files with josh :exclude clauses. Josh-proxy applies exclusions at the transport layer — excluded files never appear in the subrepo. Preflight checks that generated filter files are committed. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- VERSION | 2 +- bin/josh-sync | 22 ++++++++++++++ docs/guide.md | 63 +++++++++++++++++++++++++++++++++++++++ lib/config.sh | 47 ++++++++++++++++++++++++++++- schema/config-schema.json | 6 ++++ 5 files changed, 138 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 9084fa2..26aaba0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.2.0 diff --git a/bin/josh-sync b/bin/josh-sync index f3dbd3b..898defe 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -422,6 +422,28 @@ cmd_preflight() { warn ".gitea/workflows/josh-sync-reverse.yml not found (optional)" fi + # 4. Exclude filter files + local has_excludes + has_excludes=$(echo "$JOSH_SYNC_TARGETS" | jq '[.[] | select((.exclude // []) | length > 0)] | length') + + if [ "$has_excludes" -gt 0 ]; then + echo "" + echo "4. Exclude filter files" + + while IFS= read -r target_name; do + local filter_file=".josh-filters/${target_name}.josh" + if [ -f "$filter_file" ]; then + if git ls-files --error-unmatch "$filter_file" >/dev/null 2>&1; then + pass "${filter_file} exists and is tracked" + else + warn "${filter_file} exists but is NOT committed — run: git add ${filter_file}" + fi + else + fail "${filter_file} not generated — check parse_config" + fi + done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[] | select((.exclude // []) | length > 0) | .name') + fi + # Summary echo "" echo "=============================" diff --git a/docs/guide.md b/docs/guide.md index 2a7349c..a663978 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -91,6 +91,9 @@ targets: branches: main: main # mono_branch: subrepo_branch forward_only: [] + exclude: # files excluded from subrepo (optional) + - ".monorepo/" # monorepo-only config dir + - "**/internal/" # internal dirs at any depth - name: "auth" subfolder: "services/auth" @@ -515,6 +518,66 @@ Bot commits include a git trailer like `Josh-Sync-Origin: forward/main/2024-02-1 Sync state is stored as JSON files on an orphan branch (`josh-sync-state`), one file per target/branch. This tracks the last-synced commit SHAs and timestamps to avoid re-syncing the same changes. +## Excluding Files from Sync + +Some files in the monorepo subfolder may not belong in the subrepo (e.g., monorepo-specific CI configs, internal tooling). The `exclude` config field removes these at the josh-proxy layer — excluded files never appear in the subrepo. + +### Configuration + +Add an `exclude` list to any target: + +```yaml +targets: + - name: "billing" + subfolder: "services/billing" + subrepo_url: "git@host:org/billing.git" + exclude: + - ".monorepo/" # directory at subfolder root + - "**/internal/" # directory at any depth + - "*.secret" # files by extension + branches: + main: main +``` + +### How it works + +When `exclude` is present, josh-sync generates a `.josh-filters/<target>.josh` file containing a [josh stored filter](https://josh-project.github.io/josh/reference/filters.html) with `:exclude` clauses: + +``` +:/services/billing:exclude[ + ::.monorepo/ + ::**/internal/ + ::*.secret +] +``` + +Josh-proxy reads this file from the monorepo and applies the filter at the transport layer. This means: +- **Forward sync**: the filtered clone already excludes the files +- **Reverse sync**: pushes through josh also respect the exclusion +- **Reset**: the subrepo history never contains excluded files +- **Tree comparison**: `skip` detection works correctly (excluded files are not in the diff) + +### Pattern syntax + +Josh uses `::` patterns inside `:exclude[...]`: + +| Pattern | Matches | +|---------|---------| +| `dir/` | Directory at subfolder root | +| `file` | File at subfolder root | +| `**/dir/` | Directory at any depth | +| `**/file` | File at any depth | +| `*.ext` | Glob pattern (single `*` only) | + +### Setup + +1. Add `exclude` to the target in `.josh-sync.yml` +2. Run `josh-sync preflight` — this generates `.josh-filters/<target>.josh` +3. Commit the generated file: `git add .josh-filters/ && git commit` +4. Forward sync will now exclude the specified files + +If you change the `exclude` list, re-run `preflight` and commit the updated `.josh-filters/` file. + ## Adding a New Target To add a new subrepo after initial setup: diff --git a/lib/config.sh b/lib/config.sh index e11214c..2b1c017 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -7,6 +7,45 @@ # # Requires: lib/core.sh sourced first, yq and jq on PATH +# ─── Josh Filter File Generation ────────────────────────────────── +# Generates .josh-filters/<target>.josh for targets with exclude patterns. +# These files must be committed to the monorepo — josh-proxy reads them +# from the repo at clone time via the :+ stored filter syntax. + +_generate_josh_filters() { + local has_excludes + has_excludes=$(echo "$JOSH_SYNC_TARGETS" | jq '[.[] | select((.exclude // []) | length > 0)] | length') + + if [ "$has_excludes" -eq 0 ]; then + return + fi + + mkdir -p .josh-filters + + local target_name subfolder exclude_patterns filter_content + while IFS= read -r target_name; do + subfolder=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \ + '.[] | select(.name == $n) | .subfolder') + exclude_patterns=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \ + '.[] | select(.name == $n) | .exclude | map(" ::" + .) | join("\n")') + + filter_content=":/${subfolder}:exclude[ +${exclude_patterns} +]" + + local filter_file=".josh-filters/${target_name}.josh" + local existing="" + if [ -f "$filter_file" ]; then + existing=$(cat "$filter_file") + fi + + if [ "$filter_content" != "$existing" ]; then + echo "$filter_content" > "$filter_file" + log "WARN" "Generated ${filter_file} — commit this file to the monorepo" + fi + done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[] | select((.exclude // []) | length > 0) | .name') +} + # ─── Phase 1: Parse Config ───────────────────────────────────────── parse_config() { @@ -36,7 +75,10 @@ parse_config() { export JOSH_SYNC_TARGETS JOSH_SYNC_TARGETS=$(echo "$config_json" | jq '[.targets[] | . + # Auto-derive josh_filter from subfolder if not set - (if (.josh_filter // "") == "" then + # When exclude patterns are present, use a stored josh filter (:+.josh-filters/<name>) + (if (.exclude // [] | length) > 0 then + {josh_filter: (":+.josh-filters/" + .name)} + elif (.josh_filter // "") == "" then {josh_filter: (":/" + .subfolder)} else {} end) + # Derive gitea_host and subrepo_repo_path from subrepo_url @@ -56,6 +98,9 @@ parse_config() { ) ]') + # Generate .josh-filters/*.josh for targets with exclude patterns + _generate_josh_filters + # Load .env credentials (if present, not required — CI sets these via secrets) if [ -f .env ]; then # shellcheck source=/dev/null diff --git a/schema/config-schema.json b/schema/config-schema.json index bb222e9..d81ab28 100644 --- a/schema/config-schema.json +++ b/schema/config-schema.json @@ -70,6 +70,12 @@ "items": { "type": "string" }, "default": [], "description": "Branches that only sync mono → subrepo (never reverse)" + }, + "exclude": { + "type": "array", + "items": { "type": "string" }, + "default": [], + "description": "File/directory patterns to exclude from sync via josh :exclude filter. Josh pattern syntax: 'dir/' for directories, '*.ext' for globs, '**/dir/' for nested matches. Generates a .josh-filters/<target>.josh file that must be committed." } } } From 5929585d6c75f326d0ec76723300f341cd342671 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Sat, 14 Feb 2026 09:47:16 +0300 Subject: [PATCH 18/37] Fix josh-proxy rejecting stored filter path with slash Josh-proxy's parser treats "/" in :+ paths as a filter separator, so :+.josh-filters/backend fails. Use flat naming at repo root: .josh-filter-<target>.josh referenced as :+.josh-filter-<target>. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- bin/josh-sync | 2 +- docs/guide.md | 10 ++++++---- lib/config.sh | 14 +++++++------- schema/config-schema.json | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/bin/josh-sync b/bin/josh-sync index 898defe..7cd14c6 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -431,7 +431,7 @@ cmd_preflight() { echo "4. Exclude filter files" while IFS= read -r target_name; do - local filter_file=".josh-filters/${target_name}.josh" + local filter_file=".josh-filter-${target_name}.josh" if [ -f "$filter_file" ]; then if git ls-files --error-unmatch "$filter_file" >/dev/null 2>&1; then pass "${filter_file} exists and is tracked" diff --git a/docs/guide.md b/docs/guide.md index a663978..664327a 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -541,7 +541,7 @@ targets: ### How it works -When `exclude` is present, josh-sync generates a `.josh-filters/<target>.josh` file containing a [josh stored filter](https://josh-project.github.io/josh/reference/filters.html) with `:exclude` clauses: +When `exclude` is present, josh-sync generates a `.josh-filter-<target>.josh` file at the monorepo root containing a [josh stored filter](https://josh-project.github.io/josh/reference/filters.html) with `:exclude` clauses: ``` :/services/billing:exclude[ @@ -551,6 +551,8 @@ When `exclude` is present, josh-sync generates a `.josh-filters/<target>.josh` f ] ``` +The file is referenced in the josh-proxy URL as `:+.josh-filter-<target>` (flat naming — josh-proxy's parser treats `/` in `:+` paths as a filter separator, so subdirectory paths don't work). + Josh-proxy reads this file from the monorepo and applies the filter at the transport layer. This means: - **Forward sync**: the filtered clone already excludes the files - **Reverse sync**: pushes through josh also respect the exclusion @@ -572,11 +574,11 @@ Josh uses `::` patterns inside `:exclude[...]`: ### Setup 1. Add `exclude` to the target in `.josh-sync.yml` -2. Run `josh-sync preflight` — this generates `.josh-filters/<target>.josh` -3. Commit the generated file: `git add .josh-filters/ && git commit` +2. Run `josh-sync preflight` — this generates `.josh-filter-<target>.josh` +3. Commit the generated file: `git add .josh-filter-*.josh && git commit` 4. Forward sync will now exclude the specified files -If you change the `exclude` list, re-run `preflight` and commit the updated `.josh-filters/` file. +If you change the `exclude` list, re-run `preflight` and commit the updated `.josh-filter-*.josh` file. ## Adding a New Target diff --git a/lib/config.sh b/lib/config.sh index 2b1c017..fe22fdc 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -8,9 +8,11 @@ # Requires: lib/core.sh sourced first, yq and jq on PATH # ─── Josh Filter File Generation ────────────────────────────────── -# Generates .josh-filters/<target>.josh for targets with exclude patterns. +# Generates .josh-filter-<target>.josh for targets with exclude patterns. # These files must be committed to the monorepo — josh-proxy reads them # from the repo at clone time via the :+ stored filter syntax. +# Files are at the repo root (flat naming) because josh-proxy's parser +# treats "/" in :+ paths as a filter separator. _generate_josh_filters() { local has_excludes @@ -20,8 +22,6 @@ _generate_josh_filters() { return fi - mkdir -p .josh-filters - local target_name subfolder exclude_patterns filter_content while IFS= read -r target_name; do subfolder=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \ @@ -33,7 +33,7 @@ _generate_josh_filters() { ${exclude_patterns} ]" - local filter_file=".josh-filters/${target_name}.josh" + local filter_file=".josh-filter-${target_name}.josh" local existing="" if [ -f "$filter_file" ]; then existing=$(cat "$filter_file") @@ -75,9 +75,9 @@ parse_config() { export JOSH_SYNC_TARGETS JOSH_SYNC_TARGETS=$(echo "$config_json" | jq '[.targets[] | . + # Auto-derive josh_filter from subfolder if not set - # When exclude patterns are present, use a stored josh filter (:+.josh-filters/<name>) + # When exclude patterns are present, use a stored josh filter (:+.josh-filter-<name>) (if (.exclude // [] | length) > 0 then - {josh_filter: (":+.josh-filters/" + .name)} + {josh_filter: (":+.josh-filter-" + .name)} elif (.josh_filter // "") == "" then {josh_filter: (":/" + .subfolder)} else {} end) + @@ -98,7 +98,7 @@ parse_config() { ) ]') - # Generate .josh-filters/*.josh for targets with exclude patterns + # Generate .josh-filter-*.josh for targets with exclude patterns _generate_josh_filters # Load .env credentials (if present, not required — CI sets these via secrets) diff --git a/schema/config-schema.json b/schema/config-schema.json index d81ab28..c86c19b 100644 --- a/schema/config-schema.json +++ b/schema/config-schema.json @@ -75,7 +75,7 @@ "type": "array", "items": { "type": "string" }, "default": [], - "description": "File/directory patterns to exclude from sync via josh :exclude filter. Josh pattern syntax: 'dir/' for directories, '*.ext' for globs, '**/dir/' for nested matches. Generates a .josh-filters/<target>.josh file that must be committed." + "description": "File/directory patterns to exclude from sync via josh :exclude filter. Josh pattern syntax: 'dir/' for directories, '*.ext' for globs, '**/dir/' for nested matches. Generates a .josh-filter-<target>.josh file that must be committed." } } } From d7f8618b3852a5ed24b8c862d01791ba7daeb030 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Sat, 14 Feb 2026 10:19:41 +0300 Subject: [PATCH 19/37] Use inline :exclude in josh-proxy URL instead of stored filter files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The :+ stored filter syntax doesn't work in josh-proxy URLs. Inline :exclude[::p1,::p2] works directly — no files to generate or commit, no extra dependencies. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- bin/josh-sync | 22 ------------------- docs/guide.md | 19 +++++----------- lib/config.sh | 46 ++------------------------------------- schema/config-schema.json | 2 +- 4 files changed, 9 insertions(+), 80 deletions(-) diff --git a/bin/josh-sync b/bin/josh-sync index 7cd14c6..f3dbd3b 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -422,28 +422,6 @@ cmd_preflight() { warn ".gitea/workflows/josh-sync-reverse.yml not found (optional)" fi - # 4. Exclude filter files - local has_excludes - has_excludes=$(echo "$JOSH_SYNC_TARGETS" | jq '[.[] | select((.exclude // []) | length > 0)] | length') - - if [ "$has_excludes" -gt 0 ]; then - echo "" - echo "4. Exclude filter files" - - while IFS= read -r target_name; do - local filter_file=".josh-filter-${target_name}.josh" - if [ -f "$filter_file" ]; then - if git ls-files --error-unmatch "$filter_file" >/dev/null 2>&1; then - pass "${filter_file} exists and is tracked" - else - warn "${filter_file} exists but is NOT committed — run: git add ${filter_file}" - fi - else - fail "${filter_file} not generated — check parse_config" - fi - done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[] | select((.exclude // []) | length > 0) | .name') - fi - # Summary echo "" echo "=============================" diff --git a/docs/guide.md b/docs/guide.md index 664327a..b3f0435 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -541,19 +541,13 @@ targets: ### How it works -When `exclude` is present, josh-sync generates a `.josh-filter-<target>.josh` file at the monorepo root containing a [josh stored filter](https://josh-project.github.io/josh/reference/filters.html) with `:exclude` clauses: +When `exclude` is present, josh-sync appends an inline `:exclude` filter to the josh-proxy URL. For the example above, the josh filter becomes: ``` -:/services/billing:exclude[ - ::.monorepo/ - ::**/internal/ - ::*.secret -] +:/services/billing:exclude[::.monorepo/,::**/internal/,::*.secret] ``` -The file is referenced in the josh-proxy URL as `:+.josh-filter-<target>` (flat naming — josh-proxy's parser treats `/` in `:+` paths as a filter separator, so subdirectory paths don't work). - -Josh-proxy reads this file from the monorepo and applies the filter at the transport layer. This means: +Josh-proxy applies this filter at the transport layer — no extra files to generate or commit. This means: - **Forward sync**: the filtered clone already excludes the files - **Reverse sync**: pushes through josh also respect the exclusion - **Reset**: the subrepo history never contains excluded files @@ -574,11 +568,10 @@ Josh uses `::` patterns inside `:exclude[...]`: ### Setup 1. Add `exclude` to the target in `.josh-sync.yml` -2. Run `josh-sync preflight` — this generates `.josh-filter-<target>.josh` -3. Commit the generated file: `git add .josh-filter-*.josh && git commit` -4. Forward sync will now exclude the specified files +2. Run `josh-sync preflight` to verify the filter works +3. Forward sync will now exclude the specified files -If you change the `exclude` list, re-run `preflight` and commit the updated `.josh-filter-*.josh` file. +No extra files to generate or commit — the exclusion is embedded directly in the josh-proxy URL. ## Adding a New Target diff --git a/lib/config.sh b/lib/config.sh index fe22fdc..bdf8b95 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -7,45 +7,6 @@ # # Requires: lib/core.sh sourced first, yq and jq on PATH -# ─── Josh Filter File Generation ────────────────────────────────── -# Generates .josh-filter-<target>.josh for targets with exclude patterns. -# These files must be committed to the monorepo — josh-proxy reads them -# from the repo at clone time via the :+ stored filter syntax. -# Files are at the repo root (flat naming) because josh-proxy's parser -# treats "/" in :+ paths as a filter separator. - -_generate_josh_filters() { - local has_excludes - has_excludes=$(echo "$JOSH_SYNC_TARGETS" | jq '[.[] | select((.exclude // []) | length > 0)] | length') - - if [ "$has_excludes" -eq 0 ]; then - return - fi - - local target_name subfolder exclude_patterns filter_content - while IFS= read -r target_name; do - subfolder=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \ - '.[] | select(.name == $n) | .subfolder') - exclude_patterns=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \ - '.[] | select(.name == $n) | .exclude | map(" ::" + .) | join("\n")') - - filter_content=":/${subfolder}:exclude[ -${exclude_patterns} -]" - - local filter_file=".josh-filter-${target_name}.josh" - local existing="" - if [ -f "$filter_file" ]; then - existing=$(cat "$filter_file") - fi - - if [ "$filter_content" != "$existing" ]; then - echo "$filter_content" > "$filter_file" - log "WARN" "Generated ${filter_file} — commit this file to the monorepo" - fi - done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[] | select((.exclude // []) | length > 0) | .name') -} - # ─── Phase 1: Parse Config ───────────────────────────────────────── parse_config() { @@ -75,9 +36,9 @@ parse_config() { export JOSH_SYNC_TARGETS JOSH_SYNC_TARGETS=$(echo "$config_json" | jq '[.targets[] | . + # Auto-derive josh_filter from subfolder if not set - # When exclude patterns are present, use a stored josh filter (:+.josh-filter-<name>) + # When exclude patterns are present, append inline :exclude[::p1,::p2,...] to the filter (if (.exclude // [] | length) > 0 then - {josh_filter: (":+.josh-filter-" + .name)} + {josh_filter: (":/" + .subfolder + ":exclude[" + (.exclude | map("::" + .) | join(",")) + "]")} elif (.josh_filter // "") == "" then {josh_filter: (":/" + .subfolder)} else {} end) + @@ -98,9 +59,6 @@ parse_config() { ) ]') - # Generate .josh-filter-*.josh for targets with exclude patterns - _generate_josh_filters - # Load .env credentials (if present, not required — CI sets these via secrets) if [ -f .env ]; then # shellcheck source=/dev/null diff --git a/schema/config-schema.json b/schema/config-schema.json index c86c19b..b3c9675 100644 --- a/schema/config-schema.json +++ b/schema/config-schema.json @@ -75,7 +75,7 @@ "type": "array", "items": { "type": "string" }, "default": [], - "description": "File/directory patterns to exclude from sync via josh :exclude filter. Josh pattern syntax: 'dir/' for directories, '*.ext' for globs, '**/dir/' for nested matches. Generates a .josh-filter-<target>.josh file that must be committed." + "description": "File/directory patterns to exclude from sync via josh :exclude filter. Josh pattern syntax: 'dir/' for directories, '*.ext' for globs, '**/dir/' for nested matches. Patterns are embedded inline in the josh-proxy URL." } } } From 22bd59a9d759e85e28cb839d80b5e2ebb54ae4f7 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Sat, 14 Feb 2026 10:40:08 +0300 Subject: [PATCH 20/37] Auto-reconcile subrepo history when josh filter changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the exclude list changes, josh-proxy recomputes filtered history with new SHAs, breaking common ancestry with the subrepo. Instead of requiring a manual reset (force-push), forward sync now detects the filter change and creates a reconciliation merge commit that connects the old and new histories — no force-push, no re-clone needed. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- bin/josh-sync | 17 +++++++++-- docs/guide.md | 4 +++ lib/sync.sh | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/bin/josh-sync b/bin/josh-sync index f3dbd3b..460821a 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -208,10 +208,20 @@ _sync_direction() { fi fi - # Run sync + # Check for filter change (forward only — reverse uses same filter) local result if [ "$direction" = "forward" ]; then - result=$(forward_sync) + local prev_filter + prev_filter=$(echo "$state" | jq -r '.last_forward.josh_filter // empty') + + if [ -n "$prev_filter" ] && [ "$prev_filter" != "$JOSH_FILTER" ]; then + log "WARN" "Josh filter changed — reconciling histories" + log "INFO" "Old: ${prev_filter}" + log "INFO" "New: ${JOSH_FILTER}" + result=$(reconcile_filter_change) + else + result=$(forward_sync) + fi else result=$(reverse_sync) fi @@ -240,8 +250,9 @@ _sync_direction() { --arg s_sha "${subrepo_sha_now:-}" \ --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --arg status "$result" \ + --arg filter "$JOSH_FILTER" \ --argjson prev "$state" \ - '$prev + {last_forward: {mono_sha:$m_sha, subrepo_sha:$s_sha, timestamp:$ts, status:$status}}') + '$prev + {last_forward: {mono_sha:$m_sha, subrepo_sha:$s_sha, timestamp:$ts, status:$status, josh_filter:$filter}}') else local mono_sha_now mono_sha_now=$(git rev-parse "origin/${branch}" 2>/dev/null || echo "") diff --git a/docs/guide.md b/docs/guide.md index b3f0435..24e1754 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -573,6 +573,10 @@ Josh uses `::` patterns inside `:exclude[...]`: No extra files to generate or commit — the exclusion is embedded directly in the josh-proxy URL. +### Changing the exclude list + +You can safely add or remove patterns from `exclude` at any time. When josh-sync detects that the filter has changed since the last sync, it automatically creates a reconciliation merge commit on the subrepo that connects the old and new histories — no manual reset or force-push required. Developers do not need to re-clone the subrepo. + ## Adding a New Target To add a new subrepo after initial setup: diff --git a/lib/sync.sh b/lib/sync.sh index ab1b7e8..48b2136 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -128,6 +128,85 @@ ${BOT_TRAILER}: forward/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 fi } +# ─── Filter Change Reconciliation ───────────────────────────────── +# When the josh filter changes (e.g., exclude patterns added/removed), +# josh-proxy recomputes filtered history with new SHAs. This creates a +# merge commit on the subrepo that connects old and new histories, +# re-establishing shared ancestry without a destructive force-push. +# Returns: reconciled | lease-rejected + +reconcile_filter_change() { + local mono_branch="$SYNC_BRANCH_MONO" + local subrepo_branch="$SYNC_BRANCH_SUBREPO" + local work_dir + work_dir=$(mktemp -d) + # shellcheck disable=SC2064 # Intentional early expansion — work_dir is local + trap "rm -rf '$work_dir'" EXIT + + log "INFO" "=== Filter change reconciliation: ${mono_branch} ===" + + # 1. Clone subrepo + git clone "$(subrepo_auth_url)" \ + --branch "$subrepo_branch" --single-branch \ + "${work_dir}/subrepo" || die "Failed to clone subrepo" + + cd "${work_dir}/subrepo" || exit + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + + local subrepo_head + subrepo_head=$(git rev-parse HEAD) + log "INFO" "Subrepo HEAD: ${subrepo_head:0:12}" + + # 2. Fetch josh-proxy filtered view (new filter) + git remote add josh-filtered "$(josh_auth_url)" + git fetch josh-filtered "$mono_branch" || die "Failed to fetch from josh-proxy" + + local josh_head josh_tree + josh_head=$(git rev-parse "josh-filtered/${mono_branch}") + # shellcheck disable=SC1083 # {tree} is git syntax, not shell brace expansion + josh_tree=$(git rev-parse "josh-filtered/${mono_branch}^{tree}") + log "INFO" "Josh-proxy HEAD (new filter): ${josh_head:0:12}" + + # 3. Check if trees are already identical (filter change had no effect) + local subrepo_tree + # shellcheck disable=SC1083 + subrepo_tree=$(git rev-parse "HEAD^{tree}") + if [ "$josh_tree" = "$subrepo_tree" ]; then + log "INFO" "Trees identical after filter change — no reconciliation needed" + echo "skip" + return + fi + + # 4. Create merge commit: subrepo HEAD + josh-proxy HEAD, with josh-proxy's tree + local merge_commit + merge_commit=$(git commit-tree "$josh_tree" \ + -p "$subrepo_head" \ + -p "$josh_head" \ + -m "Sync: filter configuration updated + +${BOT_TRAILER}: filter-change/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)") + + git reset --hard "$merge_commit" >&2 + log "INFO" "Created reconciliation merge: ${merge_commit:0:12}" + + # 5. Record lease and push + local subrepo_sha + subrepo_sha=$(subrepo_ls_remote "$subrepo_branch") + + if git push \ + --force-with-lease="refs/heads/${subrepo_branch}:${subrepo_sha}" \ + "$(subrepo_auth_url)" \ + "HEAD:refs/heads/${subrepo_branch}"; then + + log "INFO" "Filter change reconciled — shared ancestry re-established" + echo "reconciled" + else + log "WARN" "Force-with-lease rejected — subrepo changed during reconciliation" + echo "lease-rejected" + fi +} + # ─── Reverse Sync: Subrepo → Monorepo ────────────────────────────── # # Always creates a PR on the monorepo — never pushes directly. From c0ddb887ffb88bf566d03d35d3059cf29e4336ba Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Sat, 14 Feb 2026 13:30:24 +0300 Subject: [PATCH 21/37] Fix filter reconciliation for pre-v1.2 state and unrelated histories Three bugs found during first CI run after enabling :exclude: - Derive old filter (:/subfolder) when state has no josh_filter stored (pre-v1.2 upgrade path) - Detect unrelated histories in forward_sync() and fall back to reconcile_filter_change() instead of creating a useless conflict PR - Skip state update on conflict result (prevents storing wrong filter and mono SHA that blocks retries) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- bin/josh-sync | 20 ++++++++++++++++++++ lib/sync.sh | 11 +++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/bin/josh-sync b/bin/josh-sync index 460821a..cc5414b 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -214,6 +214,18 @@ _sync_direction() { local prev_filter prev_filter=$(echo "$state" | jq -r '.last_forward.josh_filter // empty') + # If no filter stored (pre-v1.2 state) but a previous sync exists, + # the old filter was the simple :/subfolder (before exclude was added) + if [ -z "$prev_filter" ]; then + local prev_mono_sha + prev_mono_sha=$(echo "$state" | jq -r '.last_forward.mono_sha // empty') + if [ -n "$prev_mono_sha" ]; then + local subfolder + subfolder=$(echo "$TARGET_JSON" | jq -r '.subfolder') + prev_filter=":/${subfolder}" + fi + fi + if [ -n "$prev_filter" ] && [ "$prev_filter" != "$JOSH_FILTER" ]; then log "WARN" "Josh filter changed — reconciling histories" log "INFO" "Old: ${prev_filter}" @@ -225,6 +237,13 @@ _sync_direction() { else result=$(reverse_sync) fi + # If forward sync hit unrelated histories, fall back to reconciliation + if [ "$result" = "unrelated" ]; then + log "WARN" "Unrelated histories detected — falling back to filter reconciliation" + result=$(reconcile_filter_change) + log "INFO" "Reconciliation result: ${result}" + fi + log "INFO" "Result: ${result}" # Handle warnings @@ -234,6 +253,7 @@ _sync_direction() { fi if [ "$result" = "conflict" ]; then echo "::warning::Target ${target_name}, branch ${branch}: merge conflict — PR created on subrepo" + continue fi if [ "$result" = "josh-rejected" ]; then echo "::error::Target ${target_name}, branch ${branch}: josh rejected push — check proxy logs" diff --git a/lib/sync.sh b/lib/sync.sh index 48b2136..b2ff277 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -11,7 +11,7 @@ # ─── Forward Sync: Monorepo → Subrepo ────────────────────────────── # -# Returns: fresh | skip | clean | lease-rejected | conflict +# Returns: fresh | skip | clean | lease-rejected | conflict | unrelated forward_sync() { local mono_branch="$SYNC_BRANCH_MONO" @@ -97,7 +97,14 @@ ${BOT_TRAILER}: forward/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 fi else - # Conflict! + # Check: unrelated histories (filter change) vs normal merge conflict + if ! git merge-base "subrepo/${subrepo_branch}" "$mono_head" >/dev/null 2>&1; then + log "INFO" "No common ancestor — histories are unrelated (filter change?)" + echo "unrelated" + return + fi + + # Normal merge conflict local conflicted conflicted=$(git diff --name-only --diff-filter=U 2>/dev/null || echo "(unknown)") git merge --abort From 16257f25d7fc5054807eaf8c395040ec7941f170 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Sat, 14 Feb 2026 14:19:56 +0300 Subject: [PATCH 22/37] Fix reverse sync false positive after filter reconciliation Add --ancestry-path to git log in reverse_sync() to prevent old subrepo history from leaking through reconciliation merge parents. Without this, every old subrepo commit appears as a "human commit" triggering a spurious 0-commit PR on the monorepo. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- lib/sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sync.sh b/lib/sync.sh index b2ff277..d24d454 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -244,7 +244,7 @@ reverse_sync() { # 3. Find new human commits (excludes bot commits from forward sync) local human_commits - human_commits=$(git log "mono-filtered/${mono_branch}..HEAD" \ + human_commits=$(git log --ancestry-path "mono-filtered/${mono_branch}..HEAD" \ --oneline --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") if [ -z "$human_commits" ]; then From ce53d3c1d2632bd96d4e808285d63cc83dde9d7e Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Sat, 14 Feb 2026 15:11:31 +0300 Subject: [PATCH 23/37] Fix reconciliation parent order and add reverse sync tree check - Swap parent order in reconcile_filter_change(): josh-filtered must be first parent so josh can follow first-parent traversal to map history back to the monorepo. Old subrepo history on parent 2. - Add tree comparison in reverse_sync() before commit detection: if subrepo tree matches josh-filtered tree, skip immediately. Prevents false positive PRs after reconciliation. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- lib/sync.sh | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/sync.sh b/lib/sync.sh index d24d454..03afe6b 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -185,11 +185,13 @@ reconcile_filter_change() { return fi - # 4. Create merge commit: subrepo HEAD + josh-proxy HEAD, with josh-proxy's tree + # 4. Create merge commit: josh-proxy HEAD (first parent) + subrepo HEAD, with josh-proxy's tree + # Josh follows first-parent traversal — josh-filtered MUST be first so josh can map + # the history back to the monorepo. Old subrepo history hangs off parent 2. local merge_commit merge_commit=$(git commit-tree "$josh_tree" \ - -p "$subrepo_head" \ -p "$josh_head" \ + -p "$subrepo_head" \ -m "Sync: filter configuration updated ${BOT_TRAILER}: filter-change/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)") @@ -242,7 +244,20 @@ reverse_sync() { git remote add mono-filtered "$(josh_auth_url)" git fetch mono-filtered "$mono_branch" || die "Failed to fetch from josh-proxy" - # 3. Find new human commits (excludes bot commits from forward sync) + # 3. Compare trees — skip if subrepo matches josh-filtered view + local subrepo_tree josh_tree + # shellcheck disable=SC1083 # {tree} is git syntax, not shell brace expansion + subrepo_tree=$(git rev-parse "HEAD^{tree}") + # shellcheck disable=SC1083 + josh_tree=$(git rev-parse "mono-filtered/${mono_branch}^{tree}") + + if [ "$subrepo_tree" = "$josh_tree" ]; then + log "INFO" "Subrepo tree matches josh-filtered view — nothing to sync" + echo "skip" + return + fi + + # 4. Find new human commits (excludes bot commits from forward sync) local human_commits human_commits=$(git log --ancestry-path "mono-filtered/${mono_branch}..HEAD" \ --oneline --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") From 95b83bd53877100f0c8d51d3d7271db664d3bfcd Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Sat, 14 Feb 2026 16:13:13 +0300 Subject: [PATCH 24/37] Fix PR body newlines rendering as literal \n Bash double-quoted strings don't interpret \n as newlines. Use actual newlines in the pr_body strings instead. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- lib/sync.sh | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/sync.sh b/lib/sync.sh index 03afe6b..c46bb5f 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -122,7 +122,14 @@ ${BOT_TRAILER}: forward/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)" >&2 local pr_body conflicted_list # shellcheck disable=SC2001 conflicted_list=$(echo "$conflicted" | sed 's/^/- /') - pr_body="## Sync Conflict\n\nMonorepo \`${mono_branch}\` has changes that conflict with \`${subrepo_branch}\`.\n\n**Conflicted files:**\n${conflicted_list}\n\nPlease resolve and merge this PR to complete the sync." + pr_body="## Sync Conflict + +Monorepo \`${mono_branch}\` has changes that conflict with \`${subrepo_branch}\`. + +**Conflicted files:** +${conflicted_list} + +Please resolve and merge this PR to complete the sync." create_pr "${SUBREPO_API}" "${SUBREPO_TOKEN}" \ "$subrepo_branch" "$conflict_branch" \ @@ -281,7 +288,18 @@ reverse_sync() { # 5. Create PR on monorepo (NEVER direct push) local pr_body - pr_body="## Subrepo changes\n\nNew commits from subrepo \`${subrepo_branch}\`:\n\n\`\`\`\n${human_commits}\n\`\`\`\n\n**Review checklist:**\n- [ ] Changes scoped to synced subfolder\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" + pr_body="## Subrepo changes + +New commits from subrepo \`${subrepo_branch}\`: + +\`\`\` +${human_commits} +\`\`\` + +**Review checklist:** +- [ ] Changes scoped to synced subfolder +- [ ] No leaked credentials or environment-specific config +- [ ] CI passes" create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ "$mono_branch" "$staging_branch" \ @@ -369,7 +387,14 @@ ${BOT_TRAILER}: import/${JOSH_SYNC_TARGET_NAME}/${ts}" >&2 # 5. Create PR on monorepo local pr_body - pr_body="## Initial import\n\nImporting existing subrepo \`${subrepo_branch}\` (${file_count} files) into \`${subfolder}/\`.\n\n**Review checklist:**\n- [ ] Content looks correct\n- [ ] No leaked credentials or environment-specific config\n- [ ] CI passes" + pr_body="## Initial import + +Importing existing subrepo \`${subrepo_branch}\` (${file_count} files) into \`${subfolder}/\`. + +**Review checklist:** +- [ ] Content looks correct +- [ ] No leaked credentials or environment-specific config +- [ ] CI passes" create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ "$mono_branch" "$staging_branch" \ From 8ab07b83ab9e74111f4f25fbdaf3227e5faf46c9 Mon Sep 17 00:00:00 2001 From: Slim B <slim.b.pro@gmail.com> Date: Sat, 14 Feb 2026 21:28:40 +0300 Subject: [PATCH 25/37] Update docs, changelog, examples, and add ADRs for v1.2 - Add v1.1.0 and v1.2.0 changelog entries - Add exclude field to config reference and example config - Add ADRs documenting all major design decisions - Fix step numbering in reverse_sync() - Fix action.yml to copy VERSION file - Add dist/ and .env to .gitignore - Use refs/tags/ format for Nix flake tag refs Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --- .gitignore | 3 +- CHANGELOG.md | 27 ++++++++ README.md | 14 ++-- action.yml | 1 + docs/adr/001-josh-proxy-for-sync.md | 42 ++++++++++++ docs/adr/002-state-on-orphan-branch.md | 50 ++++++++++++++ docs/adr/003-force-with-lease-forward.md | 33 +++++++++ docs/adr/004-always-pr-reverse.md | 41 +++++++++++ docs/adr/005-git-trailer-loop-prevention.md | 52 ++++++++++++++ docs/adr/006-inline-exclude-filter.md | 55 +++++++++++++++ docs/adr/007-reconciliation-merge.md | 53 ++++++++++++++ docs/adr/008-first-parent-ordering.md | 42 ++++++++++++ docs/adr/009-tree-comparison-guard.md | 53 ++++++++++++++ docs/adr/010-onboard-checkpoint-resume.md | 76 +++++++++++++++++++++ docs/adr/README.md | 18 +++++ docs/config-reference.md | 1 + docs/guide.md | 2 +- examples/devenv.nix | 22 +++--- examples/forward.yml | 2 +- examples/josh-sync.yml | 7 +- examples/reverse.yml | 4 +- lib/sync.sh | 6 +- 22 files changed, 580 insertions(+), 24 deletions(-) create mode 100644 docs/adr/001-josh-proxy-for-sync.md create mode 100644 docs/adr/002-state-on-orphan-branch.md create mode 100644 docs/adr/003-force-with-lease-forward.md create mode 100644 docs/adr/004-always-pr-reverse.md create mode 100644 docs/adr/005-git-trailer-loop-prevention.md create mode 100644 docs/adr/006-inline-exclude-filter.md create mode 100644 docs/adr/007-reconciliation-merge.md create mode 100644 docs/adr/008-first-parent-ordering.md create mode 100644 docs/adr/009-tree-comparison-guard.md create mode 100644 docs/adr/010-onboard-checkpoint-resume.md create mode 100644 docs/adr/README.md diff --git a/.gitignore b/.gitignore index 58c99d2..ac93a19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .claude/*local* - +dist/ +.env result diff --git a/CHANGELOG.md b/CHANGELOG.md index 8af14fe..dd3631e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## 1.2.0 + +### Features + +- **File exclusion**: `exclude` config field removes files/directories from the subrepo at the josh-proxy transport layer. Patterns are embedded inline in the josh-proxy URL using `:exclude[::pattern,...]` syntax — no extra files to generate or commit. +- **Filter change reconciliation**: When the josh filter changes (e.g., adding/removing exclude patterns), josh-sync automatically creates a reconciliation merge commit that connects old and new histories. No manual reset or force-push required. +- **Tree comparison guard**: Reverse sync now compares subrepo tree to josh-filtered tree before checking commit log. Skips immediately when trees are identical, avoiding false positives from reconciliation merge history. +- **Unrelated histories detection**: Forward sync detects when histories are unrelated (no common ancestor) and falls back to reconciliation instead of creating a useless conflict PR. + +### Fixes + +- Pre-v1.2 state compatibility: When upgrading from v1.0/v1.1 (no `josh_filter` stored in state), the old filter is derived from `subfolder` so reconciliation triggers correctly. +- Reconciliation merge parent order: Josh-filtered history is always first parent so josh-proxy can follow first-parent traversal back to the monorepo. +- Reverse sync `--ancestry-path` flag prevents old subrepo history from leaking through reconciliation merge parents. +- PR body `\n` now renders as actual newlines instead of literal text. +- Conflict result no longer updates sync state (added `continue` to skip state write). +- `action.yml` now copies VERSION file for correct `--version` output in CI. +- `.gitignore` now includes `dist/` and `.env`. + +## 1.1.0 + +### Features + +- **`onboard` command**: Interactive, resumable workflow for importing existing subrepos into the monorepo. Walks through: prerequisites check, import (creates PRs), wait for merge, reset (pushes josh-filtered history). Checkpoint/resume at every step. +- **`migrate-pr` command**: Migrates open PRs from an archived subrepo to the new one. Supports interactive selection, `--all` flag, and specific PR numbers. Uses `git apply --3way` for resilient patch application. +- **Onboard state tracking**: Stored on the `josh-sync-state` branch at `<target>/onboard.json`. Tracks step progress, import PR numbers, reset branches, and migrated PRs. + ## 1.0.0 Initial release. Extracted from [private-monorepo-example](https://code.itkan.io/pe/private-monorepo-example) into a standalone reusable library. diff --git a/README.md b/README.md index 6979fee..3b39733 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,12 @@ josh: targets: - name: "billing" subfolder: "services/billing" - josh_filter: ":/services/billing" subrepo_url: "git@gitea.example.com:ext/billing.git" subrepo_auth: "ssh" branches: main: main - forward_only: [] + exclude: # files excluded from subrepo (optional) + - ".monorepo/" bot: name: "josh-sync-bot" @@ -58,8 +58,10 @@ Run `josh-sync preflight` to validate your setup. ## Documentation -- **[Setup Guide](docs/guide.md)** — Step-by-step: prerequisites, importing existing subrepos, CI workflows, and troubleshooting +- **[Setup Guide](docs/guide.md)** — Step-by-step: prerequisites, importing existing subrepos, CI workflows, file exclusion, and troubleshooting - **[Configuration Reference](docs/config-reference.md)** — Full `.josh-sync.yml` field documentation +- **[Architecture Decision Records](docs/adr/)** — Design rationale and trade-offs +- **[Changelog](CHANGELOG.md)** — Version history ## CLI @@ -79,12 +81,16 @@ josh-sync state reset <target> [branch] - **Forward sync** (mono → subrepo): pushes directly if clean, creates conflict PR if not. Uses `--force-with-lease` for safety. - **Reverse sync** (subrepo → mono): always creates a PR, never pushes directly. +- **File exclusion**: `exclude` patterns are embedded inline in the josh-proxy URL. Excluded files exist only in the monorepo. +- **Filter reconciliation**: Changing the exclude list auto-creates a merge commit that connects old and new histories — no force-push needed. - **Loop prevention**: `Josh-Sync-Origin:` git trailer filters out bot commits. - **State tracking**: orphan branch `josh-sync-state` stores JSON per target/branch. ## Dependencies -`bash >=4`, `git`, `curl`, `jq`, `yq` ([mikefarah/yq](https://github.com/mikefarah/yq) v4+), `openssh` +`bash >=4`, `git`, `curl`, `jq`, `yq` ([mikefarah/yq](https://github.com/mikefarah/yq) v4+), `openssh`, `rsync` + +> The Nix flake bundles all dependencies automatically. ## License diff --git a/action.yml b/action.yml index 629cf21..30d7bb2 100644 --- a/action.yml +++ b/action.yml @@ -26,6 +26,7 @@ runs: run: | JOSH_DIR="$(mktemp -d)" cp -r "${{ github.action_path }}/bin" "${{ github.action_path }}/lib" "${JOSH_DIR}/" + cp "${{ github.action_path }}/VERSION" "${JOSH_DIR}/" 2>/dev/null || true chmod +x "${JOSH_DIR}/bin/josh-sync" echo "${JOSH_DIR}/bin" >> "$GITHUB_PATH" echo "JOSH_SYNC_ROOT=${JOSH_DIR}" >> "$GITHUB_ENV" diff --git a/docs/adr/001-josh-proxy-for-sync.md b/docs/adr/001-josh-proxy-for-sync.md new file mode 100644 index 0000000..df7cd26 --- /dev/null +++ b/docs/adr/001-josh-proxy-for-sync.md @@ -0,0 +1,42 @@ +# ADR-001: Josh-proxy for Bidirectional Sync + +**Status:** Accepted +**Date:** 2026-01 + +## Context + +We need bidirectional sync between a monorepo and N external subrepos. Each subrepo corresponds to a subfolder in the monorepo. Developers on both sides should see a clean, complete git history — not synthetic commits or squashed blobs. + +### Alternatives considered + +1. **git subtree**: Built into git. `git subtree split` extracts a subfolder into a standalone repo. However, subtree split rewrites history on every run (O(n) on total commits), creating new SHAs each time. Bidirectional sync requires manual `subtree merge` with conflict-prone history grafting. No transport-layer filtering — all content must be fetched. + +2. **git submodule**: Tracks external repos via `.gitmodules` pointer commits. Does not provide content-level integration — monorepo commits don't contain subrepo files directly. Developers must run `git submodule update`. Bidirectional sync is not a supported workflow. + +3. **Custom diff-and-patch scripts**: Compute diffs between monorepo subfolder and subrepo, apply patches in both directions. Fragile with renames, binary files, and merge conflicts. Loses authorship and commit granularity. + +4. **josh-proxy**: A git proxy that computes filtered views of repositories in real-time. Clients `git clone` through josh and receive a repo containing only the specified subfolder, with history rewritten to match. Josh maintains a persistent SHA mapping, so the same monorepo commit always produces the same filtered SHA. Bidirectional: pushing back through josh maps filtered commits to monorepo commits. + +## Decision + +Use josh-proxy as the transport layer for all sync operations. + +## Consequences + +**Positive:** +- Clean git history in both directions — no synthetic commits +- Deterministic SHA mapping — same monorepo state always produces same filtered SHA +- Bidirectional by design — push through josh maps back to monorepo +- Transport-layer filtering — content exclusion happens at clone/push time, not via generated files +- Supports any git hosting platform (Gitea, GitHub, GitLab) since it's a proxy + +**Negative:** +- Requires running a josh-proxy instance (operational overhead) +- Josh-proxy is a Rust project with a smaller community than git-native tools +- Proxy must have network access to the monorepo's git server +- Josh's SHA mapping is opaque — debugging requires understanding josh internals +- First-parent traversal behavior must be respected in merge commits (see ADR-008) + +**Risks:** +- Josh-proxy downtime blocks all sync operations +- Josh-proxy bugs could corrupt history mapping (mitigated by force-with-lease on forward, always-PR on reverse) diff --git a/docs/adr/002-state-on-orphan-branch.md b/docs/adr/002-state-on-orphan-branch.md new file mode 100644 index 0000000..925ca05 --- /dev/null +++ b/docs/adr/002-state-on-orphan-branch.md @@ -0,0 +1,50 @@ +# ADR-002: State Storage on Orphan Git Branch + +**Status:** Accepted +**Date:** 2026-01 + +## Context + +Josh-sync needs persistent state to track what has already been synced (last-synced commit SHAs, timestamps, status). This prevents re-syncing unchanged content and enables incremental operation. The state must survive CI runner teardown — runners are ephemeral containers. + +### Alternatives considered + +1. **File in the repo**: Commit a state JSON file to the monorepo. Every sync run creates a commit, polluting history. Race conditions when multiple sync jobs run concurrently. + +2. **External database/KV store**: Redis, SQLite, or a cloud KV service. Adds an infrastructure dependency. Credentials and connectivity to manage. + +3. **CI artifacts/cache**: Platform-specific (GitHub Actions cache, Gitea cache). Not portable across CI platforms. Expiry policies vary. + +4. **Orphan git branch**: A branch with no parent relationship to the main history. Stores JSON files in a simple `<target>/<branch>.json` layout. Pushed to origin, so it survives runner teardown. No external dependencies — uses git itself. + +## Decision + +Store sync state as JSON files on an orphan branch (`josh-sync-state`) in the monorepo. + +### Storage layout + +``` +origin/josh-sync-state/ + <target>/<branch>.json # sync state per target/branch + <target>/onboard.json # onboard workflow state (v1.1+) +``` + +### Implementation + +- `read_state()`: `git fetch origin josh-sync-state && git show origin/josh-sync-state:<key>.json` +- `write_state()`: Uses `git worktree` to check out the orphan branch in a temp directory, writes JSON, commits, and pushes. This avoids touching the main working tree. + +## Consequences + +**Positive:** +- Zero external dependencies — only git +- Portable across CI platforms (Gitea Actions, GitHub Actions, local) +- Human-readable JSON files — easy to inspect and debug +- Atomic updates via git commit + push +- Natural namespacing via directory structure + +**Negative:** +- Concurrent writes can race (mitigated by concurrency groups in CI workflows) +- `git worktree` adds complexity to the write path +- State branch appears in `git branch -a` output (minor clutter) +- Push failures on the state branch are non-fatal (logged as warning, sync still succeeds) diff --git a/docs/adr/003-force-with-lease-forward.md b/docs/adr/003-force-with-lease-forward.md new file mode 100644 index 0000000..d0701c1 --- /dev/null +++ b/docs/adr/003-force-with-lease-forward.md @@ -0,0 +1,33 @@ +# ADR-003: Force-with-Lease for Forward Sync + +**Status:** Accepted +**Date:** 2026-01 + +## Context + +Forward sync pushes monorepo changes to the subrepo. If someone pushes directly to the subrepo between when josh-sync reads its HEAD and when josh-sync pushes, a naive `git push` would overwrite their work. A `git push --force` would be worse — it would silently destroy concurrent changes. + +## Decision + +Use `git push --force-with-lease=refs/heads/<branch>:<expected-sha>` for all forward sync pushes. The expected SHA is recorded at the start of the sync operation (the "lease"). + +### How it works + +1. Record subrepo HEAD SHA before any operations: `subrepo_sha=$(subrepo_ls_remote "$branch")` +2. Perform merge of monorepo changes onto subrepo state +3. Push with explicit lease: `--force-with-lease=refs/heads/main:<subrepo_sha>` +4. If the subrepo HEAD changed since step 1, git rejects the push +5. Josh-sync reports `lease-rejected` and retries on the next run + +## Consequences + +**Positive:** +- Never overwrites concurrent changes — git atomically checks the expected SHA +- Explicit SHA lease (not just "current tracking ref") prevents stale-ref bugs +- Failed leases are retried on the next sync run — no data loss, just delay +- Works correctly with josh-proxy's SHA mapping + +**Negative:** +- Lease-rejected means the sync run did work that gets discarded (clone, merge, etc.) +- Persistent lease failures indicate a concurrent push pattern that needs investigation +- Requires the `--force-with-lease` flag with explicit SHA — the shorthand form (`--force-with-lease` without `=`) is unsafe because it uses the local tracking ref, which may be stale diff --git a/docs/adr/004-always-pr-reverse.md b/docs/adr/004-always-pr-reverse.md new file mode 100644 index 0000000..2adb659 --- /dev/null +++ b/docs/adr/004-always-pr-reverse.md @@ -0,0 +1,41 @@ +# ADR-004: Always-PR Policy for Reverse Sync + +**Status:** Accepted +**Date:** 2026-01 + +## Context + +Reverse sync brings subrepo changes back into the monorepo. The monorepo is the source of truth and typically has CI checks, code review requirements, and branch protection rules. Pushing directly to the monorepo's main branch would bypass these safeguards. + +### Alternatives considered + +1. **Direct push**: Fast, but bypasses all review and CI. A bad subrepo commit could break the entire monorepo with no review gate. + +2. **Always create a PR**: Pushes to a staging branch (`auto-sync/subrepo-<branch>-<timestamp>`), then creates a PR via API. Humans review and merge. + +3. **Configurable per-target**: Let users choose direct push vs PR. Adds complexity and a dangerous default. + +## Decision + +Reverse sync always creates a PR on the monorepo. Never pushes directly to the target branch. + +### Implementation + +1. Push subrepo HEAD through josh-proxy to a staging branch: `git push -o "base=main" josh://... HEAD:refs/heads/auto-sync/subrepo-main-<ts>` +2. Create PR via Gitea/GitHub API targeting the monorepo's main branch +3. PR includes a review checklist: scoped to subfolder, no leaked credentials, CI passes + +The `-o "base=main"` option tells josh-proxy which monorepo branch to map the push against. + +## Consequences + +**Positive:** +- All monorepo changes go through review — consistent with team workflow +- CI runs on the PR branch before merge +- Bad subrepo changes are caught before they affect the monorepo +- Audit trail via PR history + +**Negative:** +- Reverse sync is not instant — requires human action to merge the PR +- Stale PRs accumulate if subrepo changes frequently but PRs aren't merged promptly +- Adds API dependency (needs token with PR creation scope) diff --git a/docs/adr/005-git-trailer-loop-prevention.md b/docs/adr/005-git-trailer-loop-prevention.md new file mode 100644 index 0000000..c4b79e7 --- /dev/null +++ b/docs/adr/005-git-trailer-loop-prevention.md @@ -0,0 +1,52 @@ +# ADR-005: Git Trailer for Loop Prevention + +**Status:** Accepted +**Date:** 2026-01 + +## Context + +Bidirectional sync creates an infinite loop risk: forward sync pushes commit A to the subrepo, reverse sync sees commit A as "new" and creates a PR back to the monorepo, forward sync sees the merged PR as "new" and pushes again, etc. + +### Alternatives considered + +1. **SHA tracking only**: Compare SHAs to skip already-synced content. Breaks when josh-proxy rewrites SHAs (which it always does for filtered views). The monorepo commit SHA and the filtered/subrepo commit SHA are never the same. + +2. **Commit message prefix**: Add `[sync]` to bot commit messages. Fragile — humans might use the same prefix. Requires string matching on message content. + +3. **Git trailer**: A structured key-value pair in the commit message body (after a blank line), following the `git interpret-trailers` convention. Format: `Key: value`. Machine-parseable, unlikely to be used by humans, and supported by `git log --grep`. + +## Decision + +All bot commits include a git trailer with a configurable key (default: `Josh-Sync-Origin`). Both sync directions filter out commits containing this trailer. + +### Format + +``` +Sync from monorepo 2026-02-12T10:30:00Z + +Josh-Sync-Origin: forward/main/2026-02-12T10:30:00Z +``` + +The trailer value encodes: direction, branch, and timestamp. This aids debugging but is not parsed by the loop filter — only the trailer key presence matters. + +### Filtering + +- **Reverse sync**: `git log --invert-grep --grep="^${BOT_TRAILER}:"` excludes all commits with the trailer +- **CI loop guard**: The composite action checks if HEAD commit has the trailer before running sync at all + +### Configuration + +The trailer key is set in `.josh-sync.yml` under `bot.trailer`. This allows multiple josh-sync instances (with different bots) to operate on the same repos without interfering. + +## Consequences + +**Positive:** +- Reliable loop prevention — trailer is part of the immutable commit object +- Configurable key avoids conflicts between multiple sync bots +- Human-readable — `git log` shows the trailer in commit messages +- CI loop guard prevents unnecessary sync runs entirely + +**Negative:** +- Commits with manually-added trailers matching the key would be incorrectly filtered +- Trailer must be in the commit body (after blank line), not the subject line +- Squash-and-merge on PRs may lose the trailer if the platform doesn't preserve commit message body diff --git a/docs/adr/006-inline-exclude-filter.md b/docs/adr/006-inline-exclude-filter.md new file mode 100644 index 0000000..6940787 --- /dev/null +++ b/docs/adr/006-inline-exclude-filter.md @@ -0,0 +1,55 @@ +# ADR-006: Inline Exclude in Josh-Proxy URL + +**Status:** Accepted +**Date:** 2026-02 + +## Context + +Some files in a monorepo subfolder should not appear in the subrepo (e.g., monorepo-specific CI configs, internal tooling, secrets templates). We need a mechanism to exclude these files from sync. + +### Alternatives considered + +1. **`.josh-sync-exclude` file committed to the repo**: A gitignore-style file listing patterns. Requires generating and committing a file. Changes to the exclude list create commits. The file itself would need to be excluded from the subrepo (circular dependency). + +2. **Post-clone file deletion**: Clone through josh, then `rm -rf` excluded paths before pushing. Fragile — deletions create diff noise. Doesn't work for reverse sync (excluded files would appear as "deleted" in the subrepo). + +3. **Josh `:exclude` filter inline in the URL**: Josh-proxy supports `:exclude[::pattern1,::pattern2]` appended to the filter path. The exclusion happens at the transport layer — git objects for excluded files are never transferred. Works identically for clone (forward) and push (reverse). + +4. **Separate josh filter file**: Generate a josh filter expression and store it somewhere. Adds state management complexity. + +## Decision + +Embed exclusion patterns inline in the josh-proxy URL using josh's native `:exclude` syntax. The `exclude` config field in `.josh-sync.yml` is transformed at config parse time into the josh filter string. + +### Example + +Config: +```yaml +exclude: + - ".monorepo/" + - "**/internal/" +``` + +Produces josh filter: +``` +:/services/billing:exclude[::.monorepo/,::**/internal/] +``` + +### Implementation + +The `parse_config()` function in `lib/config.sh` uses jq to conditionally append `:exclude[...]` to the josh filter when the `exclude` array is non-empty. The enriched filter is stored in `JOSH_SYNC_TARGETS` JSON and used everywhere via `$JOSH_FILTER`. + +## Consequences + +**Positive:** +- Zero committed files — exclusion is purely in the URL +- Transport-layer filtering — excluded content never leaves the git server +- Works identically for forward sync (clone), reverse sync (push), and reset +- Tree comparison (`skip` detection) works correctly since excluded files aren't in the filtered view +- Standard josh syntax — no custom invention + +**Negative:** +- Josh's `:exclude` pattern syntax is limited (no negation, no regex — only glob-style patterns with `::` prefix) +- Long exclude lists make the URL unwieldy (though this is cosmetic — git handles long URLs fine) +- Changing the exclude list changes the josh filter, which changes all filtered SHAs (see ADR-007 for how this is handled) +- Debugging requires understanding josh's filter composition syntax diff --git a/docs/adr/007-reconciliation-merge.md b/docs/adr/007-reconciliation-merge.md new file mode 100644 index 0000000..6b17aaf --- /dev/null +++ b/docs/adr/007-reconciliation-merge.md @@ -0,0 +1,53 @@ +# ADR-007: Reconciliation Merge for Filter Changes + +**Status:** Accepted +**Date:** 2026-02 + +## Context + +When the josh filter changes (e.g., adding exclude patterns), josh-proxy recomputes the entire filtered history with new SHAs. The subrepo's existing history (based on the old filter) shares no common ancestor with the new filtered history. A naive forward sync would see "unrelated histories" and fail. + +### Alternatives considered + +1. **Force-push to subrepo**: Replace subrepo history with the new filtered view (same as `josh-sync reset`). Destructive — all local clones become invalid, open PRs are orphaned, developers must re-clone. + +2. **Cherry-pick new commits**: Identify commits that exist in the new filtered history but not the old, cherry-pick them onto the subrepo. Complex — the "same" commit has different SHAs in old vs new filtered history. No reliable way to match them. + +3. **Reconciliation merge commit**: Create a merge commit on the subrepo that has both the new filtered HEAD and the old subrepo HEAD as parents, using the new filtered tree. This establishes shared ancestry without rewriting history. + +## Decision + +When josh-sync detects a filter change (stored filter in state differs from current `$JOSH_FILTER`), create a reconciliation merge commit using `git commit-tree`. + +### How it works + +1. Clone subrepo (has old history) +2. Fetch josh-proxy filtered view (has new history) +3. If trees are identical → skip (filter change had no effect on content) +4. Create merge commit: `git commit-tree <josh-tree> -p <josh-head> -p <subrepo-head>` +5. Push with `--force-with-lease` + +The merge commit uses the josh-filtered tree (new content) and has two parents: +- **Parent 1**: josh-filtered HEAD (new filter history) — must be first (see ADR-008) +- **Parent 2**: subrepo HEAD (old filter history) — preserves old history as a side branch + +### Detection + +Filter change is detected by comparing the stored `josh_filter` in sync state with the current `$JOSH_FILTER`. For pre-v1.2 state (no filter stored), the old filter is derived as `:/<subfolder>`. + +As a reactive fallback, `forward_sync()` also detects unrelated histories via `git merge-base` and falls back to reconciliation. + +## Consequences + +**Positive:** +- Non-destructive — old history is preserved as parent 2 of the merge +- Developers don't need to re-clone the subrepo +- Open PRs on the subrepo remain valid (they're based on commits that are still ancestors) +- Automatic — no manual intervention needed when changing exclude patterns +- Force-with-lease protects against concurrent changes during reconciliation + +**Negative:** +- The merge commit is synthetic (created by bot, not a real merge of concurrent work) +- Parent ordering is critical — wrong order breaks josh's reverse mapping (see ADR-008) +- The reconciliation merge contains a bot trailer, so reverse sync correctly ignores it +- If the subrepo has diverged significantly (manual commits during filter change), the reconciliation merge may produce unexpected tree content (uses josh-filtered tree unconditionally) diff --git a/docs/adr/008-first-parent-ordering.md b/docs/adr/008-first-parent-ordering.md new file mode 100644 index 0000000..1b54155 --- /dev/null +++ b/docs/adr/008-first-parent-ordering.md @@ -0,0 +1,42 @@ +# ADR-008: First-Parent Ordering in Reconciliation Merges + +**Status:** Accepted +**Date:** 2026-02 + +## Context + +Josh-proxy uses **first-parent traversal** when mapping subrepo history back to the monorepo. When you push a commit through josh-proxy, josh walks the first-parent chain to find a commit it can map to a monorepo commit. If the first parent leads to unmappable history, josh cannot reconstruct the monorepo-side branch correctly. + +This became critical when the reconciliation merge (ADR-007) initially had the wrong parent order: old subrepo history as parent 1, josh-filtered as parent 2. Josh followed parent 1, couldn't find any mappable commit, and created a monorepo branch containing only the subrepo subfolder content — effectively deleting 1280 files from the rest of the monorepo. + +## Decision + +In reconciliation merge commits, the josh-filtered HEAD **must be parent 1** (first parent). The old subrepo HEAD is parent 2. + +```bash +git commit-tree "$josh_tree" \ + -p "$josh_head" \ # parent 1: josh-filtered — josh follows this + -p "$subrepo_head" \ # parent 2: old history — side branch, ignored by josh + -m "..." +``` + +### Why this is safe + +- The old subrepo HEAD (`subrepo_head`) is still an ancestor of the merge commit regardless of parent order — push succeeds either way +- `--ancestry-path` in reverse sync still follows `B → M → C` regardless of parent order (it traces all paths, not just first-parent) +- Josh follows first-parent and finds the josh-filtered commit, which maps cleanly back to the monorepo + +## Consequences + +**Positive:** +- Josh can map the reconciliation merge back to the monorepo correctly +- Reverse sync through josh produces correct diffs (only subrepo-scoped changes) +- `git log --first-parent` on the subrepo shows the clean josh-filtered lineage + +**Negative:** +- This is a subtle invariant — future changes to merge commit creation must preserve parent order +- The constraint is undocumented in josh-proxy's own documentation (discovered empirically) +- No automated test can verify this without a running josh-proxy instance + +**Lesson learned:** +Parent order in `git commit-tree -p` is not cosmetic. For tools that rely on first-parent traversal (josh-proxy, `git log --first-parent`), parent 1 must be the "mainline" that the tool should follow. diff --git a/docs/adr/009-tree-comparison-guard.md b/docs/adr/009-tree-comparison-guard.md new file mode 100644 index 0000000..3e6c667 --- /dev/null +++ b/docs/adr/009-tree-comparison-guard.md @@ -0,0 +1,53 @@ +# ADR-009: Tree Comparison as Sync Skip Guard + +**Status:** Accepted +**Date:** 2026-02 + +## Context + +Both forward and reverse sync need to detect "nothing to do" quickly. The primary mechanism is SHA comparison against stored state (last-synced SHA). However, this misses cases where: + +- State is reset or lost +- Reconciliation merges change SHAs without changing content +- Multiple sync runs overlap + +Additionally, reverse sync originally relied on `git log <base>..HEAD` to find new commits. After a reconciliation merge, the `..` range can leak old subrepo history through the merge's second parent, creating false positives. + +## Decision + +Add tree-level comparison as an early skip guard in both forward and reverse sync. Compare the git tree objects (which represent directory content, not commit history) to determine if there's actually any content difference. + +### Forward sync + +```bash +mono_tree=$(git rev-parse 'HEAD^{tree}') +subrepo_tree=$(git rev-parse "subrepo/${branch}^{tree}") +[ "$mono_tree" = "$subrepo_tree" ] && echo "skip" +``` + +### Reverse sync + +```bash +subrepo_tree=$(git rev-parse "HEAD^{tree}") +josh_tree=$(git rev-parse "mono-filtered/${branch}^{tree}") +[ "$subrepo_tree" = "$josh_tree" ] && echo "skip" +``` + +Tree comparison happens **before** commit log analysis. If trees are identical, there is definitionally nothing to sync, regardless of what the commit history looks like. + +### Combined with `--ancestry-path` + +For reverse sync, even when trees differ, `git log --ancestry-path` restricts the commit range to the direct lineage between the two endpoints. This prevents old history from leaking through reconciliation merge parents. + +## Consequences + +**Positive:** +- Eliminates false positives from reconciliation merges (trees are identical after reconciliation) +- Fast — tree SHA comparison is O(1), no content traversal +- Correct by definition — if trees match, content is identical +- Defense in depth — works even when state tracking has gaps + +**Negative:** +- Tree comparison alone doesn't tell you *which* commits are new (still need `git log` for PR descriptions) +- Adds an extra `git rev-parse` call per sync direction (negligible cost) +- Cannot detect file-mode-only changes if josh normalizes modes (theoretical edge case) diff --git a/docs/adr/010-onboard-checkpoint-resume.md b/docs/adr/010-onboard-checkpoint-resume.md new file mode 100644 index 0000000..f6246e2 --- /dev/null +++ b/docs/adr/010-onboard-checkpoint-resume.md @@ -0,0 +1,76 @@ +# ADR-010: Onboard Workflow with Checkpoint/Resume + +**Status:** Accepted +**Date:** 2026-02 + +## Context + +Onboarding an existing subrepo into the monorepo is a multi-step process that involves human interaction (renaming repos, merging PRs). The full flow is: + +1. Prerequisites: rename existing repo, create new empty repo +2. Import: copy subrepo content into monorepo, create import PR(s) +3. Wait: human merges the import PR(s) +4. Reset: force-push josh-filtered history to the new empty repo +5. (Optional) Migrate open PRs from archived repo + +Each step can fail or be interrupted. The process may span hours or days (waiting for PR review). If interrupted, restarting from scratch wastes work and can create duplicate PRs. + +### Alternatives considered + +1. **Single-shot script**: Run all steps in sequence. If interrupted, must restart from scratch. Duplicate PRs if import step is re-run. + +2. **Manual step-by-step commands**: `import`, then manually run `reset`. Simple but error-prone — users may forget steps or run them out of order. + +3. **Checkpoint/resume with persistent state**: Track the current step and intermediate results (PR numbers, reset branches) in persistent state. On re-run, resume from the last completed step. + +## Decision + +Implement `josh-sync onboard` as a checkpoint/resume workflow with state stored on the `josh-sync-state` branch at `<target>/onboard.json`. + +### State machine + +``` +start → importing → waiting-for-merge → resetting → complete +``` + +Each transition is persisted before proceeding. Re-running `josh-sync onboard <target>` reads the current step and resumes. + +### State schema + +```json +{ + "step": "waiting-for-merge", + "archived_api": "https://host/api/v1/repos/org/repo-archived", + "archived_url": "git@host:org/repo-archived.git", + "archived_auth": "ssh", + "import_prs": { "main": 42 }, + "reset_branches": ["main"], + "migrated_prs": [ + { "old_number": 5, "new_number": 12, "title": "Fix login" } + ], + "timestamp": "2026-02-10T14:30:00Z" +} +``` + +### Per-branch progress + +Import and reset both iterate over branches. Progress is saved after each branch, so interruption mid-iteration resumes at the next unprocessed branch. + +### PR migration + +`josh-sync migrate-pr` is a separate command that reads onboard state (for the archived repo URL) and tracks migrated PRs. It uses `git apply --3way` for resilient patch application — the subrepo's content is identical after reset, so patches apply cleanly. + +## Consequences + +**Positive:** +- Safe to interrupt at any point — no duplicate work on resume +- Per-branch tracking prevents duplicate import PRs or redundant resets +- Archived repo URL stored in state — `migrate-pr` can operate independently +- `--restart` flag allows starting over if state is corrupted +- Human-friendly — prints instructions at each step + +**Negative:** +- State management adds complexity (read/write onboard state, step validation) +- Interactive steps (`read -r`) are not suitable for fully automated pipelines +- Onboard state persists on the state branch even after completion (minor clutter) +- The step machine is linear — cannot skip steps or run them out of order diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..42e2773 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,18 @@ +# Architecture Decision Records + +This directory contains Architecture Decision Records (ADRs) for josh-sync. Each ADR documents a significant design decision, its context, the alternatives considered, and the rationale for the chosen approach. + +## Index + +| ADR | Title | Status | +|-----|-------|--------| +| [001](001-josh-proxy-for-sync.md) | Josh-proxy for bidirectional sync | Accepted | +| [002](002-state-on-orphan-branch.md) | State storage on orphan git branch | Accepted | +| [003](003-force-with-lease-forward.md) | Force-with-lease for forward sync | Accepted | +| [004](004-always-pr-reverse.md) | Always-PR policy for reverse sync | Accepted | +| [005](005-git-trailer-loop-prevention.md) | Git trailer for loop prevention | Accepted | +| [006](006-inline-exclude-filter.md) | Inline exclude in josh-proxy URL | Accepted | +| [007](007-reconciliation-merge.md) | Reconciliation merge for filter changes | Accepted | +| [008](008-first-parent-ordering.md) | First-parent ordering in reconciliation merges | Accepted | +| [009](009-tree-comparison-guard.md) | Tree comparison as sync skip guard | Accepted | +| [010](010-onboard-checkpoint-resume.md) | Onboard workflow with checkpoint/resume | Accepted | diff --git a/docs/config-reference.md b/docs/config-reference.md index 01b3af8..d13745f 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -32,6 +32,7 @@ Each target maps a monorepo subfolder to an external subrepo. | `subrepo_ssh_key_var` | string | No | `"SUBREPO_SSH_KEY"` | Name of the env var holding the SSH private key for this target. | | `branches` | object | Yes | — | Branch mapping: `mono_branch: subrepo_branch`. Each key-value pair syncs those branches bidirectionally. | | `forward_only` | string[] | No | `[]` | Branches that only sync mono → subrepo, never reverse. | +| `exclude` | string[] | No | `[]` | File/directory patterns to exclude from sync via josh `:exclude` filter. Excluded files exist only in the monorepo, never in the subrepo. See [Excluding Files](guide.md#excluding-files-from-sync). | ## `bot` Section diff --git a/docs/guide.md b/docs/guide.md index 24e1754..bb1cc3e 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -186,7 +186,7 @@ To pin to a specific version, use a tag ref in `devenv.yaml`: ```yaml josh-sync: - url: git+https://your-gitea.example.com/org/josh-sync?ref=v1.1 + url: git+https://your-gitea.example.com/org/josh-sync?ref=refs/tags/v1.2 flake: true ``` diff --git a/examples/devenv.nix b/examples/devenv.nix index e7b0c17..29f6623 100644 --- a/examples/devenv.nix +++ b/examples/devenv.nix @@ -5,12 +5,12 @@ # In devenv.yaml: # inputs: # josh-sync: -# url: github:org/josh-sync/v1.0.0 +# url: git+https://your-gitea.example.com/org/josh-sync?ref=refs/tags/v1.2 # flake: true # # Or in flake.nix: # inputs.josh-sync = { -# url = "github:org/josh-sync/v1.0.0"; +# url = "git+https://your-gitea.example.com/org/josh-sync?ref=refs/tags/v1.2"; # inputs.nixpkgs.follows = "nixpkgs"; # }; @@ -21,14 +21,16 @@ # josh-sync CLI is now available in the shell. # Commands: - # josh-sync sync --forward Forward sync (mono → subrepo) - # josh-sync sync --reverse Reverse sync (subrepo → mono) - # josh-sync preflight Validate config and connectivity - # josh-sync import <target> Initial import from subrepo - # josh-sync reset <target> Reset subrepo to josh-filtered view - # josh-sync status Show target config and sync state - # josh-sync state show <t> [b] Show state JSON - # josh-sync state reset <t> [b] Reset state + # josh-sync sync --forward Forward sync (mono → subrepo) + # josh-sync sync --reverse Reverse sync (subrepo → mono) + # josh-sync preflight Validate config and connectivity + # josh-sync import <target> Initial import from subrepo + # josh-sync reset <target> Reset subrepo to josh-filtered view + # josh-sync onboard <target> Interactive import + reset workflow + # josh-sync migrate-pr <target> Migrate PRs from archived repo + # josh-sync status Show target config and sync state + # josh-sync state show <t> [b] Show state JSON + # josh-sync state reset <t> [b] Reset state enterShell = '' echo "Josh Sync available — run 'josh-sync --help' for commands" diff --git a/examples/forward.yml b/examples/forward.yml index 97d499f..9732a3a 100644 --- a/examples/forward.yml +++ b/examples/forward.yml @@ -62,7 +62,7 @@ jobs: done | sort -u | paste -sd ',' -) echo "targets=${TARGETS}" >> "$GITHUB_OUTPUT" - - uses: https://your-gitea.example.com/org/josh-sync@v1 + - uses: https://your-gitea.example.com/org/josh-sync@v1.2 with: direction: forward target: ${{ github.event.inputs.target || steps.detect.outputs.targets }} diff --git a/examples/josh-sync.yml b/examples/josh-sync.yml index 5db4223..09bb455 100644 --- a/examples/josh-sync.yml +++ b/examples/josh-sync.yml @@ -10,17 +10,19 @@ josh: targets: - name: "billing" subfolder: "services/billing" - josh_filter: ":/services/billing" + # josh_filter auto-derived as ":/services/billing" if omitted subrepo_url: "https://gitea.example.com/ext/billing.git" subrepo_auth: "https" branches: main: main develop: develop forward_only: [] + exclude: # files excluded from subrepo (optional) + - ".monorepo/" # directory at subfolder root + - "**/internal/" # directory at any depth - name: "auth" subfolder: "services/auth" - josh_filter: ":/services/auth" subrepo_url: "git@gitea.example.com:ext/auth.git" subrepo_auth: "ssh" # Per-target credential override (reads from $AUTH_SSH_KEY instead of $SUBREPO_SSH_KEY) @@ -31,7 +33,6 @@ targets: - name: "shared-lib" subfolder: "libs/shared" - josh_filter: ":/libs/shared" subrepo_url: "https://gitea.example.com/ext/shared-lib.git" branches: main: main diff --git a/examples/reverse.yml b/examples/reverse.yml index def838a..820fdc1 100644 --- a/examples/reverse.yml +++ b/examples/reverse.yml @@ -10,7 +10,7 @@ name: "Josh Sync ← Subrepo" on: schedule: - - cron: "0 1,7,13,19 * * *" # Every 6h, offset from forward + - cron: "0 1,7,13,19 * * *" # Every 6h, offset from forward workflow_dispatch: inputs: target: @@ -40,7 +40,7 @@ jobs: curl -sL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" \ -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq - - uses: https://your-gitea.example.com/org/josh-sync@v1 + - uses: https://your-gitea.example.com/org/josh-sync@v1.2 with: direction: reverse target: ${{ github.event.inputs.target || '' }} diff --git a/lib/sync.sh b/lib/sync.sh index c46bb5f..22b9888 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -265,6 +265,8 @@ reverse_sync() { fi # 4. Find new human commits (excludes bot commits from forward sync) + # Uses --ancestry-path to restrict to the direct lineage and avoid + # leaking old history through reconciliation merge parents. local human_commits human_commits=$(git log --ancestry-path "mono-filtered/${mono_branch}..HEAD" \ --oneline --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") @@ -278,7 +280,7 @@ reverse_sync() { log "INFO" "New human commits to sync:" echo "$human_commits" >&2 - # 4. Push through josh to a staging branch + # 5. Push through josh to a staging branch local ts ts=$(date +%Y%m%d-%H%M%S) local staging_branch="auto-sync/subrepo-${subrepo_branch}-${ts}" @@ -286,7 +288,7 @@ reverse_sync() { if git push -o "base=${mono_branch}" "$(josh_auth_url)" "HEAD:refs/heads/${staging_branch}"; then log "INFO" "Pushed to staging branch via josh: ${staging_branch}" - # 5. Create PR on monorepo (NEVER direct push) + # 6. Create PR on monorepo (NEVER direct push) local pr_body pr_body="## Subrepo changes From d6f334b861ca30d13001bc4a6b325706b475d81b Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Tue, 17 Mar 2026 12:48:17 +0300 Subject: [PATCH 26/37] Add linearize fallback for reverse sync and workflow guide (v1.3) When josh-proxy rejects a reverse sync push due to unmappable merge commits, fall back to linearizing: cherry-pick regular commits individually, squash only the merge commits via cherry-pick -m 1. Also adds a recommended Git workflow section to the guide explaining where cross-branch merges should happen (monorepo) vs feature work (subrepo), and expands troubleshooting for the "josh rejected push" error with root cause analysis and prevention advice. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- CHANGELOG.md | 11 +++ VERSION | 2 +- docs/adr/011-linearize-fallback-reverse.md | 68 +++++++++++++ docs/adr/README.md | 1 + docs/guide.md | 84 +++++++++++++++- lib/sync.sh | 106 +++++++++++++++++---- 6 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 docs/adr/011-linearize-fallback-reverse.md diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3631e..98d27db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 1.3.0 + +### Features + +- **Linearize fallback for reverse sync**: When josh-proxy rejects a push due to unmappable merge commits, reverse sync automatically falls back to linearizing the history — cherry-picking regular commits individually and squashing only the problematic merge commits via `cherry-pick -m 1`. Preserves individual commit granularity while handling complex merge topologies that josh cannot map. PR body notes when linearization was used. + +### Docs + +- Expanded troubleshooting guide for "Josh rejected push" with root cause analysis and prevention advice. +- Added ADR-011: Linearize fallback for reverse sync. + ## 1.2.0 ### Features diff --git a/VERSION b/VERSION index 26aaba0..f0bb29e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.3.0 diff --git a/docs/adr/011-linearize-fallback-reverse.md b/docs/adr/011-linearize-fallback-reverse.md new file mode 100644 index 0000000..80e3261 --- /dev/null +++ b/docs/adr/011-linearize-fallback-reverse.md @@ -0,0 +1,68 @@ +# ADR-011: Linearize Fallback for Reverse Sync + +**Status:** Accepted +**Date:** 2026-03 + +## Context + +Josh-proxy rejects reverse sync pushes when the subrepo history contains merge commits whose parents it cannot map through the filter. This happens when: + +1. A long-lived branch (e.g., `stage`) is merged into `main` via a merge commit +2. That branch contains auto-sync merge commits or criss-cross merges with `main` +3. Josh encounters the merge commit, tries to map both parents through the filter, and fails with a 500 error + +This is a legitimate subrepo workflow — teams merge staging branches into main regularly. The sync tool should handle it without requiring teams to change their Git workflow. + +### Alternatives considered + +1. **Fail and log**: The pre-v1.3 behavior. Leaves the sync stuck until someone manually intervenes. Bad for unattended cron-based sync. + +2. **Squash all commits into one**: Create a single `commit-tree` with the diff between josh-filtered base and subrepo HEAD. Simple but destroys all commit granularity — 10 unrelated commits appear as one blob on the monorepo PR. + +3. **Rewrite subrepo history**: Rebase or filter-branch the subrepo to remove problematic merges. Breaks the sync relationship with the monorepo (josh's SHA mapping becomes invalid) and forces all developers to re-clone. + +4. **Linearize: cherry-pick regular commits, squash only merges**: Walk the human commits in order, cherry-pick non-merge commits as-is, and use `cherry-pick -m 1` for merge commits. Preserves individual commit granularity for regular commits; only the problematic merge commits lose their multi-parent structure. + +## Decision + +When the direct push through josh-proxy fails, fall back to option 4: linearize the history by cherry-picking onto the josh-filtered base. + +### How it works + +1. Direct `git push` through josh-proxy is attempted first (existing behavior) +2. If it fails, create a temporary branch from `mono-filtered/<branch>` +3. Walk human commits (oldest-first, bot commits excluded) from the ancestry path: + - **Regular commits** (≤1 parent): `git cherry-pick <sha>` — preserves author, date, and message + - **Merge commits** (>1 parent): `git cherry-pick -m 1 <sha>` — applies the merge's diff relative to its first parent as a single commit +4. If cherry-pick conflicts (rare — usually due to ordering issues), fall back to `git diff | git apply` with the original author metadata +5. Push the linearized branch through josh-proxy +6. PR body includes a note explaining that merge commits were squashed + +### What is preserved + +- Individual non-merge commits (author, date, message, diff) +- The net effect of each merge commit (as a squashed single commit) +- The original commit list in the PR body for reference + +### What is lost + +- The multi-parent structure of merge commits (they become single-parent) +- The distinction between "changes introduced by the merge" vs "changes from each parent" — the merge is represented as its diff from first parent + +## Consequences + +**Positive:** +- Reverse sync no longer gets stuck on merge commits — handles the common case automatically +- Non-merge commits retain full granularity (no unnecessary squashing) +- Subrepo history is untouched — no rewriting, no broken sync relationship +- The PR body documents when linearization was used, so reviewers know + +**Negative:** +- Merge commit semantics are lost on the monorepo side (they appear as regular commits) +- If a merge commit's changes conflict with a prior cherry-picked commit during linearization, the diff-apply fallback may produce a subtly different result than the original merge resolution +- Adds complexity to the reverse sync path (two code paths: direct push and linearize fallback) + +**Risk mitigation:** +- The direct push is always tried first — linearization only activates when josh rejects the push +- The monorepo PR is still reviewed by humans before merging +- The original commit SHAs are listed in the PR body for cross-referencing diff --git a/docs/adr/README.md b/docs/adr/README.md index 42e2773..eefee96 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -16,3 +16,4 @@ This directory contains Architecture Decision Records (ADRs) for josh-sync. Each | [008](008-first-parent-ordering.md) | First-parent ordering in reconciliation merges | Accepted | | [009](009-tree-comparison-guard.md) | Tree comparison as sync skip guard | Accepted | | [010](010-onboard-checkpoint-resume.md) | Onboard workflow with checkpoint/resume | Accepted | +| [011](011-linearize-fallback-reverse.md) | Linearize fallback for reverse sync | Accepted | diff --git a/docs/guide.md b/docs/guide.md index bb1cc3e..38f4a67 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -518,6 +518,60 @@ Bot commits include a git trailer like `Josh-Sync-Origin: forward/main/2024-02-1 Sync state is stored as JSON files on an orphan branch (`josh-sync-state`), one file per target/branch. This tracks the last-synced commit SHAs and timestamps to avoid re-syncing the same changes. +## Recommended Git Workflow + +Josh-proxy maps commits through a filter per-branch. It handles linear history and simple merges (short-lived feature branches) without issues. However, it **cannot map merge commits whose parents were created on the subrepo side** — because those commits were never pushed through josh and have no monorepo-side mapping. + +This means cross-branch merges (e.g., `stage` → `main`) must happen on the monorepo side, where josh can filter the result cleanly. + +### The rule + +**The monorepo owns branch topology. The subrepo owns feature development.** + +### What to do where + +| Action | Where to do it | Why | +|--------|---------------|-----| +| Feature branch → `main` | **Subrepo** (PR, any merge strategy) | Short-lived branch with clean lineage — josh handles it | +| `stage` → `main` (promotion) | **Monorepo** | Cross-branch merge — forward sync propagates the result to both subrepo branches | +| `main` → `stage` (catch-up) | **Monorepo** | Same reason — avoids criss-cross merge history on subrepo | +| Hotfix to `main` | **Either side** | Single commit or small PR — works everywhere | +| Config/CI changes (monorepo-only) | **Monorepo** | Not synced to subrepo (use `exclude` for monorepo-only files) | + +### Feature development (subrepo) + +This is the primary workflow for subrepo developers: + +1. Create a feature branch from `main` (or whichever synced branch) +2. Develop, commit, push +3. Open a PR targeting `main` on the subrepo +4. Merge the PR (merge commit, squash, or rebase — all work) +5. Reverse sync picks up the new commits and creates a PR on the monorepo + +Any merge strategy works because the feature branch lineage stays within josh's mapped history. + +### Cross-branch merges (monorepo) + +When promoting `stage` to `main`, or catching up `stage` with `main`: + +1. Open a PR on the **monorepo** merging `stage` → `main` (or `main` → `stage`) +2. Review and merge on the monorepo +3. Forward sync propagates the result to the subrepo's `main` and `stage` branches + +This works because josh does the filtering — it computes the subrepo view from the monorepo merge result, rather than trying to reconstruct a monorepo merge from subrepo commits. + +### What to avoid + +- **Don't merge `stage` into `main` on the subrepo with a merge commit.** The merge parents include commits created on the subrepo side (forward sync merges, criss-cross merges) that josh has no mapping for. Josh rejects the push with a 500 error. +- **Don't merge `main` into `stage` on the subrepo.** Creates criss-cross merge history that causes the same josh mapping failure when `stage` is later merged back. +- **Don't rebase synced branches on the subrepo.** This rewrites commit SHAs that josh has already mapped, breaking the sync relationship. + +### If you must merge cross-branch on the subrepo + +Use a **squash merge**. A squash merge produces a single commit with one parent — josh can always map it. You lose the individual commit history on the target branch, but the sync goes through cleanly. + +As a safety net, josh-sync v1.3+ automatically falls back to linearizing the history when josh rejects a push — cherry-picking regular commits individually and squashing only the problematic merge commits. See the [troubleshooting section](#josh-rejected-push-reverse-sync) and [ADR-011](adr/011-linearize-fallback-reverse.md). + ## Excluding Files from Sync Some files in the monorepo subfolder may not belong in the subrepo (e.g., monorepo-specific CI configs, internal tooling). The `exclude` config field removes these at the josh-proxy layer — excluded files never appear in the subrepo. @@ -616,7 +670,35 @@ Normal: the subrepo changed while sync was running. The next sync run will pick ### "Josh rejected push" (reverse sync) -Josh-proxy couldn't map the push back to the monorepo. Check josh-proxy logs, verify the josh filter is correct. May indicate a history divergence — consider running `josh-sync reset <target>`. +Josh-proxy couldn't map the push back to the monorepo. This has two common causes: + +#### Merge commits with unmappable parents + +**Symptom:** Josh returns `500 Internal Server Error` with a message like: + +``` +rejecting merge with 2 parents: +"Merge pull request 'stage' (#30) from stage into main" (c4fa3c9...) +1) "Merge branch 'auto-sync/import-...'" (4bf8704...) +2) "Merge branch 'main' into stage" (d021654...) +``` + +**Cause:** The subrepo has a merge commit whose parents josh-proxy cannot trace through its filter. This typically happens when: +- A long-lived branch (e.g., `stage`) is merged into `main` via a merge commit (not squash) +- That branch contains auto-sync merge commits or other history that doesn't exist in josh's filtered view +- Someone merges `main` into a feature/staging branch and then merges it back — the criss-cross parents confuse josh's mapping + +**Automatic handling (v1.3+):** josh-sync automatically falls back to linearizing the history when the direct push fails. Regular commits are cherry-picked individually (preserving authorship and messages), while merge commits are squashed into single commits via `cherry-pick -m 1`. The PR notes when this fallback was used. See [ADR-011](adr/011-linearize-fallback-reverse.md). + +**Prevention:** In josh-synced subrepos, prefer **squash merges** when merging long-lived branches (stage, develop) into the synced branch. Squash merges produce a single commit with no merge parents, which josh can always map. Regular feature branch merges (short-lived, no auto-sync history) are usually fine. + +**Manual resolution (if automatic fallback also fails):** This indicates a more fundamental history issue. Options: +1. Cherry-pick the desired changes manually onto a clean branch and push through josh +2. Run `josh-sync reset <target>` to re-establish history (destructive — all subrepo clones must re-fetch) + +#### Filter or path mismatch + +Josh-proxy couldn't map the push due to an incorrect filter. Check josh-proxy logs, verify the `josh_filter` or `subfolder` in `.josh-sync.yml` is correct, and ensure the subfolder exists in the monorepo. ### Import PR shows "No changes" diff --git a/lib/sync.sh b/lib/sync.sh index 22b9888..95c2e51 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -284,37 +284,109 @@ reverse_sync() { local ts ts=$(date +%Y%m%d-%H%M%S) local staging_branch="auto-sync/subrepo-${subrepo_branch}-${ts}" + local push_ref="HEAD" + local linearized=false - if git push -o "base=${mono_branch}" "$(josh_auth_url)" "HEAD:refs/heads/${staging_branch}"; then - log "INFO" "Pushed to staging branch via josh: ${staging_branch}" + if ! git push -o "base=${mono_branch}" "$(josh_auth_url)" "${push_ref}:refs/heads/${staging_branch}" 2>/dev/null; then + # Josh rejects pushes when the history contains merge commits whose + # parents it cannot map (e.g., merges of staging/feature branches that + # include auto-sync history). Fall back to linearizing: cherry-pick + # regular commits as-is, squash only the problematic merge commits. + # See ADR-011. + log "WARN" "Direct push failed — linearizing history (cherry-pick + squash merges)" - # 6. Create PR on monorepo (NEVER direct push) - local pr_body - pr_body="## Subrepo changes + local original_head + original_head=$(git rev-parse HEAD) + + git checkout -b "tmp-linearize-$$" "mono-filtered/${mono_branch}" 2>/dev/null \ + || die "Failed to create linearize branch" + + local commit_shas + commit_shas=$(git log --ancestry-path "mono-filtered/${mono_branch}..${original_head}" \ + --reverse --format="%H" --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") + + local sha + for sha in $commit_shas; do + local parent_count + parent_count=$(git cat-file -p "$sha" | grep -c "^parent ") + + if [ "$parent_count" -le 1 ]; then + # Regular commit — cherry-pick to preserve message and authorship + if ! git cherry-pick "$sha" 2>/dev/null; then + log "WARN" "Cherry-pick conflict on ${sha:0:7} — applying tree directly" + git cherry-pick --abort 2>/dev/null + # Fall back to applying the exact tree diff for this commit + local msg author_name author_email author_date + msg=$(git log -1 --format="%B" "$sha") + author_name=$(git log -1 --format="%an" "$sha") + author_email=$(git log -1 --format="%ae" "$sha") + author_date=$(git log -1 --format="%aI" "$sha") + git diff HEAD "$sha" | git apply --index 2>/dev/null \ + || die "Failed to apply diff for ${sha:0:7}" + GIT_AUTHOR_NAME="$author_name" GIT_AUTHOR_EMAIL="$author_email" \ + GIT_AUTHOR_DATE="$author_date" \ + git commit -m "$msg" || die "Failed to commit ${sha:0:7}" + fi + else + # Merge commit — cherry-pick relative to first parent (squashes the merge) + local msg author_name author_email author_date + msg=$(git log -1 --format="%B" "$sha") + author_name=$(git log -1 --format="%an" "$sha") + author_email=$(git log -1 --format="%ae" "$sha") + author_date=$(git log -1 --format="%aI" "$sha") + + if ! git cherry-pick -m 1 "$sha" 2>/dev/null; then + git cherry-pick --abort 2>/dev/null + git diff HEAD "$sha" | git apply --index 2>/dev/null \ + || die "Failed to apply merge diff for ${sha:0:7}" + GIT_AUTHOR_NAME="$author_name" GIT_AUTHOR_EMAIL="$author_email" \ + GIT_AUTHOR_DATE="$author_date" \ + git commit -m "$msg" || die "Failed to commit merge ${sha:0:7}" + fi + log "INFO" "Squashed merge commit ${sha:0:7}: $(echo "$msg" | head -1)" + fi + done + + push_ref="HEAD" + linearized=true + + git push -o "base=${mono_branch}" "$(josh_auth_url)" \ + "${push_ref}:refs/heads/${staging_branch}" \ + || { log "ERROR" "Josh rejected linearized push — check josh-proxy logs"; echo "josh-rejected"; return; } + fi + + log "INFO" "Pushed to staging branch via josh: ${staging_branch}${linearized:+ (linearized)}" + + # 6. Create PR on monorepo (NEVER direct push) + local pr_body + local linearize_note="" + if [ "$linearized" = true ]; then + linearize_note=" +> **Note:** Merge commits were squashed during sync because the subrepo +> history contained merges that josh-proxy cannot map (see ADR-011). +> Regular commits are preserved individually." + fi + pr_body="## Subrepo changes New commits from subrepo \`${subrepo_branch}\`: \`\`\` ${human_commits} \`\`\` - +${linearize_note} **Review checklist:** - [ ] Changes scoped to synced subfolder - [ ] No leaked credentials or environment-specific config - [ ] CI passes" - create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ - "$mono_branch" "$staging_branch" \ - "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ - "$pr_body" \ - || die "Failed to create PR on monorepo (check GITEA_TOKEN)" + create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ + "$mono_branch" "$staging_branch" \ + "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ + "$pr_body" \ + || die "Failed to create PR on monorepo (check GITEA_TOKEN)" - log "INFO" "Reverse sync PR created on monorepo" - echo "pr-created" - else - log "ERROR" "Josh rejected push — check josh-proxy logs" - echo "josh-rejected" - fi + log "INFO" "Reverse sync PR created on monorepo" + echo "pr-created" } # ─── Initial Import: Subrepo → Monorepo (first time) ─────────────── From b4eaa47ef6c8a7e9d61191e68f8662c0e6fa5d11 Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Tue, 17 Mar 2026 14:07:20 +0300 Subject: [PATCH 27/37] Fix reverse sync missing human commits when mono-filtered advances --ancestry-path returns empty when mono-filtered/main has advanced past the last forward sync (any monorepo commit causes josh to create a new filtered SHA not in the subrepo's ancestry). Use git merge-base to find the last connected point instead. Also return skip-dirty (not skip) when trees differ but no human commits found, preventing state from being updated on false skips which would permanently lose unsynced commits. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- CHANGELOG.md | 6 ++++++ bin/josh-sync | 4 ++++ lib/sync.sh | 39 ++++++++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98d27db..331843f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,16 @@ - **Linearize fallback for reverse sync**: When josh-proxy rejects a push due to unmappable merge commits, reverse sync automatically falls back to linearizing the history — cherry-picking regular commits individually and squashing only the problematic merge commits via `cherry-pick -m 1`. Preserves individual commit granularity while handling complex merge topologies that josh cannot map. PR body notes when linearization was used. +### Fixes + +- **Reverse sync human commit detection**: Fixed `--ancestry-path` returning empty when `mono-filtered` has advanced past the last forward sync. Any monorepo commit (even outside the subfolder) causes josh to create a new filtered commit, breaking the ancestry path to subrepo HEAD. Now uses `git merge-base` to find the last connected point. +- **State not updated on false skip**: When trees differ but no human commits are found, reverse sync now returns `skip-dirty` instead of `skip`. State is not updated, so the system retries on the next run instead of silently dropping commits. + ### Docs - Expanded troubleshooting guide for "Josh rejected push" with root cause analysis and prevention advice. - Added ADR-011: Linearize fallback for reverse sync. +- Added recommended Git workflow section to the guide. ## 1.2.0 diff --git a/bin/josh-sync b/bin/josh-sync index cc5414b..54f3906 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -259,6 +259,10 @@ _sync_direction() { echo "::error::Target ${target_name}, branch ${branch}: josh rejected push — check proxy logs" continue fi + if [ "$result" = "skip-dirty" ]; then + log "WARN" "Trees differ but no human commits — skipping state update (will retry)" + continue + fi # Update state (only on success) local new_state diff --git a/lib/sync.sh b/lib/sync.sh index 95c2e51..dbf2238 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -226,7 +226,9 @@ ${BOT_TRAILER}: filter-change/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)") # ─── Reverse Sync: Subrepo → Monorepo ────────────────────────────── # # Always creates a PR on the monorepo — never pushes directly. -# Returns: skip | pr-created | josh-rejected +# Returns: skip | skip-dirty | pr-created | josh-rejected +# skip — trees identical, nothing to do +# skip-dirty — trees differ but no human commits found (don't update state) reverse_sync() { local mono_branch="$SYNC_BRANCH_MONO" @@ -265,15 +267,27 @@ reverse_sync() { fi # 4. Find new human commits (excludes bot commits from forward sync) - # Uses --ancestry-path to restrict to the direct lineage and avoid - # leaking old history through reconciliation merge parents. - local human_commits - human_commits=$(git log --ancestry-path "mono-filtered/${mono_branch}..HEAD" \ - --oneline --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") + # Uses merge-base + --ancestry-path to restrict to the direct lineage. + # merge-base is needed because mono-filtered may have advanced past the + # last forward sync (any monorepo commit, even outside the subfolder, + # causes josh to create a new filtered commit). Using mono-filtered + # directly with --ancestry-path returns empty when it's not an ancestor + # of HEAD. merge-base always finds the last connected point. + local base human_commits + base=$(git merge-base "mono-filtered/${mono_branch}" HEAD 2>/dev/null || echo "") + + if [ -n "$base" ]; then + human_commits=$(git log --ancestry-path "${base}..HEAD" \ + --oneline --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") + else + # No common ancestor (first sync or fully diverged) — skip ancestry-path + human_commits=$(git log "mono-filtered/${mono_branch}..HEAD" \ + --oneline --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") + fi if [ -z "$human_commits" ]; then log "INFO" "No new human commits in subrepo — nothing to sync" - echo "skip" + echo "skip-dirty" return fi @@ -301,9 +315,16 @@ reverse_sync() { git checkout -b "tmp-linearize-$$" "mono-filtered/${mono_branch}" 2>/dev/null \ || die "Failed to create linearize branch" + # Reuse merge-base from step 4 (same reasoning: mono-filtered may have + # advanced past the last forward sync, breaking --ancestry-path). local commit_shas - commit_shas=$(git log --ancestry-path "mono-filtered/${mono_branch}..${original_head}" \ - --reverse --format="%H" --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") + if [ -n "$base" ]; then + commit_shas=$(git log --ancestry-path "${base}..${original_head}" \ + --reverse --format="%H" --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") + else + commit_shas=$(git log "mono-filtered/${mono_branch}..${original_head}" \ + --reverse --format="%H" --invert-grep --grep="^${BOT_TRAILER}:" 2>/dev/null || echo "") + fi local sha for sha in $commit_shas; do From d3799187a14ed1491674d502f52de2e0408ac57f Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Tue, 17 Mar 2026 14:33:42 +0300 Subject: [PATCH 28/37] Clean up linearize fallback output formatting Suppress git stdout/stderr noise from cherry-pick, checkout, and commit during linearization. Add delimiters and consistent indented log lines so the output is easy to scan. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- lib/sync.sh | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/sync.sh b/lib/sync.sh index dbf2238..596ae65 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -308,11 +308,12 @@ reverse_sync() { # regular commits as-is, squash only the problematic merge commits. # See ADR-011. log "WARN" "Direct push failed — linearizing history (cherry-pick + squash merges)" + log "INFO" "─── linearize start ───" local original_head original_head=$(git rev-parse HEAD) - git checkout -b "tmp-linearize-$$" "mono-filtered/${mono_branch}" 2>/dev/null \ + git checkout -b "tmp-linearize-$$" "mono-filtered/${mono_branch}" >/dev/null 2>&1 \ || die "Failed to create linearize branch" # Reuse merge-base from step 4 (same reasoning: mono-filtered may have @@ -330,23 +331,27 @@ reverse_sync() { for sha in $commit_shas; do local parent_count parent_count=$(git cat-file -p "$sha" | grep -c "^parent ") + local short_msg + short_msg=$(git log -1 --format="%s" "$sha") if [ "$parent_count" -le 1 ]; then # Regular commit — cherry-pick to preserve message and authorship - if ! git cherry-pick "$sha" 2>/dev/null; then - log "WARN" "Cherry-pick conflict on ${sha:0:7} — applying tree directly" + if git cherry-pick "$sha" >/dev/null 2>&1; then + log "INFO" " cherry-pick ${sha:0:7}: ${short_msg}" + else + log "WARN" " conflict ${sha:0:7} — applying tree directly" git cherry-pick --abort 2>/dev/null - # Fall back to applying the exact tree diff for this commit local msg author_name author_email author_date msg=$(git log -1 --format="%B" "$sha") author_name=$(git log -1 --format="%an" "$sha") author_email=$(git log -1 --format="%ae" "$sha") author_date=$(git log -1 --format="%aI" "$sha") - git diff HEAD "$sha" | git apply --index 2>/dev/null \ + git diff HEAD "$sha" | git apply --index >/dev/null 2>&1 \ || die "Failed to apply diff for ${sha:0:7}" GIT_AUTHOR_NAME="$author_name" GIT_AUTHOR_EMAIL="$author_email" \ GIT_AUTHOR_DATE="$author_date" \ - git commit -m "$msg" || die "Failed to commit ${sha:0:7}" + git commit -m "$msg" >/dev/null 2>&1 || die "Failed to commit ${sha:0:7}" + log "INFO" " applied ${sha:0:7}: ${short_msg}" fi else # Merge commit — cherry-pick relative to first parent (squashes the merge) @@ -356,18 +361,22 @@ reverse_sync() { author_email=$(git log -1 --format="%ae" "$sha") author_date=$(git log -1 --format="%aI" "$sha") - if ! git cherry-pick -m 1 "$sha" 2>/dev/null; then + if git cherry-pick -m 1 "$sha" >/dev/null 2>&1; then + : # success + else git cherry-pick --abort 2>/dev/null - git diff HEAD "$sha" | git apply --index 2>/dev/null \ + git diff HEAD "$sha" | git apply --index >/dev/null 2>&1 \ || die "Failed to apply merge diff for ${sha:0:7}" GIT_AUTHOR_NAME="$author_name" GIT_AUTHOR_EMAIL="$author_email" \ GIT_AUTHOR_DATE="$author_date" \ - git commit -m "$msg" || die "Failed to commit merge ${sha:0:7}" + git commit -m "$msg" >/dev/null 2>&1 || die "Failed to commit merge ${sha:0:7}" fi - log "INFO" "Squashed merge commit ${sha:0:7}: $(echo "$msg" | head -1)" + log "INFO" " squash ${sha:0:7}: ${short_msg}" fi done + log "INFO" "─── linearize done ────" + push_ref="HEAD" linearized=true From c9b37f2477677e30c8115b944d11d8a6d55776e7 Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Tue, 17 Mar 2026 14:41:04 +0300 Subject: [PATCH 29/37] Fix linearize fallback applying wrong diff (full tree instead of parent) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fallback for failed cherry-picks used `git diff HEAD $sha` which computes the diff between the linearize branch and the commit's full tree. This captures ALL tree differences, not just the changes the commit introduced — causing unrelated file deletions. Fix: diff from the commit's own parent (`${sha}^` for regular commits, `${sha}^1` for merge commits). Also detect and skip empty merges (where cherry-pick -m 1 produces nothing because the changes were already applied by a prior cherry-pick). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- lib/sync.sh | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/sync.sh b/lib/sync.sh index 596ae65..5c80de2 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -339,14 +339,16 @@ reverse_sync() { if git cherry-pick "$sha" >/dev/null 2>&1; then log "INFO" " cherry-pick ${sha:0:7}: ${short_msg}" else - log "WARN" " conflict ${sha:0:7} — applying tree directly" + log "WARN" " conflict ${sha:0:7} — applying patch from parent" git cherry-pick --abort 2>/dev/null local msg author_name author_email author_date msg=$(git log -1 --format="%B" "$sha") author_name=$(git log -1 --format="%an" "$sha") author_email=$(git log -1 --format="%ae" "$sha") author_date=$(git log -1 --format="%aI" "$sha") - git diff HEAD "$sha" | git apply --index >/dev/null 2>&1 \ + # Diff from the commit's own parent — NOT from HEAD (which is + # the linearize branch and may have a different base). + git diff "${sha}^" "$sha" | git apply --index >/dev/null 2>&1 \ || die "Failed to apply diff for ${sha:0:7}" GIT_AUTHOR_NAME="$author_name" GIT_AUTHOR_EMAIL="$author_email" \ GIT_AUTHOR_DATE="$author_date" \ @@ -362,16 +364,26 @@ reverse_sync() { author_date=$(git log -1 --format="%aI" "$sha") if git cherry-pick -m 1 "$sha" >/dev/null 2>&1; then - : # success + log "INFO" " squash ${sha:0:7}: ${short_msg}" else git cherry-pick --abort 2>/dev/null - git diff HEAD "$sha" | git apply --index >/dev/null 2>&1 \ - || die "Failed to apply merge diff for ${sha:0:7}" - GIT_AUTHOR_NAME="$author_name" GIT_AUTHOR_EMAIL="$author_email" \ - GIT_AUTHOR_DATE="$author_date" \ - git commit -m "$msg" >/dev/null 2>&1 || die "Failed to commit merge ${sha:0:7}" + # Diff from first parent — what the merge actually introduced. + # NOT from HEAD (which would capture all tree differences, not + # just this merge's changes). + if git diff "${sha}^1" "$sha" --quiet 2>/dev/null; then + # Empty diff from first parent — the merge introduced no new + # changes (e.g., a fast-forward merge or changes already applied + # by a prior cherry-pick). Skip silently. + log "INFO" " skip ${sha:0:7}: ${short_msg} (empty — already applied)" + else + git diff "${sha}^1" "$sha" | git apply --index >/dev/null 2>&1 \ + || die "Failed to apply merge diff for ${sha:0:7}" + GIT_AUTHOR_NAME="$author_name" GIT_AUTHOR_EMAIL="$author_email" \ + GIT_AUTHOR_DATE="$author_date" \ + git commit -m "$msg" >/dev/null 2>&1 || die "Failed to commit merge ${sha:0:7}" + log "INFO" " squash ${sha:0:7}: ${short_msg}" + fi fi - log "INFO" " squash ${sha:0:7}: ${short_msg}" fi done From 4861179c6fd89eb5c9553d2c4b303e21fb8c018e Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Tue, 17 Mar 2026 14:49:10 +0300 Subject: [PATCH 30/37] Fix linearize fallback for already-applied merge commits When cherry-pick -m 1 fails because the merge's changes were already applied by a prior cherry-pick (empty result, clean working tree), skip the commit instead of trying git-apply which also fails. Only use the diff-from-first-parent fallback for real conflicts. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- lib/sync.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/sync.sh b/lib/sync.sh index 5c80de2..4fceae3 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -366,16 +366,17 @@ reverse_sync() { if git cherry-pick -m 1 "$sha" >/dev/null 2>&1; then log "INFO" " squash ${sha:0:7}: ${short_msg}" else - git cherry-pick --abort 2>/dev/null - # Diff from first parent — what the merge actually introduced. - # NOT from HEAD (which would capture all tree differences, not - # just this merge's changes). - if git diff "${sha}^1" "$sha" --quiet 2>/dev/null; then - # Empty diff from first parent — the merge introduced no new - # changes (e.g., a fast-forward merge or changes already applied - # by a prior cherry-pick). Skip silently. - log "INFO" " skip ${sha:0:7}: ${short_msg} (empty — already applied)" + # Cherry-pick failed — distinguish empty result (changes already + # applied by a prior cherry-pick) from a real conflict. + if git diff --cached --quiet 2>/dev/null && git diff --quiet 2>/dev/null; then + # Working tree clean — the merge changes are already present + git cherry-pick --abort 2>/dev/null || git reset --hard HEAD >/dev/null 2>&1 + log "INFO" " skip ${sha:0:7}: ${short_msg} (already applied)" else + # Real conflict — apply diff from first parent (what the merge + # actually introduced, NOT from HEAD which would capture all + # tree differences). + git cherry-pick --abort 2>/dev/null git diff "${sha}^1" "$sha" | git apply --index >/dev/null 2>&1 \ || die "Failed to apply merge diff for ${sha:0:7}" GIT_AUTHOR_NAME="$author_name" GIT_AUTHOR_EMAIL="$author_email" \ From 65cc8eaf8ac71cfcabc5b677f41b8ccea0012f3c Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Tue, 17 Mar 2026 14:55:53 +0300 Subject: [PATCH 31/37] Trim overengineered docs and ADR Remove redundant step-by-step subsections from workflow guide (the table already covers it), tighten ADR-011 alternatives, and remove version pins that go stale. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --- docs/adr/011-linearize-fallback-reverse.md | 16 +++---------- docs/guide.md | 26 ++-------------------- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/docs/adr/011-linearize-fallback-reverse.md b/docs/adr/011-linearize-fallback-reverse.md index 80e3261..1974498 100644 --- a/docs/adr/011-linearize-fallback-reverse.md +++ b/docs/adr/011-linearize-fallback-reverse.md @@ -15,13 +15,9 @@ This is a legitimate subrepo workflow — teams merge staging branches into main ### Alternatives considered -1. **Fail and log**: The pre-v1.3 behavior. Leaves the sync stuck until someone manually intervenes. Bad for unattended cron-based sync. - -2. **Squash all commits into one**: Create a single `commit-tree` with the diff between josh-filtered base and subrepo HEAD. Simple but destroys all commit granularity — 10 unrelated commits appear as one blob on the monorepo PR. - -3. **Rewrite subrepo history**: Rebase or filter-branch the subrepo to remove problematic merges. Breaks the sync relationship with the monorepo (josh's SHA mapping becomes invalid) and forces all developers to re-clone. - -4. **Linearize: cherry-pick regular commits, squash only merges**: Walk the human commits in order, cherry-pick non-merge commits as-is, and use `cherry-pick -m 1` for merge commits. Preserves individual commit granularity for regular commits; only the problematic merge commits lose their multi-parent structure. +1. **Squash all commits into one**: Simple but destroys all commit granularity. +2. **Rewrite subrepo history**: Breaks josh's SHA mapping, forces all developers to re-clone. +3. **Linearize: cherry-pick regular commits, squash only merges**: Preserves individual commit granularity; only merge commits lose their multi-parent structure. ## Decision @@ -59,10 +55,4 @@ When the direct push through josh-proxy fails, fall back to option 4: linearize **Negative:** - Merge commit semantics are lost on the monorepo side (they appear as regular commits) -- If a merge commit's changes conflict with a prior cherry-picked commit during linearization, the diff-apply fallback may produce a subtly different result than the original merge resolution - Adds complexity to the reverse sync path (two code paths: direct push and linearize fallback) - -**Risk mitigation:** -- The direct push is always tried first — linearization only activates when josh rejects the push -- The monorepo PR is still reviewed by humans before merging -- The original commit SHAs are listed in the PR body for cross-referencing diff --git a/docs/guide.md b/docs/guide.md index 38f4a67..fe883c9 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -538,28 +538,6 @@ This means cross-branch merges (e.g., `stage` → `main`) must happen on the mon | Hotfix to `main` | **Either side** | Single commit or small PR — works everywhere | | Config/CI changes (monorepo-only) | **Monorepo** | Not synced to subrepo (use `exclude` for monorepo-only files) | -### Feature development (subrepo) - -This is the primary workflow for subrepo developers: - -1. Create a feature branch from `main` (or whichever synced branch) -2. Develop, commit, push -3. Open a PR targeting `main` on the subrepo -4. Merge the PR (merge commit, squash, or rebase — all work) -5. Reverse sync picks up the new commits and creates a PR on the monorepo - -Any merge strategy works because the feature branch lineage stays within josh's mapped history. - -### Cross-branch merges (monorepo) - -When promoting `stage` to `main`, or catching up `stage` with `main`: - -1. Open a PR on the **monorepo** merging `stage` → `main` (or `main` → `stage`) -2. Review and merge on the monorepo -3. Forward sync propagates the result to the subrepo's `main` and `stage` branches - -This works because josh does the filtering — it computes the subrepo view from the monorepo merge result, rather than trying to reconstruct a monorepo merge from subrepo commits. - ### What to avoid - **Don't merge `stage` into `main` on the subrepo with a merge commit.** The merge parents include commits created on the subrepo side (forward sync merges, criss-cross merges) that josh has no mapping for. Josh rejects the push with a 500 error. @@ -570,7 +548,7 @@ This works because josh does the filtering — it computes the subrepo view from Use a **squash merge**. A squash merge produces a single commit with one parent — josh can always map it. You lose the individual commit history on the target branch, but the sync goes through cleanly. -As a safety net, josh-sync v1.3+ automatically falls back to linearizing the history when josh rejects a push — cherry-picking regular commits individually and squashing only the problematic merge commits. See the [troubleshooting section](#josh-rejected-push-reverse-sync) and [ADR-011](adr/011-linearize-fallback-reverse.md). +As a safety net, josh-sync automatically falls back to linearizing the history when josh rejects a push — cherry-picking regular commits individually and squashing only the problematic merge commits. See the [troubleshooting section](#josh-rejected-push-reverse-sync) and [ADR-011](adr/011-linearize-fallback-reverse.md). ## Excluding Files from Sync @@ -688,7 +666,7 @@ rejecting merge with 2 parents: - That branch contains auto-sync merge commits or other history that doesn't exist in josh's filtered view - Someone merges `main` into a feature/staging branch and then merges it back — the criss-cross parents confuse josh's mapping -**Automatic handling (v1.3+):** josh-sync automatically falls back to linearizing the history when the direct push fails. Regular commits are cherry-picked individually (preserving authorship and messages), while merge commits are squashed into single commits via `cherry-pick -m 1`. The PR notes when this fallback was used. See [ADR-011](adr/011-linearize-fallback-reverse.md). +**Automatic handling:** josh-sync automatically falls back to linearizing the history when the direct push fails. Regular commits are cherry-picked individually (preserving authorship and messages), while merge commits are squashed into single commits via `cherry-pick -m 1`. The PR notes when this fallback was used. See [ADR-011](adr/011-linearize-fallback-reverse.md). **Prevention:** In josh-synced subrepos, prefer **squash merges** when merging long-lived branches (stage, develop) into the synced branch. Squash merges produce a single commit with no merge parents, which josh can always map. Regular feature branch merges (short-lived, no auto-sync history) are usually fine. From 479b5920761072ada8a936ac74485bfde4426fb9 Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Wed, 1 Apr 2026 00:56:33 +0300 Subject: [PATCH 32/37] Add versioning contract: semver policy, breaking change sections, schema_version guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CHANGELOG: add ### Breaking Changes section to all versions (None for 1.1–1.3) - README: add Versioning section with semver table and floating tag explanation - schema/config-schema.json: add schema_version field (const: 1, optional) - lib/config.sh: fail fast if schema_version is present and unsupported Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> --- CHANGELOG.md | 12 ++++++++++++ README.md | 12 ++++++++++++ lib/config.sh | 5 +++++ schema/config-schema.json | 5 +++++ 4 files changed, 34 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 331843f..4039d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 1.3.0 +### Breaking Changes + +_None._ + ### Features - **Linearize fallback for reverse sync**: When josh-proxy rejects a push due to unmappable merge commits, reverse sync automatically falls back to linearizing the history — cherry-picking regular commits individually and squashing only the problematic merge commits via `cherry-pick -m 1`. Preserves individual commit granularity while handling complex merge topologies that josh cannot map. PR body notes when linearization was used. @@ -19,6 +23,10 @@ ## 1.2.0 +### Breaking Changes + +_None._ + ### Features - **File exclusion**: `exclude` config field removes files/directories from the subrepo at the josh-proxy transport layer. Patterns are embedded inline in the josh-proxy URL using `:exclude[::pattern,...]` syntax — no extra files to generate or commit. @@ -38,6 +46,10 @@ ## 1.1.0 +### Breaking Changes + +_None._ + ### Features - **`onboard` command**: Interactive, resumable workflow for importing existing subrepos into the monorepo. Walks through: prerequisites check, import (creates PRs), wait for merge, reset (pushes josh-filtered history). Checkpoint/resume at every step. diff --git a/README.md b/README.md index 3b39733..0f8466c 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,18 @@ Run `josh-sync preflight` to validate your setup. - **[Architecture Decision Records](docs/adr/)** — Design rationale and trade-offs - **[Changelog](CHANGELOG.md)** — Version history +## Versioning + +josh-sync follows [semver](https://semver.org/): + +| Bump | Meaning | Action required | +|------|---------|-----------------| +| **Patch** (1.x.**y**) | Bug fixes only | Safe to upgrade | +| **Minor** (1.**y**.0) | New optional config fields, new commands, new action inputs | Safe to upgrade; new features are opt-in | +| **Major** (**y**.0.0) | Removed or renamed config fields, CLI flags, action inputs, env vars, or state format | Read the **Breaking Changes** section in the changelog before upgrading | + +CI workflows pin to a floating major tag (e.g. `@v1`). A breaking change bumps the major version and moves the tag — `@v1` workflows keep working until you explicitly update to `@v2`. + ## CLI ``` diff --git a/lib/config.sh b/lib/config.sh index bdf8b95..bc12366 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -16,6 +16,11 @@ parse_config() { local config_json config_json=$(yq -o json "$config_file") || die "Failed to parse ${config_file}" + # Schema version check (optional field — absent means v1) + local schema_version + schema_version=$(echo "$config_json" | jq -r '.schema_version // 1') + [ "$schema_version" = "1" ] || die "Unsupported schema_version ${schema_version} in ${config_file} (this version of josh-sync supports schema_version 1 — see CHANGELOG.md for migration instructions)" + # Export global values export JOSH_PROXY_URL JOSH_PROXY_URL=$(echo "$config_json" | jq -r '.josh.proxy_url') diff --git a/schema/config-schema.json b/schema/config-schema.json index b3c9675..4afeb72 100644 --- a/schema/config-schema.json +++ b/schema/config-schema.json @@ -5,6 +5,11 @@ "type": "object", "required": ["josh", "targets", "bot"], "properties": { + "schema_version": { + "type": "integer", + "const": 1, + "description": "Schema version. Bump when config fields are removed or renamed (matches josh-sync major version)." + }, "josh": { "type": "object", "required": ["proxy_url", "monorepo_path"], From 1cb23a4411a50f5827caadf8b3184d1c63b07bb9 Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Wed, 1 Apr 2026 02:46:43 +0300 Subject: [PATCH 33/37] Release 1.4.0 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> --- CHANGELOG.md | 21 +++++++++++++++++++++ VERSION | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4039d21..45bdf31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 1.4.0 + +### Breaking Changes + +_None._ + +### Features + +- **`schema_version` config field**: New optional integer field in `.josh-sync.yml`. Defaults to `1` when absent. josh-sync fails fast with a clear error if an unsupported version is present — safe path for future config migrations without silent misinterpretation. + +### Fixes + +- **Linearize fallback applying wrong diff**: Fallback for failed `cherry-pick` was diffing from `HEAD` to the commit's full tree instead of from the commit's own parent, causing unrelated file deletions. Now diffs from the commit's own parent (`${sha}^` / `${sha}^1`). +- **Linearize fallback for already-applied merge commits**: When `cherry-pick -m 1` produces an empty result because a merge's changes were already applied by a prior cherry-pick, the commit is now skipped instead of attempting a failing `git apply`. +- **Linearize fallback output**: Suppressed git noise from cherry-pick, checkout, and commit during linearization; added delimiters and consistent indented log lines. + +### Docs + +- Workflow guide: removed redundant step-by-step subsections (table already covers them); tightened ADR-011 alternatives; removed stale version pins. +- README: added Versioning section with semver policy table and floating tag explanation. + ## 1.3.0 ### Breaking Changes diff --git a/VERSION b/VERSION index f0bb29e..88c5fb8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.0 +1.4.0 From 5e9b1341686cf22700125240bd551801b6bf113e Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Sat, 4 Apr 2026 01:43:47 +0300 Subject: [PATCH 34/37] Add --force flag to reverse sync Bypasses the PR step and force-pushes the subrepo's state directly onto the monorepo branch. Also skips the skip-dirty guard. Only valid with --reverse. Returns status "forced". Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> --- CHANGELOG.md | 12 ++++++++++++ README.md | 4 ++-- VERSION | 2 +- bin/josh-sync | 13 +++++++++++++ lib/auth.sh | 10 ++++++++++ lib/sync.sh | 53 ++++++++++++++++++++++++++++++++++----------------- 6 files changed, 74 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45bdf31..80ace72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 1.5.0 + +### Breaking Changes + +_None._ + +### Features + +- **`--force` flag for reverse sync**: `josh-sync sync --reverse --force` bypasses + the PR step and force-pushes the subrepo's state directly onto the monorepo branch. + Also skips the `skip-dirty` guard. Only valid with `--reverse`. Returns status `forced`. + ## 1.4.0 ### Breaking Changes diff --git a/README.md b/README.md index 0f8466c..bd5c0dc 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ CI workflows pin to a floating major tag (e.g. `@v1`). A breaking change bumps t ## CLI ``` -josh-sync sync [--forward|--reverse] [--target NAME[,NAME]] [--branch BRANCH] +josh-sync sync [--forward|--reverse] [--force] [--target NAME[,NAME]] [--branch BRANCH] josh-sync preflight josh-sync import <target> josh-sync reset <target> @@ -92,7 +92,7 @@ josh-sync state reset <target> [branch] ## How It Works - **Forward sync** (mono → subrepo): pushes directly if clean, creates conflict PR if not. Uses `--force-with-lease` for safety. -- **Reverse sync** (subrepo → mono): always creates a PR, never pushes directly. +- **Reverse sync** (subrepo → mono): creates a PR by default. Add `--force` to bypass the PR and push directly to the mono branch (destructive). - **File exclusion**: `exclude` patterns are embedded inline in the josh-proxy URL. Excluded files exist only in the monorepo. - **Filter reconciliation**: Changing the exclude list auto-creates a merge commit that connects old and new histories — no force-push needed. - **Loop prevention**: `Josh-Sync-Origin:` git trailer filters out bot commits. diff --git a/VERSION b/VERSION index 88c5fb8..bc80560 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.0 +1.5.0 diff --git a/bin/josh-sync b/bin/josh-sync index 54f3906..2d089b3 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -88,6 +88,7 @@ Global flags: Sync flags: --forward Forward only (mono → subrepo) --reverse Reverse only (subrepo → mono) + --force Force-push subrepo state to mono branch (--reverse only, skips PR) --target NAME Filter to target(s) — comma-separated for multiple (env: JOSH_SYNC_TARGET) --branch BRANCH Filter to one branch @@ -108,6 +109,7 @@ cmd_sync() { local filter_target="${JOSH_SYNC_TARGET:-}" local filter_branch="" local config_file=".josh-sync.yml" + local force_flag=false while [ $# -gt 0 ]; do case "$1" in @@ -117,10 +119,18 @@ cmd_sync() { --branch) filter_branch="$2"; shift 2 ;; --config) config_file="$2"; shift 2 ;; --debug) export JOSH_SYNC_DEBUG=1; shift ;; + --force) force_flag=true; shift ;; *) die "Unknown flag: $1" ;; esac done + if [ "$force_flag" = true ] && [ "$direction" != "reverse" ]; then + die "--force requires --reverse" + fi + if [ "$force_flag" = true ]; then + export SYNC_REVERSE_FORCE=1 + fi + parse_config "$config_file" # Forward sync @@ -263,6 +273,9 @@ _sync_direction() { log "WARN" "Trees differ but no human commits — skipping state update (will retry)" continue fi + if [ "$result" = "forced" ]; then + log "WARN" "Target ${target_name}, branch ${branch}: force-pushed subrepo directly to mono — DESTRUCTIVE" + fi # Update state (only on success) local new_state diff --git a/lib/auth.sh b/lib/auth.sh index 851f940..fd432d4 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -28,6 +28,16 @@ subrepo_auth_url() { fi } +# ─── Monorepo Auth URL ───────────────────────────────────────────── +# Derives a push-capable HTTPS URL from MONOREPO_API. +# MONOREPO_API shape: https://<host>/api/v1/repos/<org/repo> + +mono_auth_url() { + local api_host_path + api_host_path=$(echo "$MONOREPO_API" | sed 's|https://||; s|/api/v1/repos/|/|') + echo "https://${BOT_USER}:${GITEA_TOKEN}@${api_host_path}.git" +} + # ─── Remote Queries ───────────────────────────────────────────────── subrepo_ls_remote() { diff --git a/lib/sync.sh b/lib/sync.sh index 4fceae3..bd6ce33 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -240,6 +240,8 @@ reverse_sync() { log "INFO" "=== Reverse sync: subrepo/${subrepo_branch} → mono/${mono_branch} ===" + local force_mode="${SYNC_REVERSE_FORCE:-0}" + # 1. Clone subrepo git clone "$(subrepo_auth_url)" \ --branch "$subrepo_branch" --single-branch \ @@ -286,9 +288,13 @@ reverse_sync() { fi if [ -z "$human_commits" ]; then - log "INFO" "No new human commits in subrepo — nothing to sync" - echo "skip-dirty" - return + if [ "$force_mode" = "1" ]; then + log "WARN" "No human commits found — force mode: continuing anyway" + else + log "INFO" "No new human commits in subrepo — nothing to sync" + echo "skip-dirty" + return + fi fi log "INFO" "New human commits to sync:" @@ -400,16 +406,28 @@ reverse_sync() { log "INFO" "Pushed to staging branch via josh: ${staging_branch}${linearized:+ (linearized)}" - # 6. Create PR on monorepo (NEVER direct push) - local pr_body - local linearize_note="" - if [ "$linearized" = true ]; then - linearize_note=" + # 6. Publish: force-push directly (--force) or create PR (default) + if [ "$force_mode" = "1" ]; then + log "WARN" "Force mode: pushing ${staging_branch} directly to mono/${mono_branch} — bypassing PR" + local mono_dir="${work_dir}/monorepo" + git clone "$(mono_auth_url)" \ + --branch "$staging_branch" --single-branch \ + "$mono_dir" || die "Failed to clone monorepo staging branch for force-push" + git -C "$mono_dir" push --force "$(mono_auth_url)" \ + "HEAD:refs/heads/${mono_branch}" \ + || die "Failed to force-push to mono/${mono_branch}" + log "INFO" "Force-pushed mono/${staging_branch} → mono/${mono_branch}" + echo "forced" + else + local pr_body + local linearize_note="" + if [ "$linearized" = true ]; then + linearize_note=" > **Note:** Merge commits were squashed during sync because the subrepo > history contained merges that josh-proxy cannot map (see ADR-011). > Regular commits are preserved individually." - fi - pr_body="## Subrepo changes + fi + pr_body="## Subrepo changes New commits from subrepo \`${subrepo_branch}\`: @@ -422,14 +440,15 @@ ${linearize_note} - [ ] No leaked credentials or environment-specific config - [ ] CI passes" - create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ - "$mono_branch" "$staging_branch" \ - "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ - "$pr_body" \ - || die "Failed to create PR on monorepo (check GITEA_TOKEN)" + create_pr "${MONOREPO_API}" "${GITEA_TOKEN}" \ + "$mono_branch" "$staging_branch" \ + "[Subrepo Sync] ${subrepo_branch} → ${mono_branch}" \ + "$pr_body" \ + || die "Failed to create PR on monorepo (check GITEA_TOKEN)" - log "INFO" "Reverse sync PR created on monorepo" - echo "pr-created" + log "INFO" "Reverse sync PR created on monorepo" + echo "pr-created" + fi } # ─── Initial Import: Subrepo → Monorepo (first time) ─────────────── From b28662d3b4541078bde6745a6a10e16aba90c97c Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Wed, 29 Apr 2026 06:31:21 +0300 Subject: [PATCH 35/37] =?UTF-8?q?Release=202.0.0=20=E2=80=94=20explicit=20?= =?UTF-8?q?monorepo=5Furl,=20drop=20first-target-host=20inference?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Breaking change. The monorepo's gitea host was previously inferred from the first target's subrepo_url, silently coupling the two and breaking on any multi-host setup (monorepo on host A, target on host B). v2.0.0 replaces the inference with a required josh.monorepo_url field that mirrors subrepo_url — same parsing, same SSH/HTTPS auth options, same shape. - schema_version: 2 is now required; v1 configs are rejected with a clear migration error - josh.monorepo_url is required; josh.monorepo_auth is optional (default https) - mono_auth_url() in lib/auth.sh mirrors subrepo_auth_url() - MONOREPO_API derives from monorepo_url's host instead of .[0].gitea_host; env-var override preserved as escape hatch - initial_import() and force-push call mono_auth_url() instead of building URLs inline - ADR-012 documents the rationale; CHANGELOG includes migration snippet Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- CHANGELOG.md | 37 +++++++ README.md | 4 + VERSION | 2 +- docs/adr/012-explicit-monorepo-url.md | 53 +++++++++ docs/adr/README.md | 1 + docs/config-reference.md | 17 ++- docs/guide.md | 6 +- examples/josh-sync.yml | 13 ++- lib/auth.sh | 24 +++-- lib/config.sh | 36 +++++-- lib/sync.sh | 7 +- schema/config-schema.json | 20 +++- tests/fixtures/minimal.yml | 2 + tests/fixtures/multi-target.yml | 2 + tests/unit/auth.bats | 36 +++++++ tests/unit/config.bats | 148 ++++++++++++++++++++++++++ 16 files changed, 377 insertions(+), 31 deletions(-) create mode 100644 docs/adr/012-explicit-monorepo-url.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ace72..711a774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,42 @@ # Changelog +## 2.0.0 + +### Breaking Changes + +- **`schema_version: 2` is now required.** v1 configs are rejected with a migration pointer. Add `schema_version: 2` at the top of `.josh-sync.yml`. +- **`josh.monorepo_url` is now required.** Set it to your monorepo's git clone URL — `git@host:org/repo.git` (SSH), `ssh://...`, or `https://host/org/repo.git`. Removes the silent inference of the gitea host from the first target's `subrepo_url`, which silently misbehaved when the monorepo and a target lived on different gitea instances. See [ADR-012](docs/adr/012-explicit-monorepo-url.md). +- **`MONOREPO_API` derivation changed.** The default is now `https://{monorepo_url's host}/api/v1/repos/{monorepo_path}`. The env-var override is preserved as an escape hatch for custom API endpoints. + +#### Migration + +```yaml +# Before (v1.x) +josh: + proxy_url: "https://josh.example.com" + monorepo_path: "org/monorepo" + +# After (v2.0.0) +schema_version: 2 +josh: + proxy_url: "https://josh.example.com" + monorepo_path: "org/monorepo" + monorepo_url: "git@gitea.example.com:org/monorepo.git" # required + monorepo_auth: "ssh" # optional, default "https" +``` + +If you previously set the `MONOREPO_API` env var to work around the first-target-host bug, you can remove it — the new derivation gets it right from `monorepo_url`. + +### Features + +- **Multi-host configs supported.** The monorepo and any subrepo may live on different gitea instances. Target ordering no longer affects how the monorepo's host is resolved. +- **SSH clone of the monorepo via `monorepo_auth: ssh`.** Uses the user's ssh-agent / `~/.ssh/config`; no token required for the clone itself. PR creation still uses `GITEA_TOKEN`. +- **Symmetric config shape.** Monorepo and subrepos now both declare a URL plus an auth method. + +### Fixes + +- First-target ordering no longer load-bearing on `MONOREPO_API` derivation (was `gitea_host=.[0].gitea_host` in v1.x). + ## 1.5.0 ### Breaking Changes diff --git a/README.md b/README.md index bd5c0dc..3eb74d9 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,13 @@ Bidirectional monorepo ↔ subrepo sync via [josh-proxy](https://josh-project.gi Create `.josh-sync.yml` in your monorepo root: ```yaml +schema_version: 2 + josh: proxy_url: "https://josh.example.com" monorepo_path: "org/monorepo" + monorepo_url: "git@gitea.example.com:org/monorepo.git" + monorepo_auth: "ssh" # optional, default "https" targets: - name: "billing" diff --git a/VERSION b/VERSION index bc80560..227cea2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.0 +2.0.0 diff --git a/docs/adr/012-explicit-monorepo-url.md b/docs/adr/012-explicit-monorepo-url.md new file mode 100644 index 0000000..46a40df --- /dev/null +++ b/docs/adr/012-explicit-monorepo-url.md @@ -0,0 +1,53 @@ +# ADR-012: Explicit `monorepo_url` (drop first-target-host inference) + +**Status:** Accepted +**Date:** 2026-04 + +## Context + +josh-sync v1.x had no first-class field for the monorepo's git location. To call the gitea PR API and to clone the monorepo for `initial_import` / force-push, the script derived the gitea host from the **first target's `subrepo_url`**: + +```bash +# lib/config.sh (v1.x) +gitea_host=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[0].gitea_host') +export MONOREPO_API="${MONOREPO_API:-https://${gitea_host}/api/v1/repos/${MONOREPO_PATH}}" +``` + +This worked silently for every existing config — studio, kinanah/stores, itkan/sdlc — because in each, target 0's host happens to match the monorepo's host. The portal monorepo broke the assumption: monorepo on `code.atome.ac:atome-portal/portal`, single target `bff` on `code.itkan.io:sdlc/odoo_bff_core`. The derivation produced `https://code.itkan.io/atome-portal/portal.git` (host borrowed from the wrong target), giving a 404 on `josh-sync import`. + +Two issues stack: + +1. **Implicit coupling**: target ordering becomes load-bearing in a non-obvious way. The `MONOREPO_API` env var was the only escape hatch, hidden in shell init. +2. **Asymmetry**: `subrepo_url` is declared explicitly per target with full SSH/HTTPS support; the monorepo had only a `monorepo_path` and was always cloned via HTTPS+token. Local users with SSH keys for the monorepo still had to provide a token. + +### Alternatives considered + +1. **Keep deriving, document the env override loudly.** Cheapest. Leaves the booby trap in place — still fails closed for any future cross-host config. +2. **Add `josh.gitea_host` only.** Fixes the host issue without enabling SSH cloning of the monorepo. Half a fix, and asymmetric with how subrepos are declared. +3. **Add `josh.monorepo_url` (full URL), mirror `subrepo_url` exactly.** Removes the inference; enables SSH clone; symmetric with subrepos. Breaking — every existing config needs to add the field. + +## Decision + +Add a required `josh.monorepo_url` field plus an optional `josh.monorepo_auth` (default `"https"`). Mirror the `subrepo_url` parsing and `subrepo_auth_url()` helper exactly: + +- `parse_config` extracts `MONOREPO_GITEA_HOST` and `MONOREPO_REPO_PATH` from `monorepo_url` using the same jq regex as `subrepo_url`. +- `MONOREPO_API` derives from `MONOREPO_GITEA_HOST` instead of `.[0].gitea_host`. The env-var override stays as a real escape hatch (custom API endpoints). +- `mono_auth_url()` in `lib/auth.sh` mirrors `subrepo_auth_url()`: SSH returns the bare URL, HTTPS injects `${BOT_USER}:${GITEA_TOKEN}@` (auto-converting `git@host:org/repo.git` to `https://host/org/repo.git` first). +- `initial_import()` and the force-push path in `lib/sync.sh` call `mono_auth_url()` instead of constructing the URL inline from `MONOREPO_API`. + +The change is shipped as **v2.0.0** with `schema_version: 2` required. v1 configs are rejected with a migration pointer; configs missing `monorepo_url` are rejected with the field name and an example. + +## Consequences + +**Positive:** +- No more first-target-ordering coupling. Add or reorder targets freely. +- Multi-host setups (monorepo on host A, subrepo on host B) are first-class. +- SSH clone of the monorepo works without a token (the user's ssh-agent / `~/.ssh/config` resolves the key). PR creation still needs `GITEA_TOKEN`. +- Symmetric config: monorepo and subrepos both declare a URL + auth method. + +**Negative:** +- Breaking change. Every existing config must add `schema_version: 2` and `josh.monorepo_url`. Mitigated by the validation `die`s pointing at the field name and changelog. +- Two related-but-distinct fields (`monorepo_path` for the josh-proxy filter; `monorepo_url` for the gitea clone) — kept separate because josh-proxy's path may not always equal the gitea path in custom setups. + +**Risks:** +- Users may set `monorepo_url` to a host that doesn't match their `proxy_url`'s upstream. Detection is out of scope for this ADR; a follow-up `josh-sync preflight` check could compare hosts and warn. diff --git a/docs/adr/README.md b/docs/adr/README.md index eefee96..36add1d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -17,3 +17,4 @@ This directory contains Architecture Decision Records (ADRs) for josh-sync. Each | [009](009-tree-comparison-guard.md) | Tree comparison as sync skip guard | Accepted | | [010](010-onboard-checkpoint-resume.md) | Onboard workflow with checkpoint/resume | Accepted | | [011](011-linearize-fallback-reverse.md) | Linearize fallback for reverse sync | Accepted | +| [012](012-explicit-monorepo-url.md) | Explicit `monorepo_url` (drop first-target-host inference) | Accepted | diff --git a/docs/config-reference.md b/docs/config-reference.md index d13745f..c953083 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -5,17 +5,26 @@ Full reference for `.josh-sync.yml` fields and environment variables. ## `.josh-sync.yml` Structure ```yaml -josh: # josh-proxy settings (required) -targets: # sync targets (required, at least 1) -bot: # bot identity for sync commits (required) +schema_version: 2 # required since v2.0.0 +josh: # josh-proxy + monorepo settings (required) +targets: # sync targets (required, at least 1) +bot: # bot identity for sync commits (required) ``` +## `schema_version` + +| Type | Required | Description | +|------|----------|-------------| +| integer | Yes | Must be `2` for josh-sync v2.x configs. v1 configs are rejected; see [Changelog v2.0.0](../CHANGELOG.md#200) for migration. | + ## `josh` Section | Field | Type | Required | Description | |-------|------|----------|-------------| | `proxy_url` | string | Yes | Josh-proxy URL, no trailing slash. Must start with `http://` or `https://`. | | `monorepo_path` | string | Yes | Repository path as josh-proxy sees it (e.g., `org/monorepo`). | +| `monorepo_url` | string | Yes | Monorepo git clone URL — `git@host:org/repo.git` (SSH), `ssh://...`, or `https://host/org/repo.git`. The gitea host is derived from this URL and used for the PR API and direct clones, decoupled from any target's host. | +| `monorepo_auth` | string | No (default `"https"`) | Clone auth method: `"https"` (injects `${SYNC_BOT_USER}:${SYNC_BOT_TOKEN}` into the URL) or `"ssh"` (uses the user's ssh-agent / `~/.ssh/config`). PR creation always uses `GITEA_TOKEN` regardless. | ## `targets[]` Section @@ -73,7 +82,7 @@ targets: | `JOSH_SYNC_TARGET` | Restrict sync to a single target | All targets | | `JOSH_SYNC_STATE_BRANCH` | Name of the orphan branch for state storage | `josh-sync-state` | | `JOSH_SYNC_DEBUG` | Enable verbose logging (`1` to enable) | `0` | -| `MONOREPO_API` | Override monorepo API URL | Auto-derived from first target's host | +| `MONOREPO_API` | Override the monorepo gitea PR API URL. Escape hatch for custom API endpoints; rarely needed. | `https://{monorepo_url's host}/api/v1/repos/{monorepo_path}` | ## JSON Schema diff --git a/docs/guide.md b/docs/guide.md index fe883c9..a54be32 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -78,9 +78,13 @@ git ls-remote https://josh.example.com/org/monorepo.git HEAD Create `.josh-sync.yml` at the monorepo root. Each target maps a monorepo subfolder to an external subrepo: ```yaml +schema_version: 2 + josh: proxy_url: "https://josh.example.com" # josh-proxy URL (no trailing slash) - monorepo_path: "org/monorepo" # repo path as josh sees it + monorepo_path: "org/monorepo" # repo path as josh-proxy sees it + monorepo_url: "git@gitea.example.com:org/monorepo.git" # monorepo git clone URL + monorepo_auth: "ssh" # optional, default "https" targets: - name: "billing" # unique identifier diff --git a/examples/josh-sync.yml b/examples/josh-sync.yml index 09bb455..2cb4b07 100644 --- a/examples/josh-sync.yml +++ b/examples/josh-sync.yml @@ -1,11 +1,22 @@ # .josh-sync.yml — Multi-target configuration example # Place this at the root of your monorepo. +# Required since v2.0.0. Bump alongside the josh-sync major version when +# breaking config changes ship. +schema_version: 2 + josh: # Your josh-proxy instance URL (no trailing slash) proxy_url: "https://josh.example.com" - # Repo path as josh sees it (org/repo on your Gitea/GitHub) + # Repo path as josh-proxy sees it (org/repo) monorepo_path: "org/monorepo" + # Monorepo git clone URL — git@host:org/repo.git (SSH) or https://host/org/repo.git. + # The gitea host is derived from this URL (used for the PR API and direct clones). + # The monorepo and any subrepo may live on different gitea hosts. + monorepo_url: "git@gitea.example.com:org/monorepo.git" + # Optional. Default "https". Use "ssh" to clone the monorepo via SSH (no token + # needed for the clone — PR creation still requires GITEA_TOKEN). + monorepo_auth: "ssh" targets: - name: "billing" diff --git a/lib/auth.sh b/lib/auth.sh index fd432d4..bf4ce0a 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -2,7 +2,8 @@ # lib/auth.sh — Authenticated URLs, remote queries, and PR creation # # Requires: lib/core.sh and lib/config.sh sourced first -# Expects: JOSH_PROXY_URL, MONOREPO_PATH, BOT_USER, GITEA_TOKEN, JOSH_FILTER, +# Expects: JOSH_PROXY_URL, MONOREPO_PATH, MONOREPO_URL, MONOREPO_AUTH, +# BOT_USER, GITEA_TOKEN, JOSH_FILTER, # SUBREPO_URL, SUBREPO_AUTH, SUBREPO_TOKEN (set by parse_config + load_target) # ─── Josh-Proxy Auth URL ─────────────────────────────────────────── @@ -29,13 +30,24 @@ subrepo_auth_url() { } # ─── Monorepo Auth URL ───────────────────────────────────────────── -# Derives a push-capable HTTPS URL from MONOREPO_API. -# MONOREPO_API shape: https://<host>/api/v1/repos/<org/repo> +# HTTPS: injects user:token into MONOREPO_URL (auto-converts git@ form to https) +# SSH: returns bare MONOREPO_URL (auth via user's ssh-agent / ~/.ssh/config, +# or via GIT_SSH_COMMAND if the caller wired one up for the monorepo host) mono_auth_url() { - local api_host_path - api_host_path=$(echo "$MONOREPO_API" | sed 's|https://||; s|/api/v1/repos/|/|') - echo "https://${BOT_USER}:${GITEA_TOKEN}@${api_host_path}.git" + if [ "${MONOREPO_AUTH:-https}" = "ssh" ]; then + echo "$MONOREPO_URL" + else + local https_url="$MONOREPO_URL" + # Convert git@host:org/repo.git → https://host/org/repo.git so token injection works + if [[ "$https_url" == git@* ]]; then + https_url=$(echo "$https_url" | sed -E 's|^git@([^:]+):(.+)$|https://\1/\2|') + elif [[ "$https_url" == ssh://* ]]; then + https_url=$(echo "$https_url" | sed -E 's|^ssh://[^@]*@([^/]+)/(.+)$|https://\1/\2|') + fi + # shellcheck disable=SC2001 + echo "$https_url" | sed "s|https://|https://${BOT_USER}:${GITEA_TOKEN}@|" + fi } # ─── Remote Queries ───────────────────────────────────────────────── diff --git a/lib/config.sh b/lib/config.sh index bc12366..d8655ee 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -16,16 +16,21 @@ parse_config() { local config_json config_json=$(yq -o json "$config_file") || die "Failed to parse ${config_file}" - # Schema version check (optional field — absent means v1) + # Schema version check — required, must be 2 (v1 configs are no longer supported) local schema_version - schema_version=$(echo "$config_json" | jq -r '.schema_version // 1') - [ "$schema_version" = "1" ] || die "Unsupported schema_version ${schema_version} in ${config_file} (this version of josh-sync supports schema_version 1 — see CHANGELOG.md for migration instructions)" + schema_version=$(echo "$config_json" | jq -r '.schema_version // empty') + [ -n "$schema_version" ] || die "schema_version missing in ${config_file} (required since v2.0.0 — set 'schema_version: 2' at the top of the file; see CHANGELOG.md for migration instructions)" + [ "$schema_version" = "2" ] || die "Unsupported schema_version ${schema_version} in ${config_file} (this version of josh-sync requires schema_version 2 — see CHANGELOG.md for migration instructions)" # Export global values export JOSH_PROXY_URL JOSH_PROXY_URL=$(echo "$config_json" | jq -r '.josh.proxy_url') export MONOREPO_PATH MONOREPO_PATH=$(echo "$config_json" | jq -r '.josh.monorepo_path') + export MONOREPO_URL + MONOREPO_URL=$(echo "$config_json" | jq -r '.josh.monorepo_url') + export MONOREPO_AUTH + MONOREPO_AUTH=$(echo "$config_json" | jq -r '.josh.monorepo_auth // "https"') export BOT_NAME BOT_NAME=$(echo "$config_json" | jq -r '.bot.name') export BOT_EMAIL @@ -35,8 +40,27 @@ parse_config() { [ -n "$JOSH_PROXY_URL" ] && [ "$JOSH_PROXY_URL" != "null" ] || die "josh.proxy_url missing in config" [ -n "$MONOREPO_PATH" ] && [ "$MONOREPO_PATH" != "null" ] || die "josh.monorepo_path missing in config" + [ -n "$MONOREPO_URL" ] && [ "$MONOREPO_URL" != "null" ] || die "josh.monorepo_url is required (added in schema_version 2). Set it to your monorepo's git clone URL, e.g. 'git@code.example.io:org/repo.git'." + [ "$MONOREPO_AUTH" = "ssh" ] || [ "$MONOREPO_AUTH" = "https" ] || die "josh.monorepo_auth must be 'ssh' or 'https' (got '${MONOREPO_AUTH}')" [ -n "$BOT_TRAILER" ] && [ "$BOT_TRAILER" != "null" ] || die "bot.trailer missing in config" + # Derive monorepo gitea_host and repo_path from monorepo_url (mirrors target derivation below) + local monorepo_meta + monorepo_meta=$(jq -nr --arg url "$MONOREPO_URL" ' + $url as $u | + if ($u | test("^ssh://")) then + ($u | capture("ssh://[^@]*@(?<h>[^/]+)/(?<p>.+?)(\\.git)?$") // {h: "", p: ""}) + elif ($u | test("^git@")) then + ($u | capture("git@(?<h>[^:/]+)[:/](?<p>.+?)(\\.git)?$") // {h: "", p: ""}) + elif ($u | test("^https?://")) then + ($u | capture("https?://(?<h>[^/]+)/(?<p>.+?)(\\.git)?$") // {h: "", p: ""}) + else + {h: "", p: ""} + end | "\(.h) \(.p)"') + export MONOREPO_GITEA_HOST="${monorepo_meta% *}" + export MONOREPO_REPO_PATH="${monorepo_meta#* }" + [ -n "$MONOREPO_GITEA_HOST" ] || die "Could not derive gitea host from josh.monorepo_url='${MONOREPO_URL}' (expected git@host:org/repo.git or https://host/org/repo.git)" + # Enrich targets with derived fields (gitea_host, subrepo_repo_path, auto josh_filter) export JOSH_SYNC_TARGETS JOSH_SYNC_TARGETS=$(echo "$config_json" | jq '[.targets[] | . + @@ -73,10 +97,8 @@ parse_config() { export GITEA_TOKEN="${GITEA_TOKEN:-${SYNC_BOT_TOKEN:-}}" export BOT_USER="${BOT_USER:-${SYNC_BOT_USER:-}}" - # Monorepo API URL (derived from first target's host, overridable via env) - local gitea_host - gitea_host=$(echo "$JOSH_SYNC_TARGETS" | jq -r '.[0].gitea_host') - export MONOREPO_API="${MONOREPO_API:-https://${gitea_host}/api/v1/repos/${MONOREPO_PATH}}" + # Monorepo API URL (derived from monorepo_url's host, overridable via env for custom API endpoints) + export MONOREPO_API="${MONOREPO_API:-https://${MONOREPO_GITEA_HOST}/api/v1/repos/${MONOREPO_PATH}}" log "INFO" "Config loaded: $(echo "$JOSH_SYNC_TARGETS" | jq 'length') target(s)" } diff --git a/lib/sync.sh b/lib/sync.sh index bd6ce33..045a106 100644 --- a/lib/sync.sh +++ b/lib/sync.sh @@ -475,12 +475,7 @@ initial_import() { log "INFO" "=== Initial import: subrepo/${subrepo_branch} → mono/${mono_branch}:${subfolder}/ ===" # 1. Clone monorepo (directly, not through josh — we need the real repo) - local mono_auth_url - local api_host_path - api_host_path=$(echo "$MONOREPO_API" | sed 's|https://||; s|/api/v1/repos/|/|') - mono_auth_url="https://${BOT_USER}:${GITEA_TOKEN}@${api_host_path}.git" - - git clone "$mono_auth_url" \ + git clone "$(mono_auth_url)" \ --branch "$mono_branch" --single-branch \ "${work_dir}/monorepo" || die "Failed to clone monorepo" diff --git a/schema/config-schema.json b/schema/config-schema.json index 4afeb72..eff6966 100644 --- a/schema/config-schema.json +++ b/schema/config-schema.json @@ -3,16 +3,16 @@ "title": "josh-sync configuration", "description": "Configuration for bidirectional monorepo ↔ subrepo sync via josh-proxy", "type": "object", - "required": ["josh", "targets", "bot"], + "required": ["schema_version", "josh", "targets", "bot"], "properties": { "schema_version": { "type": "integer", - "const": 1, - "description": "Schema version. Bump when config fields are removed or renamed (matches josh-sync major version)." + "const": 2, + "description": "Schema version. Required. Must be 2 for josh-sync v2.x configs." }, "josh": { "type": "object", - "required": ["proxy_url", "monorepo_path"], + "required": ["proxy_url", "monorepo_path", "monorepo_url"], "properties": { "proxy_url": { "type": "string", @@ -21,7 +21,17 @@ }, "monorepo_path": { "type": "string", - "description": "Repo path as josh sees it (org/repo)" + "description": "Repo path as josh-proxy sees it (org/repo)" + }, + "monorepo_url": { + "type": "string", + "description": "Monorepo git clone URL — git@host:org/repo.git (SSH) or https://host/org/repo.git. The gitea host is derived from this URL." + }, + "monorepo_auth": { + "type": "string", + "enum": ["https", "ssh"], + "default": "https", + "description": "Auth method for cloning the monorepo. SSH uses the user's ssh-agent / ~/.ssh/config; HTTPS injects ${SYNC_BOT_USER}:${SYNC_BOT_TOKEN} into the URL." } } }, diff --git a/tests/fixtures/minimal.yml b/tests/fixtures/minimal.yml index d767d4e..232c5d3 100644 --- a/tests/fixtures/minimal.yml +++ b/tests/fixtures/minimal.yml @@ -1,7 +1,9 @@ # Test fixture: minimal single-target config +schema_version: 2 josh: proxy_url: "https://josh.test.local" monorepo_path: "org/repo" + monorepo_url: "git@gitea.test.local:org/repo.git" targets: - name: "example" diff --git a/tests/fixtures/multi-target.yml b/tests/fixtures/multi-target.yml index d0ced19..8e89b4b 100644 --- a/tests/fixtures/multi-target.yml +++ b/tests/fixtures/multi-target.yml @@ -1,7 +1,9 @@ # Test fixture: multi-target config +schema_version: 2 josh: proxy_url: "https://josh.test.local" monorepo_path: "org/monorepo" + monorepo_url: "git@gitea.test.local:org/monorepo.git" targets: - name: "app-a" diff --git a/tests/unit/auth.bats b/tests/unit/auth.bats index e94b5cc..1517170 100644 --- a/tests/unit/auth.bats +++ b/tests/unit/auth.bats @@ -37,3 +37,39 @@ setup() { url=$(subrepo_auth_url) [ "$url" = "git@gitea.test.local:ext/app-a.git" ] } + +@test "mono_auth_url returns bare URL for SSH" { + export MONOREPO_AUTH="ssh" + export MONOREPO_URL="git@code.example.io:org/monorepo.git" + + local url + url=$(mono_auth_url) + [ "$url" = "git@code.example.io:org/monorepo.git" ] +} + +@test "mono_auth_url injects credentials for HTTPS git@-form monorepo_url" { + export MONOREPO_AUTH="https" + export MONOREPO_URL="git@code.example.io:org/monorepo.git" + + local url + url=$(mono_auth_url) + [ "$url" = "https://sync-bot:test-token-123@code.example.io/org/monorepo.git" ] +} + +@test "mono_auth_url injects credentials for HTTPS https-form monorepo_url" { + export MONOREPO_AUTH="https" + export MONOREPO_URL="https://code.example.io/org/monorepo.git" + + local url + url=$(mono_auth_url) + [ "$url" = "https://sync-bot:test-token-123@code.example.io/org/monorepo.git" ] +} + +@test "mono_auth_url defaults to HTTPS when MONOREPO_AUTH unset" { + unset MONOREPO_AUTH + export MONOREPO_URL="https://code.example.io/org/monorepo.git" + + local url + url=$(mono_auth_url) + [ "$url" = "https://sync-bot:test-token-123@code.example.io/org/monorepo.git" ] +} diff --git a/tests/unit/config.bats b/tests/unit/config.bats index 7c306d0..c995eba 100644 --- a/tests/unit/config.bats +++ b/tests/unit/config.bats @@ -104,3 +104,151 @@ setup() { run bash -c 'source "$JOSH_SYNC_ROOT/lib/core.sh"; source "$JOSH_SYNC_ROOT/lib/config.sh"; parse_config "nonexistent.yml"' [ "$status" -ne 0 ] } + +@test "parse_config fails when schema_version is missing (no v1 fallback)" { + cd "$(mktemp -d)" + cat > .josh-sync.yml <<'YAML' +josh: + proxy_url: "https://josh.test.local" + monorepo_path: "org/repo" + monorepo_url: "git@gitea.test.local:org/repo.git" +targets: + - name: "x" + subfolder: "x" + subrepo_url: "git@gitea.test.local:ext/x.git" + branches: { main: main } +bot: + name: "b" + email: "b@b" + trailer: "T" +YAML + run bash -c 'source "$JOSH_SYNC_ROOT/lib/core.sh"; source "$JOSH_SYNC_ROOT/lib/config.sh"; parse_config ".josh-sync.yml"' + [ "$status" -ne 0 ] + [[ "$output" == *"schema_version missing"* ]] +} + +@test "parse_config rejects schema_version: 1" { + cd "$(mktemp -d)" + cat > .josh-sync.yml <<'YAML' +schema_version: 1 +josh: + proxy_url: "https://josh.test.local" + monorepo_path: "org/repo" + monorepo_url: "git@gitea.test.local:org/repo.git" +targets: + - name: "x" + subfolder: "x" + subrepo_url: "git@gitea.test.local:ext/x.git" + branches: { main: main } +bot: + name: "b" + email: "b@b" + trailer: "T" +YAML + run bash -c 'source "$JOSH_SYNC_ROOT/lib/core.sh"; source "$JOSH_SYNC_ROOT/lib/config.sh"; parse_config ".josh-sync.yml"' + [ "$status" -ne 0 ] + [[ "$output" == *"Unsupported schema_version"* ]] || [[ "$output" == *"requires schema_version 2"* ]] +} + +@test "parse_config fails when josh.monorepo_url is missing" { + cd "$(mktemp -d)" + cat > .josh-sync.yml <<'YAML' +schema_version: 2 +josh: + proxy_url: "https://josh.test.local" + monorepo_path: "org/repo" +targets: + - name: "x" + subfolder: "x" + subrepo_url: "git@gitea.test.local:ext/x.git" + branches: { main: main } +bot: + name: "b" + email: "b@b" + trailer: "T" +YAML + run bash -c 'source "$JOSH_SYNC_ROOT/lib/core.sh"; source "$JOSH_SYNC_ROOT/lib/config.sh"; parse_config ".josh-sync.yml"' + [ "$status" -ne 0 ] + [[ "$output" == *"monorepo_url"* ]] +} + +@test "parse_config exports MONOREPO_URL, MONOREPO_AUTH, MONOREPO_GITEA_HOST, MONOREPO_REPO_PATH from SSH form" { + cd "$(mktemp -d)" + cp "$FIXTURES/minimal.yml" .josh-sync.yml + + parse_config ".josh-sync.yml" + + [ "$MONOREPO_URL" = "git@gitea.test.local:org/repo.git" ] + [ "$MONOREPO_AUTH" = "https" ] + [ "$MONOREPO_GITEA_HOST" = "gitea.test.local" ] + [ "$MONOREPO_REPO_PATH" = "org/repo" ] +} + +@test "parse_config derives MONOREPO_GITEA_HOST from HTTPS monorepo_url" { + cd "$(mktemp -d)" + cat > .josh-sync.yml <<'YAML' +schema_version: 2 +josh: + proxy_url: "https://josh.test.local" + monorepo_path: "org/repo" + monorepo_url: "https://gitea.example.io/org/repo.git" +targets: + - name: "x" + subfolder: "x" + subrepo_url: "git@gitea.test.local:ext/x.git" + branches: { main: main } +bot: + name: "b" + email: "b@b" + trailer: "T" +YAML + parse_config ".josh-sync.yml" + [ "$MONOREPO_GITEA_HOST" = "gitea.example.io" ] + [ "$MONOREPO_REPO_PATH" = "org/repo" ] +} + +@test "parse_config derives MONOREPO_API from monorepo_url's host (cross-host: monorepo and target on different hosts)" { + cd "$(mktemp -d)" + cat > .josh-sync.yml <<'YAML' +schema_version: 2 +josh: + proxy_url: "https://josh.example.io" + monorepo_path: "org/repo" + monorepo_url: "git@code.example.io:org/repo.git" +targets: + - name: "x" + subfolder: "x" + subrepo_url: "git@other-host.io:ext/x.git" + branches: { main: main } +bot: + name: "b" + email: "b@b" + trailer: "T" +YAML + parse_config ".josh-sync.yml" + [ "$MONOREPO_API" = "https://code.example.io/api/v1/repos/org/repo" ] +} + +@test "parse_config rejects invalid monorepo_auth value" { + cd "$(mktemp -d)" + cat > .josh-sync.yml <<'YAML' +schema_version: 2 +josh: + proxy_url: "https://josh.test.local" + monorepo_path: "org/repo" + monorepo_url: "git@gitea.test.local:org/repo.git" + monorepo_auth: "weird" +targets: + - name: "x" + subfolder: "x" + subrepo_url: "git@gitea.test.local:ext/x.git" + branches: { main: main } +bot: + name: "b" + email: "b@b" + trailer: "T" +YAML + run bash -c 'source "$JOSH_SYNC_ROOT/lib/core.sh"; source "$JOSH_SYNC_ROOT/lib/config.sh"; parse_config ".josh-sync.yml"' + [ "$status" -ne 0 ] + [[ "$output" == *"monorepo_auth"* ]] +} From f2e39be5e19e970c051538e83ee0742379d363ad Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Tue, 26 May 2026 10:38:20 +0300 Subject: [PATCH 36/37] "#" --- CHANGELOG.md | 16 ++ Makefile | 2 +- README.md | 2 + VERSION | 2 +- bin/josh-sync | 41 +++ docs/adr/013-non-destructive-adoption.md | 55 ++++ docs/adr/README.md | 1 + docs/guide.md | 59 +++- lib/adopt.sh | 335 +++++++++++++++++++++++ tests/unit/adopt.bats | 92 +++++++ 10 files changed, 588 insertions(+), 17 deletions(-) create mode 100644 docs/adr/013-non-destructive-adoption.md create mode 100644 lib/adopt.sh create mode 100644 tests/unit/adopt.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 711a774..31ad6e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 2.1.0 + +### Breaking Changes + +_None._ + +### Features + +- **`adopt` command for non-destructive existing subrepos**: `josh-sync adopt <target>` imports existing subrepo content into the monorepo, waits for the import PR to merge, then creates an adoption merge commit on the subrepo. The adoption merge uses the Josh-filtered monorepo commit as first parent and the existing subrepo commit as second parent, preserving old subrepo history while establishing Josh-compatible ancestry. +- **Resumable adoption state**: adoption progress is stored on `josh-sync-state` at `<target>/adopt.json`, separate from sync and onboard state. + +### Safety + +- Adoption requires the subrepo tree to match the Josh-filtered monorepo tree before creating the merge commit. +- Adoption pushes with a normal fast-forward push only. It never force-pushes. + ## 2.0.0 ### Breaking Changes diff --git a/Makefile b/Makefile index a748e3f..3b4de3e 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ dist/josh-sync: bin/josh-sync lib/*.sh VERSION @echo '# Generated by: make build' >> dist/josh-sync @echo '' >> dist/josh-sync @# Inline all library modules (strip shebangs and source directives) - @for f in lib/core.sh lib/config.sh lib/auth.sh lib/state.sh lib/sync.sh lib/onboard.sh; do \ + @for f in lib/core.sh lib/config.sh lib/auth.sh lib/state.sh lib/sync.sh lib/adopt.sh lib/onboard.sh; do \ echo "# --- $$f ---" >> dist/josh-sync; \ grep -v '^#!/' "$$f" | grep -v '^# shellcheck source=' >> dist/josh-sync; \ echo '' >> dist/josh-sync; \ diff --git a/README.md b/README.md index 3eb74d9..01966b2 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ CI workflows pin to a floating major tag (e.g. `@v1`). A breaking change bumps t josh-sync sync [--forward|--reverse] [--force] [--target NAME[,NAME]] [--branch BRANCH] josh-sync preflight josh-sync import <target> +josh-sync adopt <target> [--restart] josh-sync reset <target> josh-sync onboard <target> [--restart] josh-sync migrate-pr <target> [PR#...] [--all] @@ -97,6 +98,7 @@ josh-sync state reset <target> [branch] - **Forward sync** (mono → subrepo): pushes directly if clean, creates conflict PR if not. Uses `--force-with-lease` for safety. - **Reverse sync** (subrepo → mono): creates a PR by default. Add `--force` to bypass the PR and push directly to the mono branch (destructive). +- **Adoption**: existing active subrepos can be connected to josh-sync with a normal fast-forward merge commit, preserving old subrepo history and avoiding force-push. - **File exclusion**: `exclude` patterns are embedded inline in the josh-proxy URL. Excluded files exist only in the monorepo. - **Filter reconciliation**: Changing the exclude list auto-creates a merge commit that connects old and new histories — no force-push needed. - **Loop prevention**: `Josh-Sync-Origin:` git trailer filters out bot commits. diff --git a/VERSION b/VERSION index 227cea2..7ec1d6d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 +2.1.0 diff --git a/bin/josh-sync b/bin/josh-sync index 2d089b3..d126e3b 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -8,6 +8,7 @@ # sync Run forward and/or reverse sync # preflight Validate config, connectivity, auth # import <target> Initial import: pull subrepo into monorepo +# adopt <target> Adopt existing subrepo history without rewriting it # reset <target> Reset subrepo to josh-filtered view # onboard <target> Import existing subrepo into monorepo (interactive) # migrate-pr <target> [PR#...] [--all] Move PRs from archived to new subrepo @@ -41,6 +42,8 @@ source "${JOSH_LIB_DIR}/auth.sh" source "${JOSH_LIB_DIR}/state.sh" # shellcheck source=../lib/sync.sh source "${JOSH_LIB_DIR}/sync.sh" +# shellcheck source=../lib/adopt.sh +source "${JOSH_LIB_DIR}/adopt.sh" # shellcheck source=../lib/onboard.sh source "${JOSH_LIB_DIR}/onboard.sh" @@ -72,6 +75,7 @@ Commands: sync Run forward and/or reverse sync preflight Validate config, connectivity, auth, workflow coverage import <target> Initial import: pull existing subrepo into monorepo (creates PR) + adopt <target> Adopt existing subrepo history without rewriting it reset <target> Reset subrepo to josh-filtered view (after merging import PR) onboard <target> Import existing subrepo into monorepo (interactive, resumable) migrate-pr <target> [PR#...] [--all] Move PRs from archived to new subrepo @@ -530,6 +534,42 @@ cmd_import() { done } +# ─── Adopt Command ───────────────────────────────────────────────── + +cmd_adopt() { + local config_file=".josh-sync.yml" + local target_name="" + local restart=false + + while [ $# -gt 0 ]; do + case "$1" in + --config) config_file="$2"; shift 2 ;; + --debug) export JOSH_SYNC_DEBUG=1; shift ;; + --restart) restart=true; shift ;; + -*) die "Unknown flag: $1" ;; + *) target_name="$1"; shift ;; + esac + done + + if [ -z "$target_name" ]; then + echo "Usage: josh-sync adopt <target> [--restart]" >&2 + parse_config "$config_file" + echo "Available targets:" >&2 + echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2 + exit 1 + fi + + parse_config "$config_file" + + local target_json + target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)') + [ -n "$target_json" ] || die "Target '${target_name}' not found in config" + + log "INFO" "══════ Adopt target: ${target_name} ══════" + load_target "$target_json" + adopt_flow "$target_json" "$restart" +} + # ─── Reset Command ───────────────────────────────────────────────── cmd_reset() { @@ -886,6 +926,7 @@ main() { sync) cmd_sync "$@" ;; preflight) cmd_preflight "$@" ;; import) cmd_import "$@" ;; + adopt) cmd_adopt "$@" ;; reset) cmd_reset "$@" ;; onboard) cmd_onboard "$@" ;; migrate-pr) cmd_migrate_pr "$@" ;; diff --git a/docs/adr/013-non-destructive-adoption.md b/docs/adr/013-non-destructive-adoption.md new file mode 100644 index 0000000..a744b77 --- /dev/null +++ b/docs/adr/013-non-destructive-adoption.md @@ -0,0 +1,55 @@ +# ADR-013: Non-destructive adoption merge for existing subrepos + +**Status:** Accepted +**Date:** 2026-04 + +## Context + +Existing subrepos often already have real history, open PRs, and developer clones. +The original `import` then `reset` onboarding path establishes Josh-compatible +history by force-pushing the Josh-filtered monorepo history onto the subrepo. +That is correct for empty replacement repos, but it rewrites the active subrepo +branch and invalidates local clones. + +We need a path for active subrepos where history must remain available on the +same branch and where open PR branches should stay based on the existing history. + +## Decision + +Add `josh-sync adopt <target>` as a separate workflow from `onboard` and `reset`. +Adoption imports the subrepo content into the monorepo, waits for the import PR +to merge, then creates one merge commit on each configured subrepo branch: + +1. First parent: Josh-filtered monorepo HEAD +2. Second parent: existing subrepo HEAD +3. Tree: Josh-filtered monorepo tree + +Josh follows first-parent history back to the monorepo, while Git still keeps +the old subrepo history reachable through parent 2. The push is a normal +fast-forward push from the existing subrepo HEAD to the adoption merge commit. +No force-push is used. + +Before creating the adoption merge, josh-sync requires the existing subrepo tree +to match the Josh-filtered monorepo tree. If the trees differ, adoption aborts +and the user must merge the import PR or reconcile the branch contents first. + +Adoption state is stored separately at `<target>/adopt.json` on the +`josh-sync-state` branch. + +## Consequences + +**Positive:** + +- Existing subrepo history remains on the active branch. +- Existing clones can fast-forward instead of hard-resetting or re-cloning. +- Open PR branches remain based on reachable history. +- Josh-compatible ancestry is established without a destructive rewrite. + +**Negative:** + +- Adoption adds one synthetic merge commit to each adopted subrepo branch. +- Strict "no subrepo commit at all" adoption is impossible if the existing + subrepo branch must become connected to the Josh-filtered history without a + rewrite. A commit is needed to join the two histories. +- Tree equality is strict. If the monorepo import differs from the subrepo + content, adoption stops until the user resolves the mismatch. diff --git a/docs/adr/README.md b/docs/adr/README.md index 36add1d..a570fc1 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -18,3 +18,4 @@ This directory contains Architecture Decision Records (ADRs) for josh-sync. Each | [010](010-onboard-checkpoint-resume.md) | Onboard workflow with checkpoint/resume | Accepted | | [011](011-linearize-fallback-reverse.md) | Linearize fallback for reverse sync | Accepted | | [012](012-explicit-monorepo-url.md) | Explicit `monorepo_url` (drop first-target-host inference) | Accepted | +| [013](013-non-destructive-adoption.md) | Non-destructive adoption merge for existing subrepos | Accepted | diff --git a/docs/guide.md b/docs/guide.md index a54be32..540b685 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -224,14 +224,40 @@ For a new monorepo before import, preflight may warn that subfolders don't exist ## Step 5: Import Existing Subrepos -This is the critical onboarding step. There are two approaches: +This is the critical onboarding step. There are three approaches: -- **`josh-sync onboard`** (recommended) — interactive, resumable, preserves open PRs -- **Manual `import` → merge → `reset`** — lower-level, for automation or when there are no open PRs to preserve +- **`josh-sync adopt`** (recommended for active existing subrepos) — non-destructive, resumable, preserves existing subrepo history +- **`josh-sync onboard`** — destructive replacement-repo workflow for teams that intentionally archive the old repo +- **Manual `import` → merge → `reset`** — lower-level destructive path for automation or empty replacement repos -### Option A: Onboard (recommended) +### Option A: Adopt (recommended for active subrepos) -The `onboard` command walks you through the entire process interactively, with checkpoint/resume at every step. +Use `adopt` when the subrepo already exists, developers already clone it, or open PR branches should remain based on existing history. + +```bash +josh-sync adopt billing +``` + +The command will: +1. **Import** — copies current subrepo content into the monorepo and creates import PRs (one per branch) +2. **Wait for merge** — shows PR numbers and waits for you to merge them +3. **Verify trees** — requires the existing subrepo tree to match the Josh-filtered monorepo tree +4. **Adopt** — creates a merge commit on the subrepo with Josh-filtered HEAD as parent 1 and existing subrepo HEAD as parent 2 +5. **Push** — pushes the adoption merge with a normal fast-forward push, never force-push + +The adoption merge preserves the old subrepo history while giving Josh a first-parent path back to the monorepo. If interrupted, re-run `josh-sync adopt billing` to resume. Use `--restart` to start over. + +After adoption, developers can update existing clones with a normal fast-forward: + +```bash +git fetch origin +git checkout main +git merge --ff-only origin/main +``` + +### Option B: Onboard with replacement repo + +The `onboard` command walks through the destructive replacement-repo process interactively, with checkpoint/resume at every step. **Before you start:** @@ -272,13 +298,13 @@ josh-sync migrate-pr billing 5 8 12 PR migration works by fetching the diff from the archived repo's PR, applying it to the new repo, and creating a new PR. File content is identical after reset, so patches apply cleanly. -### Option B: Manual import → merge → reset +### Option C: Manual import → merge → reset -Use this when the subrepo has no open PRs to preserve, or for scripted automation. +Use this for scripted automation or when you intentionally want to replace subrepo history. > Do this **one target at a time** to keep PRs reviewable. -#### 5b-1. Import +#### 5c-1. Import ```bash josh-sync import billing @@ -293,13 +319,13 @@ This: Review the import PR — check for leaked credentials, environment-specific config, or files that shouldn't be in the monorepo. -#### 5b-2. Merge the import PR +#### 5c-2. Merge the import PR Merge the PR using your Git platform's UI. This lands the subrepo content into the monorepo's main branch. > At this point, the monorepo has the content but the histories are disconnected. Sync will **not** work until you complete the reset step. -#### 5b-3. Reset +#### 5c-3. Reset ```bash josh-sync reset billing @@ -326,7 +352,7 @@ git checkout stage && git reset --hard origin/stage # repeat for each branch Or simply delete and re-clone the subrepo. Local-only branches (not pushed to the remote) will be lost either way. -#### 5b-4. Repeat for each target +#### 5c-4. Repeat for each target ``` For each target: @@ -337,13 +363,13 @@ For each target: ### Verify -After all targets are imported and reset (whichever option you used): +After all targets are adopted or reset: ```bash # Check all targets show state josh-sync status -# Test forward sync — should return "skip" (trees are identical after reset) +# Test forward sync — should return "skip" (trees are identical after adoption/reset) josh-sync sync --forward --target billing # Test reverse sync — should return "skip" (no new human commits) @@ -620,9 +646,12 @@ To add a new subrepo after initial setup: 1. Add the target to `.josh-sync.yml` 2. Update the forward workflow's `paths:` list to include the new subfolder 3. Commit and push -4. Import the target: +4. Import or adopt the target: ```bash - # Recommended: interactive onboard (preserves open PRs) + # Recommended for an existing active subrepo + josh-sync adopt new-target + + # Replacement-repo workflow josh-sync onboard new-target # Or manual: import → merge PR → reset diff --git a/lib/adopt.sh b/lib/adopt.sh new file mode 100644 index 0000000..b779aa7 --- /dev/null +++ b/lib/adopt.sh @@ -0,0 +1,335 @@ +#!/usr/bin/env bash +# lib/adopt.sh — Non-destructive adoption of existing subrepo history +# +# Provides: +# adopt_flow() — Import → wait for merge → adoption merge per branch +# adopt_branch() — Create the adoption merge and push it fast-forward only +# +# Adopt state is stored on the josh-sync-state branch at <target>/adopt.json. +# Steps: start → importing → waiting-for-merge → adopting → complete +# +# Requires: lib/core.sh, lib/config.sh, lib/auth.sh, lib/state.sh, lib/sync.sh sourced + +# ─── Adopt State Helpers ───────────────────────────────────────── + +read_adopt_state() { + local target_name="${1:-$JOSH_SYNC_TARGET_NAME}" + git fetch origin "$STATE_BRANCH" 2>/dev/null || true + git show "origin/${STATE_BRANCH}:${target_name}/adopt.json" 2>/dev/null || echo '{}' +} + +write_adopt_state() { + local target_name="${1:-$JOSH_SYNC_TARGET_NAME}" + local state_json="$2" + local key="${target_name}/adopt" + local tmp_dir + tmp_dir=$(mktemp -d) + + if git rev-parse "origin/${STATE_BRANCH}" >/dev/null 2>&1; then + git worktree add "$tmp_dir" "origin/${STATE_BRANCH}" 2>/dev/null + else + git worktree add --detach "$tmp_dir" 2>/dev/null + (cd "$tmp_dir" && git checkout --orphan "$STATE_BRANCH" && git rm -rf . 2>/dev/null || true) + fi + + mkdir -p "$(dirname "${tmp_dir}/${key}.json")" + echo "$state_json" | jq '.' > "${tmp_dir}/${key}.json" + + ( + cd "$tmp_dir" || exit + git add -A + if ! git diff --cached --quiet 2>/dev/null; then + git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ + commit -m "adopt: update ${target_name}" + git push origin "HEAD:${STATE_BRANCH}" || log "WARN" "Failed to push adopt state" + fi + ) + + git worktree remove "$tmp_dir" 2>/dev/null || rm -rf "$tmp_dir" +} + +# ─── Adopt Branch ──────────────────────────────────────────────── +# Establishes shared ancestry without rewriting the existing subrepo branch. +# +# Preconditions: +# - The subrepo branch and josh-filtered monorepo branch must have identical trees. +# +# Resulting commit: +# - parent 1: josh-filtered HEAD, so josh can follow first-parent history +# - parent 2: existing subrepo HEAD, so old subrepo history stays reachable +# - tree: josh-filtered tree +# +# Returns: adopted | already-adopted | tree-mismatch | push-rejected | missing-branch + +adopt_branch() { + local mono_branch="$SYNC_BRANCH_MONO" + local subrepo_branch="$SYNC_BRANCH_SUBREPO" + local work_dir + work_dir=$(mktemp -d) + # shellcheck disable=SC2064 # Intentional early expansion — work_dir is local + trap "rm -rf '$work_dir'" EXIT + + log "INFO" "=== Adopt: subrepo/${subrepo_branch} ↔ mono/${mono_branch} ===" + + local remote_subrepo_sha + remote_subrepo_sha=$(git ls-remote "$(subrepo_auth_url)" "refs/heads/${subrepo_branch}" | awk '{print $1}') \ + || die "Failed to reach subrepo (check SSH key / auth)" + if [ -z "$remote_subrepo_sha" ]; then + log "ERROR" "Subrepo branch ${subrepo_branch} does not exist" + echo "missing-branch" + return + fi + + git clone "$(subrepo_auth_url)" \ + --branch "$subrepo_branch" --single-branch \ + "${work_dir}/subrepo" || die "Failed to clone subrepo" + + cd "${work_dir}/subrepo" || exit + git config user.name "$BOT_NAME" + git config user.email "$BOT_EMAIL" + + local subrepo_head subrepo_tree + subrepo_head=$(git rev-parse HEAD) + # shellcheck disable=SC1083 # {tree} is git syntax, not shell brace expansion + subrepo_tree=$(git rev-parse "HEAD^{tree}") + log "INFO" "Subrepo HEAD: ${subrepo_head:0:12}" + + git remote add josh-filtered "$(josh_auth_url)" + git fetch josh-filtered "$mono_branch" || die "Failed to fetch from josh-proxy" + + local josh_head josh_tree + josh_head=$(git rev-parse "josh-filtered/${mono_branch}") + # shellcheck disable=SC1083 + josh_tree=$(git rev-parse "josh-filtered/${mono_branch}^{tree}") + log "INFO" "Josh-filtered HEAD: ${josh_head:0:12}" + + if [ "$subrepo_tree" != "$josh_tree" ]; then + log "ERROR" "Tree mismatch: merge the import PR and ensure subrepo/${subrepo_branch} matches mono/${mono_branch} before adopting" + echo "tree-mismatch" + return + fi + + if git merge-base --is-ancestor "$josh_head" "$subrepo_head" \ + && git merge-base --is-ancestor "$subrepo_head" "$josh_head"; then + log "INFO" "Subrepo and josh-filtered histories already point to the same commit" + echo "already-adopted" + return + fi + + if git merge-base --is-ancestor "$josh_head" "$subrepo_head"; then + log "INFO" "Josh-filtered HEAD is already an ancestor of subrepo HEAD" + echo "already-adopted" + return + fi + + local merge_commit + merge_commit=$(git commit-tree "$josh_tree" \ + -p "$josh_head" \ + -p "$subrepo_head" \ + -m "Adopt ${JOSH_SYNC_TARGET_NAME} for josh-sync + +${BOT_TRAILER}: adopt/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)") + + git reset --hard "$merge_commit" >&2 + log "INFO" "Created adoption merge: ${merge_commit:0:12}" + + if git push "$(subrepo_auth_url)" "HEAD:refs/heads/${subrepo_branch}"; then + log "INFO" "Adoption complete: pushed fast-forward merge to subrepo/${subrepo_branch}" + echo "adopted" + else + log "WARN" "Fast-forward push rejected — subrepo changed during adoption" + echo "push-rejected" + fi +} + +# ─── Adopt Flow ────────────────────────────────────────────────── +# Interactive orchestrator with checkpoint/resume. +# Usage: adopt_flow <target_json> <restart> + +adopt_flow() { + local target_json="$1" + local restart="${2:-false}" + local target_name="$JOSH_SYNC_TARGET_NAME" + + local adopt_state current_step + adopt_state=$(read_adopt_state "$target_name") + current_step=$(echo "$adopt_state" | jq -r '.step // "start"') + + if [ "$restart" = true ]; then + log "INFO" "Restarting adopt from scratch" + current_step="start" + adopt_state='{}' + fi + + log "INFO" "Adopt step: ${current_step}" + + if [ "$current_step" = "start" ]; then + echo "" >&2 + echo "=== Adopting ${target_name} ===" >&2 + echo "" >&2 + echo "This keeps the existing subrepo history and adds one adoption merge commit per branch." >&2 + echo "No force-push is used. Existing open PR branches remain based on the old history." >&2 + echo "" >&2 + + adopt_state=$(jq -n \ + --arg step "importing" \ + --arg mode "adopt" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{step:$step, mode:$mode, import_prs:{}, adopted_branches:[], timestamp:$ts}') + write_adopt_state "$target_name" "$adopt_state" + current_step="importing" + fi + + if [ "$current_step" = "importing" ]; then + echo "" >&2 + log "INFO" "Step 1: Importing current subrepo content into monorepo..." + + local branches import_prs + branches=$(echo "$target_json" | jq -r '.branches | keys[]') + import_prs=$(echo "$adopt_state" | jq -r '.import_prs // {}') + + for branch in $branches; do + local mapped + mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') + [ -z "$mapped" ] && continue + + if echo "$import_prs" | jq -e --arg b "$branch" 'has($b)' >/dev/null 2>&1; then + log "INFO" "Import PR already recorded for ${branch} — skipping" + continue + fi + + export SYNC_BRANCH_MONO="$branch" + export SYNC_BRANCH_SUBREPO="$mapped" + + local result + result=$(initial_import) + log "INFO" "Import result for ${branch}: ${result}" + + if [ "$result" = "pr-created" ]; then + local prs pr_number + prs=$(list_open_prs "$MONOREPO_API" "$GITEA_TOKEN") + pr_number=$(echo "$prs" | jq -r --arg t "$target_name" --arg b "$branch" \ + '[.[] | select(.title | test("\\[Import\\] " + $t + ":")) | select(.base.ref == $b)] | .[0].number // empty') + + [ -n "$pr_number" ] || die "Could not find import PR number for ${branch}; cannot safely continue adoption" + import_prs=$(echo "$import_prs" | jq --arg b "$branch" --arg n "$pr_number" '. + {($b): ($n | tonumber)}') + log "INFO" "Import PR for ${branch}: #${pr_number}" + fi + + adopt_state=$(echo "$adopt_state" | jq --argjson prs "$import_prs" '.import_prs = $prs') + write_adopt_state "$target_name" "$adopt_state" + done + + adopt_state=$(echo "$adopt_state" | jq \ + --arg step "waiting-for-merge" \ + --argjson prs "$import_prs" \ + '.step = $step | .import_prs = $prs') + write_adopt_state "$target_name" "$adopt_state" + current_step="waiting-for-merge" + fi + + if [ "$current_step" = "waiting-for-merge" ]; then + echo "" >&2 + log "INFO" "Step 2: Waiting for import PR(s) to be merged..." + + local import_prs pr_count + import_prs=$(echo "$adopt_state" | jq -r '.import_prs') + pr_count=$(echo "$import_prs" | jq 'length') + + if [ "$pr_count" -gt 0 ]; then + echo "" >&2 + echo "Import PRs to merge:" >&2 + echo "$import_prs" | jq -r 'to_entries[] | " \(.key): PR #\(.value)"' >&2 + echo "" >&2 + echo "Merge the import PR(s) on the monorepo, then press Enter..." >&2 + read -r + + local all_merged=true + for branch in $(echo "$import_prs" | jq -r 'keys[]'); do + local pr_number pr_json merged + pr_number=$(echo "$import_prs" | jq -r --arg b "$branch" '.[$b]') + pr_json=$(get_pr "$MONOREPO_API" "$GITEA_TOKEN" "$pr_number") + merged=$(echo "$pr_json" | jq -r '.merged // false') + + if [ "$merged" = "true" ]; then + log "INFO" "PR #${pr_number} (${branch}): merged" + else + log "ERROR" "PR #${pr_number} (${branch}): NOT merged — merge it first" + all_merged=false + fi + done + + if [ "$all_merged" = false ]; then + die "Not all import PRs are merged. Re-run 'josh-sync adopt ${target_name}' after merging." + fi + else + log "INFO" "No import PRs recorded — subfolder already matched subrepo" + fi + + adopt_state=$(echo "$adopt_state" | jq '.step = "adopting"') + write_adopt_state "$target_name" "$adopt_state" + current_step="adopting" + fi + + if [ "$current_step" = "adopting" ]; then + echo "" >&2 + log "INFO" "Step 3: Creating adoption merge commit(s)..." + + local branches already_adopted + branches=$(echo "$target_json" | jq -r '.branches | keys[]') + already_adopted=$(echo "$adopt_state" | jq -r '.adopted_branches // []') + + for branch in $branches; do + if echo "$already_adopted" | jq -e --arg b "$branch" 'index($b) != null' >/dev/null 2>&1; then + log "INFO" "Branch ${branch} already adopted — skipping" + continue + fi + + local mapped + mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') + [ -z "$mapped" ] && continue + + export SYNC_BRANCH_MONO="$branch" + export SYNC_BRANCH_SUBREPO="$mapped" + + local result + result=$(adopt_branch) + log "INFO" "Adopt result for ${branch}: ${result}" + + case "$result" in + adopted|already-adopted) + adopt_state=$(echo "$adopt_state" | jq --arg b "$branch" '.adopted_branches += [$b]') + write_adopt_state "$target_name" "$adopt_state" + ;; + tree-mismatch) + die "Tree mismatch for ${branch}. Merge the import PR and make sure subrepo/${mapped} matches the Josh-filtered monorepo tree." + ;; + push-rejected) + die "Subrepo branch ${mapped} changed during adoption. Re-run 'josh-sync adopt ${target_name}' to retry." + ;; + missing-branch) + die "Subrepo branch ${mapped} does not exist." + ;; + *) + die "Unexpected adopt result: ${result}" + ;; + esac + done + + adopt_state=$(echo "$adopt_state" | jq \ + --arg step "complete" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '.step = $step | .timestamp = $ts') + write_adopt_state "$target_name" "$adopt_state" + current_step="complete" + fi + + if [ "$current_step" = "complete" ]; then + echo "" >&2 + echo "=== Adoption complete! ===" >&2 + echo "" >&2 + echo "The subrepo keeps its existing history and now has a josh-sync adoption merge." >&2 + echo "Developers can keep their clones; they only need to fast-forward active branches:" >&2 + echo " git fetch origin && git checkout main && git merge --ff-only origin/main" >&2 + fi +} diff --git a/tests/unit/adopt.bats b/tests/unit/adopt.bats new file mode 100644 index 0000000..5a3d06c --- /dev/null +++ b/tests/unit/adopt.bats @@ -0,0 +1,92 @@ +#!/usr/bin/env bats +# tests/unit/adopt.bats — Non-destructive adoption merge tests + +setup() { + export JOSH_SYNC_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + source "$JOSH_SYNC_ROOT/lib/core.sh" + source "$JOSH_SYNC_ROOT/lib/adopt.sh" + + TEST_ROOT="$(mktemp -d)" + export BOT_NAME="test-bot" + export BOT_EMAIL="test-bot@test.local" + export BOT_TRAILER="Josh-Sync-Origin" + export JOSH_SYNC_TARGET_NAME="example" + export SYNC_BRANCH_MONO="main" + export SYNC_BRANCH_SUBREPO="main" +} + +teardown() { + rm -rf "$TEST_ROOT" +} + +make_bare_repo() { + local name="$1" + local content="$2" + local message="$3" + local work="${TEST_ROOT}/${name}-work" + local bare="${TEST_ROOT}/${name}.git" + + git init "$work" >/dev/null + git -C "$work" checkout -b main >/dev/null + git -C "$work" config user.name "Test User" + git -C "$work" config user.email "test@test.local" + printf '%s\n' "$content" > "${work}/README.md" + git -C "$work" add README.md + git -C "$work" commit -m "$message" >/dev/null + + git init --bare "$bare" >/dev/null + git -C "$work" remote add origin "$bare" + git -C "$work" push origin main >/dev/null + printf '%s\n' "$bare" +} + +subrepo_auth_url() { + printf '%s\n' "$SUBREPO_BARE" +} + +josh_auth_url() { + printf '%s\n' "$JOSH_BARE" +} + +@test "adopt_branch creates merge with josh head first parent and subrepo head second parent" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + + local old_subrepo_head josh_head josh_tree result adopt_head + old_subrepo_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + josh_head=$(git --git-dir="$JOSH_BARE" rev-parse main) + josh_tree=$(git --git-dir="$JOSH_BARE" rev-parse 'main^{tree}') + + result=$(adopt_branch) + + [ "$result" = "adopted" ] + adopt_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + [ "$adopt_head" != "$old_subrepo_head" ] + [ "$(git --git-dir="$SUBREPO_BARE" rev-parse 'main^1')" = "$josh_head" ] + [ "$(git --git-dir="$SUBREPO_BARE" rev-parse 'main^2')" = "$old_subrepo_head" ] + [ "$(git --git-dir="$SUBREPO_BARE" rev-parse 'main^{tree}')" = "$josh_tree" ] +} + +@test "adopt_branch aborts when trees differ and leaves subrepo head unchanged" { + SUBREPO_BARE="$(make_bare_repo subrepo "subrepo tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "josh tree" "josh filtered history")" + + local old_subrepo_head result + old_subrepo_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + + result=$(adopt_branch) + + [ "$result" = "tree-mismatch" ] + [ "$(git --git-dir="$SUBREPO_BARE" rev-parse main)" = "$old_subrepo_head" ] +} + +@test "adopt_branch reports missing subrepo branch before cloning" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + export SYNC_BRANCH_SUBREPO="missing" + + local result + result=$(adopt_branch) + + [ "$result" = "missing-branch" ] +} From dc2cdea360396b2caea3dc1fd9597103c3507c0a Mon Sep 17 00:00:00 2001 From: SBPro <slim.b.pro@gmail.com> Date: Wed, 27 May 2026 03:32:04 +0100 Subject: [PATCH 37/37] =?UTF-8?q?Release=202.2.0=20=E2=80=94=20unified=20o?= =?UTF-8?q?nboard=20strategy=20+=20safety=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Folds `josh-sync adopt` into `josh-sync onboard <target>` as the adopt strategy alongside the original reset flow. Strategy resolves by precedence: --mode flag > targets[].history_lock config (preserve|rewrite) > auto-detect via `git ls-remote --heads`. `josh-sync adopt` is kept as a thin alias. Adds the new `targets[].history_lock` config field (validated at parse time) and folds `<target>/adopt.json` state into `<target>/onboard.json` with a `.strategy` field; legacy state files are read with a backward-compat fallback. Safety hardening (from a multi-angle review of the unification): - auto-detect distinguishes auth/network failure from empty repo - resume validates --mode against the strategy saved in state - `josh-sync adopt` rejects a conflicting --mode in the forwarded args - missing import-PR lookup dies in both strategies (was WARN+continue for reset, which could create duplicate import PRs on resume) - --restart durably removes the legacy adopt.json from the state branch - adopt_branch is now a subshell function (EXIT trap can't clobber callers) - strategy value validated after resolve/load (reset|adopt) - --mode with missing/empty value dies with a usage hint - migrate-pr against an adopt-strategy target dies with a specific hint - reset importing asserts archived_url is present (no "null" → git clone) End-to-end + CLI bats coverage added (tests/unit/adopt_e2e.bats, tests/unit/cli.bats). 72 tests, shellcheck clean. Makefile dist bundle header now correctly interpolates VERSION and line count. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- CHANGELOG.md | 28 + Makefile | 4 +- VERSION | 2 +- bin/josh-sync | 76 +-- docs/adr/013-non-destructive-adoption.md | 1 + docs/config-reference.md | 1 + docs/guide.md | 60 +- lib/adopt.sh | 250 +------- lib/config.sh | 9 + lib/onboard.sh | 467 ++++++++++----- schema/config-schema.json | 5 + tests/unit/adopt_e2e.bats | 697 +++++++++++++++++++++++ tests/unit/cli.bats | 123 ++++ 13 files changed, 1284 insertions(+), 439 deletions(-) create mode 100644 tests/unit/adopt_e2e.bats create mode 100644 tests/unit/cli.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 31ad6e1..b0318ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## 2.2.0 + +### Changes + +- **Unified onboarding via `josh-sync onboard <target>`.** The previously separate `adopt` workflow is now an onboarding strategy alongside the original `reset` flow. Strategy is resolved by precedence: `--mode={reset,adopt}` flag → `targets[].history_lock` config (`preserve` → adopt, `rewrite` → reset) → auto-detect via `git ls-remote --heads` on the subrepo (heads → adopt, empty → reset). The chosen strategy is logged at preflight (and re-announced on every resume). +- **`josh-sync adopt` kept as a back-compat CLI alias** for `josh-sync onboard --mode=adopt`. Existing in-flight adoptions (state at `<target>/adopt.json` on the `josh-sync-state` branch) continue to resume — `read_onboard_state` falls back to that legacy file with an implicit `strategy: adopt` and strips the legacy `.mode` field. Pre-unification `onboard.json` files (no `.strategy`) are read as `strategy: reset`. +- **New `targets[].history_lock` config field** (`preserve` | `rewrite`) — pins the onboarding strategy independently of the subrepo's current emptiness. Validated at config parse time. See [Config Reference](docs/config-reference.md#targets-section). +- **End-to-end bats coverage** — tree-mismatch surfacing, idempotency, merge shape (ADR-005 trailer, ADR-008 first-parent ordering), fast-forward push semantics, reverse-sync loop guard, linearize-fallback selection, resume from every checkpoint, strategy resolution precedence, legacy-state migration, and the CLI surface (flag parsing, alias forwarding, error messages). + +### Safety + +- **Auto-detect now distinguishes auth/network failures from genuinely empty repos.** A failing `git ls-remote` previously fell through to `reset` (destructive force-push); it now dies with a clear "pass --mode explicitly" message. +- **Resume validates `--mode` against the strategy already saved in state.** Conflicting flags die with a `--restart` hint instead of being silently ignored. +- **`josh-sync adopt` rejects a conflicting `--mode`.** Previously the user-supplied flag silently won. +- **Missing import-PR lookup now `die`s in both strategies.** The reset path used to `WARN` and continue without recording the PR number, leading to duplicate import PRs on resume. +- **`--restart` durably removes the legacy `<target>/adopt.json`** from the state branch so it can't silently resurrect via the read fallback. +- **`adopt_branch` is now a subshell function** — its EXIT trap can no longer clobber callers' traps (test reporting, etc.). +- **Strategy value is validated** after resolve/load (`reset|adopt`); unknown/corrupt values die with a `--restart` hint. +- **`--mode` with a missing/empty value** dies with a usage hint instead of crashing under `set -u`. +- **`migrate-pr` against an adopt-strategy target** dies with a specific message rather than the misleading "Run onboard first" loop. +- **Reset import step asserts `archived_url` is present** instead of silently flowing the literal string `"null"` into `git clone`. + +ADR-013 updated with a status note pointing at the unified command. + +### Fixes + +- Makefile `dist/josh-sync` bundle header now correctly interpolates VERSION and line count (was emitting empty values due to make's `$(...)` swallowing the shell command substitution). + ## 2.1.0 ### Breaking Changes diff --git a/Makefile b/Makefile index 3b4de3e..754132c 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dist/josh-sync: bin/josh-sync lib/*.sh VERSION @mkdir -p dist @echo "Bundling josh-sync..." @echo '#!/usr/bin/env bash' > dist/josh-sync - @echo "# josh-sync $(cat VERSION) — bundled distribution" >> dist/josh-sync + @echo "# josh-sync $$(cat VERSION) — bundled distribution" >> dist/josh-sync @echo '# Generated by: make build' >> dist/josh-sync @echo '' >> dist/josh-sync @# Inline all library modules (strip shebangs and source directives) @@ -36,7 +36,7 @@ dist/josh-sync: bin/josh-sync lib/*.sh VERSION | sed '/^JOSH_LIB_DIR=/,/^source/d' \ >> dist/josh-sync @chmod +x dist/josh-sync - @echo "Built: dist/josh-sync ($(wc -l < dist/josh-sync) lines)" + @echo "Built: dist/josh-sync ($$(wc -l < dist/josh-sync) lines)" clean: rm -rf dist/ diff --git a/VERSION b/VERSION index 7ec1d6d..ccbccc3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.0 +2.2.0 diff --git a/bin/josh-sync b/bin/josh-sync index d126e3b..bba810b 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -8,9 +8,9 @@ # sync Run forward and/or reverse sync # preflight Validate config, connectivity, auth # import <target> Initial import: pull subrepo into monorepo -# adopt <target> Adopt existing subrepo history without rewriting it +# onboard <target> Interactive onboarding (auto-picks reset|adopt strategy) +# adopt <target> Alias for `onboard --mode=adopt` # reset <target> Reset subrepo to josh-filtered view -# onboard <target> Import existing subrepo into monorepo (interactive) # migrate-pr <target> [PR#...] [--all] Move PRs from archived to new subrepo # status Show target config and sync state # state show|reset Manage sync state directly @@ -75,9 +75,13 @@ Commands: sync Run forward and/or reverse sync preflight Validate config, connectivity, auth, workflow coverage import <target> Initial import: pull existing subrepo into monorepo (creates PR) - adopt <target> Adopt existing subrepo history without rewriting it + onboard <target> Onboard a subrepo (interactive, resumable). Picks a strategy: + reset — force-push josh-filtered history (replaces subrepo) + adopt — non-destructive merge that keeps subrepo history + Precedence: --mode flag > targets[].history_lock > auto-detect + via ls-remote (heads → adopt, empty → reset). + adopt <target> Alias for 'onboard --mode=adopt' (kept for back-compat) reset <target> Reset subrepo to josh-filtered view (after merging import PR) - onboard <target> Import existing subrepo into monorepo (interactive, resumable) migrate-pr <target> [PR#...] [--all] Move PRs from archived to new subrepo status Show target config and sync state state show <target> [branch] Show sync state JSON @@ -96,6 +100,9 @@ Sync flags: --target NAME Filter to target(s) — comma-separated for multiple (env: JOSH_SYNC_TARGET) --branch BRANCH Filter to one branch +Onboard flags: + --mode={reset,adopt} Override the onboarding strategy + Environment: JOSH_SYNC_TARGET Restrict to a single target name JOSH_SYNC_STATE_BRANCH State branch name (default: josh-sync-state) @@ -534,40 +541,33 @@ cmd_import() { done } -# ─── Adopt Command ───────────────────────────────────────────────── +# ─── Adopt Command (alias) ───────────────────────────────────────── +# `josh-sync adopt` is preserved as a thin alias for the unified onboard +# command. Effectively: `josh-sync onboard --mode=adopt <args...>`. cmd_adopt() { - local config_file=".josh-sync.yml" - local target_name="" - local restart=false - - while [ $# -gt 0 ]; do - case "$1" in - --config) config_file="$2"; shift 2 ;; - --debug) export JOSH_SYNC_DEBUG=1; shift ;; - --restart) restart=true; shift ;; - -*) die "Unknown flag: $1" ;; - *) target_name="$1"; shift ;; + # Reject a conflicting --mode in the forwarded args: the alias means adopt, + # so `josh-sync adopt … --mode=reset` is contradictory. Without this guard + # the user-supplied --mode would silently win (last --mode in cmd_onboard's + # parser is the one that takes effect). + local i=1 a + while [ $i -le $# ]; do + a="${!i}" + case "$a" in + --mode=adopt) ;; + --mode=*) die "'josh-sync adopt' is an alias for --mode=adopt; conflicting '${a}' given. Use 'josh-sync onboard <target>' instead." ;; + --mode) + i=$((i + 1)) + if [ $i -gt $# ] || [ "${!i}" != "adopt" ]; then + die "'josh-sync adopt' is an alias for --mode=adopt; conflicting '--mode ${!i:-<missing>}' given. Use 'josh-sync onboard <target>' instead." + fi + ;; esac + i=$((i + 1)) done - if [ -z "$target_name" ]; then - echo "Usage: josh-sync adopt <target> [--restart]" >&2 - parse_config "$config_file" - echo "Available targets:" >&2 - echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2 - exit 1 - fi - - parse_config "$config_file" - - local target_json - target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)') - [ -n "$target_json" ] || die "Target '${target_name}' not found in config" - - log "INFO" "══════ Adopt target: ${target_name} ══════" - load_target "$target_json" - adopt_flow "$target_json" "$restart" + log "INFO" "(josh-sync adopt is now an alias for: josh-sync onboard --mode=adopt)" + cmd_onboard --mode=adopt "$@" } # ─── Reset Command ───────────────────────────────────────────────── @@ -743,19 +743,25 @@ cmd_onboard() { local config_file=".josh-sync.yml" local target_name="" local restart=false + local mode="" while [ $# -gt 0 ]; do case "$1" in --config) config_file="$2"; shift 2 ;; --debug) export JOSH_SYNC_DEBUG=1; shift ;; --restart) restart=true; shift ;; + --mode=*) mode="${1#--mode=}" + [ -n "$mode" ] || die "Empty --mode= (use --mode=reset|adopt)" + shift ;; + --mode) [ $# -ge 2 ] || die "--mode requires a value (reset|adopt)" + mode="$2"; shift 2 ;; -*) die "Unknown flag: $1" ;; *) target_name="$1"; shift ;; esac done if [ -z "$target_name" ]; then - echo "Usage: josh-sync onboard <target> [--restart]" >&2 + echo "Usage: josh-sync onboard <target> [--mode=reset|adopt] [--restart]" >&2 parse_config "$config_file" echo "Available targets:" >&2 echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2 @@ -770,7 +776,7 @@ cmd_onboard() { log "INFO" "══════ Onboard target: ${target_name} ══════" load_target "$target_json" - onboard_flow "$target_json" "$restart" + onboard_flow "$target_json" "$restart" "$mode" } # ─── Migrate PR Command ────────────────────────────────────────── diff --git a/docs/adr/013-non-destructive-adoption.md b/docs/adr/013-non-destructive-adoption.md index a744b77..7e56849 100644 --- a/docs/adr/013-non-destructive-adoption.md +++ b/docs/adr/013-non-destructive-adoption.md @@ -2,6 +2,7 @@ **Status:** Accepted **Date:** 2026-04 +**Update (2026-05):** The standalone `josh-sync adopt <target>` command described below has been folded into `josh-sync onboard <target>` as the `adopt` strategy (selectable via `--mode=adopt`, `targets[].history_lock: preserve`, or auto-detection when the subrepo has any branches). `josh-sync adopt` is retained as a thin CLI alias. The state file moved from `<target>/adopt.json` to `<target>/onboard.json` with a `.strategy` field; the legacy file is still read for backward compatibility. See the guide's Onboarding section. ## Context diff --git a/docs/config-reference.md b/docs/config-reference.md index c953083..fc1beca 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -42,6 +42,7 @@ Each target maps a monorepo subfolder to an external subrepo. | `branches` | object | Yes | — | Branch mapping: `mono_branch: subrepo_branch`. Each key-value pair syncs those branches bidirectionally. | | `forward_only` | string[] | No | `[]` | Branches that only sync mono → subrepo, never reverse. | | `exclude` | string[] | No | `[]` | File/directory patterns to exclude from sync via josh `:exclude` filter. Excluded files exist only in the monorepo, never in the subrepo. See [Excluding Files](guide.md#excluding-files-from-sync). | +| `history_lock` | string | No | (auto-detect) | Onboarding strategy hint. `"preserve"` selects the non-destructive [adopt](guide.md#onboarding) strategy (keeps subrepo history via an adoption merge — ADR-013); `"rewrite"` selects the destructive reset strategy (force-pushes josh-filtered history). When omitted, `josh-sync onboard` auto-detects: subrepos with branches → adopt, empty subrepos → reset. The CLI flag `--mode={reset,adopt}` overrides this. | ## `bot` Section diff --git a/docs/guide.md b/docs/guide.md index 540b685..fdac918 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -222,20 +222,31 @@ This validates: For a new monorepo before import, preflight may warn that subfolders don't exist yet — that's expected. -## Step 5: Import Existing Subrepos +## Step 5: Onboard Existing Subrepos -This is the critical onboarding step. There are three approaches: +`josh-sync onboard <target>` is the single entry point for connecting an existing subrepo to the monorepo. It runs interactively with checkpoint/resume and picks one of two strategies: -- **`josh-sync adopt`** (recommended for active existing subrepos) — non-destructive, resumable, preserves existing subrepo history -- **`josh-sync onboard`** — destructive replacement-repo workflow for teams that intentionally archive the old repo -- **Manual `import` → merge → `reset`** — lower-level destructive path for automation or empty replacement repos +- **adopt** (non-destructive) — keeps the subrepo's existing history and joins it to josh-filtered history via a single adoption merge commit. No force-push. Developers fast-forward; open PR branches stay valid. +- **reset** (destructive) — force-pushes josh-filtered history onto the subrepo, replacing its prior history. Use this for empty replacement repos (typically paired with renaming the old repo to `*-archived`). -### Option A: Adopt (recommended for active subrepos) +### Strategy selection -Use `adopt` when the subrepo already exists, developers already clone it, or open PR branches should remain based on existing history. +`josh-sync onboard` resolves the strategy in this order (highest precedence first): + +1. **CLI flag** — `--mode=adopt` or `--mode=reset` overrides everything. +2. **Config** — `targets[].history_lock: preserve` selects adopt; `rewrite` selects reset. +3. **Auto-detect** — `git ls-remote --heads` on the subrepo. Branches present → adopt. Empty repo → reset. + +The chosen strategy is logged at preflight so you can see what will happen before any side effects. + +`josh-sync adopt <target>` is kept as a back-compat alias for `josh-sync onboard --mode=adopt`. + +### Adopt strategy (auto-selected for active subrepos) ```bash -josh-sync adopt billing +josh-sync onboard billing # auto-detect — picks adopt for non-empty subrepo +josh-sync onboard billing --mode=adopt # force adopt explicitly +josh-sync adopt billing # equivalent back-compat alias ``` The command will: @@ -245,7 +256,7 @@ The command will: 4. **Adopt** — creates a merge commit on the subrepo with Josh-filtered HEAD as parent 1 and existing subrepo HEAD as parent 2 5. **Push** — pushes the adoption merge with a normal fast-forward push, never force-push -The adoption merge preserves the old subrepo history while giving Josh a first-parent path back to the monorepo. If interrupted, re-run `josh-sync adopt billing` to resume. Use `--restart` to start over. +The adoption merge preserves the old subrepo history while giving Josh a first-parent path back to the monorepo. If interrupted, re-run `josh-sync onboard billing` to resume. Use `--restart` to start over. See [ADR-013](adr/013-non-destructive-adoption.md) for the design rationale. After adoption, developers can update existing clones with a normal fast-forward: @@ -255,9 +266,9 @@ git checkout main git merge --ff-only origin/main ``` -### Option B: Onboard with replacement repo +### Reset strategy (auto-selected for empty replacement repos) -The `onboard` command walks through the destructive replacement-repo process interactively, with checkpoint/resume at every step. +Use the reset strategy when you intentionally want to archive the old subrepo and start fresh from a new empty repo. **Before you start:** @@ -269,7 +280,8 @@ The rename preserves the archived repo with all its history and open PRs. The ne **Run onboard:** ```bash -josh-sync onboard billing +josh-sync onboard billing # auto-detect — picks reset for empty subrepo +josh-sync onboard billing --mode=reset # force reset explicitly ``` The command will: @@ -298,13 +310,13 @@ josh-sync migrate-pr billing 5 8 12 PR migration works by fetching the diff from the archived repo's PR, applying it to the new repo, and creating a new PR. File content is identical after reset, so patches apply cleanly. -### Option C: Manual import → merge → reset +### Manual: import → merge → reset -Use this for scripted automation or when you intentionally want to replace subrepo history. +Use this for scripted automation or when you intentionally want to replace subrepo history without the interactive wrapper. > Do this **one target at a time** to keep PRs reviewable. -#### 5c-1. Import +#### Manual: Import ```bash josh-sync import billing @@ -319,13 +331,13 @@ This: Review the import PR — check for leaked credentials, environment-specific config, or files that shouldn't be in the monorepo. -#### 5c-2. Merge the import PR +#### Manual: Merge the import PR Merge the PR using your Git platform's UI. This lands the subrepo content into the monorepo's main branch. > At this point, the monorepo has the content but the histories are disconnected. Sync will **not** work until you complete the reset step. -#### 5c-3. Reset +#### Manual: Reset ```bash josh-sync reset billing @@ -352,7 +364,7 @@ git checkout stage && git reset --hard origin/stage # repeat for each branch Or simply delete and re-clone the subrepo. Local-only branches (not pushed to the remote) will be lost either way. -#### 5c-4. Repeat for each target +#### Manual: Repeat for each target ``` For each target: @@ -646,15 +658,13 @@ To add a new subrepo after initial setup: 1. Add the target to `.josh-sync.yml` 2. Update the forward workflow's `paths:` list to include the new subfolder 3. Commit and push -4. Import or adopt the target: +4. Onboard the target — `josh-sync onboard` auto-picks `adopt` for non-empty subrepos and `reset` for empty ones; override with `--mode={reset,adopt}` or pin via the target's `history_lock` config field. ```bash - # Recommended for an existing active subrepo - josh-sync adopt new-target + josh-sync onboard new-target # auto-detect strategy + josh-sync onboard new-target --mode=adopt # force adopt + josh-sync onboard new-target --mode=reset # force reset - # Replacement-repo workflow - josh-sync onboard new-target - - # Or manual: import → merge PR → reset + # Manual lower-level path (when scripting): josh-sync import new-target # merge the PR josh-sync reset new-target diff --git a/lib/adopt.sh b/lib/adopt.sh index b779aa7..70a55bc 100644 --- a/lib/adopt.sh +++ b/lib/adopt.sh @@ -1,52 +1,14 @@ #!/usr/bin/env bash -# lib/adopt.sh — Non-destructive adoption of existing subrepo history +# lib/adopt.sh — Non-destructive adoption merge primitive # # Provides: -# adopt_flow() — Import → wait for merge → adoption merge per branch -# adopt_branch() — Create the adoption merge and push it fast-forward only +# adopt_branch() — Create one adoption merge commit and push it fast-forward # -# Adopt state is stored on the josh-sync-state branch at <target>/adopt.json. -# Steps: start → importing → waiting-for-merge → adopting → complete +# The orchestration (state machine, import → wait → finalize) lives in +# lib/onboard.sh under strategy="adopt". `josh-sync adopt` is now a CLI alias +# for `josh-sync onboard --mode=adopt`; see ADR-013 for design rationale. # -# Requires: lib/core.sh, lib/config.sh, lib/auth.sh, lib/state.sh, lib/sync.sh sourced - -# ─── Adopt State Helpers ───────────────────────────────────────── - -read_adopt_state() { - local target_name="${1:-$JOSH_SYNC_TARGET_NAME}" - git fetch origin "$STATE_BRANCH" 2>/dev/null || true - git show "origin/${STATE_BRANCH}:${target_name}/adopt.json" 2>/dev/null || echo '{}' -} - -write_adopt_state() { - local target_name="${1:-$JOSH_SYNC_TARGET_NAME}" - local state_json="$2" - local key="${target_name}/adopt" - local tmp_dir - tmp_dir=$(mktemp -d) - - if git rev-parse "origin/${STATE_BRANCH}" >/dev/null 2>&1; then - git worktree add "$tmp_dir" "origin/${STATE_BRANCH}" 2>/dev/null - else - git worktree add --detach "$tmp_dir" 2>/dev/null - (cd "$tmp_dir" && git checkout --orphan "$STATE_BRANCH" && git rm -rf . 2>/dev/null || true) - fi - - mkdir -p "$(dirname "${tmp_dir}/${key}.json")" - echo "$state_json" | jq '.' > "${tmp_dir}/${key}.json" - - ( - cd "$tmp_dir" || exit - git add -A - if ! git diff --cached --quiet 2>/dev/null; then - git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ - commit -m "adopt: update ${target_name}" - git push origin "HEAD:${STATE_BRANCH}" || log "WARN" "Failed to push adopt state" - fi - ) - - git worktree remove "$tmp_dir" 2>/dev/null || rm -rf "$tmp_dir" -} +# Requires: lib/core.sh, lib/config.sh, lib/auth.sh sourced # ─── Adopt Branch ──────────────────────────────────────────────── # Establishes shared ancestry without rewriting the existing subrepo branch. @@ -61,7 +23,11 @@ write_adopt_state() { # # Returns: adopted | already-adopted | tree-mismatch | push-rejected | missing-branch -adopt_branch() { +adopt_branch() ( + # Subshell function: contains the function-level EXIT trap, the internal cd, + # and any set -e propagation so it cannot pollute the caller's shell. Without + # this, the trap below would clobber callers' EXIT handlers (bats's own test + # reporting in particular silently disappears). local mono_branch="$SYNC_BRANCH_MONO" local subrepo_branch="$SYNC_BRANCH_SUBREPO" local work_dir @@ -140,196 +106,4 @@ ${BOT_TRAILER}: adopt/${mono_branch}/$(date -u +%Y-%m-%dT%H:%M:%SZ)") log "WARN" "Fast-forward push rejected — subrepo changed during adoption" echo "push-rejected" fi -} - -# ─── Adopt Flow ────────────────────────────────────────────────── -# Interactive orchestrator with checkpoint/resume. -# Usage: adopt_flow <target_json> <restart> - -adopt_flow() { - local target_json="$1" - local restart="${2:-false}" - local target_name="$JOSH_SYNC_TARGET_NAME" - - local adopt_state current_step - adopt_state=$(read_adopt_state "$target_name") - current_step=$(echo "$adopt_state" | jq -r '.step // "start"') - - if [ "$restart" = true ]; then - log "INFO" "Restarting adopt from scratch" - current_step="start" - adopt_state='{}' - fi - - log "INFO" "Adopt step: ${current_step}" - - if [ "$current_step" = "start" ]; then - echo "" >&2 - echo "=== Adopting ${target_name} ===" >&2 - echo "" >&2 - echo "This keeps the existing subrepo history and adds one adoption merge commit per branch." >&2 - echo "No force-push is used. Existing open PR branches remain based on the old history." >&2 - echo "" >&2 - - adopt_state=$(jq -n \ - --arg step "importing" \ - --arg mode "adopt" \ - --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{step:$step, mode:$mode, import_prs:{}, adopted_branches:[], timestamp:$ts}') - write_adopt_state "$target_name" "$adopt_state" - current_step="importing" - fi - - if [ "$current_step" = "importing" ]; then - echo "" >&2 - log "INFO" "Step 1: Importing current subrepo content into monorepo..." - - local branches import_prs - branches=$(echo "$target_json" | jq -r '.branches | keys[]') - import_prs=$(echo "$adopt_state" | jq -r '.import_prs // {}') - - for branch in $branches; do - local mapped - mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') - [ -z "$mapped" ] && continue - - if echo "$import_prs" | jq -e --arg b "$branch" 'has($b)' >/dev/null 2>&1; then - log "INFO" "Import PR already recorded for ${branch} — skipping" - continue - fi - - export SYNC_BRANCH_MONO="$branch" - export SYNC_BRANCH_SUBREPO="$mapped" - - local result - result=$(initial_import) - log "INFO" "Import result for ${branch}: ${result}" - - if [ "$result" = "pr-created" ]; then - local prs pr_number - prs=$(list_open_prs "$MONOREPO_API" "$GITEA_TOKEN") - pr_number=$(echo "$prs" | jq -r --arg t "$target_name" --arg b "$branch" \ - '[.[] | select(.title | test("\\[Import\\] " + $t + ":")) | select(.base.ref == $b)] | .[0].number // empty') - - [ -n "$pr_number" ] || die "Could not find import PR number for ${branch}; cannot safely continue adoption" - import_prs=$(echo "$import_prs" | jq --arg b "$branch" --arg n "$pr_number" '. + {($b): ($n | tonumber)}') - log "INFO" "Import PR for ${branch}: #${pr_number}" - fi - - adopt_state=$(echo "$adopt_state" | jq --argjson prs "$import_prs" '.import_prs = $prs') - write_adopt_state "$target_name" "$adopt_state" - done - - adopt_state=$(echo "$adopt_state" | jq \ - --arg step "waiting-for-merge" \ - --argjson prs "$import_prs" \ - '.step = $step | .import_prs = $prs') - write_adopt_state "$target_name" "$adopt_state" - current_step="waiting-for-merge" - fi - - if [ "$current_step" = "waiting-for-merge" ]; then - echo "" >&2 - log "INFO" "Step 2: Waiting for import PR(s) to be merged..." - - local import_prs pr_count - import_prs=$(echo "$adopt_state" | jq -r '.import_prs') - pr_count=$(echo "$import_prs" | jq 'length') - - if [ "$pr_count" -gt 0 ]; then - echo "" >&2 - echo "Import PRs to merge:" >&2 - echo "$import_prs" | jq -r 'to_entries[] | " \(.key): PR #\(.value)"' >&2 - echo "" >&2 - echo "Merge the import PR(s) on the monorepo, then press Enter..." >&2 - read -r - - local all_merged=true - for branch in $(echo "$import_prs" | jq -r 'keys[]'); do - local pr_number pr_json merged - pr_number=$(echo "$import_prs" | jq -r --arg b "$branch" '.[$b]') - pr_json=$(get_pr "$MONOREPO_API" "$GITEA_TOKEN" "$pr_number") - merged=$(echo "$pr_json" | jq -r '.merged // false') - - if [ "$merged" = "true" ]; then - log "INFO" "PR #${pr_number} (${branch}): merged" - else - log "ERROR" "PR #${pr_number} (${branch}): NOT merged — merge it first" - all_merged=false - fi - done - - if [ "$all_merged" = false ]; then - die "Not all import PRs are merged. Re-run 'josh-sync adopt ${target_name}' after merging." - fi - else - log "INFO" "No import PRs recorded — subfolder already matched subrepo" - fi - - adopt_state=$(echo "$adopt_state" | jq '.step = "adopting"') - write_adopt_state "$target_name" "$adopt_state" - current_step="adopting" - fi - - if [ "$current_step" = "adopting" ]; then - echo "" >&2 - log "INFO" "Step 3: Creating adoption merge commit(s)..." - - local branches already_adopted - branches=$(echo "$target_json" | jq -r '.branches | keys[]') - already_adopted=$(echo "$adopt_state" | jq -r '.adopted_branches // []') - - for branch in $branches; do - if echo "$already_adopted" | jq -e --arg b "$branch" 'index($b) != null' >/dev/null 2>&1; then - log "INFO" "Branch ${branch} already adopted — skipping" - continue - fi - - local mapped - mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') - [ -z "$mapped" ] && continue - - export SYNC_BRANCH_MONO="$branch" - export SYNC_BRANCH_SUBREPO="$mapped" - - local result - result=$(adopt_branch) - log "INFO" "Adopt result for ${branch}: ${result}" - - case "$result" in - adopted|already-adopted) - adopt_state=$(echo "$adopt_state" | jq --arg b "$branch" '.adopted_branches += [$b]') - write_adopt_state "$target_name" "$adopt_state" - ;; - tree-mismatch) - die "Tree mismatch for ${branch}. Merge the import PR and make sure subrepo/${mapped} matches the Josh-filtered monorepo tree." - ;; - push-rejected) - die "Subrepo branch ${mapped} changed during adoption. Re-run 'josh-sync adopt ${target_name}' to retry." - ;; - missing-branch) - die "Subrepo branch ${mapped} does not exist." - ;; - *) - die "Unexpected adopt result: ${result}" - ;; - esac - done - - adopt_state=$(echo "$adopt_state" | jq \ - --arg step "complete" \ - --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '.step = $step | .timestamp = $ts') - write_adopt_state "$target_name" "$adopt_state" - current_step="complete" - fi - - if [ "$current_step" = "complete" ]; then - echo "" >&2 - echo "=== Adoption complete! ===" >&2 - echo "" >&2 - echo "The subrepo keeps its existing history and now has a josh-sync adoption merge." >&2 - echo "Developers can keep their clones; they only need to fast-forward active branches:" >&2 - echo " git fetch origin && git checkout main && git merge --ff-only origin/main" >&2 - fi -} +) diff --git a/lib/config.sh b/lib/config.sh index d8655ee..5347796 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -88,6 +88,15 @@ parse_config() { ) ]') + # Validate targets[].history_lock against the schema enum (preserve|rewrite). + # Caught at config parse time so typos don't sit silently until `onboard` runs. + local bad_lock + bad_lock=$(echo "$JOSH_SYNC_TARGETS" | jq -r ' + .[] | select(.history_lock != null) + | select(.history_lock != "preserve" and .history_lock != "rewrite") + | "\(.name): \(.history_lock)"' | head -1) + [ -z "$bad_lock" ] || die "Invalid targets[].history_lock value (must be 'preserve' or 'rewrite'): ${bad_lock}" + # Load .env credentials (if present, not required — CI sets these via secrets) if [ -f .env ]; then # shellcheck source=/dev/null diff --git a/lib/onboard.sh b/lib/onboard.sh index 1cc6916..0c21369 100644 --- a/lib/onboard.sh +++ b/lib/onboard.sh @@ -1,23 +1,56 @@ #!/usr/bin/env bash -# lib/onboard.sh — Onboard orchestration and PR migration +# lib/onboard.sh — Unified onboard orchestration (reset + adopt strategies) # # Provides: -# onboard_flow() — Interactive: import → wait for merge → reset to new repo -# migrate_one_pr() — Migrate a single PR from archived repo to new repo +# onboard_flow() — Interactive: import → wait → finalize +# finalize = subrepo_reset (rewrite) or +# adopt_branch (preserve) +# resolve_onboard_strategy() — Pick reset|adopt from --mode / config / +# ls-remote auto-detection +# migrate_one_pr() — Migrate one PR from archived to new subrepo # -# Onboard state is stored on the josh-sync-state branch at <target>/onboard.json. -# Steps: start → importing → waiting-for-merge → resetting → complete +# State is stored on the josh-sync-state branch at <target>/onboard.json: +# { step, strategy, import_prs, ... } +# read_onboard_state falls back to legacy <target>/adopt.json (with implicit +# strategy="adopt") so in-flight v2.1 adoptions continue to resume correctly. # -# Requires: lib/core.sh, lib/config.sh, lib/auth.sh, lib/state.sh, lib/sync.sh sourced -# Expects: JOSH_SYNC_TARGET_NAME, BOT_NAME, BOT_EMAIL, SUBREPO_API, SUBREPO_TOKEN, etc. +# Steps: +# start → importing → waiting-for-merge → resetting | adopting → complete +# +# Requires: lib/core.sh, lib/config.sh, lib/auth.sh, lib/state.sh, lib/sync.sh, +# lib/adopt.sh sourced +# Expects: JOSH_SYNC_TARGET_NAME, BOT_NAME, BOT_EMAIL, SUBREPO_API, +# SUBREPO_TOKEN, etc. -# ─── Onboard State Helpers ──────────────────────────────────────── -# Follow the same pattern as read_state()/write_state() in lib/state.sh. +# ─── State Helpers ──────────────────────────────────────────────── +# Same pattern as read_state()/write_state() in lib/state.sh. read_onboard_state() { local target_name="${1:-$JOSH_SYNC_TARGET_NAME}" git fetch origin "$STATE_BRANCH" 2>/dev/null || true - git show "origin/${STATE_BRANCH}:${target_name}/onboard.json" 2>/dev/null || echo '{}' + + local json + json=$(git show "origin/${STATE_BRANCH}:${target_name}/onboard.json" 2>/dev/null || echo "") + if [ -n "$json" ]; then + # Pre-unification onboard.json had no .strategy field — the only flow that + # existed was the destructive reset path; treat missing/empty .strategy as + # reset so resume doesn't silently flip via re-resolution. + echo "$json" | jq '. + {strategy: ((.strategy // "") | if . == "" then "reset" else . end)}' + return + fi + + # Backward-compat: fall back to v2.1 <target>/adopt.json so in-flight + # adoptions started before the unification continue to resume. The legacy + # schema lacks `.strategy`; inject "adopt" so resolve/dispatch downstream + # treats it correctly. Also drop the legacy `.mode` field (v2.1 carried it + # redundantly) to avoid future schema collisions. + json=$(git show "origin/${STATE_BRANCH}:${target_name}/adopt.json" 2>/dev/null || echo "") + if [ -n "$json" ]; then + echo "$json" | jq 'del(.mode) + {strategy: ((.strategy // "") | if . == "" then "adopt" else . end)}' + return + fi + + echo '{}' } write_onboard_state() { @@ -50,18 +83,97 @@ write_onboard_state() { git worktree remove "$tmp_dir" 2>/dev/null || rm -rf "$tmp_dir" } +# Remove the v2.1 <target>/adopt.json from the state branch if present, so +# that a --restart durably clears legacy state on the next read. Called by +# onboard_flow's restart path. +_onboard_state_remove_legacy() { + local target_name="${1:-$JOSH_SYNC_TARGET_NAME}" + git fetch origin "$STATE_BRANCH" 2>/dev/null || true + git show "origin/${STATE_BRANCH}:${target_name}/adopt.json" >/dev/null 2>&1 || return 0 + + local tmp_dir + tmp_dir=$(mktemp -d) + if ! git worktree add "$tmp_dir" "origin/${STATE_BRANCH}" 2>/dev/null; then + rm -rf "$tmp_dir" + return 0 + fi + ( + cd "$tmp_dir" || exit + git rm -q "${target_name}/adopt.json" 2>/dev/null || true + if ! git diff --cached --quiet 2>/dev/null; then + git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ + commit -q -m "onboard: drop legacy ${target_name}/adopt.json on --restart" + git push -q origin "HEAD:${STATE_BRANCH}" || log "WARN" "Failed to push legacy adopt.json removal" + fi + ) + git worktree remove "$tmp_dir" 2>/dev/null || rm -rf "$tmp_dir" +} + +# ─── Strategy Resolution ────────────────────────────────────────── +# Precedence (highest first): +# 1. explicit --mode flag +# 2. targets[].history_lock (preserve → adopt, rewrite → reset) +# 3. auto-detect: subrepo has any branches → adopt, otherwise reset +# +# Prints the chosen strategy (reset|adopt) on stdout; logs the source on +# stderr so the preflight line in cmd_onboard surfaces the choice. + +resolve_onboard_strategy() { + local target_json="$1" + local explicit_mode="${2:-}" + + if [ -n "$explicit_mode" ]; then + case "$explicit_mode" in + reset|adopt) ;; + *) die "Invalid --mode '${explicit_mode}' (must be 'reset' or 'adopt')" ;; + esac + log "INFO" "Strategy: ${explicit_mode} (from --mode flag)" + echo "$explicit_mode" + return + fi + + local history_lock + history_lock=$(echo "$target_json" | jq -r '.history_lock // empty') + if [ -n "$history_lock" ]; then + case "$history_lock" in + preserve) + log "INFO" "Strategy: adopt (from config history_lock: preserve)" + echo "adopt"; return ;; + rewrite) + log "INFO" "Strategy: reset (from config history_lock: rewrite)" + echo "reset"; return ;; + *) + die "Invalid history_lock '${history_lock}' in target config (must be 'preserve' or 'rewrite')" ;; + esac + fi + + # Critical: distinguish a genuinely empty subrepo from a failed ls-remote. + # Treating an auth/network failure as "empty" would silently pick reset and + # force-push a live subrepo's history away. + local heads + if heads=$(git ls-remote --heads "$(subrepo_auth_url)" 2>/dev/null); then + if [ -n "$heads" ]; then + log "INFO" "Strategy: adopt (auto-detected — subrepo has branches)" + echo "adopt" + else + log "INFO" "Strategy: reset (auto-detected — subrepo has no branches)" + echo "reset" + fi + else + die "Cannot auto-detect strategy: 'git ls-remote' against the subrepo failed (auth/network). Pass --mode=reset|adopt explicitly, or set targets[].history_lock for this target." + fi +} + # ─── Derive Archived API URL ───────────────────────────────────── # Given a URL like "git@host:org/repo-archived.git" or # "https://host/org/repo-archived.git", derive the Gitea API URL. _archived_api_from_url() { local url="$1" - # Strip .git suffix first — avoids non-greedy regex issues in POSIX ERE url="${url%.git}" local host repo_path if echo "$url" | grep -qE '^(ssh://|git@)'; then - # SSH URL if echo "$url" | grep -q '^ssh://'; then host=$(echo "$url" | sed -E 's|ssh://[^@]*@([^/]+)/.*|\1|') repo_path=$(echo "$url" | sed -E 's|ssh://[^@]*@[^/]+/(.+)$|\1|') @@ -70,7 +182,6 @@ _archived_api_from_url() { repo_path=$(echo "$url" | sed -E 's|git@[^:/]+[:/](.+)$|\1|') fi else - # HTTPS URL host=$(echo "$url" | sed -E 's|https?://([^/]+)/.*|\1|') repo_path=$(echo "$url" | sed -E 's|https?://[^/]+/(.+)$|\1|') fi @@ -80,104 +191,131 @@ _archived_api_from_url() { # ─── Onboard Flow ──────────────────────────────────────────────── # Interactive orchestrator with checkpoint/resume. -# Usage: onboard_flow <target_json> <restart> +# Usage: onboard_flow <target_json> <restart> [explicit_mode] +# explicit_mode — "reset", "adopt", or "" for auto-resolve. onboard_flow() { local target_json="$1" local restart="${2:-false}" + local explicit_mode="${3:-}" local target_name="$JOSH_SYNC_TARGET_NAME" - # Load existing onboard state (or empty) - local onboard_state - onboard_state=$(read_onboard_state "$target_name") - local current_step - current_step=$(echo "$onboard_state" | jq -r '.step // "start"') + local state current_step strategy + state=$(read_onboard_state "$target_name") + current_step=$(echo "$state" | jq -r '.step // "start"') + strategy=$(echo "$state" | jq -r '.strategy // empty') if [ "$restart" = true ]; then log "INFO" "Restarting onboard from scratch" current_step="start" - onboard_state='{}' + state='{}' + strategy="" + # Durably clear any legacy v2.1 adopt.json so the next read doesn't + # silently fall back to it. + _onboard_state_remove_legacy "$target_name" fi + if [ -z "$strategy" ]; then + strategy=$(resolve_onboard_strategy "$target_json" "$explicit_mode") + elif [ -n "$explicit_mode" ] && [ "$strategy" != "$explicit_mode" ]; then + die "Saved strategy is '${strategy}' (mid-flow); cannot switch to --mode=${explicit_mode} without --restart." + else + log "INFO" "Strategy: ${strategy} (resumed from state)" + fi + + case "$strategy" in + reset|adopt) ;; + *) die "Unknown strategy '${strategy}' in state (corrupt?). Re-run with --restart to start over." ;; + esac + log "INFO" "Onboard step: ${current_step}" - # ── Step 1: Prerequisites + archived repo info ── + # ── Step 1: Strategy-specific intro and (for reset) archived repo info ── if [ "$current_step" = "start" ]; then echo "" >&2 - echo "=== Onboarding ${target_name} ===" >&2 - echo "" >&2 - echo "Before proceeding, you should have:" >&2 - echo " 1. Renamed the existing subrepo (e.g., storefront → storefront-archived)" >&2 - echo " 2. Created a new EMPTY repo at the original URL" >&2 + echo "=== Onboarding ${target_name} (strategy: ${strategy}) ===" >&2 echo "" >&2 - # Verify the new (empty) subrepo is reachable (no HEAD ref — works on empty repos) - if git ls-remote "$(subrepo_auth_url)" >/dev/null 2>&1; then - # shellcheck disable=SC2001 # sed is clearer for URL pattern replacement - log "INFO" "New subrepo is reachable at $(echo "$SUBREPO_URL" | sed 's|://[^@]*@|://***@|')" + if [ "$strategy" = "reset" ]; then + echo "Before proceeding, you should have:" >&2 + echo " 1. Renamed the existing subrepo (e.g., storefront → storefront-archived)" >&2 + echo " 2. Created a new EMPTY repo at the original URL" >&2 + echo "" >&2 + + if git ls-remote "$(subrepo_auth_url)" >/dev/null 2>&1; then + # shellcheck disable=SC2001 # sed is clearer for URL pattern replacement + log "INFO" "New subrepo is reachable at $(echo "$SUBREPO_URL" | sed 's|://[^@]*@|://***@|')" + else + log "WARN" "New subrepo is not reachable — make sure you created the new empty repo" + fi + + echo "Enter the archived repo URL (e.g., git@host:org/repo-archived.git):" >&2 + local archived_url + read -r archived_url + [ -n "$archived_url" ] || die "Archived URL is required" + + local archived_auth="${SUBREPO_AUTH:-https}" + local archived_api + archived_api=$(_archived_api_from_url "$archived_url") + + if curl -sf -H "Authorization: token ${SUBREPO_TOKEN}" \ + "${archived_api}" >/dev/null 2>&1; then + log "INFO" "Archived repo reachable: ${archived_api}" + else + log "WARN" "Cannot reach archived repo API — check URL and token" + echo "Continue anyway? (y/N):" >&2 + local confirm + read -r confirm + [ "$confirm" = "y" ] || [ "$confirm" = "Y" ] || die "Aborted" + fi + + state=$(jq -n \ + --arg step "importing" \ + --arg strategy "reset" \ + --arg archived_api "$archived_api" \ + --arg archived_url "$archived_url" \ + --arg archived_auth "$archived_auth" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{step:$step, strategy:$strategy, archived_api:$archived_api, + archived_url:$archived_url, archived_auth:$archived_auth, + import_prs:{}, reset_branches:[], migrated_prs:[], timestamp:$ts}') else - log "WARN" "New subrepo is not reachable — make sure you created the new empty repo" + # strategy = adopt + echo "This keeps the existing subrepo history and adds one adoption merge commit per branch." >&2 + echo "No force-push is used. Existing open PR branches remain based on the old history." >&2 + echo "" >&2 + + state=$(jq -n \ + --arg step "importing" \ + --arg strategy "adopt" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{step:$step, strategy:$strategy, import_prs:{}, adopted_branches:[], timestamp:$ts}') fi - echo "Enter the archived repo URL (e.g., git@host:org/repo-archived.git):" >&2 - local archived_url - read -r archived_url - [ -n "$archived_url" ] || die "Archived URL is required" - - # Determine auth type for archived repo (same as current subrepo) - local archived_auth="${SUBREPO_AUTH:-https}" - - # Derive API URL - local archived_api - archived_api=$(_archived_api_from_url "$archived_url") - - # Verify archived repo is reachable via API - if curl -sf -H "Authorization: token ${SUBREPO_TOKEN}" \ - "${archived_api}" >/dev/null 2>&1; then - log "INFO" "Archived repo reachable: ${archived_api}" - else - log "WARN" "Cannot reach archived repo API — check URL and token" - echo "Continue anyway? (y/N):" >&2 - local confirm - read -r confirm - [ "$confirm" = "y" ] || [ "$confirm" = "Y" ] || die "Aborted" - fi - - # Save state - onboard_state=$(jq -n \ - --arg step "importing" \ - --arg archived_api "$archived_api" \ - --arg archived_url "$archived_url" \ - --arg archived_auth "$archived_auth" \ - --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{step:$step, archived_api:$archived_api, archived_url:$archived_url, - archived_auth:$archived_auth, import_prs:{}, reset_branches:[], - migrated_prs:[], timestamp:$ts}') - write_onboard_state "$target_name" "$onboard_state" + write_onboard_state "$target_name" "$state" current_step="importing" fi - # ── Step 2: Import (reuses initial_import()) ── + # ── Step 2: Import (reset uses archived clone URL; adopt uses subrepo URL) ── if [ "$current_step" = "importing" ]; then echo "" >&2 log "INFO" "Step 2: Importing subrepo content into monorepo..." - local branches + local branches import_prs branches=$(echo "$target_json" | jq -r '.branches | keys[]') + import_prs=$(echo "$state" | jq -r '.import_prs // {}') - # Load existing import_prs from state (resume support) - local import_prs - import_prs=$(echo "$onboard_state" | jq -r '.import_prs // {}') - - # Build the archived repo clone URL for initial_import(). - # The content lives in the archived repo — the new repo at SUBREPO_URL is empty. - local archived_url archived_clone_url - archived_url=$(echo "$onboard_state" | jq -r '.archived_url') - if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then - archived_clone_url="$archived_url" - else - # shellcheck disable=SC2001 - archived_clone_url=$(echo "$archived_url" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|") + local archived_clone_url="" + if [ "$strategy" = "reset" ]; then + local archived_url + archived_url=$(echo "$state" | jq -r '.archived_url // empty') + [ -n "$archived_url" ] || die "Reset strategy: archived_url missing from state (corrupt or interrupted before start step finished). Re-run with --restart." + if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then + archived_clone_url="$archived_url" + else + # shellcheck disable=SC2001 + archived_clone_url=$(echo "$archived_url" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|") + fi fi for branch in $branches; do @@ -185,7 +323,6 @@ onboard_flow() { mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') [ -z "$mapped" ] && continue - # Skip branches that already have an import PR recorded if echo "$import_prs" | jq -e --arg b "$branch" 'has($b)' >/dev/null 2>&1; then log "INFO" "Import PR already recorded for ${branch} — skipping" continue @@ -196,11 +333,14 @@ onboard_flow() { log "INFO" "Importing branch: ${branch} (subrepo: ${mapped})" local result - result=$(initial_import "$archived_clone_url") + if [ -n "$archived_clone_url" ]; then + result=$(initial_import "$archived_clone_url") + else + result=$(initial_import) + fi log "INFO" "Import result for ${branch}: ${result}" if [ "$result" = "pr-created" ]; then - # Find the import PR number via API local prs pr_number prs=$(list_open_prs "$MONOREPO_API" "$GITEA_TOKEN") pr_number=$(echo "$prs" | jq -r --arg t "$target_name" --arg b "$branch" \ @@ -210,37 +350,35 @@ onboard_flow() { import_prs=$(echo "$import_prs" | jq --arg b "$branch" --arg n "$pr_number" '. + {($b): ($n | tonumber)}') log "INFO" "Import PR for ${branch}: #${pr_number}" else - log "WARN" "Could not find import PR number for ${branch} — check monorepo PRs" + # Same hazard for both strategies: continuing without recording the + # PR number means the next resume re-runs initial_import and creates + # a duplicate import PR. Bail out and let the user reconcile. + die "Could not find import PR number for ${branch} on monorepo. Re-running would create a duplicate import PR — check the monorepo's open PRs first." fi fi - # Save progress after each branch (resume support) - onboard_state=$(echo "$onboard_state" | jq --argjson prs "$import_prs" '.import_prs = $prs') - write_onboard_state "$target_name" "$onboard_state" + state=$(echo "$state" | jq --argjson prs "$import_prs" '.import_prs = $prs') + write_onboard_state "$target_name" "$state" done - # Update state - onboard_state=$(echo "$onboard_state" | jq \ + state=$(echo "$state" | jq \ --arg step "waiting-for-merge" \ --argjson prs "$import_prs" \ '.step = $step | .import_prs = $prs') - write_onboard_state "$target_name" "$onboard_state" + write_onboard_state "$target_name" "$state" current_step="waiting-for-merge" fi - # ── Step 3: Wait for merge ── + # ── Step 3: Wait for import PR(s) to merge ── if [ "$current_step" = "waiting-for-merge" ]; then echo "" >&2 log "INFO" "Step 3: Waiting for import PR(s) to be merged..." - local import_prs - import_prs=$(echo "$onboard_state" | jq -r '.import_prs') - local pr_count + local import_prs pr_count + import_prs=$(echo "$state" | jq -r '.import_prs') pr_count=$(echo "$import_prs" | jq 'length') - if [ "$pr_count" -eq 0 ]; then - log "WARN" "No import PRs recorded — skipping merge check" - else + if [ "$pr_count" -gt 0 ]; then echo "" >&2 echo "Import PRs to merge:" >&2 echo "$import_prs" | jq -r 'to_entries[] | " \(.key): PR #\(.value)"' >&2 @@ -248,12 +386,10 @@ onboard_flow() { echo "Merge the import PR(s) on the monorepo, then press Enter..." >&2 read -r - # Verify each PR is merged local all_merged=true for branch in $(echo "$import_prs" | jq -r 'keys[]'); do - local pr_number + local pr_number pr_json merged pr_number=$(echo "$import_prs" | jq -r --arg b "$branch" '.[$b]') - local pr_json merged pr_json=$(get_pr "$MONOREPO_API" "$GITEA_TOKEN" "$pr_number") merged=$(echo "$pr_json" | jq -r '.merged // false') @@ -268,26 +404,31 @@ onboard_flow() { if [ "$all_merged" = false ]; then die "Not all import PRs are merged. Re-run 'josh-sync onboard ${target_name}' after merging." fi + else + log "INFO" "No import PRs recorded — subfolder already matched subrepo" fi - # Update state - onboard_state=$(echo "$onboard_state" | jq '.step = "resetting"') - write_onboard_state "$target_name" "$onboard_state" - current_step="resetting" + local next_step + if [ "$strategy" = "reset" ]; then + next_step="resetting" + else + next_step="adopting" + fi + state=$(echo "$state" | jq --arg s "$next_step" '.step = $s') + write_onboard_state "$target_name" "$state" + current_step="$next_step" fi - # ── Step 4: Reset (pushes josh-filtered history to new repo) ── + # ── Step 4a: Finalize via reset (force-push josh-filtered history) ── if [ "$current_step" = "resetting" ]; then echo "" >&2 log "INFO" "Step 4: Pushing josh-filtered history to new subrepo..." - local branches + local branches already_reset branches=$(echo "$target_json" | jq -r '.branches | keys[]') - local already_reset - already_reset=$(echo "$onboard_state" | jq -r '.reset_branches // []') + already_reset=$(echo "$state" | jq -r '.reset_branches // []') for branch in $branches; do - # Skip branches already reset (resume support) if echo "$already_reset" | jq -e --arg b "$branch" 'index($b) != null' >/dev/null 2>&1; then log "INFO" "Branch ${branch} already reset — skipping" continue @@ -304,18 +445,69 @@ onboard_flow() { result=$(subrepo_reset) log "INFO" "Reset result for ${branch}: ${result}" - # Track progress - onboard_state=$(echo "$onboard_state" | jq --arg b "$branch" \ - '.reset_branches += [$b]') - write_onboard_state "$target_name" "$onboard_state" + state=$(echo "$state" | jq --arg b "$branch" '.reset_branches += [$b]') + write_onboard_state "$target_name" "$state" done - # Update state - onboard_state=$(echo "$onboard_state" | jq \ + state=$(echo "$state" | jq \ --arg step "complete" \ --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ '.step = $step | .timestamp = $ts') - write_onboard_state "$target_name" "$onboard_state" + write_onboard_state "$target_name" "$state" + current_step="complete" + fi + + # ── Step 4b: Finalize via adoption merge ── + if [ "$current_step" = "adopting" ]; then + echo "" >&2 + log "INFO" "Step 4: Creating adoption merge commit(s)..." + + local branches already_adopted + branches=$(echo "$target_json" | jq -r '.branches | keys[]') + already_adopted=$(echo "$state" | jq -r '.adopted_branches // []') + + for branch in $branches; do + if echo "$already_adopted" | jq -e --arg b "$branch" 'index($b) != null' >/dev/null 2>&1; then + log "INFO" "Branch ${branch} already adopted — skipping" + continue + fi + + local mapped + mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty') + [ -z "$mapped" ] && continue + + export SYNC_BRANCH_MONO="$branch" + export SYNC_BRANCH_SUBREPO="$mapped" + + local result + result=$(adopt_branch) + log "INFO" "Adopt result for ${branch}: ${result}" + + case "$result" in + adopted|already-adopted) + state=$(echo "$state" | jq --arg b "$branch" '.adopted_branches += [$b]') + write_onboard_state "$target_name" "$state" + ;; + tree-mismatch) + die "Tree mismatch for ${branch}. Merge the import PR and make sure subrepo/${mapped} matches the Josh-filtered monorepo tree." + ;; + push-rejected) + die "Subrepo branch ${mapped} changed during adoption. Re-run 'josh-sync onboard ${target_name} --mode=adopt' to retry." + ;; + missing-branch) + die "Subrepo branch ${mapped} does not exist." + ;; + *) + die "Unexpected adopt result: ${result}" + ;; + esac + done + + state=$(echo "$state" | jq \ + --arg step "complete" \ + --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '.step = $step | .timestamp = $ts') + write_onboard_state "$target_name" "$state" current_step="complete" fi @@ -324,14 +516,20 @@ onboard_flow() { echo "" >&2 echo "=== Onboarding complete! ===" >&2 echo "" >&2 - echo "The new subrepo now has josh-filtered history." >&2 - echo "Developers should re-clone or reset their local copies:" >&2 - echo " git fetch origin && git reset --hard origin/main" >&2 - echo "" >&2 - echo "To migrate open PRs from the archived repo:" >&2 - echo " josh-sync migrate-pr ${target_name} # interactive picker" >&2 - echo " josh-sync migrate-pr ${target_name} --all # migrate all" >&2 - echo " josh-sync migrate-pr ${target_name} 5 8 12 # specific PRs" >&2 + if [ "$strategy" = "reset" ]; then + echo "The new subrepo now has josh-filtered history." >&2 + echo "Developers should re-clone or reset their local copies:" >&2 + echo " git fetch origin && git reset --hard origin/main" >&2 + echo "" >&2 + echo "To migrate open PRs from the archived repo:" >&2 + echo " josh-sync migrate-pr ${target_name} # interactive picker" >&2 + echo " josh-sync migrate-pr ${target_name} --all # migrate all" >&2 + echo " josh-sync migrate-pr ${target_name} 5 8 12 # specific PRs" >&2 + else + echo "The subrepo keeps its existing history and now has a josh-sync adoption merge." >&2 + echo "Developers can keep their clones; they only need to fast-forward active branches:" >&2 + echo " git fetch origin && git checkout main && git merge --ff-only origin/main" >&2 + fi fi } @@ -346,15 +544,17 @@ migrate_one_pr() { local pr_number="$1" local target_name="$JOSH_SYNC_TARGET_NAME" - # Read archived repo info from onboard state - local onboard_state archived_api + local onboard_state strategy archived_api onboard_state=$(read_onboard_state "$target_name") + strategy=$(echo "$onboard_state" | jq -r '.strategy // "reset"') + if [ "$strategy" = "adopt" ]; then + die "Target '${target_name}' was onboarded with the adopt strategy, which does not record an archived repo. migrate-pr only applies to the reset strategy." + fi archived_api=$(echo "$onboard_state" | jq -r '.archived_api') if [ -z "$archived_api" ] || [ "$archived_api" = "null" ]; then - die "No archived repo info found. Run 'josh-sync onboard ${target_name}' first." + die "No archived repo info found for '${target_name}'. Run 'josh-sync onboard ${target_name} --mode=reset' first." fi - # Check if this PR was already migrated local already_migrated already_migrated=$(echo "$onboard_state" | jq -r \ --argjson num "$pr_number" '.migrated_prs // [] | map(select(.old_number == $num)) | length') @@ -363,10 +563,8 @@ migrate_one_pr() { return 0 fi - # Same credentials — the repo was just renamed local archived_token="$SUBREPO_TOKEN" - # 1. Get PR metadata from archived repo local pr_json title base head body pr_json=$(get_pr "$archived_api" "$archived_token" "$pr_number") \ || die "Failed to fetch PR #${pr_number} from archived repo" @@ -377,8 +575,6 @@ migrate_one_pr() { log "INFO" "Migrating PR #${pr_number}: \"${title}\" (${base} <- ${head})" - # 2. Clone new subrepo, add archived repo as second remote - # Save cwd so we can restore it (function runs in caller's shell, not subshell) local original_dir original_dir=$(pwd) @@ -394,7 +590,6 @@ migrate_one_pr() { git config user.name "$BOT_NAME" git config user.email "$BOT_EMAIL" - # Build authenticated URL for the archived repo local archived_url archived_clone_url archived_url=$(echo "$onboard_state" | jq -r '.archived_url') if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then @@ -404,12 +599,10 @@ migrate_one_pr() { archived_clone_url=$(echo "$archived_url" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|") fi - # Fetch the PR's head and base branches from the archived repo git remote add archived "$archived_clone_url" git fetch archived "$head" "$base" 2>&1 \ || die "Failed to fetch branches from archived repo" - # 3. Compute diff locally and apply with --3way git checkout -B "$head" >&2 local diff @@ -428,13 +621,11 @@ Migrated from archived repo PR #${pr_number}" >&2 git push "$(subrepo_auth_url)" "$head" >&2 \ || die "Failed to push branch ${head}" - # 4. Create PR on new repo local new_number new_number=$(create_pr_number "$SUBREPO_API" "$SUBREPO_TOKEN" \ "$base" "$head" "$title" "$body") log "INFO" "Migrated PR #${pr_number} -> #${new_number}: \"${title}\"" - # 5. Record in onboard state cd "$original_dir" || true onboard_state=$(read_onboard_state "$target_name") onboard_state=$(echo "$onboard_state" | jq \ diff --git a/schema/config-schema.json b/schema/config-schema.json index eff6966..da8b169 100644 --- a/schema/config-schema.json +++ b/schema/config-schema.json @@ -91,6 +91,11 @@ "items": { "type": "string" }, "default": [], "description": "File/directory patterns to exclude from sync via josh :exclude filter. Josh pattern syntax: 'dir/' for directories, '*.ext' for globs, '**/dir/' for nested matches. Patterns are embedded inline in the josh-proxy URL." + }, + "history_lock": { + "type": "string", + "enum": ["rewrite", "preserve"], + "description": "Onboarding strategy override. 'preserve' selects the non-destructive adopt strategy (keeps subrepo history via an adoption merge); 'rewrite' selects the destructive reset strategy (force-pushes josh-filtered history). When omitted, `josh-sync onboard` auto-detects: subrepos with branches → adopt, empty subrepos → reset. The CLI flag `--mode={reset,adopt}` overrides this." } } } diff --git a/tests/unit/adopt_e2e.bats b/tests/unit/adopt_e2e.bats new file mode 100644 index 0000000..a566829 --- /dev/null +++ b/tests/unit/adopt_e2e.bats @@ -0,0 +1,697 @@ +#!/usr/bin/env bats + +bats_require_minimum_version 1.5.0 + +# tests/unit/adopt_e2e.bats — End-to-end adopt behaviour and ADR cross-cuts +# +# Covers the goal-1 scenarios from the adopt unification plan: +# - tree-equality rejection (clear hint + no commit on subrepo) +# - already-adopted idempotency +# - merge commit shape (ADR-005 trailer, ADR-008 first-parent walk) +# - fast-forward push semantics (success + push-rejected surfacing) +# - reverse-sync loop guard (ADR-005) +# - linearize-fallback selection invariant (ADR-011) +# - checkpoint/resume at each adopt_flow step (ADR-010) +# +# Tests run against real local bare repos — no mocking of the git layer. + +setup() { + export JOSH_SYNC_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + source "$JOSH_SYNC_ROOT/lib/core.sh" + source "$JOSH_SYNC_ROOT/lib/state.sh" + source "$JOSH_SYNC_ROOT/lib/auth.sh" + source "$JOSH_SYNC_ROOT/lib/adopt.sh" + source "$JOSH_SYNC_ROOT/lib/onboard.sh" + + TEST_ROOT="$(mktemp -d)" + export BOT_NAME="test-bot" + export BOT_EMAIL="test-bot@test.local" + export BOT_TRAILER="Josh-Sync-Origin" + export JOSH_SYNC_TARGET_NAME="example" + export SYNC_BRANCH_MONO="main" + export SYNC_BRANCH_SUBREPO="main" + + # Test-local auth shims must be defined AFTER sourcing lib/auth.sh, otherwise + # the lib's real subrepo_auth_url / josh_auth_url (which need SUBREPO_URL, + # BOT_USER, etc.) overrides our bare-repo shortcut. + subrepo_auth_url() { printf '%s\n' "${SUBREPO_BARE:-}"; } + josh_auth_url() { printf '%s\n' "${JOSH_BARE:-}"; } +} + +teardown() { + # Restore cwd before the rm — setup_state_monorepo cd's into a dir that's + # about to be deleted, and bats doesn't reset cwd between tests. + cd "$BATS_TEST_DIRNAME" 2>/dev/null || true + rm -rf "$TEST_ROOT" +} + +# ─── Helpers ──────────────────────────────────────────────────────── + +# Build a fresh bare repo seeded with one commit; returns the bare path on stdout. +make_bare_repo() { + local name="$1" + local content="$2" + local message="$3" + local work="${TEST_ROOT}/${name}-work" + local bare="${TEST_ROOT}/${name}.git" + + git init "$work" >/dev/null + git -C "$work" checkout -b main >/dev/null + git -C "$work" config user.name "Test User" + git -C "$work" config user.email "test@test.local" + printf '%s\n' "$content" > "${work}/README.md" + git -C "$work" add README.md + git -C "$work" commit -m "$message" >/dev/null + + git init --bare "$bare" >/dev/null + # Make HEAD on the bare point at main so later clones don't land on master. + git -C "$bare" symbolic-ref HEAD refs/heads/main + git -C "$work" remote add origin "$bare" + git -C "$work" push origin main >/dev/null + printf '%s\n' "$bare" +} + +# Append a commit to an existing bare repo. If $3 is empty, makes an empty commit +# (tree unchanged) so first-parent walks have something to observe. +add_commit_to_bare() { + local bare="$1" + local message="$2" + local content="${3:-}" + local work="${TEST_ROOT}/add-${RANDOM}-$$" + git clone "$bare" "$work" >/dev/null 2>&1 + git -C "$work" config user.name "Test User" + git -C "$work" config user.email "test@test.local" + if [ -n "$content" ]; then + printf '%s\n' "$content" >> "${work}/README.md" + git -C "$work" commit -am "$message" >/dev/null + else + git -C "$work" commit --allow-empty -m "$message" >/dev/null + fi + git -C "$work" push origin main >/dev/null 2>&1 +} + +# Stand up a local working "monorepo" with an `origin` bare remote and cd into +# it. Gives lib/state.sh helpers (worktree + push to origin) something real to +# work against. +setup_state_monorepo() { + local mono_work="${TEST_ROOT}/mono" + local mono_bare="${TEST_ROOT}/mono.git" + git init -q "$mono_work" + git -C "$mono_work" checkout -q -b main + git -C "$mono_work" config user.name "Test User" + git -C "$mono_work" config user.email "test@test.local" + printf 'mono\n' > "${mono_work}/README.md" + git -C "$mono_work" add README.md + git -C "$mono_work" commit -q -m "initial" + git init -q --bare "$mono_bare" + git -C "$mono_work" remote add origin "$mono_bare" + git -C "$mono_work" push -q origin main + cd "$mono_work" +} + +# adopt_branch is defined as a subshell function (`adopt_branch() ( ... )`) in +# lib/adopt.sh so its EXIT trap stays contained — no helper needed here. +adopt_branch_quietly() { + adopt_branch >/dev/null +} + +# ─── Tree-mismatch surfacing ──────────────────────────────────────── + +@test "adopt_branch tree-mismatch error message points the user at the import PR" { + SUBREPO_BARE="$(make_bare_repo subrepo "subrepo content" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "josh content" "josh filtered history")" + + local combined + combined=$(adopt_branch 2>&1) + + [[ "$combined" == *"merge the import PR"* ]] +} + +@test "adopt_branch tree-mismatch creates no commit on the subrepo (clean abort)" { + SUBREPO_BARE="$(make_bare_repo subrepo "subrepo content" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "josh content" "josh filtered history")" + + local before_count before_head after_count after_head + before_count=$(git --git-dir="$SUBREPO_BARE" rev-list --count main) + before_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + + adopt_branch_quietly 2>&1 + + after_count=$(git --git-dir="$SUBREPO_BARE" rev-list --count main) + after_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + [ "$before_count" = "$after_count" ] + [ "$before_head" = "$after_head" ] +} + +# ─── Idempotency ──────────────────────────────────────────────────── + +@test "adopt_branch is idempotent — second invocation returns already-adopted, no new commit" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + + local first_result first_head first_count + first_result=$(adopt_branch) + first_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + first_count=$(git --git-dir="$SUBREPO_BARE" rev-list --count main) + + local second_result second_head second_count + second_result=$(adopt_branch) + second_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + second_count=$(git --git-dir="$SUBREPO_BARE" rev-list --count main) + + [ "$first_result" = "adopted" ] + [ "$second_result" = "already-adopted" ] + [ "$first_head" = "$second_head" ] + [ "$first_count" = "$second_count" ] +} + +# ─── Merge commit shape ───────────────────────────────────────────── + +@test "adopt_branch merge carries the configured bot trailer (ADR-005)" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + + adopt_branch_quietly + + local merge_msg + merge_msg=$(git --git-dir="$SUBREPO_BARE" log -1 --format=%B main) + [[ "$merge_msg" == *"${BOT_TRAILER}: adopt/${SYNC_BRANCH_MONO}/"* ]] +} + +@test "adopt_branch merge places josh-filtered HEAD as parent 1 — first-parent walk reaches monorepo (ADR-008)" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo c1")" + add_commit_to_bare "$SUBREPO_BARE" "subrepo c2" + + JOSH_BARE="$(make_bare_repo josh "same tree" "josh c1")" + add_commit_to_bare "$JOSH_BARE" "josh c2" + + adopt_branch_quietly + + # First-parent walk from the adoption merge follows the josh chain (parent 1) + # and never visits subrepo-only history. If parent ordering ever flipped, the + # subrepo commits would appear here — which is exactly the failure mode that + # corrupted the monorepo branch in ADR-008. + local first_parent_msgs + first_parent_msgs=$(git --git-dir="$SUBREPO_BARE" log --first-parent --format=%s main) + [[ "$first_parent_msgs" == *"josh c1"* ]] + [[ "$first_parent_msgs" == *"josh c2"* ]] + [[ "$first_parent_msgs" != *"subrepo c1"* ]] + [[ "$first_parent_msgs" != *"subrepo c2"* ]] +} + +# ─── Fast-forward push semantics ──────────────────────────────────── + +@test "adopt_branch fast-forwards without --force, leaving old subrepo HEAD reachable" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + + local old_head result new_head + old_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + result=$(adopt_branch) + new_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + + [ "$result" = "adopted" ] + [ "$new_head" != "$old_head" ] + # The original subrepo HEAD must still be an ancestor of the new HEAD — + # nothing was rewritten. This is what makes the push a real fast-forward. + git --git-dir="$SUBREPO_BARE" merge-base --is-ancestor "$old_head" "$new_head" +} + +@test "adopt_branch surfaces push-rejected when the bare subrepo rejects the push (simulated 'subrepo moved')" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + + # Equivalent observable behaviour to "subrepo HEAD changed mid-flight": the + # fast-forward push fails. Install a pre-receive hook that rejects + # deterministically — adopt_branch's else-branch should surface push-rejected. + printf '%s\n' \ + '#!/usr/bin/env bash' \ + 'echo "subrepo changed during adoption" >&2' \ + 'exit 1' \ + > "${SUBREPO_BARE}/hooks/pre-receive" + chmod +x "${SUBREPO_BARE}/hooks/pre-receive" + + local before_head result after_head + before_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + result=$(adopt_branch 2>/dev/null) + after_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + + [ "$result" = "push-rejected" ] + [ "$before_head" = "$after_head" ] +} + +# ─── Reverse-sync loop guard (ADR-005) ────────────────────────────── + +@test "adoption merge is invisible to reverse-sync bot-trailer filter (no re-ingestion loop)" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + + local josh_head + josh_head=$(git --git-dir="$JOSH_BARE" rev-parse main) + + adopt_branch_quietly + + # Mirror reverse_sync's exact query (lib/sync.sh): + # base = merge-base(mono-filtered/<branch>, HEAD) + # human_commits = git log --ancestry-path "$base..HEAD" \ + # --invert-grep --grep="^${BOT_TRAILER}:" + # --ancestry-path is critical: without it, the subrepo's parent-2 orphan + # commit (no trailer, real user commit) would show up as a "human commit" + # and reverse sync would try to re-ingest the adoption merge's other side. + local merge_sha base human_commits + merge_sha=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + base=$(git --git-dir="$SUBREPO_BARE" merge-base "$josh_head" "$merge_sha") + human_commits=$(git --git-dir="$SUBREPO_BARE" log \ + --ancestry-path "${base}..${merge_sha}" \ + --invert-grep --grep="^${BOT_TRAILER}:" \ + --oneline) + + [ -z "$human_commits" ] +} + +# ─── Linearize fallback (ADR-011) ─────────────────────────────────── + +@test "linearize-fallback commit selection excludes the adopt merge but keeps human commits on top" { + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + + local josh_head + josh_head=$(git --git-dir="$JOSH_BARE" rev-parse main) + + adopt_branch_quietly + + # Real human commit on top of the adoption merge. + add_commit_to_bare "$SUBREPO_BARE" "human: add note" "human change" + + # Replicate reverse_sync's linearize walk exactly (lib/sync.sh): + # base = merge-base(mono-filtered/<branch>, HEAD) + # commits = git log --ancestry-path "$base..HEAD" --reverse \ + # --format=%H --invert-grep --grep="^${BOT_TRAILER}:" + # With messy upstream (adopt-merge in the path), linearize must skip the + # merge (trailer-filtered) and cherry-pick only the human commit. + local work base picked + work="${TEST_ROOT}/linearize-walk" + git clone "$SUBREPO_BARE" "$work" >/dev/null 2>&1 + base=$(git -C "$work" merge-base "$josh_head" HEAD) + picked=$(git -C "$work" log --ancestry-path "${base}..HEAD" \ + --reverse --format=%s --invert-grep --grep="^${BOT_TRAILER}:") + + [[ "$picked" == *"human: add note"* ]] + [[ "$picked" != *"Adopt ${JOSH_SYNC_TARGET_NAME} for josh-sync"* ]] +} + +# ─── Checkpoint / resume (ADR-010) — strategy=adopt path through onboard_flow + +# Stubs that run inside $(...) subshells can't increment shell variables in +# the parent. Track invocations via append-only files instead and count bytes. +call_log() { printf '.' >> "${TEST_ROOT}/${1}.log"; } +call_count() { + local f="${TEST_ROOT}/${1}.log" + if [ -f "$f" ]; then wc -c < "$f" | tr -d ' '; else echo 0; fi +} + +@test "write_onboard_state then read_onboard_state round-trips the JSON" { + setup_state_monorepo + + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"adopting","strategy":"adopt","import_prs":{"main":42},"adopted_branches":[]}' + + local got + got=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + [ "$(echo "$got" | jq -r '.step')" = "adopting" ] + [ "$(echo "$got" | jq -r '.strategy')" = "adopt" ] + [ "$(echo "$got" | jq -r '.import_prs.main')" = "42" ] +} + +@test "read_onboard_state falls back to legacy <target>/adopt.json with implicit strategy=adopt" { + setup_state_monorepo + + # Simulate a state branch where only the v2.1-era adopt.json exists (e.g., + # an adoption started before the unification). Write directly to the state + # branch under the legacy filename, without the new strategy field. + local tmp + tmp=$(mktemp -d) + git worktree add --detach "$tmp" >/dev/null 2>&1 + ( cd "$tmp" && git checkout -q --orphan "$STATE_BRANCH" && { git rm -rfq . 2>/dev/null || true; } + mkdir -p "$JOSH_SYNC_TARGET_NAME" + printf '%s\n' '{"step":"adopting","import_prs":{},"adopted_branches":[]}' \ + > "${JOSH_SYNC_TARGET_NAME}/adopt.json" + git add -A + git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ + commit -q -m "legacy adopt state" + git push -q origin "HEAD:${STATE_BRANCH}" ) + git worktree remove "$tmp" 2>/dev/null || rm -rf "$tmp" + + local got + got=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + [ "$(echo "$got" | jq -r '.step')" = "adopting" ] + [ "$(echo "$got" | jq -r '.strategy')" = "adopt" ] +} + +@test "onboard_flow resumes from step=importing (strategy=adopt) — runs import, falls through wait, finalizes via adopt_branch" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"importing","strategy":"adopt","import_prs":{},"adopted_branches":[]}' + + initial_import() { call_log import; echo "skip"; } + adopt_branch() { call_log adopt; echo "adopted"; } + subrepo_reset() { call_log reset; echo "reset"; } + get_pr() { echo '{"merged":true}'; } + list_open_prs() { echo '[]'; } + + local target_json='{"name":"example","branches":{"main":"main"}}' + onboard_flow "$target_json" "false" "" >/dev/null 2>&1 + + [ "$(call_count import)" = "1" ] + [ "$(call_count adopt)" = "1" ] + [ "$(call_count reset)" = "0" ] + local final + final=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + [ "$(echo "$final" | jq -r '.step')" = "complete" ] + [ "$(echo "$final" | jq -r '.strategy')" = "adopt" ] +} + +@test "onboard_flow resumes from step=waiting-for-merge (strategy=adopt) — confirms merged PR, finalizes via adopt_branch" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"waiting-for-merge","strategy":"adopt","import_prs":{"main":42},"adopted_branches":[]}' + + initial_import() { call_log import; echo "skip"; } + adopt_branch() { call_log adopt; echo "adopted"; } + subrepo_reset() { call_log reset; echo "reset"; } + get_pr() { echo '{"merged":true}'; } + list_open_prs() { echo '[]'; } + export MONOREPO_API="http://stub" GITEA_TOKEN="stub" + + local target_json='{"name":"example","branches":{"main":"main"}}' + onboard_flow "$target_json" "false" "" <<< "" >/dev/null 2>&1 + + [ "$(call_count import)" = "0" ] + [ "$(call_count adopt)" = "1" ] + [ "$(call_count reset)" = "0" ] + local final + final=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + [ "$(echo "$final" | jq -r '.step')" = "complete" ] +} + +@test "onboard_flow resumes from step=adopting (strategy=adopt) — skips importing and waiting-for-merge entirely" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"adopting","strategy":"adopt","import_prs":{},"adopted_branches":[]}' + + initial_import() { call_log import; echo "skip"; } + adopt_branch() { call_log adopt; echo "adopted"; } + subrepo_reset() { call_log reset; echo "reset"; } + get_pr() { echo '{"merged":true}'; } + list_open_prs() { echo '[]'; } + + local target_json='{"name":"example","branches":{"main":"main"}}' + onboard_flow "$target_json" "false" "" >/dev/null 2>&1 + + [ "$(call_count import)" = "0" ] + [ "$(call_count adopt)" = "1" ] + [ "$(call_count reset)" = "0" ] + local final + final=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + [ "$(echo "$final" | jq -r '.step')" = "complete" ] +} + +@test "onboard_flow with step=complete is a no-op (no import, no finalize)" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"complete","strategy":"adopt","import_prs":{},"adopted_branches":["main"]}' + + initial_import() { call_log import; echo "skip"; } + adopt_branch() { call_log adopt; echo "adopted"; } + subrepo_reset() { call_log reset; echo "reset"; } + + local target_json='{"name":"example","branches":{"main":"main"}}' + onboard_flow "$target_json" "false" "" >/dev/null 2>&1 + + [ "$(call_count import)" = "0" ] + [ "$(call_count adopt)" = "0" ] + [ "$(call_count reset)" = "0" ] +} + +@test "onboard_flow --restart with explicit strategy=adopt wipes state and re-runs the full sequence" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"complete","strategy":"adopt","import_prs":{"main":7},"adopted_branches":["main"]}' + + initial_import() { call_log import; echo "skip"; } + adopt_branch() { call_log adopt; echo "adopted"; } + subrepo_reset() { call_log reset; echo "reset"; } + get_pr() { echo '{"merged":true}'; } + list_open_prs() { echo '[]'; } + + local target_json='{"name":"example","branches":{"main":"main"}}' + onboard_flow "$target_json" "true" "adopt" >/dev/null 2>&1 + + [ "$(call_count import)" = "1" ] + [ "$(call_count adopt)" = "1" ] + [ "$(call_count reset)" = "0" ] +} + +# ─── Goal 2: strategy resolution precedence ───────────────────────── + +@test "resolve_onboard_strategy honours explicit --mode flag above config and auto-detect" { + # Config says preserve (would map to adopt); flag should win. + local target_json='{"name":"x","history_lock":"preserve"}' + local picked + picked=$(resolve_onboard_strategy "$target_json" "reset") + [ "$picked" = "reset" ] +} + +@test "resolve_onboard_strategy maps history_lock=preserve to adopt" { + local target_json='{"name":"x","history_lock":"preserve"}' + local picked + picked=$(resolve_onboard_strategy "$target_json" "") + [ "$picked" = "adopt" ] +} + +@test "resolve_onboard_strategy maps history_lock=rewrite to reset" { + local target_json='{"name":"x","history_lock":"rewrite"}' + local picked + picked=$(resolve_onboard_strategy "$target_json" "") + [ "$picked" = "reset" ] +} + +@test "resolve_onboard_strategy auto-detects adopt when subrepo has any branches" { + SUBREPO_BARE="$(make_bare_repo subrepo "any" "any")" + local target_json='{"name":"x"}' + local picked + picked=$(resolve_onboard_strategy "$target_json" "") + [ "$picked" = "adopt" ] +} + +@test "resolve_onboard_strategy auto-detects reset when subrepo has no branches (empty repo)" { + SUBREPO_BARE="${TEST_ROOT}/empty.git" + git init -q --bare "$SUBREPO_BARE" + local target_json='{"name":"x"}' + local picked + picked=$(resolve_onboard_strategy "$target_json" "") + [ "$picked" = "reset" ] +} + +@test "resolve_onboard_strategy rejects unknown --mode value" { + local target_json='{"name":"x"}' + run resolve_onboard_strategy "$target_json" "weird" + [ "$status" -ne 0 ] + [[ "$output" == *"Invalid --mode"* ]] +} + +@test "resolve_onboard_strategy rejects unknown history_lock value" { + local target_json='{"name":"x","history_lock":"freeze"}' + run resolve_onboard_strategy "$target_json" "" + [ "$status" -ne 0 ] + [[ "$output" == *"Invalid history_lock"* ]] +} + +# ─── Goal 2: onboard_flow with strategy=reset (regression-preserving) ── + +@test "onboard_flow resumes from step=resetting (strategy=reset) — finalizes via subrepo_reset" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"resetting","strategy":"reset","import_prs":{},"reset_branches":[]}' + + initial_import() { call_log import; echo "skip"; } + adopt_branch() { call_log adopt; echo "adopted"; } + subrepo_reset() { call_log reset; echo "reset"; } + + local target_json='{"name":"example","branches":{"main":"main"}}' + onboard_flow "$target_json" "false" "" >/dev/null 2>&1 + + [ "$(call_count import)" = "0" ] + [ "$(call_count adopt)" = "0" ] + [ "$(call_count reset)" = "1" ] + local final + final=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + [ "$(echo "$final" | jq -r '.step')" = "complete" ] + [ "$(echo "$final" | jq -r '.strategy')" = "reset" ] +} + +@test "onboard_flow finalize step (strategy=adopt) drives the real adopt_branch end-to-end against bare repos" { + # No stubbing of adopt_branch: this test exercises the full unified + # dispatch by pre-seeding state at step=adopting and letting onboard_flow + # call adopt_branch for real against local bare repos via the auth-URL + # shims. Closes the spec's end-to-end criterion for `onboard --mode=adopt`. + setup_state_monorepo + SUBREPO_BARE="$(make_bare_repo subrepo "same tree" "subrepo history")" + JOSH_BARE="$(make_bare_repo josh "same tree" "josh filtered history")" + + local old_subrepo_head josh_head + old_subrepo_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + josh_head=$(git --git-dir="$JOSH_BARE" rev-parse main) + + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"adopting","strategy":"adopt","import_prs":{},"adopted_branches":[]}' + + local target_json='{"name":"example","branches":{"main":"main"}}' + onboard_flow "$target_json" "false" "" >/dev/null 2>&1 + + # 1. Subrepo HEAD advanced to a real adoption merge. + local new_subrepo_head + new_subrepo_head=$(git --git-dir="$SUBREPO_BARE" rev-parse main) + [ "$new_subrepo_head" != "$old_subrepo_head" ] + + # 2. Merge shape matches ADR-013: parent 1 = josh_head, parent 2 = old subrepo head. + [ "$(git --git-dir="$SUBREPO_BARE" rev-parse 'main^1')" = "$josh_head" ] + [ "$(git --git-dir="$SUBREPO_BARE" rev-parse 'main^2')" = "$old_subrepo_head" ] + + # 3. Merge message carries the ADR-005 bot trailer. + local merge_msg + merge_msg=$(git --git-dir="$SUBREPO_BARE" log -1 --format=%B main) + [[ "$merge_msg" == *"${BOT_TRAILER}: adopt/${SYNC_BRANCH_MONO}/"* ]] + + # 4. State advanced to complete and recorded the adopted branch. + local final + final=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + [ "$(echo "$final" | jq -r '.step')" = "complete" ] + [ "$(echo "$final" | jq -r '.strategy')" = "adopt" ] + [ "$(echo "$final" | jq -r '.adopted_branches[0]')" = "main" ] +} + +# ─── Goal 2: legacy adopt.json → onboard.json migration on write ──── + +@test "after read_onboard_state falls back to legacy adopt.json, the next write lands in onboard.json" { + setup_state_monorepo + + # Seed the state branch with ONLY the legacy file (no onboard.json yet). + local tmp + tmp=$(mktemp -d) + git worktree add --detach "$tmp" >/dev/null 2>&1 + ( cd "$tmp" && git checkout -q --orphan "$STATE_BRANCH" && { git rm -rfq . 2>/dev/null || true; } + mkdir -p "$JOSH_SYNC_TARGET_NAME" + printf '%s\n' '{"step":"adopting","import_prs":{},"adopted_branches":[]}' \ + > "${JOSH_SYNC_TARGET_NAME}/adopt.json" + git add -A + git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ + commit -q -m "legacy adopt state" + git push -q origin "HEAD:${STATE_BRANCH}" ) + git worktree remove "$tmp" 2>/dev/null || rm -rf "$tmp" + + # Read → mutate → write back via the unified helpers. + local state + state=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + state=$(echo "$state" | jq '.step = "complete"') + write_onboard_state "$JOSH_SYNC_TARGET_NAME" "$state" + + # Fetch a fresh copy of the state branch to see what's actually committed. + git fetch origin "$STATE_BRANCH" >/dev/null 2>&1 + + # onboard.json now exists with the updated step, strategy preserved. + local onboard_json + onboard_json=$(git show "origin/${STATE_BRANCH}:${JOSH_SYNC_TARGET_NAME}/onboard.json") + [ "$(echo "$onboard_json" | jq -r '.step')" = "complete" ] + [ "$(echo "$onboard_json" | jq -r '.strategy')" = "adopt" ] + + # Legacy adopt.json is left in place (harmless; read fallback still resolves). + git show "origin/${STATE_BRANCH}:${JOSH_SYNC_TARGET_NAME}/adopt.json" >/dev/null +} + +@test "onboard_flow importing step dies (does not warn-and-continue) when pr_number lookup fails — prevents duplicate import PR on resume" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"importing","strategy":"adopt","import_prs":{},"adopted_branches":[]}' + + # initial_import claims a PR was created, but list_open_prs returns nothing + # matching → pr_number ends up empty. The fix turns the old WARN-and-continue + # into a die so a subsequent resume can't re-import and duplicate the PR. + initial_import() { echo "pr-created"; } + list_open_prs() { echo '[]'; } + adopt_branch() { echo "adopted"; } + export MONOREPO_API="http://stub" GITEA_TOKEN="stub" + + local target_json='{"name":"example","branches":{"main":"main"}}' + run --separate-stderr onboard_flow "$target_json" "false" "" + [ "$status" -ne 0 ] + [[ "$stderr" == *"Could not find import PR number"* ]] + [[ "$stderr" == *"duplicate import PR"* ]] +} + +@test "resolve_onboard_strategy dies (does not silently pick reset) when ls-remote against the subrepo fails" { + # Point auth at a nonexistent local path so ls-remote fails (exit non-zero). + SUBREPO_BARE="${TEST_ROOT}/this-does-not-exist.git" + local target_json='{"name":"x"}' + run --separate-stderr resolve_onboard_strategy "$target_json" "" + [ "$status" -ne 0 ] + [[ "$stderr" == *"Cannot auto-detect strategy"* ]] +} + +@test "onboard_flow rejects --mode that conflicts with a strategy already saved in state" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"adopting","strategy":"adopt","import_prs":{},"adopted_branches":[]}' + + initial_import() { echo "skip"; } + adopt_branch() { echo "adopted"; } + subrepo_reset() { echo "reset"; } + + local target_json='{"name":"example","branches":{"main":"main"}}' + run --separate-stderr onboard_flow "$target_json" "false" "reset" + [ "$status" -ne 0 ] + [[ "$stderr" == *"Saved strategy is 'adopt'"* ]] + [[ "$stderr" == *"--restart"* ]] +} + +@test "read_onboard_state injects strategy=reset for legacy v2.1 onboard.json (no .strategy field)" { + setup_state_monorepo + + # Seed only a legacy-format onboard.json (no .strategy field). + local tmp + tmp=$(mktemp -d) + git worktree add --detach "$tmp" >/dev/null 2>&1 + ( cd "$tmp" && git checkout -q --orphan "$STATE_BRANCH" && { git rm -rfq . 2>/dev/null || true; } + mkdir -p "$JOSH_SYNC_TARGET_NAME" + printf '%s\n' '{"step":"resetting","import_prs":{"main":42},"reset_branches":["main"]}' \ + > "${JOSH_SYNC_TARGET_NAME}/onboard.json" + git add -A + git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \ + commit -q -m "legacy onboard state" + git push -q origin "HEAD:${STATE_BRANCH}" ) + git worktree remove "$tmp" 2>/dev/null || rm -rf "$tmp" + + local got + got=$(read_onboard_state "$JOSH_SYNC_TARGET_NAME") + [ "$(echo "$got" | jq -r '.step')" = "resetting" ] + [ "$(echo "$got" | jq -r '.strategy')" = "reset" ] +} + +@test "onboard_flow resumes from step=waiting-for-merge (strategy=reset) — transitions to resetting" { + setup_state_monorepo + write_onboard_state "$JOSH_SYNC_TARGET_NAME" \ + '{"step":"waiting-for-merge","strategy":"reset","import_prs":{"main":42},"reset_branches":[],"archived_url":"git@host:org/old.git","archived_api":"https://host/api/v1/repos/org/old","archived_auth":"https"}' + + initial_import() { call_log import; echo "skip"; } + adopt_branch() { call_log adopt; echo "adopted"; } + subrepo_reset() { call_log reset; echo "reset"; } + get_pr() { echo '{"merged":true}'; } + list_open_prs() { echo '[]'; } + export MONOREPO_API="http://stub" GITEA_TOKEN="stub" + + local target_json='{"name":"example","branches":{"main":"main"}}' + onboard_flow "$target_json" "false" "" <<< "" >/dev/null 2>&1 + + [ "$(call_count adopt)" = "0" ] + [ "$(call_count reset)" = "1" ] +} diff --git a/tests/unit/cli.bats b/tests/unit/cli.bats new file mode 100644 index 0000000..7f08b8e --- /dev/null +++ b/tests/unit/cli.bats @@ -0,0 +1,123 @@ +#!/usr/bin/env bats + +bats_require_minimum_version 1.5.0 + +# tests/unit/cli.bats — bin/josh-sync CLI surface tests +# +# Invokes the CLI as a subprocess (not by sourcing libs) so flag parsing, +# command dispatch, help/usage text, and the `adopt → onboard --mode=adopt` +# alias regress cleanly. These tests deliberately stay at the parse/dispatch +# layer — anything that would reach out to git remotes is set up to fail at +# target-lookup time, before any side effects. +# +# Uses `run --separate-stderr` so we can assert on stderr separately (most of +# josh-sync's output is logged via core.sh's `log()` which writes to stderr). + +setup() { + JOSH_SYNC_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + JOSH_BIN="${JOSH_SYNC_ROOT}/bin/josh-sync" + FIXTURES="${JOSH_SYNC_ROOT}/tests/fixtures" + TEST_ROOT="$(mktemp -d)" +} + +teardown() { + cd "$BATS_TEST_DIRNAME" 2>/dev/null || true + rm -rf "$TEST_ROOT" +} + +# Drop the minimal fixture config into a clean cwd. Most cmd_* paths read +# .josh-sync.yml from the working directory. +seed_minimal_config() { + cp "${FIXTURES}/minimal.yml" "${TEST_ROOT}/.josh-sync.yml" + cd "$TEST_ROOT" +} + +# ─── Global flags ─────────────────────────────────────────────────── + +@test "--version exits 0 and prints the version" { + run "$JOSH_BIN" --version + [ "$status" -eq 0 ] + [[ "$output" == *"josh-sync"* ]] +} + +@test "--help exits 0 and advertises onboard, the adopt alias, and --mode" { + run "$JOSH_BIN" --help + [ "$status" -eq 0 ] + # Help text is printed to stderr via `cat >&2`, so combine with --separate. + run --separate-stderr "$JOSH_BIN" --help + [ "$status" -eq 0 ] + [[ "$stderr" == *"onboard <target>"* ]] + [[ "$stderr" == *"adopt <target>"* ]] + [[ "$stderr" == *"Alias for 'onboard --mode=adopt'"* ]] + [[ "$stderr" == *"--mode={reset,adopt}"* ]] +} + +# ─── onboard flag parsing & dispatch ──────────────────────────────── + +@test "onboard with no target exits non-zero and prints usage including --mode" { + seed_minimal_config + run --separate-stderr "$JOSH_BIN" onboard + [ "$status" -ne 0 ] + [[ "$stderr" == *"Usage: josh-sync onboard"* ]] + [[ "$stderr" == *"--mode=reset|adopt"* ]] +} + +@test "onboard --mode=invalid is rejected with a clear error" { + seed_minimal_config + run --separate-stderr "$JOSH_BIN" onboard --mode=invalid example + [ "$status" -ne 0 ] + [[ "$stderr" == *"Invalid --mode"* ]] +} + +@test "onboard --mode=adopt against an unknown target reports target-not-found" { + seed_minimal_config + # Proves --mode parses correctly: parse_config succeeds, then cmd_onboard + # fails at target lookup, before any git/network activity. + run --separate-stderr "$JOSH_BIN" onboard --mode=adopt nonexistent + [ "$status" -ne 0 ] + [[ "$stderr" == *"not found"* ]] +} + +# ─── adopt alias ──────────────────────────────────────────────────── + +@test "adopt alias forwards to onboard --mode=adopt (visible via alias log line + target lookup)" { + seed_minimal_config + run --separate-stderr "$JOSH_BIN" adopt nonexistent + [ "$status" -ne 0 ] + # Alias notice from cmd_adopt: + [[ "$stderr" == *"alias for: josh-sync onboard --mode=adopt"* ]] + # And the forwarded call lands in cmd_onboard's target lookup: + [[ "$stderr" == *"not found"* ]] +} + +@test "onboard usage listing shows --restart and --mode flags together" { + seed_minimal_config + run --separate-stderr "$JOSH_BIN" onboard + [ "$status" -ne 0 ] + [[ "$stderr" == *"--restart"* ]] + [[ "$stderr" == *"--mode"* ]] +} + +# ─── --mode value-parsing guards ──────────────────────────────────── + +@test "onboard --mode (no value) fails with a helpful message instead of crashing on unbound \$2" { + seed_minimal_config + run --separate-stderr "$JOSH_BIN" onboard example --mode + [ "$status" -ne 0 ] + [[ "$stderr" == *"--mode requires a value"* ]] +} + +@test "onboard --mode= (empty after equals) is rejected explicitly, not silently ignored" { + seed_minimal_config + run --separate-stderr "$JOSH_BIN" onboard example --mode= + [ "$status" -ne 0 ] + [[ "$stderr" == *"Empty --mode="* ]] +} + +@test "adopt alias dies when a conflicting --mode=reset is also passed (instead of silently honouring reset)" { + seed_minimal_config + run --separate-stderr "$JOSH_BIN" adopt example --mode=reset + [ "$status" -ne 0 ] + [[ "$stderr" == *"alias for --mode=adopt"* ]] + [[ "$stderr" == *"conflicting"* ]] +}