Compare commits

..

11 Commits

Author SHA1 Message Date
401d0e87a4 Show [migrated] marker and summary in migrate-pr
Interactive picker now marks already-migrated PRs. All modes (--all,
explicit numbers, interactive) track and display success/fail/skip
counts at the end.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 21:08:23 +03:00
fbacec7f6f Improve PR migration: fetch branches locally + 3-way merge
Instead of fetching the API diff (which has context-sensitive patches
that break after josh-filtered reset), fetch the archived repo's
branches directly as a second remote and compute the diff locally.
Apply with git apply --3way for resilience against context mismatches.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 20:51:22 +03:00
553f006174 Fix onboard import cloning from empty new repo instead of archived repo
initial_import() now accepts an optional clone URL override parameter.
onboard_flow() passes the archived repo URL so content is cloned from
the right source.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 19:48:46 +03:00
cb14cf9bd4 Add docs for updating josh-sync version in Nix devenv
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 18:38:44 +03:00
0363b0ee77 Fix VERSION not included in Nix package and Makefile bundle
- flake.nix: copy VERSION file to $out/ so josh_sync_version() finds it
- Makefile: add lib/onboard.sh to the bundle loop

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 18:38:07 +03:00
72430714af Update docs for onboard and migrate-pr commands
- README: add onboard and migrate-pr to CLI reference
- Guide Step 5: add onboard as recommended Option A, move manual
  import/reset to Option B, document migrate-pr usage
- Guide "Adding a New Target": mention onboard as preferred path

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 18:33:53 +03:00
105216a27e Add onboard and migrate-pr commands (v1.1.0)
New commands for safely onboarding existing subrepos into the monorepo
without losing open PRs:

- josh-sync onboard <target>: interactive, resumable 5-step flow
  (import → wait for merge → reset to new repo)
- josh-sync migrate-pr <target> [PR#...] [--all]: migrate PRs from
  archived repo to new repo via patch application

Also refactors create_pr() to wrap create_pr_number(), eliminating
duplicated curl/jq logic.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 12:41:44 +03:00
405e5f4535 Update guide.md 2026-02-13 09:31:41 +03:00
cc08c530d1 Update sync.sh 2026-02-12 20:02:34 +03:00
5758987942 Update sync.sh 2026-02-12 19:32:21 +03:00
0edbdae558 "#" 2026-02-12 18:00:08 +03:00
11 changed files with 1360 additions and 19 deletions

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
.claude/*local*
result result

View File

@@ -23,7 +23,7 @@ dist/josh-sync: bin/josh-sync lib/*.sh VERSION
@echo '# Generated by: make build' >> dist/josh-sync @echo '# Generated by: make build' >> dist/josh-sync
@echo '' >> dist/josh-sync @echo '' >> dist/josh-sync
@# Inline all library modules (strip shebangs and source directives) @# Inline all library modules (strip shebangs and source directives)
@for f in lib/core.sh lib/config.sh lib/auth.sh lib/state.sh lib/sync.sh; do \ @for f in lib/core.sh lib/config.sh lib/auth.sh lib/state.sh lib/sync.sh lib/onboard.sh; do \
echo "# --- $$f ---" >> dist/josh-sync; \ echo "# --- $$f ---" >> dist/josh-sync; \
grep -v '^#!/' "$$f" | grep -v '^# shellcheck source=' >> dist/josh-sync; \ grep -v '^#!/' "$$f" | grep -v '^# shellcheck source=' >> dist/josh-sync; \
echo '' >> dist/josh-sync; \ echo '' >> dist/josh-sync; \

View File

@@ -56,6 +56,11 @@ Add josh-sync as a flake input, then:
Run `josh-sync preflight` to validate your setup. Run `josh-sync preflight` to validate your setup.
## Documentation
- **[Setup Guide](docs/guide.md)** — Step-by-step: prerequisites, importing existing subrepos, CI workflows, and troubleshooting
- **[Configuration Reference](docs/config-reference.md)** — Full `.josh-sync.yml` field documentation
## CLI ## CLI
``` ```
@@ -63,6 +68,8 @@ josh-sync sync [--forward|--reverse] [--target NAME[,NAME]] [--branch BRANCH]
josh-sync preflight josh-sync preflight
josh-sync import <target> josh-sync import <target>
josh-sync reset <target> josh-sync reset <target>
josh-sync onboard <target> [--restart]
josh-sync migrate-pr <target> [PR#...] [--all]
josh-sync status josh-sync status
josh-sync state show <target> [branch] josh-sync state show <target> [branch]
josh-sync state reset <target> [branch] josh-sync state reset <target> [branch]

View File

@@ -1 +1 @@
1.0.0 1.1.0

View File

@@ -9,6 +9,8 @@
# preflight Validate config, connectivity, auth # preflight Validate config, connectivity, auth
# import <target> Initial import: pull subrepo into monorepo # import <target> Initial import: pull subrepo into monorepo
# reset <target> Reset subrepo to josh-filtered view # reset <target> Reset subrepo to josh-filtered view
# onboard <target> Import existing subrepo into monorepo (interactive)
# migrate-pr <target> [PR#...] [--all] Move PRs from archived to new subrepo
# status Show target config and sync state # status Show target config and sync state
# state show|reset Manage sync state directly # state show|reset Manage sync state directly
# #
@@ -39,6 +41,8 @@ source "${JOSH_LIB_DIR}/auth.sh"
source "${JOSH_LIB_DIR}/state.sh" source "${JOSH_LIB_DIR}/state.sh"
# shellcheck source=../lib/sync.sh # shellcheck source=../lib/sync.sh
source "${JOSH_LIB_DIR}/sync.sh" source "${JOSH_LIB_DIR}/sync.sh"
# shellcheck source=../lib/onboard.sh
source "${JOSH_LIB_DIR}/onboard.sh"
# ─── Version ──────────────────────────────────────────────────────── # ─── Version ────────────────────────────────────────────────────────
@@ -69,6 +73,8 @@ Commands:
preflight Validate config, connectivity, auth, workflow coverage preflight Validate config, connectivity, auth, workflow coverage
import <target> Initial import: pull existing subrepo into monorepo (creates PR) import <target> Initial import: pull existing subrepo into monorepo (creates PR)
reset <target> Reset subrepo to josh-filtered view (after merging import PR) reset <target> Reset subrepo to josh-filtered view (after merging import PR)
onboard <target> Import existing subrepo into monorepo (interactive, resumable)
migrate-pr <target> [PR#...] [--all] Move PRs from archived to new subrepo
status Show target config and sync state status Show target config and sync state
state show <target> [branch] Show sync state JSON state show <target> [branch] Show sync state JSON
state reset <target> [branch] Reset sync state to {} state reset <target> [branch] Reset sync state to {}
@@ -643,6 +649,173 @@ cmd_state() {
esac esac
} }
# ─── Onboard Command ──────────────────────────────────────────────
cmd_onboard() {
local config_file=".josh-sync.yml"
local target_name=""
local restart=false
while [ $# -gt 0 ]; do
case "$1" in
--config) config_file="$2"; shift 2 ;;
--debug) export JOSH_SYNC_DEBUG=1; shift ;;
--restart) restart=true; shift ;;
-*) die "Unknown flag: $1" ;;
*) target_name="$1"; shift ;;
esac
done
if [ -z "$target_name" ]; then
echo "Usage: josh-sync onboard <target> [--restart]" >&2
parse_config "$config_file"
echo "Available targets:" >&2
echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2
exit 1
fi
parse_config "$config_file"
local target_json
target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)')
[ -n "$target_json" ] || die "Target '${target_name}' not found in config"
log "INFO" "══════ Onboard target: ${target_name} ══════"
load_target "$target_json"
onboard_flow "$target_json" "$restart"
}
# ─── Migrate PR Command ──────────────────────────────────────────
cmd_migrate_pr() {
local config_file=".josh-sync.yml"
local target_name=""
local all=false
local pr_numbers=()
while [ $# -gt 0 ]; do
case "$1" in
--config) config_file="$2"; shift 2 ;;
--debug) export JOSH_SYNC_DEBUG=1; shift ;;
--all) all=true; shift ;;
-*) die "Unknown flag: $1" ;;
*)
if [ -z "$target_name" ]; then
target_name="$1"
else
pr_numbers+=("$1")
fi
shift ;;
esac
done
if [ -z "$target_name" ]; then
echo "Usage: josh-sync migrate-pr <target> [PR#...] [--all]" >&2
parse_config "$config_file"
echo "Available targets:" >&2
echo "$JOSH_SYNC_TARGETS" | jq -r '.[].name' | sed 's/^/ /' >&2
exit 1
fi
parse_config "$config_file"
local target_json
target_json=$(echo "$JOSH_SYNC_TARGETS" | jq -c --arg n "$target_name" '.[] | select(.name == $n)')
[ -n "$target_json" ] || die "Target '${target_name}' not found in config"
load_target "$target_json"
# Load archived repo info from onboard state
local onboard_state archived_api
onboard_state=$(read_onboard_state "$target_name")
archived_api=$(echo "$onboard_state" | jq -r '.archived_api')
if [ -z "$archived_api" ] || [ "$archived_api" = "null" ]; then
die "No archived repo info found. Run 'josh-sync onboard ${target_name}' first."
fi
log "INFO" "Archived repo: ${archived_api}"
# Load already-migrated PR numbers for skip detection and display
local migrated_numbers
migrated_numbers=$(echo "$onboard_state" | jq -r '[.migrated_prs // [] | .[].old_number] | map(tostring) | .[]')
# Counters for summary
local migrated=0 failed=0 skipped=0
# Helper: attempt migration of one PR with counting
_try_migrate() {
local num="$1"
if echo "$migrated_numbers" | grep -qx "$num"; then
log "INFO" "PR #${num} already migrated — skipping"
skipped=$((skipped + 1))
elif migrate_one_pr "$num"; then
migrated=$((migrated + 1))
else
failed=$((failed + 1))
fi
}
if [ "$all" = true ]; then
# Migrate all open PRs from archived repo
local prs
prs=$(list_open_prs "$archived_api" "$SUBREPO_TOKEN") \
|| die "Failed to list PRs on archived repo"
local count
count=$(echo "$prs" | jq 'length')
log "INFO" "Found ${count} open PR(s) on archived repo"
while read -r num; do
_try_migrate "$num"
done < <(echo "$prs" | jq -r '.[] | .number')
elif [ ${#pr_numbers[@]} -gt 0 ]; then
# Migrate specific PR numbers
for num in "${pr_numbers[@]}"; do
_try_migrate "$num"
done
else
# Interactive: list open PRs, let user pick
local prs
prs=$(list_open_prs "$archived_api" "$SUBREPO_TOKEN") \
|| die "Failed to list PRs on archived repo"
local count
count=$(echo "$prs" | jq 'length')
if [ "$count" -eq 0 ]; then
log "INFO" "No open PRs on archived repo"
return
fi
# Display PRs with [migrated] marker for already-processed ones
echo "" >&2
echo "Open PRs on archived repo:" >&2
while IFS=$'\t' read -r num title base_ref head_ref; do
if echo "$migrated_numbers" | grep -qx "$num"; then
echo " #${num}: ${title} (${base_ref} <- ${head_ref}) [migrated]" >&2
else
echo " #${num}: ${title} (${base_ref} <- ${head_ref})" >&2
fi
done < <(echo "$prs" | jq -r '.[] | "\(.number)\t\(.title)\t\(.base.ref)\t\(.head.ref)"')
echo "" >&2
echo "Enter PR numbers to migrate (space-separated), or 'all':" >&2
local selection
read -r selection
if [ "$selection" = "all" ]; then
while read -r num; do
_try_migrate "$num"
done < <(echo "$prs" | jq -r '.[] | .number')
else
for num in $selection; do
_try_migrate "$num"
done
fi
fi
log "INFO" "Migration complete: ${migrated} migrated, ${failed} failed, ${skipped} skipped"
}
# ─── Main ─────────────────────────────────────────────────────────── # ─── Main ───────────────────────────────────────────────────────────
main() { main() {
@@ -666,6 +839,8 @@ main() {
preflight) cmd_preflight "$@" ;; preflight) cmd_preflight "$@" ;;
import) cmd_import "$@" ;; import) cmd_import "$@" ;;
reset) cmd_reset "$@" ;; reset) cmd_reset "$@" ;;
onboard) cmd_onboard "$@" ;;
migrate-pr) cmd_migrate_pr "$@" ;;
status) cmd_status "$@" ;; status) cmd_status "$@" ;;
state) cmd_state "$@" ;; state) cmd_state "$@" ;;
*) *)

79
docs/config-reference.md Normal file
View File

@@ -0,0 +1,79 @@
# Configuration Reference
Full reference for `.josh-sync.yml` fields and environment variables.
## `.josh-sync.yml` Structure
```yaml
josh: # josh-proxy settings (required)
targets: # sync targets (required, at least 1)
bot: # bot identity for sync commits (required)
```
## `josh` Section
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `proxy_url` | string | Yes | Josh-proxy URL, no trailing slash. Must start with `http://` or `https://`. |
| `monorepo_path` | string | Yes | Repository path as josh-proxy sees it (e.g., `org/monorepo`). |
## `targets[]` Section
Each target maps a monorepo subfolder to an external subrepo.
| Field | Type | Required | Default | Description |
|-------|------|----------|---------|-------------|
| `name` | string | Yes | — | Unique target identifier. Used in CLI commands and state tracking. |
| `subfolder` | string | Yes | — | Monorepo subfolder path (e.g., `services/billing`). |
| `josh_filter` | string | No | `:/<subfolder>` | Josh filter expression. Auto-derived from `subfolder` if omitted. Must start with `:`. |
| `subrepo_url` | string | Yes | — | External subrepo Git URL. Supports HTTPS (`https://...`), SSH (`git@host:path`), and `ssh://` formats. |
| `subrepo_auth` | string | No | `"https"` | Auth method: `"https"` or `"ssh"`. |
| `subrepo_token_var` | string | No | `"SUBREPO_TOKEN"` | Name of the env var holding the HTTPS token for this target. |
| `subrepo_ssh_key_var` | string | No | `"SUBREPO_SSH_KEY"` | Name of the env var holding the SSH private key for this target. |
| `branches` | object | Yes | — | Branch mapping: `mono_branch: subrepo_branch`. Each key-value pair syncs those branches bidirectionally. |
| `forward_only` | string[] | No | `[]` | Branches that only sync mono → subrepo, never reverse. |
## `bot` Section
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `name` | string | Yes | Git author name for sync commits. |
| `email` | string | Yes | Git author email for sync commits. |
| `trailer` | string | Yes | Git trailer key for loop prevention (e.g., `Josh-Sync-Origin`). |
## Environment Variables
### Credentials
| Variable | Purpose | Default |
|----------|---------|---------|
| `SYNC_BOT_USER` | Bot's Git username | — |
| `SYNC_BOT_TOKEN` | API token for monorepo access and josh-proxy HTTPS auth | — |
| `SUBREPO_TOKEN` | HTTPS token for subrepo access | Falls back to `SYNC_BOT_TOKEN` |
| `SUBREPO_SSH_KEY` | SSH private key content (not a file path) for subrepo access | — |
### Per-target credential overrides
Set `subrepo_token_var` or `subrepo_ssh_key_var` in a target's config to read credentials from a different env var:
```yaml
targets:
- name: "auth"
subrepo_token_var: "AUTH_REPO_TOKEN" # reads from $AUTH_REPO_TOKEN
subrepo_ssh_key_var: "AUTH_SSH_KEY" # reads from $AUTH_SSH_KEY
```
**Resolution order:** per-target env var → default env var (`SUBREPO_TOKEN` / `SUBREPO_SSH_KEY`) → `SYNC_BOT_TOKEN` fallback.
### Runtime
| Variable | Purpose | Default |
|----------|---------|---------|
| `JOSH_SYNC_TARGET` | Restrict sync to a single target | All targets |
| `JOSH_SYNC_STATE_BRANCH` | Name of the orphan branch for state storage | `josh-sync-state` |
| `JOSH_SYNC_DEBUG` | Enable verbose logging (`1` to enable) | `0` |
| `MONOREPO_API` | Override monorepo API URL | Auto-derived from first target's host |
## JSON Schema
The config file can be validated against the JSON Schema at [`schema/config-schema.json`](../schema/config-schema.json).

593
docs/guide.md Normal file
View File

@@ -0,0 +1,593 @@
# Setup Guide
Step-by-step guide to setting up josh-sync for a new monorepo with existing subrepos.
## Overview
josh-sync provides bidirectional sync between a monorepo and N external subrepos via [josh-proxy](https://josh-project.github.io/josh/):
```
MONOREPO SUBREPOS
├── services/billing/ ──── forward ────► billing-repo/
├── services/auth/ (push or cron) auth-repo/
└── libs/shared/ ◄──── reverse ───── shared-lib-repo/
(cron → always PR)
via josh-proxy (filtered git views)
```
**Key safety properties:**
- Forward sync (mono → subrepo) uses `--force-with-lease` — never overwrites concurrent changes
- Reverse sync (subrepo → mono) always creates a PR — never pushes directly
- Git trailers (`Josh-Sync-Origin:`) prevent infinite sync loops
- State tracked on an orphan branch (`josh-sync-state`) — survives CI runner teardown
## Prerequisites
Before you begin, you need:
### josh-proxy instance
A running [josh-proxy](https://josh-project.github.io/josh/) that can access your monorepo's Git server. Verify connectivity:
```bash
git ls-remote https://josh.example.com/org/monorepo.git HEAD
```
### Bot account
A dedicated Git user (e.g., `josh-sync-bot`) with:
- Write access to the monorepo
- Write access to all subrepos
- Ability to create PRs on both monorepo and subrepo platforms
### Credentials
| Variable | Purpose | Required |
|----------|---------|----------|
| `SYNC_BOT_USER` | Bot's Git username | Yes |
| `SYNC_BOT_TOKEN` | API token with repo scope (monorepo + josh-proxy auth) | Yes |
| `SUBREPO_SSH_KEY` | SSH private key for subrepo access (if using SSH auth) | If SSH |
| `SUBREPO_TOKEN` | HTTPS token for subrepo access (defaults to `SYNC_BOT_TOKEN`) | No |
Per-target credential overrides are supported — see [Configuration Reference](config-reference.md).
### Tool dependencies
`bash >=4`, `git`, `curl`, `jq`, `yq` ([mikefarah/yq](https://github.com/mikefarah/yq) v4+), `openssh`, `rsync`
> The Nix flake bundles all dependencies automatically.
## Step 1: Create the Monorepo
Create a new repository on your Git server (e.g., `org/monorepo`). Create subdirectories for each subrepo you want to sync:
```bash
mkdir -p services/billing services/auth libs/shared
```
These directories will be populated during the import step. They can be empty or contain `.gitkeep` files for now.
Verify josh-proxy can see the monorepo:
```bash
git ls-remote https://josh.example.com/org/monorepo.git HEAD
```
## Step 2: Configure `.josh-sync.yml`
Create `.josh-sync.yml` at the monorepo root. Each target maps a monorepo subfolder to an external subrepo:
```yaml
josh:
proxy_url: "https://josh.example.com" # josh-proxy URL (no trailing slash)
monorepo_path: "org/monorepo" # repo path as josh sees it
targets:
- name: "billing" # unique identifier
subfolder: "services/billing" # monorepo subfolder
# josh_filter auto-derived as ":/services/billing" if omitted
subrepo_url: "git@gitea.example.com:ext/billing.git"
subrepo_auth: "ssh" # "https" (default) or "ssh"
branches:
main: main # mono_branch: subrepo_branch
forward_only: []
- name: "auth"
subfolder: "services/auth"
subrepo_url: "https://gitea.example.com/ext/auth.git"
subrepo_auth: "https"
subrepo_token_var: "AUTH_REPO_TOKEN" # per-target credential override
branches:
main: main
develop: develop # multiple branches supported
forward_only: []
- name: "shared-lib"
subfolder: "libs/shared"
subrepo_url: "https://gitea.example.com/ext/shared-lib.git"
branches:
main: main
forward_only: [main] # one-way: mono → subrepo only
bot:
name: "josh-sync-bot"
email: "josh-sync-bot@example.com"
trailer: "Josh-Sync-Origin" # git trailer for loop prevention
```
For the full field reference, see [Configuration Reference](config-reference.md).
## Step 3: Set Up Local Dev Environment
### Option A: Nix + devenv (recommended)
**`devenv.yaml`** — declare josh-sync as a flake input:
```yaml
inputs:
nixpkgs:
url: github:cachix/devenv-nixpkgs/rolling
josh-sync:
url: git+https://your-gitea.example.com/org/josh-sync?ref=main
flake: true
```
**`devenv.nix`** — import the josh-sync module:
```nix
{ inputs, ... }:
{
imports = [ inputs.josh-sync.devenvModules.default ];
name = "my-monorepo";
# .env contains secrets, not devenv config
dotenv.disableHint = true;
}
```
**`.envrc`** — activate devenv automatically:
```bash
DEVENV_WARN_TIMEOUT=20
use devenv
```
**`.env`** — local credentials (add to `.gitignore`):
```bash
SYNC_BOT_USER=sync-bot
SYNC_BOT_TOKEN=<your-api-token>
SUBREPO_SSH_KEY="-----BEGIN OPENSSH PRIVATE KEY-----
...
-----END OPENSSH PRIVATE KEY-----"
# Per-target overrides:
# AUTH_REPO_TOKEN=<auth-specific-token>
```
### Updating josh-sync in devenv
To update to the latest version:
```bash
devenv update josh-sync
```
Or with plain Nix flakes:
```bash
nix flake lock --update-input josh-sync
```
To pin to a specific version, use a tag ref in `devenv.yaml`:
```yaml
josh-sync:
url: git+https://your-gitea.example.com/org/josh-sync?ref=v1.1
flake: true
```
After updating, verify the version:
```bash
josh-sync --version
```
### Option B: Manual installation
Install the required tools, then either:
- Clone the josh-sync repo and add `bin/` to your `PATH`
- Run `make build` to create a single bundled script at `dist/josh-sync`
## Step 4: Validate with Preflight
```bash
josh-sync preflight
```
This validates:
- Config syntax and required fields
- josh-proxy connectivity (via `git ls-remote` through josh)
- Subrepo connectivity and authentication
- Branch mappings
- CI workflow path coverage (checks if `.gitea/workflows/josh-sync-forward.yml` paths match target subfolders)
For a new monorepo before import, preflight may warn that subfolders don't exist yet — that's expected.
## Step 5: Import Existing Subrepos
This is the critical onboarding step. There are two approaches:
- **`josh-sync onboard`** (recommended) — interactive, resumable, preserves open PRs
- **Manual `import` → merge → `reset`** — lower-level, for automation or when there are no open PRs to preserve
### Option A: Onboard (recommended)
The `onboard` command walks you through the entire process interactively, with checkpoint/resume at every step.
**Before you start:**
1. **Rename** the existing subrepo on your Git server (e.g., `stores/storefront``stores/storefront-archived`)
2. **Create a new empty repo** at the original path (e.g., a new `stores/storefront` with no commits)
The rename preserves the archived repo with all its history and open PRs. The new empty repo will receive josh-filtered history.
**Run onboard:**
```bash
josh-sync onboard billing
```
The command will:
1. **Verify prerequisites** — checks the new empty repo is reachable, asks for the archived repo URL
2. **Import** — copies subrepo content into monorepo and creates import PRs (one per branch)
3. **Wait for merge** — shows PR numbers and waits for you to merge them
4. **Reset** — pushes josh-filtered history to the new subrepo (per-branch, with resume)
5. **Done** — prints instructions for developers and PR migration
If the process is interrupted at any point, re-run `josh-sync onboard billing` to resume from where it left off. Use `--restart` to start over.
**Migrate open PRs:**
After onboard completes, migrate PRs from the archived repo to the new one:
```bash
# Interactive — lists open PRs and lets you pick
josh-sync migrate-pr billing
# Migrate all open PRs at once
josh-sync migrate-pr billing --all
# Migrate specific PRs by number
josh-sync migrate-pr billing 5 8 12
```
PR migration works by fetching the diff from the archived repo's PR, applying it to the new repo, and creating a new PR. File content is identical after reset, so patches apply cleanly.
### Option B: Manual import → merge → reset
Use this when the subrepo has no open PRs to preserve, or for scripted automation.
> Do this **one target at a time** to keep PRs reviewable.
#### 5b-1. Import
```bash
josh-sync import billing
```
This:
1. Clones the monorepo directly (not through josh)
2. Clones the subrepo
3. Copies subrepo content into the monorepo subfolder via `rsync`
4. Creates a branch `auto-sync/import-billing-<timestamp>`
5. Pushes it and creates a PR on the monorepo
Review the import PR — check for leaked credentials, environment-specific config, or files that shouldn't be in the monorepo.
#### 5b-2. Merge the import PR
Merge the PR using your Git platform's UI. This lands the subrepo content into the monorepo's main branch.
> At this point, the monorepo has the content but the histories are disconnected. Sync will **not** work until you complete the reset step.
#### 5b-3. Reset
```bash
josh-sync reset billing
```
> **You do NOT need to `git pull` locally before running reset.** The reset command clones fresh from josh-proxy — it never uses your local working copy.
This:
1. Clones the monorepo through josh-proxy with the josh filter (the "filtered view")
2. Force-pushes that filtered view to the subrepo, replacing its history
This establishes **shared commit ancestry** between josh's filtered view and the subrepo. Without this, josh-proxy can't compute diffs between the two.
> **Warning:** This is a destructive force-push that replaces the subrepo's history. Back up any important branches or tags in the subrepo beforehand. Merge or close all open pull requests on the subrepo first — they will be invalidated.
After reset, **every developer with a local clone of the subrepo** must update their local copy to match the new history:
```bash
cd /path/to/local-subrepo
git fetch origin
git checkout main && git reset --hard origin/main
git checkout stage && git reset --hard origin/stage # repeat for each branch
```
Or simply delete and re-clone the subrepo. Local-only branches (not pushed to the remote) will be lost either way.
#### 5b-4. Repeat for each target
```
For each target:
1. josh-sync import <target>
2. Review and merge the import PR on the monorepo
3. josh-sync reset <target>
```
### Verify
After all targets are imported and reset (whichever option you used):
```bash
# Check all targets show state
josh-sync status
# Test forward sync — should return "skip" (trees are identical after reset)
josh-sync sync --forward --target billing
# Test reverse sync — should return "skip" (no new human commits)
josh-sync sync --reverse --target billing
```
## Step 6: Set Up CI Workflows
### Forward sync (mono → subrepo)
Create `.gitea/workflows/josh-sync-forward.yml`:
```yaml
name: "Josh Sync → Subrepo"
on:
push:
branches: [main]
paths:
# List ALL target subfolders:
- "services/billing/**"
- "services/auth/**"
- "libs/shared/**"
schedule:
- cron: "0 */6 * * *" # every 6 hours as fallback
workflow_dispatch:
inputs:
target:
description: "Target to sync (empty = detect from push or all)"
required: false
default: ""
branch:
description: "Branch to sync (empty = triggered branch or all)"
required: false
default: ""
concurrency:
group: josh-sync-fwd-${{ github.ref_name }}
cancel-in-progress: false
jobs:
sync:
runs-on: docker
container: node:20-bookworm
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2 # needed for target detection
- name: Install tools
run: |
apt-get update -qq && apt-get install -y -qq jq curl git openssh-client >/dev/null 2>&1
curl -sL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" \
-o /usr/local/bin/yq && chmod +x /usr/local/bin/yq
- name: Detect changed target
if: github.event_name == 'push'
id: detect
run: |
CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "")
TARGETS=$(yq -o json '.targets' .josh-sync.yml \
| jq -r '.[] | "\(.name):\(.subfolder)"' \
| while IFS=: read -r name prefix; do
echo "$CHANGED" | grep -q "^${prefix}/" && echo "$name"
done | sort -u | paste -sd ',' -)
echo "targets=${TARGETS}" >> "$GITHUB_OUTPUT"
- uses: https://your-gitea.example.com/org/josh-sync@v1
with:
direction: forward
target: ${{ github.event.inputs.target || steps.detect.outputs.targets }}
branch: ${{ github.event.inputs.branch || github.ref_name }}
env:
SYNC_BOT_USER: ${{ secrets.SYNC_BOT_USER }}
SYNC_BOT_TOKEN: ${{ secrets.SYNC_BOT_TOKEN }}
SUBREPO_TOKEN: ${{ secrets.SUBREPO_TOKEN || secrets.SYNC_BOT_TOKEN }}
SUBREPO_SSH_KEY: ${{ secrets.SUBREPO_SSH_KEY }}
```
### Reverse sync (subrepo → mono)
Create `.gitea/workflows/josh-sync-reverse.yml`:
```yaml
name: "Josh Sync ← Subrepo"
on:
schedule:
- cron: "0 1,7,13,19 * * *" # every 6h, offset from forward
workflow_dispatch:
inputs:
target:
description: "Target to sync (empty = all)"
required: false
default: ""
branch:
description: "Branch to sync (empty = all eligible)"
required: false
default: ""
concurrency:
group: josh-sync-rev-${{ github.event.inputs.target || 'all' }}
cancel-in-progress: false
jobs:
sync:
runs-on: docker
container: node:20-bookworm
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Install tools
run: |
apt-get update -qq && apt-get install -y -qq jq curl git openssh-client >/dev/null 2>&1
curl -sL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_amd64" \
-o /usr/local/bin/yq && chmod +x /usr/local/bin/yq
- uses: https://your-gitea.example.com/org/josh-sync@v1
with:
direction: reverse
target: ${{ github.event.inputs.target || '' }}
branch: ${{ github.event.inputs.branch || '' }}
env:
SYNC_BOT_USER: ${{ secrets.SYNC_BOT_USER }}
SYNC_BOT_TOKEN: ${{ secrets.SYNC_BOT_TOKEN }}
SUBREPO_TOKEN: ${{ secrets.SUBREPO_TOKEN || secrets.SYNC_BOT_TOKEN }}
SUBREPO_SSH_KEY: ${{ secrets.SUBREPO_SSH_KEY }}
```
### Required CI secrets
| Secret | Purpose |
|--------|---------|
| `SYNC_BOT_USER` | Bot username |
| `SYNC_BOT_TOKEN` | Bot API token (monorepo access + josh-proxy auth) |
| `SUBREPO_SSH_KEY` | SSH private key for subrepo push (if using SSH auth) |
| `SUBREPO_TOKEN` | Optional separate subrepo token (defaults to `SYNC_BOT_TOKEN`) |
> **GitHub Actions note:** These examples target Gitea Actions. For GitHub Actions, change the `uses:` reference to a GitHub repo (e.g., `org/josh-sync@v1`) and `runs-on:` to a GitHub runner (e.g., `ubuntu-latest`).
## How Ongoing Sync Works
Once set up, sync runs automatically:
### Forward sync (mono → subrepo)
Triggered by pushes to target subfolders or on a cron schedule:
1. Clones the monorepo through josh-proxy (filtered view of the subfolder)
2. Fetches the subrepo branch for comparison
3. If trees are identical → skip
4. If subrepo branch doesn't exist → fresh push
5. Merges mono changes on top of subrepo state
6. If clean merge → pushes with `--force-with-lease` (protects against concurrent changes)
7. If lease rejected → retries on next run (subrepo changed during sync)
8. If merge conflict → creates a conflict PR on the subrepo
### Reverse sync (subrepo → mono)
Runs on a cron schedule (never triggered by subrepo pushes):
1. Clones the subrepo
2. Fetches the monorepo's josh-filtered view for comparison
3. Finds new human commits (filters out bot commits by checking for the `Josh-Sync-Origin:` trailer)
4. If no new human commits → skip
5. Pushes through josh-proxy to a staging branch
6. Creates a PR on the monorepo — **never pushes directly**
### Loop prevention
Bot commits include a git trailer like `Josh-Sync-Origin: forward/main/2024-02-12T10:30:00Z`. Each sync direction filters out commits with this trailer, preventing changes from bouncing back and forth. The CI action also has a loop guard that skips entirely if the HEAD commit has the trailer.
### State tracking
Sync state is stored as JSON files on an orphan branch (`josh-sync-state`), one file per target/branch. This tracks the last-synced commit SHAs and timestamps to avoid re-syncing the same changes.
## Adding a New Target
To add a new subrepo after initial setup:
1. Add the target to `.josh-sync.yml`
2. Update the forward workflow's `paths:` list to include the new subfolder
3. Commit and push
4. Import the target:
```bash
# Recommended: interactive onboard (preserves open PRs)
josh-sync onboard new-target
# Or manual: import → merge PR → reset
josh-sync import new-target
# merge the PR
josh-sync reset new-target
```
5. Verify with `josh-sync status`
## Troubleshooting
### "Failed to clone through josh-proxy"
- Check josh-proxy is running and accessible
- Verify `monorepo_path` matches what josh-proxy expects
- Test manually: `git ls-remote https://<user>:<token>@josh.example.com/org/repo.git:/services/app.git`
### SSH authentication failures
- `SUBREPO_SSH_KEY` must contain the actual key content, not a file path
- For per-target keys, ensure `subrepo_ssh_key_var` in config matches the env var name
- Check the key has write access to the subrepo
### "Force-with-lease rejected"
Normal: the subrepo changed while sync was running. The next sync run will pick it up. If persistent, check for another process pushing to the subrepo simultaneously.
### "Josh rejected push" (reverse sync)
Josh-proxy couldn't map the push back to the monorepo. Check josh-proxy logs, verify the josh filter is correct. May indicate a history divergence — consider running `josh-sync reset <target>`.
### Import PR shows "No changes"
The subfolder already contains the same content as the subrepo. This is fine — the import is a no-op.
### Duplicate/looping commits
Verify `bot.trailer` in config matches what's in commit messages. Check the loop guard in the CI workflow is active.
### "cannot lock ref" or "expected X but got Y"
**After reset (subrepo):** The subrepo's history was replaced by force-push. Local clones still have the old history:
```bash
cd /path/to/subrepo
git fetch origin
git checkout main && git reset --hard origin/main
```
Or simply delete and re-clone.
**After import/reset cycle (monorepo):** The import and reset steps create and update branches rapidly (`auto-sync/import-*`, `josh-sync-state`). If your local clone fetched partway through, tracking refs go stale:
```bash
git remote prune origin && git pull
```
### State issues
```bash
# View current state
josh-sync state show <target> [branch]
# Reset state (forces next sync to run regardless of SHA comparison)
josh-sync state reset <target> [branch]
```

View File

@@ -28,6 +28,7 @@
installPhase = '' installPhase = ''
mkdir -p $out/{bin,lib} mkdir -p $out/{bin,lib}
cp VERSION $out/
cp lib/*.sh $out/lib/ cp lib/*.sh $out/lib/
cp bin/josh-sync $out/bin/ cp bin/josh-sync $out/bin/
chmod +x $out/bin/josh-sync chmod +x $out/bin/josh-sync

View File

@@ -39,16 +39,15 @@ subrepo_ls_remote() {
} }
# ─── PR Creation ──────────────────────────────────────────────────── # ─── PR Creation ────────────────────────────────────────────────────
# Shared helper for creating PRs on Gitea/GitHub API. # Shared helpers for creating PRs on Gitea/GitHub API.
# Usage: create_pr <api_url> <token> <base> <head> <title> <body> # Usage: create_pr <api_url> <token> <base> <head> <title> <body>
# number=$(create_pr_number <api_url> <token> <base> <head> <title> <body>)
#
# create_pr — fire-and-forget (stdout suppressed, safe inside sync functions)
# create_pr_number — returns the new PR number via stdout
create_pr() { create_pr_number() {
local api_url="$1" local api_url="$1" token="$2" base="$3" head="$4" title="$5" body="$6"
local token="$2"
local base="$3"
local head="$4"
local title="$5"
local body="$6"
curl -sf -X POST \ curl -sf -X POST \
-H "Authorization: token ${token}" \ -H "Authorization: token ${token}" \
@@ -59,5 +58,36 @@ create_pr() {
--arg title "$title" \ --arg title "$title" \
--arg body "$body" \ --arg body "$body" \
'{base:$base, head:$head, title:$title, body:$body}')" \ '{base:$base, head:$head, title:$title, body:$body}')" \
"${api_url}/pulls" >/dev/null "${api_url}/pulls" | jq -r '.number'
}
create_pr() {
create_pr_number "$@" >/dev/null
}
# ─── PR API Helpers ──────────────────────────────────────────────
# Used by onboard and migrate-pr commands.
# List open PRs on a repo. Returns JSON array.
# Usage: list_open_prs <api_url> <token>
list_open_prs() {
local api_url="$1" token="$2"
curl -sf -H "Authorization: token ${token}" \
"${api_url}/pulls?state=open&limit=50"
}
# Get PR diff as plain text.
# Usage: get_pr_diff <api_url> <token> <pr_number>
get_pr_diff() {
local api_url="$1" token="$2" pr_number="$3"
curl -sf -H "Authorization: token ${token}" \
"${api_url}/pulls/${pr_number}.diff"
}
# Get single PR as JSON (for checking merge status, metadata, etc.).
# Usage: get_pr <api_url> <token> <pr_number>
get_pr() {
local api_url="$1" token="$2" pr_number="$3"
curl -sf -H "Authorization: token ${token}" \
"${api_url}/pulls/${pr_number}"
} }

451
lib/onboard.sh Normal file
View File

@@ -0,0 +1,451 @@
#!/usr/bin/env bash
# lib/onboard.sh — Onboard orchestration and PR migration
#
# Provides:
# onboard_flow() — Interactive: import → wait for merge → reset to new repo
# migrate_one_pr() — Migrate a single PR from archived repo to new repo
#
# Onboard state is stored on the josh-sync-state branch at <target>/onboard.json.
# Steps: start → importing → waiting-for-merge → resetting → complete
#
# Requires: lib/core.sh, lib/config.sh, lib/auth.sh, lib/state.sh, lib/sync.sh sourced
# Expects: JOSH_SYNC_TARGET_NAME, BOT_NAME, BOT_EMAIL, SUBREPO_API, SUBREPO_TOKEN, etc.
# ─── Onboard State Helpers ────────────────────────────────────────
# Follow the same pattern as read_state()/write_state() in lib/state.sh.
read_onboard_state() {
local target_name="${1:-$JOSH_SYNC_TARGET_NAME}"
git fetch origin "$STATE_BRANCH" 2>/dev/null || true
git show "origin/${STATE_BRANCH}:${target_name}/onboard.json" 2>/dev/null || echo '{}'
}
write_onboard_state() {
local target_name="${1:-$JOSH_SYNC_TARGET_NAME}"
local state_json="$2"
local key="${target_name}/onboard"
local tmp_dir
tmp_dir=$(mktemp -d)
if git rev-parse "origin/${STATE_BRANCH}" >/dev/null 2>&1; then
git worktree add "$tmp_dir" "origin/${STATE_BRANCH}" 2>/dev/null
else
git worktree add --detach "$tmp_dir" 2>/dev/null
(cd "$tmp_dir" && git checkout --orphan "$STATE_BRANCH" && { git rm -rf . 2>/dev/null || true; })
fi
mkdir -p "$(dirname "${tmp_dir}/${key}.json")"
echo "$state_json" | jq '.' > "${tmp_dir}/${key}.json"
(
cd "$tmp_dir" || exit
git add -A
if ! git diff --cached --quiet 2>/dev/null; then
git -c user.name="$BOT_NAME" -c user.email="$BOT_EMAIL" \
commit -m "onboard: update ${target_name}"
git push origin "HEAD:${STATE_BRANCH}" || log "WARN" "Failed to push onboard state"
fi
)
git worktree remove "$tmp_dir" 2>/dev/null || rm -rf "$tmp_dir"
}
# ─── Derive Archived API URL ─────────────────────────────────────
# Given a URL like "git@host:org/repo-archived.git" or
# "https://host/org/repo-archived.git", derive the Gitea API URL.
_archived_api_from_url() {
local url="$1"
# Strip .git suffix first — avoids non-greedy regex issues in POSIX ERE
url="${url%.git}"
local host repo_path
if echo "$url" | grep -qE '^(ssh://|git@)'; then
# SSH URL
if echo "$url" | grep -q '^ssh://'; then
host=$(echo "$url" | sed -E 's|ssh://[^@]*@([^/]+)/.*|\1|')
repo_path=$(echo "$url" | sed -E 's|ssh://[^@]*@[^/]+/(.+)$|\1|')
else
host=$(echo "$url" | sed -E 's|git@([^:/]+)[:/].*|\1|')
repo_path=$(echo "$url" | sed -E 's|git@[^:/]+[:/](.+)$|\1|')
fi
else
# HTTPS URL
host=$(echo "$url" | sed -E 's|https?://([^/]+)/.*|\1|')
repo_path=$(echo "$url" | sed -E 's|https?://[^/]+/(.+)$|\1|')
fi
echo "https://${host}/api/v1/repos/${repo_path}"
}
# ─── Onboard Flow ────────────────────────────────────────────────
# Interactive orchestrator with checkpoint/resume.
# Usage: onboard_flow <target_json> <restart>
onboard_flow() {
local target_json="$1"
local restart="${2:-false}"
local target_name="$JOSH_SYNC_TARGET_NAME"
# Load existing onboard state (or empty)
local onboard_state
onboard_state=$(read_onboard_state "$target_name")
local current_step
current_step=$(echo "$onboard_state" | jq -r '.step // "start"')
if [ "$restart" = true ]; then
log "INFO" "Restarting onboard from scratch"
current_step="start"
onboard_state='{}'
fi
log "INFO" "Onboard step: ${current_step}"
# ── Step 1: Prerequisites + archived repo info ──
if [ "$current_step" = "start" ]; then
echo "" >&2
echo "=== Onboarding ${target_name} ===" >&2
echo "" >&2
echo "Before proceeding, you should have:" >&2
echo " 1. Renamed the existing subrepo (e.g., storefront → storefront-archived)" >&2
echo " 2. Created a new EMPTY repo at the original URL" >&2
echo "" >&2
# Verify the new (empty) subrepo is reachable (no HEAD ref — works on empty repos)
if git ls-remote "$(subrepo_auth_url)" >/dev/null 2>&1; then
# shellcheck disable=SC2001 # sed is clearer for URL pattern replacement
log "INFO" "New subrepo is reachable at $(echo "$SUBREPO_URL" | sed 's|://[^@]*@|://***@|')"
else
log "WARN" "New subrepo is not reachable — make sure you created the new empty repo"
fi
echo "Enter the archived repo URL (e.g., git@host:org/repo-archived.git):" >&2
local archived_url
read -r archived_url
[ -n "$archived_url" ] || die "Archived URL is required"
# Determine auth type for archived repo (same as current subrepo)
local archived_auth="${SUBREPO_AUTH:-https}"
# Derive API URL
local archived_api
archived_api=$(_archived_api_from_url "$archived_url")
# Verify archived repo is reachable via API
if curl -sf -H "Authorization: token ${SUBREPO_TOKEN}" \
"${archived_api}" >/dev/null 2>&1; then
log "INFO" "Archived repo reachable: ${archived_api}"
else
log "WARN" "Cannot reach archived repo API — check URL and token"
echo "Continue anyway? (y/N):" >&2
local confirm
read -r confirm
[ "$confirm" = "y" ] || [ "$confirm" = "Y" ] || die "Aborted"
fi
# Save state
onboard_state=$(jq -n \
--arg step "importing" \
--arg archived_api "$archived_api" \
--arg archived_url "$archived_url" \
--arg archived_auth "$archived_auth" \
--arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
'{step:$step, archived_api:$archived_api, archived_url:$archived_url,
archived_auth:$archived_auth, import_prs:{}, reset_branches:[],
migrated_prs:[], timestamp:$ts}')
write_onboard_state "$target_name" "$onboard_state"
current_step="importing"
fi
# ── Step 2: Import (reuses initial_import()) ──
if [ "$current_step" = "importing" ]; then
echo "" >&2
log "INFO" "Step 2: Importing subrepo content into monorepo..."
local branches
branches=$(echo "$target_json" | jq -r '.branches | keys[]')
# Load existing import_prs from state (resume support)
local import_prs
import_prs=$(echo "$onboard_state" | jq -r '.import_prs // {}')
# Build the archived repo clone URL for initial_import().
# The content lives in the archived repo — the new repo at SUBREPO_URL is empty.
local archived_url archived_clone_url
archived_url=$(echo "$onboard_state" | jq -r '.archived_url')
if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then
archived_clone_url="$archived_url"
else
# shellcheck disable=SC2001
archived_clone_url=$(echo "$archived_url" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|")
fi
for branch in $branches; do
local mapped
mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty')
[ -z "$mapped" ] && continue
# Skip branches that already have an import PR recorded
if echo "$import_prs" | jq -e --arg b "$branch" 'has($b)' >/dev/null 2>&1; then
log "INFO" "Import PR already recorded for ${branch} — skipping"
continue
fi
export SYNC_BRANCH_MONO="$branch"
export SYNC_BRANCH_SUBREPO="$mapped"
log "INFO" "Importing branch: ${branch} (subrepo: ${mapped})"
local result
result=$(initial_import "$archived_clone_url")
log "INFO" "Import result for ${branch}: ${result}"
if [ "$result" = "pr-created" ]; then
# Find the import PR number via API
local prs pr_number
prs=$(list_open_prs "$MONOREPO_API" "$GITEA_TOKEN")
pr_number=$(echo "$prs" | jq -r --arg t "$target_name" --arg b "$branch" \
'[.[] | select(.title | test("\\[Import\\] " + $t + ":")) | select(.base.ref == $b)] | .[0].number // empty')
if [ -n "$pr_number" ]; then
import_prs=$(echo "$import_prs" | jq --arg b "$branch" --arg n "$pr_number" '. + {($b): ($n | tonumber)}')
log "INFO" "Import PR for ${branch}: #${pr_number}"
else
log "WARN" "Could not find import PR number for ${branch} — check monorepo PRs"
fi
fi
# Save progress after each branch (resume support)
onboard_state=$(echo "$onboard_state" | jq --argjson prs "$import_prs" '.import_prs = $prs')
write_onboard_state "$target_name" "$onboard_state"
done
# Update state
onboard_state=$(echo "$onboard_state" | jq \
--arg step "waiting-for-merge" \
--argjson prs "$import_prs" \
'.step = $step | .import_prs = $prs')
write_onboard_state "$target_name" "$onboard_state"
current_step="waiting-for-merge"
fi
# ── Step 3: Wait for merge ──
if [ "$current_step" = "waiting-for-merge" ]; then
echo "" >&2
log "INFO" "Step 3: Waiting for import PR(s) to be merged..."
local import_prs
import_prs=$(echo "$onboard_state" | jq -r '.import_prs')
local pr_count
pr_count=$(echo "$import_prs" | jq 'length')
if [ "$pr_count" -eq 0 ]; then
log "WARN" "No import PRs recorded — skipping merge check"
else
echo "" >&2
echo "Import PRs to merge:" >&2
echo "$import_prs" | jq -r 'to_entries[] | " \(.key): PR #\(.value)"' >&2
echo "" >&2
echo "Merge the import PR(s) on the monorepo, then press Enter..." >&2
read -r
# Verify each PR is merged
local all_merged=true
for branch in $(echo "$import_prs" | jq -r 'keys[]'); do
local pr_number
pr_number=$(echo "$import_prs" | jq -r --arg b "$branch" '.[$b]')
local pr_json merged
pr_json=$(get_pr "$MONOREPO_API" "$GITEA_TOKEN" "$pr_number")
merged=$(echo "$pr_json" | jq -r '.merged // false')
if [ "$merged" = "true" ]; then
log "INFO" "PR #${pr_number} (${branch}): merged"
else
log "ERROR" "PR #${pr_number} (${branch}): NOT merged — merge it first"
all_merged=false
fi
done
if [ "$all_merged" = false ]; then
die "Not all import PRs are merged. Re-run 'josh-sync onboard ${target_name}' after merging."
fi
fi
# Update state
onboard_state=$(echo "$onboard_state" | jq '.step = "resetting"')
write_onboard_state "$target_name" "$onboard_state"
current_step="resetting"
fi
# ── Step 4: Reset (pushes josh-filtered history to new repo) ──
if [ "$current_step" = "resetting" ]; then
echo "" >&2
log "INFO" "Step 4: Pushing josh-filtered history to new subrepo..."
local branches
branches=$(echo "$target_json" | jq -r '.branches | keys[]')
local already_reset
already_reset=$(echo "$onboard_state" | jq -r '.reset_branches // []')
for branch in $branches; do
# Skip branches already reset (resume support)
if echo "$already_reset" | jq -e --arg b "$branch" 'index($b) != null' >/dev/null 2>&1; then
log "INFO" "Branch ${branch} already reset — skipping"
continue
fi
local mapped
mapped=$(echo "$target_json" | jq -r --arg b "$branch" '.branches[$b] // empty')
[ -z "$mapped" ] && continue
export SYNC_BRANCH_MONO="$branch"
export SYNC_BRANCH_SUBREPO="$mapped"
local result
result=$(subrepo_reset)
log "INFO" "Reset result for ${branch}: ${result}"
# Track progress
onboard_state=$(echo "$onboard_state" | jq --arg b "$branch" \
'.reset_branches += [$b]')
write_onboard_state "$target_name" "$onboard_state"
done
# Update state
onboard_state=$(echo "$onboard_state" | jq \
--arg step "complete" \
--arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
'.step = $step | .timestamp = $ts')
write_onboard_state "$target_name" "$onboard_state"
current_step="complete"
fi
# ── Step 5: Done ──
if [ "$current_step" = "complete" ]; then
echo "" >&2
echo "=== Onboarding complete! ===" >&2
echo "" >&2
echo "The new subrepo now has josh-filtered history." >&2
echo "Developers should re-clone or reset their local copies:" >&2
echo " git fetch origin && git reset --hard origin/main" >&2
echo "" >&2
echo "To migrate open PRs from the archived repo:" >&2
echo " josh-sync migrate-pr ${target_name} # interactive picker" >&2
echo " josh-sync migrate-pr ${target_name} --all # migrate all" >&2
echo " josh-sync migrate-pr ${target_name} 5 8 12 # specific PRs" >&2
fi
}
# ─── Migrate One PR ──────────────────────────────────────────────
# Fetches the PR's branch from the archived repo, computes a local diff,
# and applies it to the new subrepo with --3way for resilience.
# Usage: migrate_one_pr <pr_number>
#
# Expects: JOSH_SYNC_TARGET_NAME, SUBREPO_API, SUBREPO_TOKEN, BOT_NAME, BOT_EMAIL loaded
migrate_one_pr() {
local pr_number="$1"
local target_name="$JOSH_SYNC_TARGET_NAME"
# Read archived repo info from onboard state
local onboard_state archived_api
onboard_state=$(read_onboard_state "$target_name")
archived_api=$(echo "$onboard_state" | jq -r '.archived_api')
if [ -z "$archived_api" ] || [ "$archived_api" = "null" ]; then
die "No archived repo info found. Run 'josh-sync onboard ${target_name}' first."
fi
# Check if this PR was already migrated
local already_migrated
already_migrated=$(echo "$onboard_state" | jq -r \
--argjson num "$pr_number" '.migrated_prs // [] | map(select(.old_number == $num)) | length')
if [ "$already_migrated" -gt 0 ]; then
log "INFO" "PR #${pr_number} already migrated — skipping"
return 0
fi
# Same credentials — the repo was just renamed
local archived_token="$SUBREPO_TOKEN"
# 1. Get PR metadata from archived repo
local pr_json title base head body
pr_json=$(get_pr "$archived_api" "$archived_token" "$pr_number") \
|| die "Failed to fetch PR #${pr_number} from archived repo"
title=$(echo "$pr_json" | jq -r '.title')
base=$(echo "$pr_json" | jq -r '.base.ref')
head=$(echo "$pr_json" | jq -r '.head.ref')
body=$(echo "$pr_json" | jq -r '.body // ""')
log "INFO" "Migrating PR #${pr_number}: \"${title}\" (${base} <- ${head})"
# 2. Clone new subrepo, add archived repo as second remote
# Save cwd so we can restore it (function runs in caller's shell, not subshell)
local original_dir
original_dir=$(pwd)
local work_dir
work_dir=$(mktemp -d)
# shellcheck disable=SC2064 # Intentional early expansion
trap "cd '$original_dir' 2>/dev/null; rm -rf '$work_dir'" RETURN
git clone "$(subrepo_auth_url)" --branch "$base" --single-branch \
"${work_dir}/subrepo" 2>&1 || die "Failed to clone new subrepo (branch: ${base})"
cd "${work_dir}/subrepo" || exit
git config user.name "$BOT_NAME"
git config user.email "$BOT_EMAIL"
# Build authenticated URL for the archived repo
local archived_url archived_clone_url
archived_url=$(echo "$onboard_state" | jq -r '.archived_url')
if [ "${SUBREPO_AUTH:-https}" = "ssh" ]; then
archived_clone_url="$archived_url"
else
# shellcheck disable=SC2001
archived_clone_url=$(echo "$archived_url" | sed "s|https://|https://${BOT_USER}:${SUBREPO_TOKEN}@|")
fi
# Fetch the PR's head and base branches from the archived repo
git remote add archived "$archived_clone_url"
git fetch archived "$head" "$base" 2>&1 \
|| die "Failed to fetch branches from archived repo"
# 3. Compute diff locally and apply with --3way
git checkout -B "$head" >&2
local diff
diff=$(git diff "archived/${base}..archived/${head}")
if [ -z "$diff" ]; then
log "WARN" "Empty diff for PR #${pr_number} — skipping"
return 1
fi
if echo "$diff" | git apply --3way 2>&1; then
git add -A
git commit -m "${title}
Migrated from archived repo PR #${pr_number}" >&2
git push "$(subrepo_auth_url)" "$head" >&2 \
|| die "Failed to push branch ${head}"
# 4. Create PR on new repo
local new_number
new_number=$(create_pr_number "$SUBREPO_API" "$SUBREPO_TOKEN" \
"$base" "$head" "$title" "$body")
log "INFO" "Migrated PR #${pr_number} -> #${new_number}: \"${title}\""
# 5. Record in onboard state
cd "$original_dir" || true
onboard_state=$(read_onboard_state "$target_name")
onboard_state=$(echo "$onboard_state" | jq \
--argjson old "$pr_number" \
--argjson new_num "${new_number}" \
--arg title "$title" \
'.migrated_prs += [{"old_number":$old, "new_number":$new_num, "title":$title}]')
write_onboard_state "$target_name" "$onboard_state"
else
log "ERROR" "Could not apply changes for PR #${pr_number} even with 3-way merge"
log "ERROR" "Manual migration needed: branch '${head}' from archived repo"
return 1
fi
}

View File

@@ -200,9 +200,13 @@ reverse_sync() {
# #
# Used when a subrepo already has content and you're adding it to the # Used when a subrepo already has content and you're adding it to the
# monorepo for the first time. Creates a PR. # monorepo for the first time. Creates a PR.
# Usage: initial_import [clone_url_override]
# clone_url_override — if set, clone from this URL instead of subrepo_auth_url()
# (used by onboard to clone from the archived repo)
# Returns: skip | pr-created # Returns: skip | pr-created
initial_import() { initial_import() {
local clone_url="${1:-$(subrepo_auth_url)}"
local mono_branch="$SYNC_BRANCH_MONO" local mono_branch="$SYNC_BRANCH_MONO"
local subrepo_branch="$SYNC_BRANCH_SUBREPO" local subrepo_branch="$SYNC_BRANCH_SUBREPO"
local subfolder local subfolder
@@ -225,8 +229,8 @@ initial_import() {
--branch "$mono_branch" --single-branch \ --branch "$mono_branch" --single-branch \
"${work_dir}/monorepo" || die "Failed to clone monorepo" "${work_dir}/monorepo" || die "Failed to clone monorepo"
# 2. Clone subrepo # 2. Clone subrepo (or archived repo when clone_url is overridden)
git clone "$(subrepo_auth_url)" \ git clone "$clone_url" \
--branch "$subrepo_branch" --single-branch \ --branch "$subrepo_branch" --single-branch \
"${work_dir}/subrepo" || die "Failed to clone subrepo" "${work_dir}/subrepo" || die "Failed to clone subrepo"