diff --git a/VERSION b/VERSION index 9084fa2..26aaba0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.2.0 diff --git a/bin/josh-sync b/bin/josh-sync index f3dbd3b..898defe 100755 --- a/bin/josh-sync +++ b/bin/josh-sync @@ -422,6 +422,28 @@ cmd_preflight() { warn ".gitea/workflows/josh-sync-reverse.yml not found (optional)" fi + # 4. Exclude filter files + local has_excludes + has_excludes=$(echo "$JOSH_SYNC_TARGETS" | jq '[.[] | select((.exclude // []) | length > 0)] | length') + + if [ "$has_excludes" -gt 0 ]; then + echo "" + echo "4. Exclude filter files" + + while IFS= read -r target_name; do + local filter_file=".josh-filters/${target_name}.josh" + if [ -f "$filter_file" ]; then + if git ls-files --error-unmatch "$filter_file" >/dev/null 2>&1; then + pass "${filter_file} exists and is tracked" + else + warn "${filter_file} exists but is NOT committed — run: git add ${filter_file}" + fi + else + fail "${filter_file} not generated — check parse_config" + fi + done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[] | select((.exclude // []) | length > 0) | .name') + fi + # Summary echo "" echo "=============================" diff --git a/docs/guide.md b/docs/guide.md index 2a7349c..a663978 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -91,6 +91,9 @@ targets: branches: main: main # mono_branch: subrepo_branch forward_only: [] + exclude: # files excluded from subrepo (optional) + - ".monorepo/" # monorepo-only config dir + - "**/internal/" # internal dirs at any depth - name: "auth" subfolder: "services/auth" @@ -515,6 +518,66 @@ Bot commits include a git trailer like `Josh-Sync-Origin: forward/main/2024-02-1 Sync state is stored as JSON files on an orphan branch (`josh-sync-state`), one file per target/branch. This tracks the last-synced commit SHAs and timestamps to avoid re-syncing the same changes. +## Excluding Files from Sync + +Some files in the monorepo subfolder may not belong in the subrepo (e.g., monorepo-specific CI configs, internal tooling). The `exclude` config field removes these at the josh-proxy layer — excluded files never appear in the subrepo. + +### Configuration + +Add an `exclude` list to any target: + +```yaml +targets: + - name: "billing" + subfolder: "services/billing" + subrepo_url: "git@host:org/billing.git" + exclude: + - ".monorepo/" # directory at subfolder root + - "**/internal/" # directory at any depth + - "*.secret" # files by extension + branches: + main: main +``` + +### How it works + +When `exclude` is present, josh-sync generates a `.josh-filters/.josh` file containing a [josh stored filter](https://josh-project.github.io/josh/reference/filters.html) with `:exclude` clauses: + +``` +:/services/billing:exclude[ + ::.monorepo/ + ::**/internal/ + ::*.secret +] +``` + +Josh-proxy reads this file from the monorepo and applies the filter at the transport layer. This means: +- **Forward sync**: the filtered clone already excludes the files +- **Reverse sync**: pushes through josh also respect the exclusion +- **Reset**: the subrepo history never contains excluded files +- **Tree comparison**: `skip` detection works correctly (excluded files are not in the diff) + +### Pattern syntax + +Josh uses `::` patterns inside `:exclude[...]`: + +| Pattern | Matches | +|---------|---------| +| `dir/` | Directory at subfolder root | +| `file` | File at subfolder root | +| `**/dir/` | Directory at any depth | +| `**/file` | File at any depth | +| `*.ext` | Glob pattern (single `*` only) | + +### Setup + +1. Add `exclude` to the target in `.josh-sync.yml` +2. Run `josh-sync preflight` — this generates `.josh-filters/.josh` +3. Commit the generated file: `git add .josh-filters/ && git commit` +4. Forward sync will now exclude the specified files + +If you change the `exclude` list, re-run `preflight` and commit the updated `.josh-filters/` file. + ## Adding a New Target To add a new subrepo after initial setup: diff --git a/lib/config.sh b/lib/config.sh index e11214c..2b1c017 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -7,6 +7,45 @@ # # Requires: lib/core.sh sourced first, yq and jq on PATH +# ─── Josh Filter File Generation ────────────────────────────────── +# Generates .josh-filters/.josh for targets with exclude patterns. +# These files must be committed to the monorepo — josh-proxy reads them +# from the repo at clone time via the :+ stored filter syntax. + +_generate_josh_filters() { + local has_excludes + has_excludes=$(echo "$JOSH_SYNC_TARGETS" | jq '[.[] | select((.exclude // []) | length > 0)] | length') + + if [ "$has_excludes" -eq 0 ]; then + return + fi + + mkdir -p .josh-filters + + local target_name subfolder exclude_patterns filter_content + while IFS= read -r target_name; do + subfolder=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \ + '.[] | select(.name == $n) | .subfolder') + exclude_patterns=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \ + '.[] | select(.name == $n) | .exclude | map(" ::" + .) | join("\n")') + + filter_content=":/${subfolder}:exclude[ +${exclude_patterns} +]" + + local filter_file=".josh-filters/${target_name}.josh" + local existing="" + if [ -f "$filter_file" ]; then + existing=$(cat "$filter_file") + fi + + if [ "$filter_content" != "$existing" ]; then + echo "$filter_content" > "$filter_file" + log "WARN" "Generated ${filter_file} — commit this file to the monorepo" + fi + done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[] | select((.exclude // []) | length > 0) | .name') +} + # ─── Phase 1: Parse Config ───────────────────────────────────────── parse_config() { @@ -36,7 +75,10 @@ parse_config() { export JOSH_SYNC_TARGETS JOSH_SYNC_TARGETS=$(echo "$config_json" | jq '[.targets[] | . + # Auto-derive josh_filter from subfolder if not set - (if (.josh_filter // "") == "" then + # When exclude patterns are present, use a stored josh filter (:+.josh-filters/) + (if (.exclude // [] | length) > 0 then + {josh_filter: (":+.josh-filters/" + .name)} + elif (.josh_filter // "") == "" then {josh_filter: (":/" + .subfolder)} else {} end) + # Derive gitea_host and subrepo_repo_path from subrepo_url @@ -56,6 +98,9 @@ parse_config() { ) ]') + # Generate .josh-filters/*.josh for targets with exclude patterns + _generate_josh_filters + # Load .env credentials (if present, not required — CI sets these via secrets) if [ -f .env ]; then # shellcheck source=/dev/null diff --git a/schema/config-schema.json b/schema/config-schema.json index bb222e9..d81ab28 100644 --- a/schema/config-schema.json +++ b/schema/config-schema.json @@ -70,6 +70,12 @@ "items": { "type": "string" }, "default": [], "description": "Branches that only sync mono → subrepo (never reverse)" + }, + "exclude": { + "type": "array", + "items": { "type": "string" }, + "default": [], + "description": "File/directory patterns to exclude from sync via josh :exclude filter. Josh pattern syntax: 'dir/' for directories, '*.ext' for globs, '**/dir/' for nested matches. Generates a .josh-filters/.josh file that must be committed." } } }