Add file exclusion via josh stored filters (v1.2.0)

New `exclude` config field per target generates .josh-filters/<name>.josh files with josh :exclude clauses. Josh-proxy applies exclusions at the transport layer — excluded files never appear in the subrepo. Preflight checks that generated filter files are committed. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 22:45:13 +03:00
parent 401d0e87a4
commit 187a9ead14
5 changed files with 138 additions and 2 deletions
--- a/2
+++ b/2
@@ -1 +1 @@
-1.1.0
+1.2.0
--- a/bin/josh-sync
+++ b/bin/josh-sync
@@ -422,6 +422,28 @@ cmd_preflight() {
    warn ".gitea/workflows/josh-sync-reverse.yml not found (optional)"
  fi

+  # 4. Exclude filter files
+  local has_excludes
+  has_excludes=$(echo "$JOSH_SYNC_TARGETS" | jq '[.[] | select((.exclude // []) | length > 0)] | length')
+
+  if [ "$has_excludes" -gt 0 ]; then
+    echo ""
+    echo "4. Exclude filter files"
+
+    while IFS= read -r target_name; do
+      local filter_file=".josh-filters/${target_name}.josh"
+      if [ -f "$filter_file" ]; then
+        if git ls-files --error-unmatch "$filter_file" >/dev/null 2>&1; then
+          pass "${filter_file} exists and is tracked"
+        else
+          warn "${filter_file} exists but is NOT committed — run: git add ${filter_file}"
+        fi
+      else
+        fail "${filter_file} not generated — check parse_config"
+      fi
+    done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[] | select((.exclude // []) | length > 0) | .name')
+  fi
+
  # Summary
  echo ""
  echo "============================="
--- a/docs/guide.md
+++ b/docs/guide.md
@@ -91,6 +91,9 @@ targets:
    branches:
      main: main                             # mono_branch: subrepo_branch
    forward_only: []
+    exclude:                                 # files excluded from subrepo (optional)
+      - ".monorepo/"                         # monorepo-only config dir
+      - "**/internal/"                       # internal dirs at any depth

  - name: "auth"
    subfolder: "services/auth"
@@ -515,6 +518,66 @@ Bot commits include a git trailer like `Josh-Sync-Origin: forward/main/2024-02-1

 Sync state is stored as JSON files on an orphan branch (`josh-sync-state`), one file per target/branch. This tracks the last-synced commit SHAs and timestamps to avoid re-syncing the same changes.

+## Excluding Files from Sync
+
+Some files in the monorepo subfolder may not belong in the subrepo (e.g., monorepo-specific CI configs, internal tooling). The `exclude` config field removes these at the josh-proxy layer — excluded files never appear in the subrepo.
+
+### Configuration
+
+Add an `exclude` list to any target:
+
+```yaml
+targets:
+  - name: "billing"
+    subfolder: "services/billing"
+    subrepo_url: "git@host:org/billing.git"
+    exclude:
+      - ".monorepo/"          # directory at subfolder root
+      - "**/internal/"        # directory at any depth
+      - "*.secret"            # files by extension
+    branches:
+      main: main
+```
+
+### How it works
+
+When `exclude` is present, josh-sync generates a `.josh-filters/<target>.josh` file containing a [josh stored filter](https://josh-project.github.io/josh/reference/filters.html) with `:exclude` clauses:
+
+```
+:/services/billing:exclude[
+  ::.monorepo/
+  ::**/internal/
+  ::*.secret
+]
+```
+
+Josh-proxy reads this file from the monorepo and applies the filter at the transport layer. This means:
+- **Forward sync**: the filtered clone already excludes the files
+- **Reverse sync**: pushes through josh also respect the exclusion
+- **Reset**: the subrepo history never contains excluded files
+- **Tree comparison**: `skip` detection works correctly (excluded files are not in the diff)
+
+### Pattern syntax
+
+Josh uses `::` patterns inside `:exclude[...]`:
+
+| Pattern | Matches |
+|---------|---------|
+| `dir/` | Directory at subfolder root |
+| `file` | File at subfolder root |
+| `**/dir/` | Directory at any depth |
+| `**/file` | File at any depth |
+| `*.ext` | Glob pattern (single `*` only) |
+
+### Setup
+
+1. Add `exclude` to the target in `.josh-sync.yml`
+2. Run `josh-sync preflight` — this generates `.josh-filters/<target>.josh`
+3. Commit the generated file: `git add .josh-filters/ && git commit`
+4. Forward sync will now exclude the specified files
+
+If you change the `exclude` list, re-run `preflight` and commit the updated `.josh-filters/` file.
+
 ## Adding a New Target

 To add a new subrepo after initial setup:
--- a/lib/config.sh
+++ b/lib/config.sh
@@ -7,6 +7,45 @@
 #
 # Requires: lib/core.sh sourced first, yq and jq on PATH

+# ─── Josh Filter File Generation ──────────────────────────────────
+# Generates .josh-filters/<target>.josh for targets with exclude patterns.
+# These files must be committed to the monorepo — josh-proxy reads them
+# from the repo at clone time via the :+ stored filter syntax.
+
+_generate_josh_filters() {
+  local has_excludes
+  has_excludes=$(echo "$JOSH_SYNC_TARGETS" | jq '[.[] | select((.exclude // []) | length > 0)] | length')
+
+  if [ "$has_excludes" -eq 0 ]; then
+    return
+  fi
+
+  mkdir -p .josh-filters
+
+  local target_name subfolder exclude_patterns filter_content
+  while IFS= read -r target_name; do
+    subfolder=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \
+      '.[] | select(.name == $n) | .subfolder')
+    exclude_patterns=$(echo "$JOSH_SYNC_TARGETS" | jq -r --arg n "$target_name" \
+      '.[] | select(.name == $n) | .exclude | map("  ::" + .) | join("\n")')
+
+    filter_content=":/${subfolder}:exclude[
+${exclude_patterns}
+]"
+
+    local filter_file=".josh-filters/${target_name}.josh"
+    local existing=""
+    if [ -f "$filter_file" ]; then
+      existing=$(cat "$filter_file")
+    fi
+
+    if [ "$filter_content" != "$existing" ]; then
+      echo "$filter_content" > "$filter_file"
+      log "WARN" "Generated ${filter_file} — commit this file to the monorepo"
+    fi
+  done < <(echo "$JOSH_SYNC_TARGETS" | jq -r '.[] | select((.exclude // []) | length > 0) | .name')
+}
+
 # ─── Phase 1: Parse Config ─────────────────────────────────────────

 parse_config() {
@@ -36,7 +75,10 @@ parse_config() {
  export JOSH_SYNC_TARGETS
  JOSH_SYNC_TARGETS=$(echo "$config_json" | jq '[.targets[] | . +
    # Auto-derive josh_filter from subfolder if not set
-    (if (.josh_filter // "") == "" then
+    # When exclude patterns are present, use a stored josh filter (:+.josh-filters/<name>)
+    (if (.exclude // [] | length) > 0 then
+      {josh_filter: (":+.josh-filters/" + .name)}
+    elif (.josh_filter // "") == "" then
      {josh_filter: (":/" + .subfolder)}
    else {} end) +
    # Derive gitea_host and subrepo_repo_path from subrepo_url
@@ -56,6 +98,9 @@ parse_config() {
    )
  ]')

+  # Generate .josh-filters/*.josh for targets with exclude patterns
+  _generate_josh_filters
+
  # Load .env credentials (if present, not required — CI sets these via secrets)
  if [ -f .env ]; then
    # shellcheck source=/dev/null
--- a/schema/config-schema.json
+++ b/schema/config-schema.json
@@ -70,6 +70,12 @@
            "items": { "type": "string" },
            "default": [],
            "description": "Branches that only sync mono → subrepo (never reverse)"
+          },
+          "exclude": {
+            "type": "array",
+            "items": { "type": "string" },
+            "default": [],
+            "description": "File/directory patterns to exclude from sync via josh :exclude filter. Josh pattern syntax: 'dir/' for directories, '*.ext' for globs, '**/dir/' for nested matches. Generates a .josh-filters/<target>.josh file that must be committed."
          }
        }
      }
@@ -1 +1 @@
 .1.0
 .2.0