kubeflow-pipelines/.github/workflows/deploy-pipelines.yaml

name: Deploy Kubeflow Pipelines

on:
  push:
    branches: [master, main]
    paths:
      - 'pipelines/**/*.py'
  workflow_dispatch:
    inputs:
      pipeline_path:
        description: 'Path to specific pipeline (e.g., pipelines/examples/hello_world.py)'
        required: false
        default: ''

env:
  KUBEFLOW_ENDPOINT: https://kubeflow.walleye-frog.ts.net/pipeline

jobs:
  compile-and-deploy:
    runs-on: [self-hosted, kubeflow]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install kfp
        run: pip install kfp --quiet

      - name: Get changed pipeline files
        id: changed
        run: |
          if [ -n "${{ github.event.inputs.pipeline_path }}" ]; then
            echo "files=${{ github.event.inputs.pipeline_path }}" >> $GITHUB_OUTPUT
          else
            CHANGED=$(git diff --name-only HEAD~1 HEAD -- 'pipelines/**/*.py' 2>/dev/null || find pipelines -name "*.py" | head -5)
            echo "files=$CHANGED" >> $GITHUB_OUTPUT
          fi

      - name: Compile and upload pipelines
        if: steps.changed.outputs.files != ''
        run: |
          for file in ${{ steps.changed.outputs.files }}; do
            if [ -f "$file" ]; then
              echo "🔧 Processing $file..."
              
              # Extract pipeline name from filename
              PIPELINE_NAME=$(basename "$file" .py | tr '_' '-')
              YAML_FILE="${PIPELINE_NAME}.yaml"
              
              # Compile pipeline
              echo "📦 Compiling $file..."
              kfp dsl compile --py "$file" --output "$YAML_FILE"
              
              # Upload to Kubeflow
              echo "🚀 Uploading $PIPELINE_NAME to Kubeflow..."
              kfp --endpoint ${{ env.KUBEFLOW_ENDPOINT }} pipeline create \
                -p "$PIPELINE_NAME" \
                "$YAML_FILE" || echo "Pipeline may already exist, trying upload..."
              
              echo "✅ Done: $PIPELINE_NAME"
            fi
          done

      - name: Summary
        run: |
          echo "## 🎯 Pipeline Deployment Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Endpoint:** ${{ env.KUBEFLOW_ENDPOINT }}" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Processed Files:" >> $GITHUB_STEP_SUMMARY
          for f in ${{ steps.changed.outputs.files }}; do
            [ -f "$f" ] && echo "- ✅ $f" >> $GITHUB_STEP_SUMMARY
          done