Use self-hosted runner on compute-01 for full GitOps

.github/workflows/deploy-pipelines.yaml

@@ -17,7 +17,7 @@ env:
 
 jobs:
   compile-and-deploy:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, kubeflow]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -30,7 +30,7 @@ jobs:
           python-version: '3.11'
 
       - name: Install kfp
-        run: pip install kfp
+        run: pip install kfp --quiet
 
       - name: Get changed pipeline files
         id: changed
@@ -38,48 +38,42 @@ jobs:
           if [ -n "${{ github.event.inputs.pipeline_path }}" ]; then
             echo "files=${{ github.event.inputs.pipeline_path }}" >> $GITHUB_OUTPUT
           else
-            CHANGED=$(git diff --name-only HEAD~1 HEAD -- 'pipelines/**/*.py' | tr '\n' ' ')
+            CHANGED=$(git diff --name-only HEAD~1 HEAD -- 'pipelines/**/*.py' 2>/dev/null || find pipelines -name "*.py" | head -5)
             echo "files=$CHANGED" >> $GITHUB_OUTPUT
           fi
 
       - name: Compile and upload pipelines
         if: steps.changed.outputs.files != ''
-        env:
-          KFP_ENDPOINT: ${{ env.KUBEFLOW_ENDPOINT }}
         run: |
           for file in ${{ steps.changed.outputs.files }}; do
             if [ -f "$file" ]; then
-              echo "Processing $file..."
+              echo "🔧 Processing $file..."
               
               # Extract pipeline name from filename
               PIPELINE_NAME=$(basename "$file" .py | tr '_' '-')
               YAML_FILE="${PIPELINE_NAME}.yaml"
               
               # Compile pipeline
-              echo "Compiling $file to $YAML_FILE..."
+              echo "📦 Compiling $file..."
               kfp dsl compile --py "$file" --output "$YAML_FILE"
               
-              # Upload to Kubeflow (requires auth configured)
-              # Note: This requires Kubeflow to be accessible from GHA runners
-              # For private clusters, use self-hosted runners or Tailscale
-              echo "Compiled: $YAML_FILE"
-              echo "Upload manually or configure runner access to Kubeflow"
+              # Upload to Kubeflow
+              echo "🚀 Uploading $PIPELINE_NAME to Kubeflow..."
+              kfp --endpoint ${{ env.KUBEFLOW_ENDPOINT }} pipeline create \
+                -p "$PIPELINE_NAME" \
+                "$YAML_FILE" || echo "Pipeline may already exist, trying upload..."
+              
+              echo "✅ Done: $PIPELINE_NAME"
             fi
           done
 
-      - name: Upload compiled pipelines as artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: compiled-pipelines
-          path: '*.yaml'
-          retention-days: 7
-
       - name: Summary
         run: |
-          echo "## Compiled Pipelines" >> $GITHUB_STEP_SUMMARY
+          echo "## 🎯 Pipeline Deployment Summary" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          for f in *.yaml; do
-            [ -f "$f" ] && echo "- $f" >> $GITHUB_STEP_SUMMARY
+          echo "**Endpoint:** ${{ env.KUBEFLOW_ENDPOINT }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Processed Files:" >> $GITHUB_STEP_SUMMARY
+          for f in ${{ steps.changed.outputs.files }}; do
+            [ -f "$f" ] && echo "- ✅ $f" >> $GITHUB_STEP_SUMMARY
           done
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "Download artifacts and upload to Kubeflow, or configure self-hosted runner with Tailscale access." >> $GITHUB_STEP_SUMMARY