diff --git a/.github/workflows/deploy-pipelines.yaml b/.github/workflows/deploy-pipelines.yaml new file mode 100644 index 0000000..dabaeb3 --- /dev/null +++ b/.github/workflows/deploy-pipelines.yaml @@ -0,0 +1,85 @@ +name: Deploy Kubeflow Pipelines + +on: + push: + branches: [master, main] + paths: + - 'pipelines/**/*.py' + workflow_dispatch: + inputs: + pipeline_path: + description: 'Path to specific pipeline (e.g., pipelines/examples/hello_world.py)' + required: false + default: '' + +env: + KUBEFLOW_ENDPOINT: https://kubeflow.walleye-frog.ts.net/pipeline + +jobs: + compile-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install kfp + run: pip install kfp + + - name: Get changed pipeline files + id: changed + run: | + if [ -n "${{ github.event.inputs.pipeline_path }}" ]; then + echo "files=${{ github.event.inputs.pipeline_path }}" >> $GITHUB_OUTPUT + else + CHANGED=$(git diff --name-only HEAD~1 HEAD -- 'pipelines/**/*.py' | tr '\n' ' ') + echo "files=$CHANGED" >> $GITHUB_OUTPUT + fi + + - name: Compile and upload pipelines + if: steps.changed.outputs.files != '' + env: + KFP_ENDPOINT: ${{ env.KUBEFLOW_ENDPOINT }} + run: | + for file in ${{ steps.changed.outputs.files }}; do + if [ -f "$file" ]; then + echo "Processing $file..." + + # Extract pipeline name from filename + PIPELINE_NAME=$(basename "$file" .py | tr '_' '-') + YAML_FILE="${PIPELINE_NAME}.yaml" + + # Compile pipeline + echo "Compiling $file to $YAML_FILE..." + kfp dsl compile --py "$file" --output "$YAML_FILE" + + # Upload to Kubeflow (requires auth configured) + # Note: This requires Kubeflow to be accessible from GHA runners + # For private clusters, use self-hosted runners or Tailscale + echo "Compiled: $YAML_FILE" + echo "Upload manually or configure runner access to Kubeflow" + fi + done + + - name: Upload compiled pipelines as artifacts + uses: actions/upload-artifact@v4 + with: + name: compiled-pipelines + path: '*.yaml' + retention-days: 7 + + - name: Summary + run: | + echo "## Compiled Pipelines" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + for f in *.yaml; do + [ -f "$f" ] && echo "- $f" >> $GITHUB_STEP_SUMMARY + done + echo "" >> $GITHUB_STEP_SUMMARY + echo "Download artifacts and upload to Kubeflow, or configure self-hosted runner with Tailscale access." >> $GITHUB_STEP_SUMMARY diff --git a/hello_world.yaml b/hello_world.yaml new file mode 100644 index 0000000..094df61 --- /dev/null +++ b/hello_world.yaml @@ -0,0 +1,128 @@ +# PIPELINE DEFINITION +# Name: hello-world-pipeline +# Description: A simple hello world pipeline to test Kubeflow setup +# Inputs: +# name: str [Default: 'Kubeflow User'] +components: + comp-process-greeting: + executorLabel: exec-process-greeting + inputDefinitions: + parameters: + greeting: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-say-hello: + executorLabel: exec-say-hello + inputDefinitions: + parameters: + name: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING +deploymentSpec: + executors: + exec-process-greeting: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - process_greeting + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef process_greeting(greeting: str) -> str:\n \"\"\"Process the\ + \ greeting message.\"\"\"\n processed = greeting.upper()\n print(f\"\ + Processed: {processed}\")\n return processed\n\n" + image: python:3.11-slim + exec-say-hello: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - say_hello + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.15.2'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef say_hello(name: str) -> str:\n \"\"\"Simple component that\ + \ returns a greeting.\"\"\"\n message = f\"Hello, {name}! Welcome to\ + \ Kubeflow Pipelines.\"\n print(message)\n return message\n\n" + image: python:3.11-slim +pipelineInfo: + description: A simple hello world pipeline to test Kubeflow setup + name: hello-world-pipeline +root: + dag: + tasks: + process-greeting: + cachingOptions: + enableCache: true + componentRef: + name: comp-process-greeting + dependentTasks: + - say-hello + inputs: + parameters: + greeting: + taskOutputParameter: + outputParameterKey: Output + producerTask: say-hello + taskInfo: + name: process-greeting + say-hello: + cachingOptions: + enableCache: true + componentRef: + name: comp-say-hello + inputs: + parameters: + name: + componentInputParameter: name + taskInfo: + name: say-hello + inputDefinitions: + parameters: + name: + defaultValue: Kubeflow User + isOptional: true + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.15.2