ghndrx/file-transformer-s3
1
0
Fork 0
mirror of https://github.com/ghndrx/file-transformer-s3.git synced 2026-02-10 06:45:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit: File Transformer S3 project with React dashboard and Knative functions

Browse Source
This commit is contained in:
greg
2025-07-04 08:01:46 -07:00
commit fd9abd0210
54 changed files with 5584 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
k8s/dashboard.yaml Normal file
View File

@@ -0,0 +1,82 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: file-transformer-dashboard
namespace: file-transformer
labels:
app: file-transformer-dashboard
spec:
replicas: 2
selector:
matchLabels:
app: file-transformer-dashboard
template:
metadata:
labels:
app: file-transformer-dashboard
spec:
containers:
- name: dashboard
image: file-transformer/dashboard:latest
ports:
- containerPort: 3000
env:
- name: REACT_APP_API_BASE_URL
value: "http://api-gateway.file-transformer.svc.cluster.local:8080"
- name: REACT_APP_MINIO_ENDPOINT
value: "http://minio.file-transformer.svc.cluster.local:9000"
- name: REACT_APP_MINIO_CONSOLE
value: "http://minio.file-transformer.svc.cluster.local:9001"
resources:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "256Mi"
cpu: "200m"
livenessProbe:
httpGet:
path: /
port: 3000
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /
port: 3000
initialDelaySeconds: 5
periodSeconds: 5
---
apiVersion: v1
kind: Service
metadata:
name: file-transformer-dashboard
namespace: file-transformer
spec:
selector:
app: file-transformer-dashboard
ports:
- port: 80
targetPort: 3000
type: ClusterIP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: file-transformer-dashboard-ingress
namespace: file-transformer
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/ssl-redirect: "false"
spec:
rules:
- host: dashboard.file-transformer.local
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: file-transformer-dashboard
port:
number: 80

85
k8s/functions/download-service.yaml Normal file
View File

@@ -0,0 +1,85 @@
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
name: function-download
namespace: file-transformer
labels:
app: file-transformer
function: download
spec:
template:
metadata:
annotations:
autoscaling.knative.dev/minScale: "0"
autoscaling.knative.dev/maxScale: "10"
autoscaling.knative.dev/target: "1"
spec:
containerConcurrency: 10
timeoutSeconds: 300
containers:
- image: function-download:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5000
env:
- name: MINIO_ENDPOINT
value: "minio-service:9000"
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: access-key
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: secret-key
- name: MINIO_BUCKET_NAME
value: "file-transformer-bucket"
- name: MINIO_USE_SSL
value: "false"
- name: POSTGRES_URL
valueFrom:
secretKeyRef:
name: postgres-secret
key: database-url
resources:
requests:
memory: "256Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
readinessProbe:
httpGet:
path: /health
port: 5000
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /health
port: 5000
initialDelaySeconds: 15
periodSeconds: 30
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: function-download-ingress
namespace: file-transformer
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/ssl-redirect: "false"
spec:
rules:
- host: download.file-transformer.local
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: function-download
port:
number: 80

85
k8s/functions/metadata-service.yaml Normal file
View File

@@ -0,0 +1,85 @@
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
name: function-metadata
namespace: file-transformer
labels:
app: file-transformer
function: metadata
spec:
template:
metadata:
annotations:
autoscaling.knative.dev/minScale: "0"
autoscaling.knative.dev/maxScale: "10"
autoscaling.knative.dev/target: "1"
spec:
containerConcurrency: 10
timeoutSeconds: 300
containers:
- image: function-metadata:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5000
env:
- name: MINIO_ENDPOINT
value: "minio-service:9000"
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: access-key
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: secret-key
- name: MINIO_BUCKET_NAME
value: "file-transformer-bucket"
- name: MINIO_USE_SSL
value: "false"
- name: POSTGRES_URL
valueFrom:
secretKeyRef:
name: postgres-secret
key: database-url
resources:
requests:
memory: "256Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
readinessProbe:
httpGet:
path: /health
port: 5000
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /health
port: 5000
initialDelaySeconds: 15
periodSeconds: 30
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: function-metadata-ingress
namespace: file-transformer
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/ssl-redirect: "false"
spec:
rules:
- host: metadata.file-transformer.local
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: function-metadata
port:
number: 80

63
k8s/functions/transform-service.yaml Normal file
View File

@@ -0,0 +1,63 @@
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
name: file-transform-service
namespace: file-transformer
spec:
template:
metadata:
annotations:
autoscaling.knative.dev/minScale: "0"
autoscaling.knative.dev/maxScale: "10"
autoscaling.knative.dev/target: "1"
spec:
containerConcurrency: 5
timeoutSeconds: 600
containers:
- image: file-transformer/transform-function:latest
ports:
- containerPort: 5000
env:
- name: MINIO_ENDPOINT
value: "minio.file-transformer.svc.cluster.local:9000"
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: accesskey
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: secretkey
- name: MINIO_BUCKET_NAME
valueFrom:
configMapKeyRef:
name: minio-config
key: MINIO_BUCKET_NAME
- name: POSTGRES_URL
value: "postgresql://file_user:$(POSTGRES_PASSWORD)@postgres.file-transformer.svc.cluster.local:5432/file_transformer"
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-password
resources:
requests:
memory: "512Mi"
cpu: "500m"
limits:
memory: "1Gi"
cpu: "1000m"
livenessProbe:
httpGet:
path: /health
port: 5000
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /health
port: 5000
initialDelaySeconds: 5
periodSeconds: 5

63
k8s/functions/upload-service.yaml Normal file
View File

@@ -0,0 +1,63 @@
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
name: file-upload-service
namespace: file-transformer
spec:
template:
metadata:
annotations:
autoscaling.knative.dev/minScale: "0"
autoscaling.knative.dev/maxScale: "10"
autoscaling.knative.dev/target: "1"
spec:
containerConcurrency: 10
timeoutSeconds: 300
containers:
- image: file-transformer/upload-function:latest
ports:
- containerPort: 5000
env:
- name: MINIO_ENDPOINT
value: "minio.file-transformer.svc.cluster.local:9000"
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: accesskey
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: minio-secret
key: secretkey
- name: MINIO_BUCKET_NAME
valueFrom:
configMapKeyRef:
name: minio-config
key: MINIO_BUCKET_NAME
- name: POSTGRES_URL
value: "postgresql://file_user:$(POSTGRES_PASSWORD)@postgres.file-transformer.svc.cluster.local:5432/file_transformer"
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-password
resources:
requests:
memory: "256Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
livenessProbe:
httpGet:
path: /health
port: 5000
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /health
port: 5000
initialDelaySeconds: 5
periodSeconds: 5

112
k8s/minio.yaml Normal file
View File

@@ -0,0 +1,112 @@
apiVersion: v1
kind: Secret
metadata:
name: minio-secret
namespace: file-transformer
type: Opaque
data:
accesskey: bWluaW9hZG1pbg== # minioadmin
secretkey: bWluaW9hZG1pbjEyMw== # minioadmin123
---
apiVersion: v1
kind: ConfigMap
metadata:
name: minio-config
namespace: file-transformer
data:
MINIO_BUCKET_NAME: file-transformer-bucket
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio
namespace: file-transformer
labels:
app: minio
spec:
replicas: 1
selector:
matchLabels:
app: minio
template:
metadata:
labels:
app: minio
spec:
containers:
- name: minio
image: minio/minio:latest
ports:
- containerPort: 9000
name: api
- containerPort: 9001
name: console
env:
- name: MINIO_ROOT_USER
valueFrom:
secretKeyRef:
name: minio-secret
key: accesskey
- name: MINIO_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-secret
key: secretkey
- name: MINIO_BUCKET_NAME
valueFrom:
configMapKeyRef:
name: minio-config
key: MINIO_BUCKET_NAME
command:
- /bin/bash
- -c
- |
minio server /data --console-address ":9001" &
sleep 10
mc alias set local http://localhost:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD
mc mb local/$MINIO_BUCKET_NAME --ignore-existing
mc policy set public local/$MINIO_BUCKET_NAME
wait
volumeMounts:
- name: minio-storage
mountPath: /data
resources:
requests:
memory: "256Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
volumes:
- name: minio-storage
persistentVolumeClaim:
claimName: minio-pvc
---
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: file-transformer
spec:
selector:
app: minio
ports:
- name: api
port: 9000
targetPort: 9000
- name: console
port: 9001
targetPort: 9001
type: ClusterIP
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: minio-pvc
namespace: file-transformer
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi

7
k8s/namespace.yaml Normal file
View File

@@ -0,0 +1,7 @@
apiVersion: v1
kind: Namespace
metadata:
name: file-transformer
labels:
name: file-transformer
app: file-transformer-s3

100
k8s/postgres.yaml Normal file
View File

@@ -0,0 +1,100 @@
apiVersion: v1
kind: Secret
metadata:
name: postgres-secret
namespace: file-transformer
type: Opaque
data:
postgres-password: c2VjdXJlX3Bhc3N3b3JkXzEyMw== # secure_password_123
---
apiVersion: v1
kind: ConfigMap
metadata:
name: postgres-config
namespace: file-transformer
data:
POSTGRES_DB: file_transformer
POSTGRES_USER: file_user
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgres
namespace: file-transformer
labels:
app: postgres
spec:
replicas: 1
selector:
matchLabels:
app: postgres
template:
metadata:
labels:
app: postgres
spec:
containers:
- name: postgres
image: postgres:15-alpine
ports:
- containerPort: 5432
env:
- name: POSTGRES_DB
valueFrom:
configMapKeyRef:
name: postgres-config
key: POSTGRES_DB
- name: POSTGRES_USER
valueFrom:
configMapKeyRef:
name: postgres-config
key: POSTGRES_USER
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-secret
key: postgres-password
volumeMounts:
- name: postgres-storage
mountPath: /var/lib/postgresql/data
- name: postgres-init
mountPath: /docker-entrypoint-initdb.d
resources:
requests:
memory: "256Mi"
cpu: "250m"
limits:
memory: "512Mi"
cpu: "500m"
volumes:
- name: postgres-storage
persistentVolumeClaim:
claimName: postgres-pvc
- name: postgres-init
configMap:
name: postgres-init-script
---
apiVersion: v1
kind: Service
metadata:
name: postgres
namespace: file-transformer
spec:
selector:
app: postgres
ports:
- port: 5432
targetPort: 5432
type: ClusterIP
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: postgres-pvc
namespace: file-transformer
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi