diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c716585 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: CI Pipeline + +on: + push: + branches: + - "**" + - "!main" + paths-ignore: + - "helm" + workflow_call: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Install python + uses: actions/setup-python@v5.3.0 + with: + python-version: "3.12" + + - name: Checkout code + uses: actions/checkout@v4.2.2 + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Run pytest + run: | + python tests/test.py + continue-on-error: false + + - name: Run SonarTest + run: | + echo "It's OK" + continue-on-error: false + + - name: Run security scan + run: | + echo "It's OK" + continue-on-error: false diff --git a/.gitignore b/.gitignore index 75ec3f0..5817f13 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -.vscode/* \ No newline at end of file +.vscode/* +.idea/ +.venv +__pycache__/ \ No newline at end of file diff --git a/.helm/Chart.yaml b/.helm/Chart.yaml new file mode 100644 index 0000000..65843a0 --- /dev/null +++ b/.helm/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: tradebyte-tha +description: A Helm chart for Tradebyte home task +type: application +version: 0.1.0 +appVersion: "1.16.0" +dependencies: + - name: redis + version: 20.4.0 + repository: "oci://registry-1.docker.io/bitnamicharts" \ No newline at end of file diff --git a/.helm/templates/configmap.yaml b/.helm/templates/configmap.yaml new file mode 100644 index 0000000..b21c848 --- /dev/null +++ b/.helm/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + REDIS_HOST: "{{ .Release.Name }}-redis-master" \ No newline at end of file diff --git a/.helm/templates/deployment.yaml b/.helm/templates/deployment.yaml new file mode 100644 index 0000000..81fa461 --- /dev/null +++ b/.helm/templates/deployment.yaml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + labels: + app: {{ .Chart.Name }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.scaling.dev.minReplicas }} + strategy: + type: {{ .Values.strategy.type }} + {{- if eq .Values.strategy.type "RollingUpdate"}} + rollingUpdate: + maxSurge: {{ .Values.strategy.maxSurge }} + maxUnavailable: {{ .Values.strategy.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + name: {{ .Chart.Name }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: {{ .Chart.Name }} + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: {{ .Chart.Name }} + weight: 100 + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + - name: {{ .Values.imagePullSecrets }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.application.image }}:{{ .Values.application.imageTag }} + imagePullPolicy: {{ .Values.imagePullPolicy | default "Always"}} + env: + - name: REDIS_HOST + valueFrom: + configMapKeyRef: + key: REDIS_HOST + name: {{ .Chart.Name }} + - name: REDIS_PORT + valueFrom: + secretKeyRef: + key: REDIS_PORT + name: {{ .Chart.Name }} + - name: REDIS_DB + valueFrom: + secretKeyRef: + key: REDIS_DB + name: {{ .Chart.Name }} + - name: ENVIRONMENT + value: {{ .Values.env | quote }} + - name: HOST + value: {{ .Values.application.appHost | quote }} + - name: PORT + value: {{ .Values.application.port | quote }} + resources: + {{- if eq .Values.env "prod" }} + {{- toYaml .Values.resources.prod | nindent 12 }} + {{- else if eq .Values.env "staging" }} + {{- toYaml .Values.resources.staging | nindent 12 }} + {{- else }} + {{- toYaml .Values.resources.dev | nindent 12 }} + {{- end }} + ports: + - containerPort: {{ .Values.application.port }} + protocol: TCP + livenessProbe: + httpGet: + port: {{ .Values.application.port }} + path: {{ .Values.livenessProbe.path }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + readinessProbe: + httpGet: + port: {{ .Values.application.port }} + path: {{ .Values.readinessProbe.path }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + restartPolicy: Always diff --git a/.helm/templates/hpa.yaml b/.helm/templates/hpa.yaml new file mode 100644 index 0000000..20dfedd --- /dev/null +++ b/.helm/templates/hpa.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ .Chart.Name }} + labels: + app: {{ .Chart.Name }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Chart.Name }} + {{- if eq .Values.env "prod" }} + {{- toYaml .Values.scaling.prod | nindent 2 }} + {{- else if eq .Values.env "staging" }} + {{- toYaml .Values.scaling.staging | nindent 2 }} + {{- else }} + {{- toYaml .Values.scaling.dev | nindent 2 }} + {{- end }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.scaling.cpuTh }} diff --git a/.helm/templates/ingress.yaml b/.helm/templates/ingress.yaml new file mode 100644 index 0000000..c2ea7e4 --- /dev/null +++ b/.helm/templates/ingress.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Chart.Name }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + rules: + - host: {{ .Values.application.appHost }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ .Chart.Name }} + port: + number: {{ .Values.application.port }} \ No newline at end of file diff --git a/.helm/templates/pdb.yaml b/.helm/templates/pdb.yaml new file mode 100644 index 0000000..7970c5e --- /dev/null +++ b/.helm/templates/pdb.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .Chart.Name }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + minAvailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow + selector: + matchLabels: + app: {{ .Chart.Name }} diff --git a/.helm/templates/secret.yaml b/.helm/templates/secret.yaml new file mode 100644 index 0000000..77ae9fb --- /dev/null +++ b/.helm/templates/secret.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Chart.Name }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + REDIS_PORT: NjM3OQ== + REDIS_DB: MA== diff --git a/.helm/templates/service.yaml b/.helm/templates/service.yaml new file mode 100644 index 0000000..b7c44a9 --- /dev/null +++ b/.helm/templates/service.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + app: {{ .Chart.Name }} + ports: + - protocol: TCP + port: {{ .Values.application.port }} + targetPort: {{ .Values.application.port }} \ No newline at end of file diff --git a/.helm/values.yaml b/.helm/values.yaml new file mode 100644 index 0000000..0be60fd --- /dev/null +++ b/.helm/values.yaml @@ -0,0 +1,63 @@ +env: staging +annotations: + "project": "Tradebyte Home Challenge" +application: + image: cardiffc/tradebyte + imageTag: 4 + port: 8000 + appHost: tradebyte.test +scaling: + dev: + minReplicas: 3 + maxReplicas: 3 + staging: + minReplicas: 3 + maxReplicas: 6 + prod: + minReplicas: 3 + maxReplicas: 10 + cpuTh: 80 +strategy: + type: RollingUpdate + maxSurge: 2 + maxUnavailable: 2 +resources: + dev: + limits: + cpu: 200m + memory: 1Gi + requests: + cpu: 10m + memory: 500Mi + staging: + limits: + cpu: 200m + memory: 2Gi + requests: + cpu: 50m + memory: 1Gi + prod: + limits: + cpu: 1000m + memory: 8Gi + requests: + cpu: 100m + memory: 1Gi +livenessProbe: + path: /live + initialDelaySeconds: 15 + periodSeconds: 10 + failureThreshold: 3 +readinessProbe: + path: /ready + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 +redis: + replica: + replicaCount: 1 + auth: + enabled: false + + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fbf3e3b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.12-alpine3.20 +WORKDIR /app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY static/ ./static +COPY templates/ ./templates +COPY hello.py ./ +CMD ["python", "hello.py"] diff --git a/SOLUTION.md b/SOLUTION.md new file mode 100644 index 0000000..23743a2 --- /dev/null +++ b/SOLUTION.md @@ -0,0 +1,52 @@ +# Solution + +## What Has Been Done +1. The Python code has been modified: + 1.1. Readiness and liveness probe endpoints have been added. + 1.2. Dependencies in the `requirements.txt` file have been updated. +2. Dockerfile has been created. +3. A Helm chart has been created in the `.helm` directory. +4. A simple script, `deploy.sh`, has been created to emulate the deployment pipeline. +5. Simple CI pipeline to run python tests on any branch but master has been added to .github/workflows. It's just emulation. + +##How to run + +### Prerequisites +1. You need to install a Minikube cluster (it should also work on EKS/AKS with the NGINX ingress controller) with the metrics and ingress addons. +2. Run a port-forward to be able to communicate with the API exposed via ingress: + ```zsh + kubectl port-forward -n kube-system service/ingress-nginx-controller 8080:80 -n ingress-nginx + ``` +3. Add a record to the `/etc/hosts` file with a fake FQDN (e.g., `127.0.0.1 tradebyte.test`). +4. You need to be authorized against the container registry to push publicly available Docker images. +5. Your `~/.kube/config` must be configured, and the proper context should be selected. + +"Usage: $0 $1 $3 $4 $5 " + +### Deploying +**Simply run the `deploy.sh` script with the following parameters:** +1. `image_name` - Docker Hub repo name (e.g., `cardiffc/tradebyte`). +2. `image_tag` - Tag of the new image. +3. `project_name` - Name of the project. +4. `base_url` - FQDN you want to use to expose application. (e.g. `tradebyte.test`) +4. `env` - Name of the environment (`dev` or `staging` or `prod`). + + Example: + ```zsh + ./deploy.sh cardiffc/tradebyte 1 tradebyte tradebyte.test dev + ``` + +**It will:** +1. Build and push Docker images using the provided image name and tag. +2. Render the Helm chart. +3. Install the Helm chart to the Kubernetes cluster in a namespace named after the project, using the previously built Docker images for the provided environment. +4. Install simple redis instance. + +### What Could Have Been Done Better & some tradeoffs +1. For this test task, I have created secret with some values and commited to git. I'd never do so in the real life. In a real-world scenario, I would use an external secrets storage (like AWS SSM or HashiCorp Vault) and a Kubernetes operator to sync them. +2. For this test task, I am using HTTP instead of HTTPS for the exposed application. In a real-world scenario, it should be HTTPS. +3. The Helm chart could be further tuned to allow for Service Accounts,NetworkPolicies (if needed), and so on. This tuning could be done based on the real world requirements understanding. +4. The `deploy.sh` script should be replaced with a proper CI/CD pipeline. +5. It is preferable to use Redis as a managed service provided by AWS or any other cloud platform. +6. Authentication should be enabled in Redis, but further improvements to the Python code are needed. +7. Using Gunicorn with a Tornado worker should be considered, but further code improvements are needed. diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..7b8e96c --- /dev/null +++ b/deploy.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +if [ "$#" -ne 5 ]; then + echo "Usage: $0 $1 $3 $4 $5 " + exit 1 +fi + +IMAGE_NAME=$1 +IMAGE_TAG=$2 +PROJECT_NAME=$3 +BASE_URL=$4 +ENV=$5 +CONTEXT="." + +echo "Building container image" +docker build --platform linux/amd64 -t "$IMAGE_NAME:$IMAGE_TAG" "$CONTEXT" +if [ $? -eq 0 ]; then + echo "Image build $IMAGE_NAME:$IMAGE_TAG finished" +else + echo "Failed to build image $IMAGE_NAME:$IMAGE_TAG" + exit 1 +fi + +echo "Pushing image to dockerhub" +docker push $IMAGE_NAME:$IMAGE_TAG +if [ $? -eq 0 ]; then + echo "Image $IMAGE_NAME:$IMAGE_TAG pushed" +else + echo "Failed to push image" + exit 1 +fi + +echo "Updating Helm dependencies" +helm dependency update .helm +if [ $? -eq 0 ]; then + echo "Helm dependencies updated" +else + echo "Failed to update Helm dependencies" + exit 1 +fi + +echo "Deploying image release $PROJECT_NAME" +helm -n $PROJECT_NAME upgrade --install $PROJECT_NAME .helm --set application.image=$IMAGE_NAME --set application.imageTag=$IMAGE_TAG --set env=$ENV --set application.appHost=$BASE_URL --create-namespace +if [ $? -eq 0 ]; then + echo "Application deployed" +else + echo "Failed to deploy application" +fi + + diff --git a/hello.py b/hello.py index 765265b..178bdf9 100644 --- a/hello.py +++ b/hello.py @@ -28,10 +28,21 @@ def get(self): dict={"environment": environment, "counter": r.incr("counter", 1)}, ) +class LiveHandler(tornado.web.RequestHandler): + def get(self): + response = {"status": "live"} + self.set_header("Content-Type", "application/json") + self.write(response) + +class ReadyHandler(tornado.web.RequestHandler): + def get(self): + response = {"status": "ready"} + self.set_header("Content-Type", "application/json") + self.write(response) class Application(tornado.web.Application): def __init__(self): - handlers = [(r"/", MainHandler)] + handlers = [(r"/", MainHandler), (r"/live", LiveHandler), (r"/ready", ReadyHandler)] settings = { "template_path": os.path.join( os.path.dirname(os.path.abspath(__file__)), "templates" diff --git a/requirements.txt b/requirements.txt index 57e7c89..f8985f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -redis==3.0.1 -tornado==5.1.1 +redis==5.2.0 +tornado==6.4.2