add test and ci on github and gitlab

.github/workflows/test_ci.yml

@@ -9,12 +9,22 @@ on:
     branches: [ main ]
 
 jobs:
+  script_lambda_package:
+    name: Test script lambda package
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build Lambda package
+      run: |
+        sh rony/base_files/scripts/build_lambda_package.sh
+
   build_test_lambda:
     name: Build and test Lambda
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-      
+
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
@@ -25,15 +35,37 @@ jobs:
         pip install -U pip
         pip install -r rony/base_files/etl/lambda_requirements.txt
         pip install pytest boto3
-    
+
     - name: Run tests on lambda function
       env:
         BUCKET: "my-bucket"
         REGION: "my-region"
       run: |
-        cd rony/base_files
-        python -m pytest
-
+        cd rony/base_files/tests
+        python -m pytest test_lambda.py
+
+  build_test_dags:
+    name: Build and test dags
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+#    - name: Set up Python 3.8
+#      uses: actions/setup-python@v2
+#      with:
+#        python-version: 3.8
+
+    - name: 'Airflow - Setup Airflow'
+      uses: jayamanikharyono/airflow-dag-action@v0.1-alpha
+      with:
+        dagPaths: rony/base_files/dags
+
+#    - name: 'Validate DAGs'
+#      run: |
+#        pip install -U pip
+#        pip install pytest
+#        cd rony/base_files/tests
+#        python -m pytest test_dags.py
 
   build_docker:
     name: Build and test docker image
@@ -46,7 +78,6 @@ jobs:
         docker build -t "${PWD##*/}" .
         docker run "${PWD##*/}"
 
-
   terraform_validate:
     name: Terraform validation
     runs-on: ubuntu-latest
@@ -59,19 +90,41 @@ jobs:
     - name: create zipfile for validation
       run: |
         touch rony/base_files/infrastructure/lambda_function_payload.zip
-
     - name: Terraform validate
       run: |
         cd rony/base_files/infrastructure
         terraform init
         terraform validate
 
+  terragrunt_validate:
+    name: Terragrunt validation
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: HashiCorp - Setup Terraform
+      uses: hashicorp/setup-terraform@v1.2.1
+
+    - name: HashiCorp - Setup Terragrunt
+      uses: autero1/action-terragrunt@v1.0.0
+      with:
+        terragrunt_version: 0.28.2
+
+    - name: create zipfile for validation
+      run: |
+        touch rony/base_files/infrastructure/lambda_function_payload.zip
+
+    - name: Terragrunt validate
+      run: |
+        cd rony/base_files/infrastructure
+        terragrunt validate
+
   package_install:
     name: Test package install
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -90,14 +143,13 @@ jobs:
         cd rony/tests
         coverage run -m pytest && coverage report -m
 
-
   test_project:
     name: Test project structure
     runs-on: ubuntu-latest
     needs: [package_install]
     steps:
     - uses: actions/checkout@v2
-      
+
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
@@ -108,12 +160,11 @@ jobs:
         pip install --upgrade pip
         pip install .
         rony info
-
     - name: Test Project structure
       run: |
         rony new projex
         cd projex
         cat etl/Dockerfile
         cat dags/titanic_example.py
         cat infrastructure/s3.tf
-        cat scripts/build_lambda_package.sh
+        cat scripts/build_lambda_package.sh

.gitlab-ci.yml

@@ -0,0 +1,153 @@
+image:
+  name: ubuntu:latest
+
+stages:
+  - Test package
+  - Test structure
+
+.execute_pipeline:
+  only:
+      - branches
+      - merge_requests
+
+.package_test_py:
+  before_script:
+    - apt-get -qy update && apt-get -qy install python$PYTHON_VERSION python3-pip
+    - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip
+  script:
+    - python -m pip install --upgrade pip
+    - pip install pytest
+    - pip install coverage
+    - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - pip install .
+    - cd rony/tests
+    - coverage run -m pytest && coverage report -m
+  extends: .execute_pipeline
+
+Test script package lambda:
+  stage: Test package
+  before_script:
+    - apt-get -qy update && apt-get -qy install python3.8 python3-pip zip
+    - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip
+  script:
+    - pip install -U pip
+    - pip install pytest
+    - cd rony/base_files/tests
+    - python -m pytest test_build_lambda_package.py
+
+Build and test Lambda:
+  stage: Test package
+  variables:
+    BUCKET: "my-bucket"
+    REGION: "xx-xxxx-x"
+  before_script:
+    - apt-get -qy update && apt-get -qy install python3.8 python3-pip
+    - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip
+  script:
+    - pip install -U pip
+    - pip install -r rony/base_files/etl/lambda_requirements.txt
+    - pip install pytest boto3
+    - cd rony/base_files/tests
+    - python -m pytest test_lambda.py
+  extends: .execute_pipeline
+
+Build and test dags:
+  image: puckel/docker-airflow
+  stage: Test package
+  before_script:
+    - python -m pip install --upgrade pip
+    - pip install pytest
+  script:
+    - cd rony/base_files/tests
+    - python -m pytest test_dags.py
+  extends: .execute_pipeline
+
+Build and test docker image:
+  image: docker:latest
+  stage: Test package
+  services:
+    - docker:dind
+  script:
+    - cd rony/base_files/etl
+    - docker build -t "${PWD##*/}" .
+    - docker run "${PWD##*/}"
+  extends: .execute_pipeline
+
+Terraform validation:
+  stage: Test package
+  before_script:
+    - apt-get -qy update && apt-get -qy install curl zip
+    - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform
+    - terraform --version
+  script:
+    - touch rony/base_files/infrastructure/lambda_function_payload.zip
+    - cd rony/base_files/infrastructure
+    - terraform init
+    - terraform validate
+  extends: .execute_pipeline
+
+Terragrunt validation:
+  stage: Test package
+  before_script:
+    - apt-get -qy update && apt-get -qy install curl zip
+    - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform
+    - curl -sS -L -O https://github.com/gruntwork-io/terragrunt/releases/download/v0.28.2/terragrunt_linux_amd64 && mv terragrunt_linux_amd64 /usr/local/bin/terragrunt && chmod u+x /usr/local/bin/terragrunt
+    - terragrunt --version
+  script:
+    - touch rony/base_files/infrastructure/lambda_function_payload.zip
+    - cd rony/base_files/infrastructure
+    - terragrunt validate
+  extends: .execute_pipeline
+
+Test package install (3.6):
+  stage: Test package
+  variables:
+    PYTHON_VERSION: "3.6"
+  extends: .package_test_py
+
+Test package install (3.7):
+  stage: Test package
+  variables:
+    PYTHON_VERSION: "3.7"
+  extends: .package_test_py
+
+Test package install (3.8):
+  stage: Test package
+  variables:
+    PYTHON_VERSION: "3.8"
+  extends: .package_test_py
+
+Test package install (3.9):
+  stage: Test package
+  variables:
+    PYTHON_VERSION: "3.9"
+  extends: .package_test_py
+
+
+Test project structure:
+  stage: Test structure
+  dependencies:
+    - Build and test Lambda
+    - Build and test dags
+    - Build and test docker image
+    - Terraform validation
+    - Terragrunt validation
+    - Test package install (3.6)
+    - Test package install (3.7)
+    - Test package install (3.8)
+    - Test package install (3.9)
+  before_script:
+    - apt-get -qy update && apt-get -qy install python3.8 python3-pip
+    - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip
+  script:
+    - pip install --upgrade pip
+    - pip install .
+    - rony info
+    - rony new projex
+    - cd projex
+    - cat etl/Dockerfile
+    - cat dags/titanic_example.py
+    - cat infrastructure/s3.tf
+    - cat scripts/build_lambda_package.sh
+  extends: .execute_pipeline
+

rony/base_files/ci/github/github_ci.yml

@@ -62,9 +62,13 @@ jobs:
 
     - name: HashiCorp - Setup Terraform
       uses: hashicorp/setup-terraform@v1.2.1
+      with:
+        terraform_version: 0.14.6
 
     - name: HashiCorp - Setup Terragrunt
       uses: autero1/action-terragrunt@v1.0.0
+      with:
+        terragrunt_version: 0.28.2
 
     - name: Configure AWS credentials
       uses: aws-actions/configure-aws-credentials@v1

rony/base_files/ci/gitlab/.gitlab-ci.yml

@@ -17,7 +17,7 @@ cache:
 
 before_script:
   - apt-get -qy update && apt-get -qy install curl python3.8 python3-pip zip
-  - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.5/terraform_0.14.5_linux_amd64.zip && unzip terraform_0.14.5_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform
+  - curl -sS -L -O https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip && unzip terraform_0.14.6_linux_amd64.zip && mv terraform /usr/local/bin/terraform && chmod u+x /usr/local/bin/terraform
   - curl -sS -L -O https://github.com/gruntwork-io/terragrunt/releases/download/v0.28.2/terragrunt_linux_amd64 && mv terragrunt_linux_amd64 /usr/local/bin/terragrunt && chmod u+x /usr/local/bin/terragrunt
   - ln -s python3 /usr/bin/python && ln -s pip3 /usr/bin/pip
 
@@ -26,7 +26,7 @@ stages:
   - plan
   - build
 
-test-lambda:
+build_lambda:
   stage: test
   script:
     - pip install -U pip
@@ -36,7 +36,7 @@ test-lambda:
   only:
     - branches
 
-test-iac:
+build_dl:
   stage: test
   script:
     - sh scripts/build_lambda_package.sh
@@ -46,6 +46,18 @@ test-iac:
   only:
     - branches
 
+build_docker:
+  image: docker:latest
+  stage: test
+  services:
+    - docker:dind
+  script:
+    - cd etl
+    - docker build -t "${PWD##*/}" .
+    - docker run "${PWD##*/}"
+  except:
+    - master
+
 plan:
   stage: plan
   script:

rony/base_files/infrastructure/terragrunt.hcl

@@ -7,17 +7,17 @@ provider "aws" {
 }
 EOF
 }
-remote_state {
-  backend = "s3"
-  generate = {
-    path      = "backend.tf"
-    if_exists = "overwrite"
-  }
-  config = {
-    bucket = "my-bucket-state-backend"
-
-    key = "${path_relative_to_include()}/terraform.tfstate"
-    region         = "us-east-1"
-    encrypt        = true
-  }
-}
+//remote_state {
+//  backend = "s3"
+//  generate = {
+//    path      = "backend.tf"
+//    if_exists = "overwrite"
+//  }
+//  config = {
+//    bucket = "my-bucket-state-backend"
+//
+//    key = "${path_relative_to_include()}/terraform.tfstate"
+//    region         = "us-east-1"
+//    encrypt        = true
+//  }
+//}