Using Gitlab integrated CICD for Python project on Windows#

----------- coverage: platform win32, python 3.7.0-final-0 -----------
Name                                      Stmts   Miss  Cover
-------------------------------------------------------------
python_project\__init__.py                    6      0   100%
python_project\ctx_fetcher.py                15      0   100%
python_project\extras\__init__.py             0      0   100%
python_project\extras\celery.py              18     18     0%
python_project\filters.py                     6      2    67%
python_project\parser.py                     26      0   100%
python_project\request_id.py                 42      1    98%
-------------------------------------------------------------
TOTAL                                       113     21    81%

coverage: '/^TOTAL.*\s+(\d+\%)$/'

stages:
    - venv
    - test
    - build
    - deploy

before_script:
  - $gitApiUrl = 'https://gitlab.copdips.local/api/v4'
  # will save git api token more securely later.
  - $gitApiToken = $env:GitApiToken
  - $gitApiHeader = @{"PRIVATE-TOKEN" = $gitApiToken}
  - $cicdReportsFolderPath = Join-Path (Get-Location) "cicd_reports"
  - $venvPath = "$env:temp/venv/$($env:CI_PROJECT_NAME)"
  - >
    function Set-SecurityProtocolType {
        # [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
        # $AllProtocols = [System.Net.SecurityProtocolType]'Ssl3,Tls,Tls11,Tls12'
        $AllProtocols = [System.Net.SecurityProtocolType]'Tls12'
        [System.Net.ServicePointManager]::SecurityProtocol = $AllProtocols
    }
  - >
    function Write-PythonPath {
        $pythonPath = $(Get-Command python | % source)
        Write-Output "The python path is at: '$pythonPath'"
    }
  - >
    function Get-UpstreamProject {
        $apiParam = @{
          Headers = $gitApiHeader
          Uri = "$gitApiUrl/projects?search=$($env:CI_PROJECT_NAME)"
        }
        if ($PSVersionTable.PSVersion.Major -gt 5) {
          $apiParam.SkipCertificateCheck = $true
        }
        $projectList = Invoke-RestMethod @apiParam
        $upstreamProject = $projectList | ? forked_from_project -eq $null
        return $upstreamProject
    }
  - >
    function Get-UpstreamProjectId {
        $upstreamProject = Get-UpstreamProject
        return $upstreamProject.id
    }

  - >
    function Test-CreateVenv {
        param($VenvPath, $GitCommitSHA)
        $gitShowCommand = "git show $GitCommitSHA --name-only"
        $gitShowResult = Invoke-Expression $gitShowCommand
        Write-Host "$gitShowCommand`n"
        $gitShowResult | ForEach-Object {Write-Host $_}
        $changedFiles = Invoke-Expression "git diff-tree --no-commit-id --name-only -r $GitCommitSHA"
        $requirementsFiles = @()
        $requirementsFiles += "requirements.txt"
        foreach ($requirements in $requirementsFiles) {
            if ($requirements -in $changedFiles) {
                Write-Host "`nFound $requirements in the changed files, need to create venv."
                return $True
            }
        }
        if (-not (Test-Path $VenvPath)) {
            Write-Host "`nCannot found venv at $VenvPath, need to create venv."
            return $True
        }

        Write-Host "`nNo need to create venv."
        return $False
    }
  - >
    function Enable-Venv {
        param($VenvPath)

        Invoke-Expression (Join-Path $VenvPath "Scripts/activate.ps1")
        Write-Host "venv enabled at: $VenvPath"
        Write-PythonPath
    }
  - >
    function Create-Venv {
        param($VenvPath)

        Write-Output "Creating venv at $venvPath ."
        python -m venv $VenvPath
        Write-Output "venv created at $venvPath ."
    }
  - >
    function Install-PythonRequirements {
        param($VenvPath)

        Enable-Venv $VenvPath
        python -m pip install -U pip setuptools wheel
        pip install -r requirements.txt
    }
  - >
    function Remove-Venv {
        param($VenvPath)

        if (Test-Path $VenvPath) {
            Remove-Item $VenvPath -Recurse -Force
            Write-Host "venv removed from: $VenvPath"
        } else {
            Write-Host "venv not found at: $VenvPath"
        }
    }
  - Get-Location
  - git --version
  - python --version
  - Write-PythonPath
  - $PSVersionTable | ft -a
  - Get-ChildItem env:\ | Select-Object Name, Value | ft -a

venv:
  stage: venv
  script:
    - >
      if (Test-CreateVenv $venvPath $env:CI_COMMIT_SHA) {
          Remove-Venv $venvPath
          Create-Venv $venvPath
      }
      Install-PythonRequirements $venvPath

pytest:
  stage: test
  script:
    - $reportFolder = Join-Path $cicdReportsFolderPath "pytest"
    - New-Item -Path $reportFolder -Type Directory -Force
    - $upstreamProjectId = Get-UpstreamProjectId
    - Write-Output "upstreamProjectId = $upstreamProjectId"
    # TODO: add check master last commit coverage
    - Enable-Venv $venvPath
    - pytest --cov=flask_log_request_id --cov-report=html:$reportFolder
    - $coverageLine = (Get-Content (Join-Path $reportFolder index.html) | Select-String "pc_cov").line
    - $coverageString = ($coverageLine -replace "<[^>]*>", "").trim()
    - Write-Output "Total Coverage = $coverageString"
  coverage: '/^(?i)(TOTAL).*\s+(\d+\%)$/'


nosetests:
  stage: test
  script:
    - Enable-Venv $venvPath
    - nosetests.exe
  coverage: '/^TOTAL.*\s+(\d+\%)$/'

flake8:
  stage: test
  script:
    - Enable-Venv $venvPath
    - flake8.exe .\flask_log_request_id

mypy:
  stage: test
  script:
    - Enable-Venv $venvPath
    - $reportFolder = Join-Path $cicdReportsFolderPath "mypy"
    - New-Item -Path $reportFolder -Type Directory -Force
    - $mypyResult = mypy ./flask_log_request_id --ignore-missing-imports --html-report $reportFolder --xml-report $reportFolder
    - Write-Output "MyPy result = `""
    - $mypyResult | % { Write-Output $_}
    - Write-Output "`"`nEnd of MyPy result."
    - if ($mypyResult.count -gt 2) {
          return $False
      }