Remove workspace var. can be replaced with internal variable terraform.workspace

Fix check on env vars

See merge request spengreb/atmos!4

.drone.yml

@@ -0,0 +1,20 @@
+kind: pipeline
+name: default
+
+steps:
+- name: test
+  image: python
+  commands:
+  - ./atmos.py --help
+- name: docker
+  image: plugins/docker
+  settings:
+    username:
+      from_secret: docker_user
+    password:
+      from_secret: docker_password
+    dockerfile: Dockerfile
+    repo: spengreb/atmos
+    tags: 
+      - latest
+      - "0.12.20"

.gitlab-ci.yml

@@ -0,0 +1,39 @@
+stages: 
+  - 🤞 test
+  - 🤞 test docker build
+  - 🚀 publish
+
+test:
+  stage: 🤞 test
+  script:
+    - python3 -m unittest
+
+test-docker-build:
+  stage: 🤞 test docker build
+  image: docker:latest
+  services:
+    - docker:dind
+  before_script:
+    - apk add jq curl
+  script:
+    - TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')
+    - docker build --pull -t "$CI_REGISTRY_IMAGE:$TERRAFORM_VERSION" .
+  except:
+    - master
+
+publish:
+  stage: 🚀 publish
+  image: docker:latest
+  services:
+    - docker:dind  
+  before_script:
+    - apk add jq curl
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  script:
+    - TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')
+    - docker build --pull -t "$CI_REGISTRY_IMAGE:$TERRAFORM_VERSION" .
+    - docker build --pull -t "$CI_REGISTRY_IMAGE" .
+    - docker push "$CI_REGISTRY_IMAGE:$TERRAFORM_VERSION"
+    - docker push "$CI_REGISTRY_IMAGE"
+  only:
+    - master

Dockerfile

@@ -1,9 +1,12 @@
-FROM alpine
+FROM python:latest
 
-RUN apk add python3
+RUN apt update && apt install -y jq
-RUN wget -O /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.13/terraform_0.11.13_linux_amd64.zip 
+RUN wget -O /tmp/terraform.zip `echo "https://releases.hashicorp.com/terraform/$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')/terraform_$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')_linux_amd64.zip"`
 RUN unzip /tmp/terraform.zip 
 RUN mv terraform /usr/bin/
 
-COPY shared-creds /root/.aws/credentials
+COPY git-askpass-helper.sh /usr/bin/git-pass
-COPY atmos.py /usr/bin/atmos
+
+RUN mkdir /atmos
+COPY atmos.py credentials.py workspaces.py /atmos/
+RUN ln -s /atmos/atmos.py /usr/bin/atmos

README.md

@@ -1,10 +1,24 @@
 [![CircleCI](https://circleci.com/gh/Spengreb/atmos.svg?style=svg)](https://circleci.com/gh/Spengreb/atmos)
+[![Build Status](https://cloud.drone.io/api/badges/Spengreb/atmos/status.svg)](https://cloud.drone.io/Spengreb/atmos)
 
-# Terraform Atmosphere
+# Terraform Atmosphere :earth_africa:
-Atmos is a thin wrapper for managing Terraform Workspaces easily. Using the workspace name it will select the correct .tfvar file, defaulting to a qa var file for any other workspace. This is primarily for pipelines but works just as well from the command line. It can process all terraform commands and parameters passing them on directly.
+Atmos is a thin wrapper for managing Terraform Workspaces easily. Using the workspace name it will select the correct .tfvar file, defaulting to a qa var file for any other workspace. This is primarily for pipelines but works just as well from the command line. It can process all terraform commands and parameters passing them on directly. Atmos will automatically switch workspaces per git branches if it discovers its in a git repository
 
 # Quick Start
 
+## Local Use
+
+Atmos requires terraform to be installed on your system. 
+
+- Clone this atmos project
+- Symlink atmos.py to your /usr/bin/ `$ ln -s $(pwd)/atmos.py /usr/bin/atmos`
+- Set up your `~/.aws/credentials` to include a `[default]` stanza which is where your S3 backend storage is. 
+- Setup other stanzas in your credentials file for each environment you want. For example `[dev]` with your dev account IAM credentials
+- You can also setup environment variables and use the -e flag. See below for more.
+- Use `$ atmos apply/plan/destroy` to run terraform apply whilst maintaining environment context
+
+## CI/CD
+
 - Build the atmos image
 - Use atmos as the build image in your CI/CD
 - Include switching/creating terraform workspaces
@@ -46,11 +60,11 @@ variable "workspace" {
 }
 ```
 
-This will make Terraform lookup AWS credentials from the `~/.aws/credentials` file using the workspace name as the stanza name. For example the credentials file would look like the shared-creds file in this repo.
+This will make Terraform lookup AWS credentials from the `~/.aws/credentials` file using the workspace name as the stanza name. 
 
-## atmos -t
+## atmos -e
 
-Adding the `-t` flag to atmos will make it generate a new `~/.aws/credentials` file from environment variables. You must first include the `default` access key ID & secret access key like this:
+Adding the `-e` flag to atmos will make it generate a new `~/.aws/credentials` file from environment variables. You must first include the `default` access key ID & secret access key like this:
 
 ```
 DEFAULT_ACCESS_KEY_ID=id
@@ -67,4 +81,26 @@ QA_ACCESS_KEY_ID=id
 QA_SECRET_ACCESS_KEY=key
 ```
 
-Note: Atmos will override your default credentials file as this functionality is for use in a docker container or in situations where you would rather use variables.
+# atmos -m
+
+Adding `-m` flag will set to manual mode. It will not try to automatically switch workspace per branch. It will adhere to whatever you last set the workspace to.
+
+# atmos -p 
+
+Adding `-p` flag will set the project prefix when looking for credentials. 
+
+Example:
+` $ atmos -e -p PROJ plan`
+
+Will make atmos look for environment vars with the prefix 'VER' selecting the following env vars. 
+
+```
+PROJ_DEV_ACCESS_KEY_ID
+PROJ_DEV_SECRET_ACCESS_KEY
+```
+
+Note this also works on the `.aws/credentials` file
+
+# atmos -v
+
+Verbose output mode, will show the vars atmos has selected and some environment context

atmos.py

@@ -1,66 +1,58 @@
 #!/usr/bin/env python3
 
 import argparse, subprocess, shlex, sys, os, glob
-from jinja2 import Environment, FileSystemLoader
+import workspaces
+import credentials
 
 def main(argv):
     parser = argparse.ArgumentParser(description='Control Terraform Workspaces.')
     g = parser.add_mutually_exclusive_group()
     g.add_argument("command", help="Send commands to terraform with workspace variable context", nargs='?', default=False)
-    parser.add_argument("-t", help="Template mode, gather shared-creds from environment variables (Dont use this flag if you dont want your ~/.aws/credentials replaced. This is for CI/CD", action='store_true', default=False)
+    parser.add_argument("-e", help="Gather shared-creds from environment variables. This is for CI/CD", action='store_true', default=False)
+    parser.add_argument("-m", help="Prevents workspace from changing with git branches automatically", action='store_true', default=False)
+    parser.add_argument("-n", help="Atmos will not add -var-file or -var args to terraform", action='store_true', default=False)
+    parser.add_argument("-p", "--project", help="Add a project prefix for env vars", nargs='?', default="")
+    parser.add_argument("-v", "--verbose", help="Debug mode", action="store_true", default=False)
     args, params = parser.parse_known_args()
     if args.command:
         determine_actions(args, params)
 
 def determine_actions(args, params):
-    workspace = get_env()
+    aws_creds_file = "$HOME/.aws/credentials"
-    env_actions = ["plan", "apply", "destroy"] # Commands that require env context
+    if (is_git_directory()) and not (args.m):
+        # if (args.e):
+        #     aws_creds_file = aws_creds_file + "-atmos"
+        workspaces.workspace_manager()
+
+    workspace = workspaces.get_env()
+    workspace_vars = workspace
+    if (args.project) and workspace != 'default':
+        workspace = args.project + "-" + workspace
+
+    env_actions = ["init", "plan", "apply", "destroy"] # Commands that require env context
     cmd = 'terraform {args}'.format(args=args.command)
 
+    if (args.command in env_actions) and not (args.n): # Append with env context
+        cmd = cmd + ' -var-file=vars/{env}.tfvars'.format(env=workspace_vars) 
+
     for param in params: # Pass terraform params directly through
         cmd = cmd + ' ' + param
 
-    if (args.command in env_actions) and (workspace != "default"): # Append with env context
+    if (args.e):
-        cmd = cmd + ' -var-file=vars/{env}.tfvars -var "workspace={env}"'.format(env=workspace)
+        credentials.generate(args)
 
-    if (args.t):
+    if (args.verbose):
-        generate_creds()
+        print("Atmos will run: " + cmd)
+    print('Terraform {args} using env vars in {env}'.format(args=args.command, env=workspace_vars))
+    run_cmd(cmd)
 
-    print('Terraform {args} using env vars in {env}'.format(args=args.command, env=workspace))
+def run_cmd(cmd):
     with subprocess.Popen(shlex.split(cmd)) as proc:
         exit # Start process but kill py program
 
-def generate_creds():
+def is_git_directory():
-    current_workspace = get_env()
+    return subprocess.call(['git', 'branch'], stderr=subprocess.STDOUT, stdout = open(os.devnull, 'w')) == 0
-    workspaces = ['default']
 
-    if current_workspace != 'default':
-        workspaces.append(current_workspace)
-    
-    contents = ""
-    for workspace in workspaces:
-        contents = contents + "[{workspace}]\n".format(workspace=workspace)
-        contents = contents + "access_key_id=" + os.environ.get(workspace.upper() + '_ACCESS_KEY_ID') + "\n"
-        contents = contents + "secret_access_key=" + os.environ.get(workspace.upper() + '_SECRET_ACCESS_KEY') + "\n"
-    with open(os.path.expanduser('~/.aws/credentials'), 'w+') as f:
-        f.write(contents)
-
-def get_valid_envs():
-    try:
-        # Use var files when present, otherwise default to qa
-        return [os.path.splitext(os.path.basename(x))[0] for x in glob.glob("vars/*.tfvars")]
-    except FileNotFoundError:
-        return False
-
-def get_env():
-    try:
-        tf_env = open('.terraform/environment', 'r').read()
-    except:
-        return("default")
-    if str(tf_env) in get_valid_envs():
-        return(tf_env)
-    else:
-        return("qa")
 
 if __name__ == "__main__":
-    main(sys.argv)
+    main(sys.argv)

credentials.py

@@ -0,0 +1,53 @@
+import workspaces, sys, os
+
+def generate(args):
+    current_workspace = workspaces.get_env()
+    workspaces_names = ['default']
+    aws_creds_dir = '~/.aws'
+    aws_creds_file = 'credentials'
+    aws_creds_full = aws_creds_dir + '/' + aws_creds_file
+
+    if os.path.isfile(os.path.expanduser(aws_creds_full)):
+        answer = input(f"[WARNING] File {aws_creds_full} already exists. Atmos will generate a new credentials file from your env vars. \nDo you want to override {aws_creds_full}? [y/N]")
+        if not answer or answer[0].lower() != 'y':
+            print("File not changed. This flag is for CI/CD only")
+            exit(1)
+    else:
+        if not os.path.isdir(os.path.expanduser(aws_creds_dir)):
+            os.makedirs(os.path.expanduser(aws_creds_dir))
+
+    if current_workspace != 'default':
+        workspaces_names.append(current_workspace)
+
+    project_name = ""
+    if (args.project):
+        delimeter = "_"
+        project_name = args.project.upper() + delimeter
+
+    contents = ""
+    for workspace in workspaces_names:
+        access_key_name = project_name + workspace.upper() + '_ACCESS_KEY_ID'
+        secret_key_name = project_name + workspace.upper() + '_SECRET_ACCESS_KEY'
+
+        if (args.verbose):
+            print(access_key_name)
+            print(secret_key_name)
+
+        if (workspace == 'default'):
+            contents = contents + "[{workspace}]\n".format(workspace=(workspace).lower())
+        else:
+            contents = contents + "[{workspace}]\n".format(workspace=(project_name.replace("_", "-") + workspace).lower())
+
+        try:
+            contents = contents + "aws_access_key_id=" + os.environ.get(access_key_name) + "\n"
+        except:
+            print("[ERROR]: Env Variable " + access_key_name + " not found.")
+            sys.exit(1)
+        try:
+            contents = contents + "aws_secret_access_key=" + os.environ.get(secret_key_name) + "\n"
+        except:
+            print("[ERROR]: Env Variable " + secret_key_name + " not found.")
+            sys.exit(1)
+
+    with open(os.path.expanduser(aws_creds_full), 'w+') as f:
+        f.write(contents)

git-askpass-helper.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo ${GIT_PASSWORD}

shared-creds

@@ -1,7 +0,0 @@
-[default]
-
-[preprod]
-
-[production]
-
-[qa]

tests.py

@@ -0,0 +1,69 @@
+import unittest
+from unittest.mock import MagicMock, patch
+import argparse
+import atmos
+
+class DetermineActionsTests(unittest.TestCase):
+
+    def setUp(self):
+        self.input_args = argparse.Namespace(command="mytestcommand", e=False, m=False, n=False, project="", verbose=False)
+        atmos.is_git_directory = MagicMock(return_value=False)
+        atmos.run_cmd = MagicMock()
+
+    def test_whenCalledWithNoAdditionalArgs_shouldRunTheTerraformCommand(self):
+        """Simplest case where atmos is only called with a command and no further arguments"""
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with("terraform mytestcommand")
+
+    def test_whenCalledWithParams_theyAreAppended(self):
+        """Should append all params after the command"""
+        atmos.determine_actions(self.input_args, ["--myparam", "myvalue"])
+        atmos.run_cmd.assert_called_with("terraform mytestcommand --myparam myvalue")
+
+    @patch("workspaces.get_env")
+    def test_whenCalledWithInitCommand_shouldAppendVarsAndCreds(self, mocked_get_env):
+        """Case where var-file, -var workpace=xyz is appended"""
+        self.input_args.command = "init"
+        mocked_get_env.return_value = "mytestenv"
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with('terraform init -var-file=vars/mytestenv.tfvars -var "workspace=mytestenv"')
+
+    @patch("workspaces.get_env")
+    def test_whenCalledWithPlanCommand_shouldAppendVarsAndCreds(self, mocked_get_env):
+        """Case where var-file, -var workpace=xyz is appended"""
+        self.input_args.command = "plan"
+        mocked_get_env.return_value = "mytestenv"
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with('terraform plan -var-file=vars/mytestenv.tfvars -var "workspace=mytestenv"')
+
+    @patch("workspaces.get_env")
+    def test_whenCalledWithApplyCommand_shouldAppendVarsAndCreds(self, mocked_get_env):
+        """Case where var-file, -var workpace=xyz is appended"""
+        self.input_args.command = "apply"
+        mocked_get_env.return_value = "mytestenv"
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with('terraform apply -var-file=vars/mytestenv.tfvars -var "workspace=mytestenv"')
+
+    @patch("workspaces.get_env")
+    def test_whenCalledWithDestroyCommand_shouldAppendVarsAndCreds(self, mocked_get_env):
+        """Case where var-file, -var workpace=xyz is appended"""
+        self.input_args.command = "destroy"
+        mocked_get_env.return_value = "mytestenv"
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with('terraform destroy -var-file=vars/mytestenv.tfvars -var "workspace=mytestenv"')
+
+    @patch("workspaces.workspace_manager")
+    def test_whenInAGitRepo_andManualArgIsNotGiven_andEnvironmentArgIsNotGiven_shouldCallTheWorkspaceManager(self, mocked_workspace_manager):
+        atmos.is_git_directory.return_value = True
+        atmos.determine_actions(self.input_args, [])
+        mocked_workspace_manager.assert_called_once()
+
+    @patch("credentials.generate")
+    def test_whenEnvironmentArgIsGiven_shouldGenerateCredentials(self, mocked_generate):
+        self.input_args.e = True
+        atmos.determine_actions(self.input_args, [])
+        mocked_generate.assert_called_once()
+
+
+if __name__ == '__main__':
+    unittest.main()

workspaces.py

@@ -0,0 +1,33 @@
+import subprocess, os, glob
+
+def workspace_manager():
+    branch = subprocess.getoutput("git rev-parse --abbrev-ref HEAD")
+    if branch == "master":
+        branch = "default"
+    else:
+        if branch not in get_valid_envs():
+            branch = "default"
+
+    if get_env() != branch:
+        print("[INFO]: Terraform workspace & git branch have diverged. Changing workspace to git branch...")
+        subprocess.call(["terraform", "workspace", "new", branch], stderr=subprocess.STDOUT, stdout=open(os.devnull, 'w'))
+        subprocess.call(["terraform", "workspace", "select", branch], stderr=subprocess.STDOUT, stdout=open(os.devnull, 'w'))
+
+def get_valid_envs():
+    try:
+        # Use var files when present, otherwise default to default
+        return [os.path.splitext(os.path.basename(x))[0] for x in glob.glob("vars/*.tfvars")]
+    except FileNotFoundError:
+        return False
+
+def get_env():
+    try:
+        tf_env = ""
+        with open('.terraform/environment', 'r') as f:
+            tf_env = f.readline()
+    except:
+        return("default")
+    if str(tf_env) in get_valid_envs():
+        return(tf_env)
+    else:
+        return("default")