Remove workspace var. can be replaced with internal variable terraform.workspace

Fix check on env vars

See merge request spengreb/atmos!4

.drone.yml

@@ -15,4 +15,6 @@ steps:
       from_secret: docker_password
     dockerfile: Dockerfile
     repo: spengreb/atmos
-    tags: latest
+    tags: 
+      - latest
+      - "0.12.20"

.gitlab-ci.yml

@@ -0,0 +1,39 @@
+stages: 
+  - 🤞 test
+  - 🤞 test docker build
+  - 🚀 publish
+
+test:
+  stage: 🤞 test
+  script:
+    - python3 -m unittest
+
+test-docker-build:
+  stage: 🤞 test docker build
+  image: docker:latest
+  services:
+    - docker:dind
+  before_script:
+    - apk add jq curl
+  script:
+    - TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')
+    - docker build --pull -t "$CI_REGISTRY_IMAGE:$TERRAFORM_VERSION" .
+  except:
+    - master
+
+publish:
+  stage: 🚀 publish
+  image: docker:latest
+  services:
+    - docker:dind  
+  before_script:
+    - apk add jq curl
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+  script:
+    - TERRAFORM_VERSION=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')
+    - docker build --pull -t "$CI_REGISTRY_IMAGE:$TERRAFORM_VERSION" .
+    - docker build --pull -t "$CI_REGISTRY_IMAGE" .
+    - docker push "$CI_REGISTRY_IMAGE:$TERRAFORM_VERSION"
+    - docker push "$CI_REGISTRY_IMAGE"
+  only:
+    - master

Dockerfile

@@ -1,9 +1,12 @@
-FROM ubuntu:xenial
+FROM python:latest
 
-RUN apt-get update -y && apt-get install -y python3 wget unzip git
-RUN wget -O /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.12.0/terraform_0.12.0_linux_amd64.zip
+RUN apt update && apt install -y jq
+RUN wget -O /tmp/terraform.zip `echo "https://releases.hashicorp.com/terraform/$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')/terraform_$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')_linux_amd64.zip"`
 RUN unzip /tmp/terraform.zip 
 RUN mv terraform /usr/bin/
 
-COPY shared-creds /root/.aws/credentials
-COPY atmos.py /usr/bin/atmos
+COPY git-askpass-helper.sh /usr/bin/git-pass
+
+RUN mkdir /atmos
+COPY atmos.py credentials.py workspaces.py /atmos/
+RUN ln -s /atmos/atmos.py /usr/bin/atmos

README.md

@@ -6,6 +6,19 @@ Atmos is a thin wrapper for managing Terraform Workspaces easily. Using the work
 
 # Quick Start
 
+## Local Use
+
+Atmos requires terraform to be installed on your system. 
+
+- Clone this atmos project
+- Symlink atmos.py to your /usr/bin/ `$ ln -s $(pwd)/atmos.py /usr/bin/atmos`
+- Set up your `~/.aws/credentials` to include a `[default]` stanza which is where your S3 backend storage is. 
+- Setup other stanzas in your credentials file for each environment you want. For example `[dev]` with your dev account IAM credentials
+- You can also setup environment variables and use the -e flag. See below for more.
+- Use `$ atmos apply/plan/destroy` to run terraform apply whilst maintaining environment context
+
+## CI/CD
+
 - Build the atmos image
 - Use atmos as the build image in your CI/CD
 - Include switching/creating terraform workspaces
@@ -36,7 +49,6 @@ To get the most out of Terraform workspaces it is recommended that the AWS provi
 provider "aws" {
     region = "${var.region}"
     profile = "${var.workspace}"
-    shared_credentials_file = ${var.shared_credentials_file}
 }
 ```
 
@@ -48,11 +60,11 @@ variable "workspace" {
 }
 ```
 
-This will make Terraform lookup AWS credentials from the `~/.aws/credentials` file using the workspace name as the stanza name. For example the credentials file would look like the shared-creds file in this repo.
+This will make Terraform lookup AWS credentials from the `~/.aws/credentials` file using the workspace name as the stanza name. 
 
 ## atmos -e
 
-Adding the `-e` flag to atmos will make it generate a new `~/.aws/credentials-atmos` file from environment variables. You must first include the `default` access key ID & secret access key like this:
+Adding the `-e` flag to atmos will make it generate a new `~/.aws/credentials` file from environment variables. You must first include the `default` access key ID & secret access key like this:
 
 ```
 DEFAULT_ACCESS_KEY_ID=id
@@ -69,8 +81,26 @@ QA_ACCESS_KEY_ID=id
 QA_SECRET_ACCESS_KEY=key
 ```
 
-This requires a `shared_credentials_file` variable on the top level. To support standard Terraform workflows its recommened to default this to the default shared credentials file location `$HOME/.aws/credentials`. Atmos will then handle the overriding safely in the background
-
 # atmos -m
 
-Adding `-m` flag will set to manual mode. It will not try to automatically switch workspace per branch. It will adhere to whatever you last set the workspace to.
+Adding `-m` flag will set to manual mode. It will not try to automatically switch workspace per branch. It will adhere to whatever you last set the workspace to.
+
+# atmos -p 
+
+Adding `-p` flag will set the project prefix when looking for credentials. 
+
+Example:
+` $ atmos -e -p PROJ plan`
+
+Will make atmos look for environment vars with the prefix 'VER' selecting the following env vars. 
+
+```
+PROJ_DEV_ACCESS_KEY_ID
+PROJ_DEV_SECRET_ACCESS_KEY
+```
+
+Note this also works on the `.aws/credentials` file
+
+# atmos -v
+
+Verbose output mode, will show the vars atmos has selected and some environment context

atmos.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python3
 
 import argparse, subprocess, shlex, sys, os, glob
+import workspaces
+import credentials
 
 def main(argv):
     parser = argparse.ArgumentParser(description='Control Terraform Workspaces.')
     g = parser.add_mutually_exclusive_group()
     g.add_argument("command", help="Send commands to terraform with workspace variable context", nargs='?', default=False)
-    parser.add_argument("-e", help="Gather shared-creds from environment variables (Dont use this flag if you dont want your ~/.aws/credentials replaced. This is for CI/CD", action='store_true', default=False)
+    parser.add_argument("-e", help="Gather shared-creds from environment variables. This is for CI/CD", action='store_true', default=False)
     parser.add_argument("-m", help="Prevents workspace from changing with git branches automatically", action='store_true', default=False)
     parser.add_argument("-n", help="Atmos will not add -var-file or -var args to terraform", action='store_true', default=False)
     parser.add_argument("-p", "--project", help="Add a project prefix for env vars", nargs='?', default="")
@@ -18,102 +20,39 @@ def main(argv):
 def determine_actions(args, params):
     aws_creds_file = "$HOME/.aws/credentials"
     if (is_git_directory()) and not (args.m):
-        if (args.e):
-            aws_creds_file = aws_creds_file + "-atmos"
-        workspace_manager()
+        # if (args.e):
+        #     aws_creds_file = aws_creds_file + "-atmos"
+        workspaces.workspace_manager()
 
-    workspace = get_env()
+    workspace = workspaces.get_env()
     workspace_vars = workspace
-    if (args.project):
-        workspace = args.project + "_" + workspace
+    if (args.project) and workspace != 'default':
+        workspace = args.project + "-" + workspace
 
     env_actions = ["init", "plan", "apply", "destroy"] # Commands that require env context
     cmd = 'terraform {args}'.format(args=args.command)
 
     if (args.command in env_actions) and not (args.n): # Append with env context
         cmd = cmd + ' -var-file=vars/{env}.tfvars'.format(env=workspace_vars) 
-        cmd = cmd + ' -var "workspace={env}"'.format(env=workspace)
-        cmd = cmd + ' -var "shared_credentials_file={aws_creds_file}"'.format(aws_creds_file=aws_creds_file)
 
     for param in params: # Pass terraform params directly through
         cmd = cmd + ' ' + param
 
     if (args.e):
-        generate_creds(args)
+        credentials.generate(args)
 
     if (args.verbose):
         print("Atmos will run: " + cmd)
-
     print('Terraform {args} using env vars in {env}'.format(args=args.command, env=workspace_vars))
+    run_cmd(cmd)
+
+def run_cmd(cmd):
     with subprocess.Popen(shlex.split(cmd)) as proc:
         exit # Start process but kill py program
 
-def is_git_directory(path = '.'):
-    return subprocess.call(['git', '-C', path, 'status'], stderr=subprocess.STDOUT, stdout = open(os.devnull, 'w')) == 0
+def is_git_directory():
+    return subprocess.call(['git', 'branch'], stderr=subprocess.STDOUT, stdout = open(os.devnull, 'w')) == 0
 
-def workspace_manager():
-    branch = subprocess.getoutput("git rev-parse --abbrev-ref HEAD")
-    if branch == "master":
-        branch = "default"
-    else:
-        if branch not in get_valid_envs():
-            branch = "qa"
-
-    if get_env() != branch:
-        print("[INFO]: Terraform workspace & git branch have diverged. Changing workspace to git branch...")
-        subprocess.call(["terraform", "workspace", "new", branch], stderr=subprocess.STDOUT, stdout=open(os.devnull, 'w'))
-        subprocess.call(["terraform", "workspace", "select", branch], stderr=subprocess.STDOUT, stdout=open(os.devnull, 'w'))
-
-def generate_creds(args):
-    current_workspace = get_env()
-    workspaces = ['default']
-
-    if current_workspace != 'default':
-        workspaces.append(current_workspace)
-
-    project_name = ""
-    if (args.project):
-        project_name = args.project.upper() + "_"
-
-    contents = ""
-    for workspace in workspaces:
-        access_key_name = project_name + workspace.upper() + '_ACCESS_KEY_ID'
-        secret_key_name = project_name + workspace.upper() + '_SECRET_ACCESS_KEY'
-
-        if (args.verbose):
-            print(access_key_name)
-            print(secret_key_name)
-
-        contents = contents + "[{workspace}]\n".format(workspace=workspace)
-        try:
-            contents = contents + "aws_access_key_id=" + os.environ.get(access_key_name) + "\n"
-        except:
-            print("[ERROR]: Env Variable " + access_key_name + " not found.")
-            sys.exit(1)
-        try:
-            contents = contents + "aws_secret_access_key=" + os.environ.get(secret_key_name) + "\n"
-        except:
-            print("[ERROR]: Env Variable " + secret_key_name + " not found.")
-            sys.exit(1)
-    with open(os.path.expanduser('~/.aws/credentials-atmos'), 'w+') as f:
-        f.write(contents)
-
-def get_valid_envs():
-    try:
-        # Use var files when present, otherwise default to qa
-        return [os.path.splitext(os.path.basename(x))[0] for x in glob.glob("vars/*.tfvars")]
-    except FileNotFoundError:
-        return False
-
-def get_env():
-    try:
-        tf_env = open('.terraform/environment', 'r').read()
-    except:
-        return("default")
-    if str(tf_env) in get_valid_envs():
-        return(tf_env)
-    else:
-        return("qa")
 
 if __name__ == "__main__":
-    main(sys.argv)
+    main(sys.argv)

credentials.py

@@ -0,0 +1,53 @@
+import workspaces, sys, os
+
+def generate(args):
+    current_workspace = workspaces.get_env()
+    workspaces_names = ['default']
+    aws_creds_dir = '~/.aws'
+    aws_creds_file = 'credentials'
+    aws_creds_full = aws_creds_dir + '/' + aws_creds_file
+
+    if os.path.isfile(os.path.expanduser(aws_creds_full)):
+        answer = input(f"[WARNING] File {aws_creds_full} already exists. Atmos will generate a new credentials file from your env vars. \nDo you want to override {aws_creds_full}? [y/N]")
+        if not answer or answer[0].lower() != 'y':
+            print("File not changed. This flag is for CI/CD only")
+            exit(1)
+    else:
+        if not os.path.isdir(os.path.expanduser(aws_creds_dir)):
+            os.makedirs(os.path.expanduser(aws_creds_dir))
+
+    if current_workspace != 'default':
+        workspaces_names.append(current_workspace)
+
+    project_name = ""
+    if (args.project):
+        delimeter = "_"
+        project_name = args.project.upper() + delimeter
+
+    contents = ""
+    for workspace in workspaces_names:
+        access_key_name = project_name + workspace.upper() + '_ACCESS_KEY_ID'
+        secret_key_name = project_name + workspace.upper() + '_SECRET_ACCESS_KEY'
+
+        if (args.verbose):
+            print(access_key_name)
+            print(secret_key_name)
+
+        if (workspace == 'default'):
+            contents = contents + "[{workspace}]\n".format(workspace=(workspace).lower())
+        else:
+            contents = contents + "[{workspace}]\n".format(workspace=(project_name.replace("_", "-") + workspace).lower())
+
+        try:
+            contents = contents + "aws_access_key_id=" + os.environ.get(access_key_name) + "\n"
+        except:
+            print("[ERROR]: Env Variable " + access_key_name + " not found.")
+            sys.exit(1)
+        try:
+            contents = contents + "aws_secret_access_key=" + os.environ.get(secret_key_name) + "\n"
+        except:
+            print("[ERROR]: Env Variable " + secret_key_name + " not found.")
+            sys.exit(1)
+
+    with open(os.path.expanduser(aws_creds_full), 'w+') as f:
+        f.write(contents)

git-askpass-helper.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo ${GIT_PASSWORD}

shared-creds

@@ -1,7 +0,0 @@
-[default]
-
-[preprod]
-
-[production]
-
-[qa]

tests.py

@@ -0,0 +1,69 @@
+import unittest
+from unittest.mock import MagicMock, patch
+import argparse
+import atmos
+
+class DetermineActionsTests(unittest.TestCase):
+
+    def setUp(self):
+        self.input_args = argparse.Namespace(command="mytestcommand", e=False, m=False, n=False, project="", verbose=False)
+        atmos.is_git_directory = MagicMock(return_value=False)
+        atmos.run_cmd = MagicMock()
+
+    def test_whenCalledWithNoAdditionalArgs_shouldRunTheTerraformCommand(self):
+        """Simplest case where atmos is only called with a command and no further arguments"""
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with("terraform mytestcommand")
+
+    def test_whenCalledWithParams_theyAreAppended(self):
+        """Should append all params after the command"""
+        atmos.determine_actions(self.input_args, ["--myparam", "myvalue"])
+        atmos.run_cmd.assert_called_with("terraform mytestcommand --myparam myvalue")
+
+    @patch("workspaces.get_env")
+    def test_whenCalledWithInitCommand_shouldAppendVarsAndCreds(self, mocked_get_env):
+        """Case where var-file, -var workpace=xyz is appended"""
+        self.input_args.command = "init"
+        mocked_get_env.return_value = "mytestenv"
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with('terraform init -var-file=vars/mytestenv.tfvars -var "workspace=mytestenv"')
+
+    @patch("workspaces.get_env")
+    def test_whenCalledWithPlanCommand_shouldAppendVarsAndCreds(self, mocked_get_env):
+        """Case where var-file, -var workpace=xyz is appended"""
+        self.input_args.command = "plan"
+        mocked_get_env.return_value = "mytestenv"
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with('terraform plan -var-file=vars/mytestenv.tfvars -var "workspace=mytestenv"')
+
+    @patch("workspaces.get_env")
+    def test_whenCalledWithApplyCommand_shouldAppendVarsAndCreds(self, mocked_get_env):
+        """Case where var-file, -var workpace=xyz is appended"""
+        self.input_args.command = "apply"
+        mocked_get_env.return_value = "mytestenv"
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with('terraform apply -var-file=vars/mytestenv.tfvars -var "workspace=mytestenv"')
+
+    @patch("workspaces.get_env")
+    def test_whenCalledWithDestroyCommand_shouldAppendVarsAndCreds(self, mocked_get_env):
+        """Case where var-file, -var workpace=xyz is appended"""
+        self.input_args.command = "destroy"
+        mocked_get_env.return_value = "mytestenv"
+        atmos.determine_actions(self.input_args, [])
+        atmos.run_cmd.assert_called_with('terraform destroy -var-file=vars/mytestenv.tfvars -var "workspace=mytestenv"')
+
+    @patch("workspaces.workspace_manager")
+    def test_whenInAGitRepo_andManualArgIsNotGiven_andEnvironmentArgIsNotGiven_shouldCallTheWorkspaceManager(self, mocked_workspace_manager):
+        atmos.is_git_directory.return_value = True
+        atmos.determine_actions(self.input_args, [])
+        mocked_workspace_manager.assert_called_once()
+
+    @patch("credentials.generate")
+    def test_whenEnvironmentArgIsGiven_shouldGenerateCredentials(self, mocked_generate):
+        self.input_args.e = True
+        atmos.determine_actions(self.input_args, [])
+        mocked_generate.assert_called_once()
+
+
+if __name__ == '__main__':
+    unittest.main()

workspaces.py

@@ -0,0 +1,33 @@
+import subprocess, os, glob
+
+def workspace_manager():
+    branch = subprocess.getoutput("git rev-parse --abbrev-ref HEAD")
+    if branch == "master":
+        branch = "default"
+    else:
+        if branch not in get_valid_envs():
+            branch = "default"
+
+    if get_env() != branch:
+        print("[INFO]: Terraform workspace & git branch have diverged. Changing workspace to git branch...")
+        subprocess.call(["terraform", "workspace", "new", branch], stderr=subprocess.STDOUT, stdout=open(os.devnull, 'w'))
+        subprocess.call(["terraform", "workspace", "select", branch], stderr=subprocess.STDOUT, stdout=open(os.devnull, 'w'))
+
+def get_valid_envs():
+    try:
+        # Use var files when present, otherwise default to default
+        return [os.path.splitext(os.path.basename(x))[0] for x in glob.glob("vars/*.tfvars")]
+    except FileNotFoundError:
+        return False
+
+def get_env():
+    try:
+        tf_env = ""
+        with open('.terraform/environment', 'r') as f:
+            tf_env = f.readline()
+    except:
+        return("default")
+    if str(tf_env) in get_valid_envs():
+        return(tf_env)
+    else:
+        return("default")