Merge pull request hashicorp#11286 from hashicorp/f-remote-backend

core: introduce "backends" to replace "remote state" (superset) and fix UX

Author: mitchellh

.gitignore

@@ -25,3 +25,7 @@ website/node_modules
 *.iml
 
 website/vendor
+
+# Test exclusions
+!command/test-fixtures/**/*.tfstate
+!command/test-fixtures/**/.terraform/

Makefile

@@ -38,8 +38,10 @@ plugin-dev: generate
 	mv $(GOPATH)/bin/$(PLUGIN) $(GOPATH)/bin/terraform-$(PLUGIN)
 
 # test runs the unit tests
-test: fmtcheck errcheck generate
-	TF_ACC= go test $(TEST) $(TESTARGS) -timeout=30s -parallel=4
+test:# fmtcheck errcheck generate
+	go test -i $(TEST) || exit 1
+	echo $(TEST) | \
+		xargs -t -n4 go test $(TESTARGS) -timeout=30s -parallel=4
 
 # testacc runs acceptance tests
 testacc: fmtcheck generate

backend/backend.go

@@ -0,0 +1,127 @@
+// Package backend provides interfaces that the CLI uses to interact with
+// Terraform. A backend provides the abstraction that allows the same CLI
+// to simultaneously support both local and remote operations for seamlessly
+// using Terraform in a team environment.
+package backend
+
+import (
+	"context"
+
+	"github.com/hashicorp/terraform/config/module"
+	"github.com/hashicorp/terraform/state"
+	"github.com/hashicorp/terraform/terraform"
+)
+
+// Backend is the minimal interface that must be implemented to enable Terraform.
+type Backend interface {
+	// Ask for input and configure the backend. Similar to
+	// terraform.ResourceProvider.
+	Input(terraform.UIInput, *terraform.ResourceConfig) (*terraform.ResourceConfig, error)
+	Validate(*terraform.ResourceConfig) ([]string, []error)
+	Configure(*terraform.ResourceConfig) error
+
+	// State returns the current state for this environment. This state may
+	// not be loaded locally: the proper APIs should be called on state.State
+	// to load the state.
+	State() (state.State, error)
+}
+
+// Enhanced implements additional behavior on top of a normal backend.
+//
+// Enhanced backends allow customizing the behavior of Terraform operations.
+// This allows Terraform to potentially run operations remotely, load
+// configurations from external sources, etc.
+type Enhanced interface {
+	Backend
+
+	// Operation performs a Terraform operation such as refresh, plan, apply.
+	// It is up to the implementation to determine what "performing" means.
+	// This DOES NOT BLOCK. The context returned as part of RunningOperation
+	// should be used to block for completion.
+	Operation(context.Context, *Operation) (*RunningOperation, error)
+}
+
+// Local implements additional behavior on a Backend that allows local
+// operations in addition to remote operations.
+//
+// This enables more behaviors of Terraform that require more data such
+// as `console`, `import`, `graph`. These require direct access to
+// configurations, variables, and more. Not all backends may support this
+// so we separate it out into its own optional interface.
+type Local interface {
+	// Context returns a runnable terraform Context. The operation parameter
+	// doesn't need a Type set but it needs other options set such as Module.
+	Context(*Operation) (*terraform.Context, state.State, error)
+}
+
+// An operation represents an operation for Terraform to execute.
+//
+// Note that not all fields are supported by all backends and can result
+// in an error if set. All backend implementations should show user-friendly
+// errors explaining any incorrectly set values. For example, the local
+// backend doesn't support a PlanId being set.
+//
+// The operation options are purposely designed to have maximal compatibility
+// between Terraform and Terraform Servers (a commercial product offered by
+// HashiCorp). Therefore, it isn't expected that other implementation support
+// every possible option. The struct here is generalized in order to allow
+// even partial implementations to exist in the open, without walling off
+// remote functionality 100% behind a commercial wall. Anyone can implement
+// against this interface and have Terraform interact with it just as it
+// would with HashiCorp-provided Terraform Servers.
+type Operation struct {
+	// Type is the operation to perform.
+	Type OperationType
+
+	// PlanId is an opaque value that backends can use to execute a specific
+	// plan for an apply operation.
+	//
+	// PlanOutBackend is the backend to store with the plan. This is the
+	// backend that will be used when applying the plan.
+	PlanId         string
+	PlanRefresh    bool   // PlanRefresh will do a refresh before a plan
+	PlanOutPath    string // PlanOutPath is the path to save the plan
+	PlanOutBackend *terraform.BackendState
+
+	// Module settings specify the root module to use for operations.
+	Module *module.Tree
+
+	// Plan is a plan that was passed as an argument. This is valid for
+	// plan and apply arguments but may not work for all backends.
+	Plan *terraform.Plan
+
+	// The options below are more self-explanatory and affect the runtime
+	// behavior of the operation.
+	Destroy   bool
+	Targets   []string
+	Variables map[string]interface{}
+
+	// Input/output/control options.
+	UIIn  terraform.UIInput
+	UIOut terraform.UIOutput
+}
+
+// RunningOperation is the result of starting an operation.
+type RunningOperation struct {
+	// Context should be used to track Done and Err for errors.
+	//
+	// For implementers of a backend, this context should not wrap the
+	// passed in context. Otherwise, canceling the parent context will
+	// immediately mark this context as "done" but those aren't the semantics
+	// we want: we want this context to be done only when the operation itself
+	// is fully done.
+	context.Context
+
+	// Err is the error of the operation. This is populated after
+	// the operation has completed.
+	Err error
+
+	// PlanEmpty is populated after a Plan operation completes without error
+	// to note whether a plan is empty or has changes.
+	PlanEmpty bool
+
+	// State is the final state after the operation completed. Persisting
+	// this state is managed by the backend. This should only be read
+	// after the operation completes to avoid read/write races.
+	State *terraform.State
+}

backend/cli.go

@@ -0,0 +1,70 @@
+package backend
+
+import (
+	"github.com/hashicorp/terraform/terraform"
+	"github.com/mitchellh/cli"
+	"github.com/mitchellh/colorstring"
+)
+
+// CLI is an optional interface that can be implemented to be initialized
+// with information from the Terraform CLI. If this is implemented, this
+// initialization function will be called with data to help interact better
+// with a CLI.
+//
+// This interface was created to improve backend interaction with the
+// official Terraform CLI while making it optional for API users to have
+// to provide full CLI interaction to every backend.
+//
+// If you're implementing a Backend, it is acceptable to require CLI
+// initialization. In this case, your backend should be coded to error
+// on other methods (such as State, Operation) if CLI initialization was not
+// done with all required fields.
+type CLI interface {
+	Backend
+
+	// CLIIinit is called once with options. The options passed to this
+	// function may not be modified after calling this since they can be
+	// read/written at any time by the Backend implementation.
+	CLIIinit(*CLIOpts) error
+}
+
+// CLIOpts are the options passed into CLIInit for the CLI interface.
+//
+// These options represent the functionality the CLI exposes and often
+// maps to meta-flags available on every CLI (such as -input).
+//
+// When implementing a backend, it isn't expected that every option applies.
+// Your backend should be documented clearly to explain to end users what
+// options have an affect and what won't. In some cases, it may even make sense
+// to error in your backend when an option is set so that users don't make
+// a critically incorrect assumption about behavior.
+type CLIOpts struct {
+	// CLI and Colorize control the CLI output. If CLI is nil then no CLI
+	// output will be done. If CLIColor is nil then no coloring will be done.
+	CLI      cli.Ui
+	CLIColor *colorstring.Colorize
+
+	// StatePath is the local path where state is read from.
+	//
+	// StateOutPath is the local path where the state will be written.
+	// If this is empty, it will default to StatePath.
+	//
+	// StateBackupPath is the local path where a backup file will be written.
+	// If this is empty, no backup will be taken.
+	StatePath       string
+	StateOutPath    string
+	StateBackupPath string
+
+	// ContextOpts are the base context options to set when initializing a
+	// Terraform context. Many of these will be overridden or merged by
+	// Operation. See Operation for more details.
+	ContextOpts *terraform.ContextOpts
+
+	// Input will ask for necessary input prior to performing any operations.
+	//
+	// Validation will perform validation prior to running an operation. The
+	// variable naming doesn't match the style of others since we have a func
+	// Validate.
+	Input      bool
+	Validation bool
+}

backend/legacy/backend.go

@@ -0,0 +1,62 @@
+package legacy
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/terraform/state"
+	"github.com/hashicorp/terraform/state/remote"
+	"github.com/hashicorp/terraform/terraform"
+	"github.com/mitchellh/mapstructure"
+)
+
+// Backend is an implementation of backend.Backend for legacy remote state
+// clients.
+type Backend struct {
+	// Type is the type of remote state client to support
+	Type string
+
+	// client is set after Configure is called and client is initialized.
+	client remote.Client
+}
+
+func (b *Backend) Input(
+	ui terraform.UIInput, c *terraform.ResourceConfig) (*terraform.ResourceConfig, error) {
+	// Return the config as-is, legacy doesn't support input
+	return c, nil
+}
+
+func (b *Backend) Validate(*terraform.ResourceConfig) ([]string, []error) {
+	// No validation was supported for old clients
+	return nil, nil
+}
+
+func (b *Backend) Configure(c *terraform.ResourceConfig) error {
+	// Legacy remote state was only map[string]string config
+	var conf map[string]string
+	if err := mapstructure.Decode(c.Raw, &conf); err != nil {
+		return fmt.Errorf(
+			"Failed to decode %q configuration: %s\n\n"+
+				"This backend expects all configuration keys and values to be\n"+
+				"strings. Please verify your configuration and try again.",
+			b.Type, err)
+	}
+
+	client, err := remote.NewClient(b.Type, conf)
+	if err != nil {
+		return fmt.Errorf(
+			"Failed to configure remote backend %q: %s",
+			b.Type, err)
+	}
+
+	// Set our client
+	b.client = client
+	return nil
+}
+
+func (b *Backend) State() (state.State, error) {
+	if b.client == nil {
+		panic("State called with nil remote state client")
+	}
+
+	return &remote.State{Client: b.client}, nil
+}

backend/legacy/backend_test.go

@@ -0,0 +1,49 @@
+package legacy
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/config"
+	"github.com/hashicorp/terraform/state"
+	"github.com/hashicorp/terraform/terraform"
+)
+
+func TestBackend_impl(t *testing.T) {
+	var _ backend.Backend = new(Backend)
+}
+
+func TestBackend(t *testing.T) {
+	td, err := ioutil.TempDir("", "tf")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.RemoveAll(td)
+
+	b := &Backend{Type: "local"}
+	conf := terraform.NewResourceConfig(config.TestRawConfig(t, map[string]interface{}{
+		"path": filepath.Join(td, "data"),
+	}))
+
+	// Config
+	if err := b.Configure(conf); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Grab state
+	s, err := b.State()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if s == nil {
+		t.Fatalf("state is nil")
+	}
+
+	// Test it
+	s.WriteState(state.TestStateInitial())
+	s.PersistState()
+	state.TestState(t, s)
+}

backend/legacy/legacy.go

@@ -0,0 +1,28 @@
+// Package legacy contains a backend implementation that can be used
+// with the legacy remote state clients.
+package legacy
+
+import (
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/state/remote"
+)
+
+// Init updates the backend/init package map of initializers to support
+// all the remote state types.
+//
+// If a type is already in the map, it will not be added. This will allow
+// us to slowly convert the legacy types to first-class backends.
+func Init(m map[string]func() backend.Backend) {
+	for k, _ := range remote.BuiltinClients {
+		if _, ok := m[k]; !ok {
+			// Copy the "k" value since the variable "k" is reused for
+			// each key (address doesn't change).
+			typ := k
+
+			// Build the factory function to return a backend of typ
+			m[k] = func() backend.Backend {
+				return &Backend{Type: typ}
+			}
+		}
+	}
+}

backend/legacy/legacy_test.go

@@ -0,0 +1,34 @@
+package legacy
+
+import (
+	"testing"
+
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/state/remote"
+)
+
+func TestInit(t *testing.T) {
+	m := make(map[string]func() backend.Backend)
+	Init(m)
+
+	for k, _ := range remote.BuiltinClients {
+		b, ok := m[k]
+		if !ok {
+			t.Fatalf("missing: %s", k)
+		}
+
+		if typ := b().(*Backend).Type; typ != k {
+			t.Fatalf("bad type: %s", typ)
+		}
+	}
+}
+
+func TestInit_ignoreExisting(t *testing.T) {
+	m := make(map[string]func() backend.Backend)
+	m["local"] = nil
+	Init(m)
+
+	if v, ok := m["local"]; !ok || v != nil {
+		t.Fatalf("bad: %#v", m)
+	}
+}