helper/schema: schema versioning & migration

Providers get a per-resource SchemaVersion integer that they can bump
when a resource's schema changes format. Each InstanceState with an
older recorded SchemaVersion than the cureent one is yielded to a
`MigrateSchema` function to be transformed such that it can be addressed
by the current version of the resource's Schema.

Author: Paul Hinze

helper/schema/resource.go

@@ -3,6 +3,7 @@ package schema
 import (
 	"errors"
 	"fmt"
+	"strconv"
 
 	"github.com/hashicorp/terraform/terraform"
 )
@@ -24,6 +25,31 @@ type Resource struct {
 	// resource.
 	Schema map[string]*Schema
 
+	// SchemaVersion is the version number for this resource's Schema
+	// definition. The current SchemaVersion stored in the state for each
+	// resource. Provider authors can increment this version number
+	// when Schema semantics change. If the State's SchemaVersion is less than
+	// the current SchemaVersion, the InstanceState is yielded to the
+	// MigrateState callback, where the provider can make whatever changes it
+	// needs to update the state to be compatible to the latest version of the
+	// Schema.
+	//
+	// When unset, SchemaVersion defaults to 0, so provider authors can start
+	// their Versioning at any integer >= 1
+	SchemaVersion int
+
+	// MigrateState is responsible for updating an InstanceState with an old
+	// version to the format expected by the current version of the Schema.
+	//
+	// It is called during Refresh if the State's stored SchemaVersion is less
+	// than the current SchemaVersion of the Resource.
+	//
+	// The function is yielded the state's stored SchemaVersion and a pointer to
+	// the InstanceState that needs updating, as well as the configured
+	// provider's configured meta interface{}, in case the migration process
+	// needs to make any remote API calls.
+	MigrateState StateMigrateFunc
+
 	// The functions below are the CRUD operations for this resource.
 	//
 	// The only optional operation is Update. If Update is not implemented,
@@ -69,6 +95,10 @@ type DeleteFunc func(*ResourceData, interface{}) error
 // See Resource documentation.
 type ExistsFunc func(*ResourceData, interface{}) (bool, error)
 
+// See Resource documentation.
+type StateMigrateFunc func(
+	int, *terraform.InstanceState, interface{}) (*terraform.InstanceState, error)
+
 // Apply creates, updates, and/or deletes a resource.
 func (r *Resource) Apply(
 	s *terraform.InstanceState,
@@ -158,6 +188,14 @@ func (r *Resource) Refresh(
 		}
 	}
 
+	needsMigration, stateSchemaVersion := r.checkSchemaVersion(s)
+	if needsMigration && r.MigrateState != nil {
+		s, err := r.MigrateState(stateSchemaVersion, s, meta)
+		if err != nil {
+			return s, err
+		}
+	}
+
 	data, err := schemaMap(r.Schema).Data(s, nil)
 	if err != nil {
 		return s, err
@@ -169,6 +207,13 @@ func (r *Resource) Refresh(
 		state = nil
 	}
 
+	if state != nil && r.SchemaVersion > 0 {
+		if state.Meta == nil {
+			state.Meta = make(map[string]string)
+		}
+		state.Meta["schema_version"] = strconv.Itoa(r.SchemaVersion)
+	}
+
 	return state, err
 }
 
@@ -189,3 +234,10 @@ func (r *Resource) InternalValidate() error {
 
 	return schemaMap(r.Schema).InternalValidate()
 }
+
+// Determines if a given InstanceState needs to be migrated by checking the
+// stored version number with the current SchemaVersion
+func (r *Resource) checkSchemaVersion(is *terraform.InstanceState) (bool, int) {
+	stateSchemaVersion, _ := strconv.Atoi(is.Meta["schema_version"])
+	return stateSchemaVersion < r.SchemaVersion, stateSchemaVersion
+}

helper/schema/resource_test.go

@@ -3,6 +3,7 @@ package schema
 import (
 	"fmt"
 	"reflect"
+	"strconv"
 	"testing"
 
 	"github.com/hashicorp/terraform/terraform"
@@ -478,3 +479,218 @@ func TestResourceRefresh_noExists(t *testing.T) {
 		t.Fatalf("should have no state")
 	}
 }
+
+func TestResourceRefresh_needsMigration(t *testing.T) {
+	// Schema v2 it deals only in newfoo, which tracks foo as an int
+	r := &Resource{
+		SchemaVersion: 2,
+		Schema: map[string]*Schema{
+			"newfoo": &Schema{
+				Type:     TypeInt,
+				Optional: true,
+			},
+		},
+	}
+
+	r.Read = func(d *ResourceData, m interface{}) error {
+		return d.Set("newfoo", d.Get("newfoo").(int)+1)
+	}
+
+	r.MigrateState = func(
+		v int,
+		s *terraform.InstanceState,
+		meta interface{}) (*terraform.InstanceState, error) {
+		// Real state migration functions will probably switch on this value,
+		// but we'll just assert on it for now.
+		if v != 1 {
+			t.Fatalf("Expected StateSchemaVersion to be 1, got %d", v)
+		}
+
+		if meta != 42 {
+			t.Fatal("Expected meta to be passed through to the migration function")
+		}
+
+		oldfoo, err := strconv.ParseFloat(s.Attributes["oldfoo"], 64)
+		if err != nil {
+			t.Fatalf("err: %#v", err)
+		}
+		s.Attributes["newfoo"] = strconv.Itoa((int(oldfoo * 10)))
+		delete(s.Attributes, "oldfoo")
+
+		return s, nil
+	}
+
+	// State is v1 and deals in oldfoo, which tracked foo as a float at 1/10th
+	// the scale of newfoo
+	s := &terraform.InstanceState{
+		ID: "bar",
+		Attributes: map[string]string{
+			"oldfoo": "1.2",
+		},
+		Meta: map[string]string{
+			"schema_version": "1",
+		},
+	}
+
+	actual, err := r.Refresh(s, 42)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &terraform.InstanceState{
+		ID: "bar",
+		Attributes: map[string]string{
+			"id":     "bar",
+			"newfoo": "13",
+		},
+		Meta: map[string]string{
+			"schema_version": "2",
+		},
+	}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatalf("bad:\n\nexpected: %#v\ngot: %#v", expected, actual)
+	}
+}
+
+func TestResourceRefresh_noMigrationNeeded(t *testing.T) {
+	r := &Resource{
+		SchemaVersion: 2,
+		Schema: map[string]*Schema{
+			"newfoo": &Schema{
+				Type:     TypeInt,
+				Optional: true,
+			},
+		},
+	}
+
+	r.Read = func(d *ResourceData, m interface{}) error {
+		return d.Set("newfoo", d.Get("newfoo").(int)+1)
+	}
+
+	r.MigrateState = func(
+		v int,
+		s *terraform.InstanceState,
+		meta interface{}) (*terraform.InstanceState, error) {
+		t.Fatal("Migrate function shouldn't be called!")
+		return nil, nil
+	}
+
+	s := &terraform.InstanceState{
+		ID: "bar",
+		Attributes: map[string]string{
+			"newfoo": "12",
+		},
+		Meta: map[string]string{
+			"schema_version": "2",
+		},
+	}
+
+	actual, err := r.Refresh(s, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &terraform.InstanceState{
+		ID: "bar",
+		Attributes: map[string]string{
+			"id":     "bar",
+			"newfoo": "13",
+		},
+		Meta: map[string]string{
+			"schema_version": "2",
+		},
+	}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatalf("bad:\n\nexpected: %#v\ngot: %#v", expected, actual)
+	}
+}
+
+func TestResourceRefresh_stateSchemaVersionUnset(t *testing.T) {
+	r := &Resource{
+		// Version 1 > Version 0
+		SchemaVersion: 1,
+		Schema: map[string]*Schema{
+			"newfoo": &Schema{
+				Type:     TypeInt,
+				Optional: true,
+			},
+		},
+	}
+
+	r.Read = func(d *ResourceData, m interface{}) error {
+		return d.Set("newfoo", d.Get("newfoo").(int)+1)
+	}
+
+	r.MigrateState = func(
+		v int,
+		s *terraform.InstanceState,
+		meta interface{}) (*terraform.InstanceState, error) {
+		s.Attributes["newfoo"] = s.Attributes["oldfoo"]
+		return s, nil
+	}
+
+	s := &terraform.InstanceState{
+		ID: "bar",
+		Attributes: map[string]string{
+			"oldfoo": "12",
+		},
+	}
+
+	actual, err := r.Refresh(s, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &terraform.InstanceState{
+		ID: "bar",
+		Attributes: map[string]string{
+			"id":     "bar",
+			"newfoo": "13",
+		},
+		Meta: map[string]string{
+			"schema_version": "1",
+		},
+	}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatalf("bad:\n\nexpected: %#v\ngot: %#v", expected, actual)
+	}
+}
+
+func TestResourceRefresh_migrateStateErr(t *testing.T) {
+	r := &Resource{
+		SchemaVersion: 2,
+		Schema: map[string]*Schema{
+			"newfoo": &Schema{
+				Type:     TypeInt,
+				Optional: true,
+			},
+		},
+	}
+
+	r.Read = func(d *ResourceData, m interface{}) error {
+		t.Fatal("Read should never be called!")
+		return nil
+	}
+
+	r.MigrateState = func(
+		v int,
+		s *terraform.InstanceState,
+		meta interface{}) (*terraform.InstanceState, error) {
+		return s, fmt.Errorf("triggering an error")
+	}
+
+	s := &terraform.InstanceState{
+		ID: "bar",
+		Attributes: map[string]string{
+			"oldfoo": "12",
+		},
+	}
+
+	_, err := r.Refresh(s, nil)
+	if err == nil {
+		t.Fatal("expected error, but got none!")
+	}
+}

terraform/state.go

@@ -832,6 +832,11 @@ type InstanceState struct {
 	// that is necessary for the Terraform run to complete, but is not
 	// persisted to a state file.
 	Ephemeral EphemeralState `json:"-"`
+
+	// Meta is a simple K/V map that is persisted to the State but otherwise
+	// ignored by Terraform core. It's meant to be used for accounting by
+	// external client code.
+	Meta map[string]string `json:"meta,omitempty"`
 }
 
 func (i *InstanceState) init() {