Update dependencies

Among others, this updates minio-go, so that the new "eu-west-3" zone
for AWS is supported.

vendor/cloud.google.com/go/bigtable/admin.go

@@ -18,28 +18,36 @@ package bigtable
 
 import (
 	"fmt"
+	"math"
 	"regexp"
 	"strings"
+	"time"
 
+	"cloud.google.com/go/bigtable/internal/gax"
 	btopt "cloud.google.com/go/bigtable/internal/option"
 	"cloud.google.com/go/longrunning"
 	lroauto "cloud.google.com/go/longrunning/autogen"
+	"github.com/golang/protobuf/ptypes"
+	durpb "github.com/golang/protobuf/ptypes/duration"
 	"golang.org/x/net/context"
+	"google.golang.org/api/cloudresourcemanager/v1"
+	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 	gtransport "google.golang.org/api/transport/grpc"
 	btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
-	"google.golang.org/grpc/codes"
 )
 
 const adminAddr = "bigtableadmin.googleapis.com:443"
 
 // AdminClient is a client type for performing admin operations within a specific instance.
 type AdminClient struct {
-	conn    *grpc.ClientConn
-	tClient btapb.BigtableTableAdminClient
+	conn      *grpc.ClientConn
+	tClient   btapb.BigtableTableAdminClient
+	lroClient *lroauto.OperationsClient
 
 	project, instance string
 
@@ -53,17 +61,32 @@ func NewAdminClient(ctx context.Context, project, instance string, opts ...optio
 	if err != nil {
 		return nil, err
 	}
+	// Need to add scopes for long running operations (for create table & snapshots)
+	o = append(o, option.WithScopes(cloudresourcemanager.CloudPlatformScope))
 	o = append(o, opts...)
 	conn, err := gtransport.Dial(ctx, o...)
 	if err != nil {
 		return nil, fmt.Errorf("dialing: %v", err)
 	}
+
+	lroClient, err := lroauto.NewOperationsClient(ctx, option.WithGRPCConn(conn))
+	if err != nil {
+		// This error "should not happen", since we are just reusing old connection
+		// and never actually need to dial.
+		// If this does happen, we could leak conn. However, we cannot close conn:
+		// If the user invoked the function with option.WithGRPCConn,
+		// we would close a connection that's still in use.
+		// TODO(pongad): investigate error conditions.
+		return nil, err
+	}
+
 	return &AdminClient{
-		conn:     conn,
-		tClient:  btapb.NewBigtableTableAdminClient(conn),
-		project:  project,
-		instance: instance,
-		md:       metadata.Pairs(resourcePrefixHeader, fmt.Sprintf("projects/%s/instances/%s", project, instance)),
+		conn:      conn,
+		tClient:   btapb.NewBigtableTableAdminClient(conn),
+		lroClient: lroClient,
+		project:   project,
+		instance:  instance,
+		md:        metadata.Pairs(resourcePrefixHeader, fmt.Sprintf("projects/%s/instances/%s", project, instance)),
 	}, nil
 }
 
@@ -245,6 +268,283 @@ func (ac *AdminClient) DropRowRange(ctx context.Context, table, rowKeyPrefix str
 	return err
 }
 
+// CreateTableFromSnapshot creates a table from snapshot.
+// The table will be created in the same cluster as the snapshot.
+//
+// This is a private alpha release of Cloud Bigtable snapshots. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (ac *AdminClient) CreateTableFromSnapshot(ctx context.Context, table, cluster, snapshot string) error {
+	ctx = mergeOutgoingMetadata(ctx, ac.md)
+	prefix := ac.instancePrefix()
+	snapshotPath := prefix + "/clusters/" + cluster + "/snapshots/" + snapshot
+
+	req := &btapb.CreateTableFromSnapshotRequest{
+		Parent:         prefix,
+		TableId:        table,
+		SourceSnapshot: snapshotPath,
+	}
+	op, err := ac.tClient.CreateTableFromSnapshot(ctx, req)
+	if err != nil {
+		return err
+	}
+	resp := btapb.Table{}
+	return longrunning.InternalNewOperation(ac.lroClient, op).Wait(ctx, &resp)
+}
+
+const DefaultSnapshotDuration time.Duration = 0
+
+// Creates a new snapshot in the specified cluster from the specified source table.
+// Setting the ttl to `DefaultSnapshotDuration` will use the server side default for the duration.
+//
+// This is a private alpha release of Cloud Bigtable snapshots. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (ac *AdminClient) SnapshotTable(ctx context.Context, table, cluster, snapshot string, ttl time.Duration) error {
+	ctx = mergeOutgoingMetadata(ctx, ac.md)
+	prefix := ac.instancePrefix()
+
+	var ttlProto *durpb.Duration
+
+	if ttl > 0 {
+		ttlProto = ptypes.DurationProto(ttl)
+	}
+
+	req := &btapb.SnapshotTableRequest{
+		Name:       prefix + "/tables/" + table,
+		Cluster:    prefix + "/clusters/" + cluster,
+		SnapshotId: snapshot,
+		Ttl:        ttlProto,
+	}
+
+	op, err := ac.tClient.SnapshotTable(ctx, req)
+	if err != nil {
+		return err
+	}
+	resp := btapb.Snapshot{}
+	return longrunning.InternalNewOperation(ac.lroClient, op).Wait(ctx, &resp)
+}
+
+// Returns a SnapshotIterator for iterating over the snapshots in a cluster.
+// To list snapshots across all of the clusters in the instance specify "-" as the cluster.
+//
+// This is a private alpha release of Cloud Bigtable snapshots. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (ac *AdminClient) ListSnapshots(ctx context.Context, cluster string) *SnapshotIterator {
+	ctx = mergeOutgoingMetadata(ctx, ac.md)
+	prefix := ac.instancePrefix()
+	clusterPath := prefix + "/clusters/" + cluster
+
+	it := &SnapshotIterator{}
+	req := &btapb.ListSnapshotsRequest{
+		Parent: clusterPath,
+	}
+
+	fetch := func(pageSize int, pageToken string) (string, error) {
+		req.PageToken = pageToken
+		if pageSize > math.MaxInt32 {
+			req.PageSize = math.MaxInt32
+		} else {
+			req.PageSize = int32(pageSize)
+		}
+
+		resp, err := ac.tClient.ListSnapshots(ctx, req)
+		if err != nil {
+			return "", err
+		}
+		for _, s := range resp.Snapshots {
+			snapshotInfo, err := newSnapshotInfo(s)
+			if err != nil {
+				return "", fmt.Errorf("Failed to parse snapshot proto %v", err)
+			}
+			it.items = append(it.items, snapshotInfo)
+		}
+		return resp.NextPageToken, nil
+	}
+	bufLen := func() int { return len(it.items) }
+	takeBuf := func() interface{} { b := it.items; it.items = nil; return b }
+
+	it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, bufLen, takeBuf)
+
+	return it
+}
+
+func newSnapshotInfo(snapshot *btapb.Snapshot) (*SnapshotInfo, error) {
+	nameParts := strings.Split(snapshot.Name, "/")
+	name := nameParts[len(nameParts)-1]
+	tablePathParts := strings.Split(snapshot.SourceTable.Name, "/")
+	tableId := tablePathParts[len(tablePathParts)-1]
+
+	createTime, err := ptypes.Timestamp(snapshot.CreateTime)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid createTime: %v", err)
+	}
+
+	deleteTime, err := ptypes.Timestamp(snapshot.DeleteTime)
+	if err != nil {
+		return nil, fmt.Errorf("Invalid deleteTime: %v", err)
+	}
+
+	return &SnapshotInfo{
+		Name:        name,
+		SourceTable: tableId,
+		DataSize:    snapshot.DataSizeBytes,
+		CreateTime:  createTime,
+		DeleteTime:  deleteTime,
+	}, nil
+}
+
+// An EntryIterator iterates over log entries.
+//
+// This is a private alpha release of Cloud Bigtable snapshots. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+type SnapshotIterator struct {
+	items    []*SnapshotInfo
+	pageInfo *iterator.PageInfo
+	nextFunc func() error
+}
+
+// PageInfo supports pagination. See https://godoc.org/google.golang.org/api/iterator package for details.
+func (it *SnapshotIterator) PageInfo() *iterator.PageInfo {
+	return it.pageInfo
+}
+
+// Next returns the next result. Its second return value is iterator.Done
+// (https://godoc.org/google.golang.org/api/iterator) if there are no more
+// results. Once Next returns Done, all subsequent calls will return Done.
+func (it *SnapshotIterator) Next() (*SnapshotInfo, error) {
+	if err := it.nextFunc(); err != nil {
+		return nil, err
+	}
+	item := it.items[0]
+	it.items = it.items[1:]
+	return item, nil
+}
+
+type SnapshotInfo struct {
+	Name        string
+	SourceTable string
+	DataSize    int64
+	CreateTime  time.Time
+	DeleteTime  time.Time
+}
+
+// Get snapshot metadata.
+//
+// This is a private alpha release of Cloud Bigtable snapshots. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (ac *AdminClient) SnapshotInfo(ctx context.Context, cluster, snapshot string) (*SnapshotInfo, error) {
+	ctx = mergeOutgoingMetadata(ctx, ac.md)
+	prefix := ac.instancePrefix()
+	clusterPath := prefix + "/clusters/" + cluster
+	snapshotPath := clusterPath + "/snapshots/" + snapshot
+
+	req := &btapb.GetSnapshotRequest{
+		Name: snapshotPath,
+	}
+
+	resp, err := ac.tClient.GetSnapshot(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	return newSnapshotInfo(resp)
+}
+
+// Delete a snapshot in a cluster.
+//
+// This is a private alpha release of Cloud Bigtable snapshots. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (ac *AdminClient) DeleteSnapshot(ctx context.Context, cluster, snapshot string) error {
+	ctx = mergeOutgoingMetadata(ctx, ac.md)
+	prefix := ac.instancePrefix()
+	clusterPath := prefix + "/clusters/" + cluster
+	snapshotPath := clusterPath + "/snapshots/" + snapshot
+
+	req := &btapb.DeleteSnapshotRequest{
+		Name: snapshotPath,
+	}
+	_, err := ac.tClient.DeleteSnapshot(ctx, req)
+	return err
+}
+
+// getConsistencyToken gets the consistency token for a table.
+func (ac *AdminClient) getConsistencyToken(ctx context.Context, tableName string) (string, error) {
+	req := &btapb.GenerateConsistencyTokenRequest{
+		Name: tableName,
+	}
+	resp, err := ac.tClient.GenerateConsistencyToken(ctx, req)
+	if err != nil {
+		return "", err
+	}
+	return resp.GetConsistencyToken(), nil
+}
+
+// isConsistent checks if a token is consistent for a table.
+func (ac *AdminClient) isConsistent(ctx context.Context, tableName, token string) (bool, error) {
+	req := &btapb.CheckConsistencyRequest{
+		Name:             tableName,
+		ConsistencyToken: token,
+	}
+	var resp *btapb.CheckConsistencyResponse
+
+	// Retry calls on retryable errors to avoid losing the token gathered before.
+	err := gax.Invoke(ctx, func(ctx context.Context) error {
+		var err error
+		resp, err = ac.tClient.CheckConsistency(ctx, req)
+		return err
+	}, retryOptions...)
+	if err != nil {
+		return false, err
+	}
+	return resp.GetConsistent(), nil
+}
+
+// WaitForReplication waits until all the writes committed before the call started have been propagated to all the clusters in the instance via replication.
+//
+// This is a private alpha release of Cloud Bigtable replication. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (ac *AdminClient) WaitForReplication(ctx context.Context, table string) error {
+	// Get the token.
+	prefix := ac.instancePrefix()
+	tableName := prefix + "/tables/" + table
+	token, err := ac.getConsistencyToken(ctx, tableName)
+	if err != nil {
+		return err
+	}
+
+	// Periodically check if the token is consistent.
+	timer := time.NewTicker(time.Second * 10)
+	defer timer.Stop()
+	for {
+		consistent, err := ac.isConsistent(ctx, tableName, token)
+		if err != nil {
+			return err
+		}
+		if consistent {
+			return nil
+		}
+		// Sleep for a bit or until the ctx is cancelled.
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+		}
+	}
+}
+
 const instanceAdminAddr = "bigtableadmin.googleapis.com:443"
 
 // InstanceAdminClient is a client type for performing admin operations on instances.
@@ -336,24 +636,55 @@ type InstanceConf struct {
 	InstanceType InstanceType
 }
 
+// InstanceWithClustersConfig contains the information necessary to create an Instance
+type InstanceWithClustersConfig struct {
+	InstanceID, DisplayName string
+	Clusters                []ClusterConfig
+	InstanceType            InstanceType
+}
+
 var instanceNameRegexp = regexp.MustCompile(`^projects/([^/]+)/instances/([a-z][-a-z0-9]*)$`)
 
 // CreateInstance creates a new instance in the project.
 // This method will return when the instance has been created or when an error occurs.
 func (iac *InstanceAdminClient) CreateInstance(ctx context.Context, conf *InstanceConf) error {
-	ctx = mergeOutgoingMetadata(ctx, iac.md)
-	req := &btapb.CreateInstanceRequest{
-		Parent:     "projects/" + iac.project,
-		InstanceId: conf.InstanceId,
-		Instance:   &btapb.Instance{DisplayName: conf.DisplayName, Type: btapb.Instance_Type(conf.InstanceType)},
-		Clusters: map[string]*btapb.Cluster{
-			conf.ClusterId: {
-				ServeNodes:         conf.NumNodes,
-				DefaultStorageType: conf.StorageType.proto(),
-				Location:           "projects/" + iac.project + "/locations/" + conf.Zone,
+	newConfig := InstanceWithClustersConfig{
+		InstanceID:   conf.InstanceId,
+		DisplayName:  conf.DisplayName,
+		InstanceType: conf.InstanceType,
+		Clusters: []ClusterConfig{
+			{
+				InstanceID:  conf.InstanceId,
+				ClusterID:   conf.ClusterId,
+				Zone:        conf.Zone,
+				NumNodes:    conf.NumNodes,
+				StorageType: conf.StorageType,
 			},
 		},
 	}
+	return iac.CreateInstanceWithClusters(ctx, &newConfig)
+}
+
+// CreateInstance creates a new instance with configured clusters in the project.
+// This method will return when the instance has been created or when an error occurs.
+//
+// Instances with multiple clusters are part of a private alpha release of Cloud Bigtable replication.
+// This feature is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (iac *InstanceAdminClient) CreateInstanceWithClusters(ctx context.Context, conf *InstanceWithClustersConfig) error {
+	ctx = mergeOutgoingMetadata(ctx, iac.md)
+	clusters := make(map[string]*btapb.Cluster)
+	for _, cluster := range conf.Clusters {
+		clusters[cluster.ClusterID] = cluster.proto(iac.project)
+	}
+
+	req := &btapb.CreateInstanceRequest{
+		Parent:     "projects/" + iac.project,
+		InstanceId: conf.InstanceID,
+		Instance:   &btapb.Instance{DisplayName: conf.DisplayName, Type: btapb.Instance_Type(conf.InstanceType)},
+		Clusters:   clusters,
+	}
 
 	lro, err := iac.iClient.CreateInstance(ctx, req)
 	if err != nil {
@@ -421,3 +752,99 @@ func (iac *InstanceAdminClient) InstanceInfo(ctx context.Context, instanceId str
 		DisplayName: res.DisplayName,
 	}, nil
 }
+
+// ClusterConfig contains the information necessary to create a cluster
+type ClusterConfig struct {
+	InstanceID, ClusterID, Zone string
+	NumNodes                    int32
+	StorageType                 StorageType
+}
+
+func (cc *ClusterConfig) proto(project string) *btapb.Cluster {
+	return &btapb.Cluster{
+		ServeNodes:         cc.NumNodes,
+		DefaultStorageType: cc.StorageType.proto(),
+		Location:           "projects/" + project + "/locations/" + cc.Zone,
+	}
+}
+
+// ClusterInfo represents information about a cluster.
+type ClusterInfo struct {
+	Name       string // name of the cluster
+	Zone       string // GCP zone of the cluster (e.g. "us-central1-a")
+	ServeNodes int    // number of allocated serve nodes
+	State      string // state of the cluster
+}
+
+// CreateCluster creates a new cluster in an instance.
+// This method will return when the cluster has been created or when an error occurs.
+//
+// This is a private alpha release of Cloud Bigtable replication. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (iac *InstanceAdminClient) CreateCluster(ctx context.Context, conf *ClusterConfig) error {
+	ctx = mergeOutgoingMetadata(ctx, iac.md)
+
+	req := &btapb.CreateClusterRequest{
+		Parent:    "projects/" + iac.project + "/instances/" + conf.InstanceID,
+		ClusterId: conf.ClusterID,
+		Cluster:   conf.proto(iac.project),
+	}
+
+	lro, err := iac.iClient.CreateCluster(ctx, req)
+	if err != nil {
+		return err
+	}
+	resp := btapb.Cluster{}
+	return longrunning.InternalNewOperation(iac.lroClient, lro).Wait(ctx, &resp)
+}
+
+// DeleteCluster deletes a cluster from an instance.
+//
+// This is a private alpha release of Cloud Bigtable replication. This feature
+// is not currently available to most Cloud Bigtable customers. This feature
+// might be changed in backward-incompatible ways and is not recommended for
+// production use. It is not subject to any SLA or deprecation policy.
+func (iac *InstanceAdminClient) DeleteCluster(ctx context.Context, instanceId, clusterId string) error {
+	ctx = mergeOutgoingMetadata(ctx, iac.md)
+	req := &btapb.DeleteClusterRequest{"projects/" + iac.project + "/instances/" + instanceId + "/clusters/" + clusterId}
+	_, err := iac.iClient.DeleteCluster(ctx, req)
+	return err
+}
+
+// UpdateCluster updates attributes of a cluster
+func (iac *InstanceAdminClient) UpdateCluster(ctx context.Context, instanceId, clusterId string, serveNodes int32) error {
+	ctx = mergeOutgoingMetadata(ctx, iac.md)
+	cluster := &btapb.Cluster{
+		Name:       "projects/" + iac.project + "/instances/" + instanceId + "/clusters/" + clusterId,
+		ServeNodes: serveNodes}
+	lro, err := iac.iClient.UpdateCluster(ctx, cluster)
+	if err != nil {
+		return err
+	}
+	return longrunning.InternalNewOperation(iac.lroClient, lro).Wait(ctx, nil)
+}
+
+// Clusters lists the clusters in an instance.
+func (iac *InstanceAdminClient) Clusters(ctx context.Context, instanceId string) ([]*ClusterInfo, error) {
+	ctx = mergeOutgoingMetadata(ctx, iac.md)
+	req := &btapb.ListClustersRequest{Parent: "projects/" + iac.project + "/instances/" + instanceId}
+	res, err := iac.iClient.ListClusters(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	// TODO(garyelliott): Deal with failed_locations.
+	var cis []*ClusterInfo
+	for _, c := range res.Clusters {
+		nameParts := strings.Split(c.Name, "/")
+		locParts := strings.Split(c.Location, "/")
+		cis = append(cis, &ClusterInfo{
+			Name:       nameParts[len(nameParts)-1],
+			Zone:       locParts[len(locParts)-1],
+			ServeNodes: int(c.ServeNodes),
+			State:      c.State.String(),
+		})
+	}
+	return cis, nil
+}