Update vendored library cloud.google.com/go

This commit is contained in:
Alexander Neumann committed 2018-03-30 11:41:12 +02:00
1 parent a4ff591165
commit a951e7b126
221 files changed
+19911 -2075

No files matched your search

+148
View File
@@ -203,6 +203,30 @@ func (s *mockSpannerServer) Rollback(ctx context.Context, req *spannerpb.Rollbac
return s.resps[0].(*emptypb.Empty), nil
}
func (s *mockSpannerServer) PartitionQuery(ctx context.Context, req *spannerpb.PartitionQueryRequest) (*spannerpb.PartitionResponse, error) {
md, _ := metadata.FromIncomingContext(ctx)
if xg := md["x-goog-api-client"]; len(xg) == 0 || !strings.Contains(xg[0], "gl-go/") {
return nil, fmt.Errorf("x-goog-api-client = %v, expected gl-go key", xg)
}
s.reqs = append(s.reqs, req)
if s.err != nil {
return nil, s.err
}
return s.resps[0].(*spannerpb.PartitionResponse), nil
}
func (s *mockSpannerServer) PartitionRead(ctx context.Context, req *spannerpb.PartitionReadRequest) (*spannerpb.PartitionResponse, error) {
md, _ := metadata.FromIncomingContext(ctx)
if xg := md["x-goog-api-client"]; len(xg) == 0 || !strings.Contains(xg[0], "gl-go/") {
return nil, fmt.Errorf("x-goog-api-client = %v, expected gl-go key", xg)
}
s.reqs = append(s.reqs, req)
if s.err != nil {
return nil, s.err
}
return s.resps[0].(*spannerpb.PartitionResponse), nil
}
// clientOpt is the option tests should use to connect to the test server.
// It is initialized by TestMain.
var clientOpt option.ClientOption
@@ -935,3 +959,127 @@ func TestSpannerRollbackError(t *testing.T) {
t.Errorf("got error code %q, want %q", c, errCode)
}
}
func TestSpannerPartitionQuery(t *testing.T) {
var expectedResponse *spannerpb.PartitionResponse = &spannerpb.PartitionResponse{}
mockSpanner.err = nil
mockSpanner.reqs = nil
mockSpanner.resps = append(mockSpanner.resps[:0], expectedResponse)
var formattedSession string = fmt.Sprintf("projects/%s/instances/%s/databases/%s/sessions/%s", "[PROJECT]", "[INSTANCE]", "[DATABASE]", "[SESSION]")
var sql string = "sql114126"
var request = &spannerpb.PartitionQueryRequest{
Session: formattedSession,
Sql: sql,
}
c, err := NewClient(context.Background(), clientOpt)
if err != nil {
t.Fatal(err)
}
resp, err := c.PartitionQuery(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if want, got := request, mockSpanner.reqs[0]; !proto.Equal(want, got) {
t.Errorf("wrong request %q, want %q", got, want)
}
if want, got := expectedResponse, resp; !proto.Equal(want, got) {
t.Errorf("wrong response %q, want %q)", got, want)
}
}
func TestSpannerPartitionQueryError(t *testing.T) {
errCode := codes.PermissionDenied
mockSpanner.err = gstatus.Error(errCode, "test error")
var formattedSession string = fmt.Sprintf("projects/%s/instances/%s/databases/%s/sessions/%s", "[PROJECT]", "[INSTANCE]", "[DATABASE]", "[SESSION]")
var sql string = "sql114126"
var request = &spannerpb.PartitionQueryRequest{
Session: formattedSession,
Sql: sql,
}
c, err := NewClient(context.Background(), clientOpt)
if err != nil {
t.Fatal(err)
}
resp, err := c.PartitionQuery(context.Background(), request)
if st, ok := gstatus.FromError(err); !ok {
t.Errorf("got error %v, expected grpc error", err)
} else if c := st.Code(); c != errCode {
t.Errorf("got error code %q, want %q", c, errCode)
}
_ = resp
}
func TestSpannerPartitionRead(t *testing.T) {
var expectedResponse *spannerpb.PartitionResponse = &spannerpb.PartitionResponse{}
mockSpanner.err = nil
mockSpanner.reqs = nil
mockSpanner.resps = append(mockSpanner.resps[:0], expectedResponse)
var formattedSession string = fmt.Sprintf("projects/%s/instances/%s/databases/%s/sessions/%s", "[PROJECT]", "[INSTANCE]", "[DATABASE]", "[SESSION]")
var table string = "table110115790"
var keySet *spannerpb.KeySet = &spannerpb.KeySet{}
var request = &spannerpb.PartitionReadRequest{
Session: formattedSession,
Table: table,
KeySet: keySet,
}
c, err := NewClient(context.Background(), clientOpt)
if err != nil {
t.Fatal(err)
}
resp, err := c.PartitionRead(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if want, got := request, mockSpanner.reqs[0]; !proto.Equal(want, got) {
t.Errorf("wrong request %q, want %q", got, want)
}
if want, got := expectedResponse, resp; !proto.Equal(want, got) {
t.Errorf("wrong response %q, want %q)", got, want)
}
}
func TestSpannerPartitionReadError(t *testing.T) {
errCode := codes.PermissionDenied
mockSpanner.err = gstatus.Error(errCode, "test error")
var formattedSession string = fmt.Sprintf("projects/%s/instances/%s/databases/%s/sessions/%s", "[PROJECT]", "[INSTANCE]", "[DATABASE]", "[SESSION]")
var table string = "table110115790"
var keySet *spannerpb.KeySet = &spannerpb.KeySet{}
var request = &spannerpb.PartitionReadRequest{
Session: formattedSession,
Table: table,
KeySet: keySet,
}
c, err := NewClient(context.Background(), clientOpt)
if err != nil {
t.Fatal(err)
}
resp, err := c.PartitionRead(context.Background(), request)
if st, ok := gstatus.FromError(err); !ok {
t.Errorf("got error %v, expected grpc error", err)
} else if c := st.Code(); c != errCode {
t.Errorf("got error code %q, want %q", c, errCode)
}
_ = resp
}
+50
View File
@@ -45,6 +45,8 @@ type CallOptions struct {
BeginTransaction []gax.CallOption
Commit []gax.CallOption
Rollback []gax.CallOption
PartitionQuery []gax.CallOption
PartitionRead []gax.CallOption
}
func defaultClientOptions() []option.ClientOption {
@@ -92,6 +94,8 @@ func defaultCallOptions() *CallOptions {
BeginTransaction: retry[[2]string{"default", "idempotent"}],
Commit: retry[[2]string{"long_running", "long_running"}],
Rollback: retry[[2]string{"default", "idempotent"}],
PartitionQuery: retry[[2]string{"default", "idempotent"}],
PartitionRead: retry[[2]string{"default", "idempotent"}],
}
}
@@ -405,6 +409,52 @@ func (c *Client) Rollback(ctx context.Context, req *spannerpb.RollbackRequest, o
return err
}
// PartitionQuery creates a set of partition tokens that can be used to execute a query
// operation in parallel. Each of the returned partition tokens can be used
// by [ExecuteStreamingSql][google.spanner.v1.Spanner.ExecuteStreamingSql] to specify a subset
// of the query result to read. The same session and read-only transaction
// must be used by the PartitionQueryRequest used to create the
// partition tokens and the ExecuteSqlRequests that use the partition tokens.
// Partition tokens become invalid when the session used to create them
// is deleted or begins a new transaction.
func (c *Client) PartitionQuery(ctx context.Context, req *spannerpb.PartitionQueryRequest, opts ...gax.CallOption) (*spannerpb.PartitionResponse, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.PartitionQuery[0:len(c.CallOptions.PartitionQuery):len(c.CallOptions.PartitionQuery)], opts...)
var resp *spannerpb.PartitionResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
resp, err = c.client.PartitionQuery(ctx, req, settings.GRPC...)
return err
}, opts...)
if err != nil {
return nil, err
}
return resp, nil
}
// PartitionRead creates a set of partition tokens that can be used to execute a read
// operation in parallel. Each of the returned partition tokens can be used
// by [StreamingRead][google.spanner.v1.Spanner.StreamingRead] to specify a subset of the read
// result to read. The same session and read-only transaction must be used by
// the PartitionReadRequest used to create the partition tokens and the
// ReadRequests that use the partition tokens.
// Partition tokens become invalid when the session used to create them
// is deleted or begins a new transaction.
func (c *Client) PartitionRead(ctx context.Context, req *spannerpb.PartitionReadRequest, opts ...gax.CallOption) (*spannerpb.PartitionResponse, error) {
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.PartitionRead[0:len(c.CallOptions.PartitionRead):len(c.CallOptions.PartitionRead)], opts...)
var resp *spannerpb.PartitionResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
resp, err = c.client.PartitionRead(ctx, req, settings.GRPC...)
return err
}, opts...)
if err != nil {
return nil, err
}
return resp, nil
}
// SessionIterator manages a stream of *spannerpb.Session.
type SessionIterator struct {
items []*spannerpb.Session
@@ -252,3 +252,39 @@ func ExampleClient_Rollback() {
// TODO: Handle error.
}
}
func ExampleClient_PartitionQuery() {
ctx := context.Background()
c, err := spanner.NewClient(ctx)
if err != nil {
// TODO: Handle error.
}
req := &spannerpb.PartitionQueryRequest{
// TODO: Fill request struct fields.
}
resp, err := c.PartitionQuery(ctx, req)
if err != nil {
// TODO: Handle error.
}
// TODO: Use resp.
_ = resp
}
func ExampleClient_PartitionRead() {
ctx := context.Background()
c, err := spanner.NewClient(ctx)
if err != nil {
// TODO: Handle error.
}
req := &spannerpb.PartitionReadRequest{
// TODO: Fill request struct fields.
}
resp, err := c.PartitionRead(ctx, req)
if err != nil {
// TODO: Handle error.
}
// TODO: Use resp.
_ = resp
}
+345
View File
@@ -0,0 +1,345 @@
/*
Copyright 2018 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spanner
import (
"bytes"
"encoding/gob"
"log"
"time"
"github.com/golang/protobuf/proto"
"golang.org/x/net/context"
sppb "google.golang.org/genproto/googleapis/spanner/v1"
)
// BatchReadOnlyTransaction is a ReadOnlyTransaction that allows for exporting
// arbitrarily large amounts of data from Cloud Spanner databases.
// BatchReadOnlyTransaction partitions a read/query request. Read/query request
// can then be executed independently over each partition while observing the
// same snapshot of the database. BatchReadOnlyTransaction can also be shared
// across multiple clients by passing around the BatchReadOnlyTransactionID and
// then recreating the transaction using Client.BatchReadOnlyTransactionFromID.
//
// Note: if a client is used only to run partitions, you can
// create it using a ClientConfig with both MinOpened and MaxIdle set to
// zero to avoid creating unnecessary sessions. You can also avoid excess
// gRPC channels by setting ClientConfig.NumChannels to the number of
// concurrently active BatchReadOnlyTransactions you expect to have.
type BatchReadOnlyTransaction struct {
ReadOnlyTransaction
ID BatchReadOnlyTransactionID
}
// BatchReadOnlyTransactionID is a unique identifier for a
// BatchReadOnlyTransaction. It can be used to re-create a
// BatchReadOnlyTransaction on a different machine or process by calling
// Client.BatchReadOnlyTransactionFromID.
type BatchReadOnlyTransactionID struct {
// unique ID for the transaction.
tid transactionID
// sid is the id of the Cloud Spanner session used for this transaction.
sid string
// rts is the read timestamp of this transaction.
rts time.Time
}
// Partition defines a segment of data to be read in a batch read or query. A
// partition can be serialized and processed across several different machines
// or processes.
type Partition struct {
pt []byte
qreq *sppb.ExecuteSqlRequest
rreq *sppb.ReadRequest
}
// PartitionOptions specifies options for a PartitionQueryRequest and
// PartitionReadRequest. See
// https://godoc.org/google.golang.org/genproto/googleapis/spanner/v1#PartitionOptions
// for more details.
type PartitionOptions struct {
// The desired data size for each partition generated.
PartitionBytes int64
// The desired maximum number of partitions to return.
MaxPartitions int64
}
// toProto converts a spanner.PartitionOptions into a sppb.PartitionOptions
func (opt PartitionOptions) toProto() *sppb.PartitionOptions {
return &sppb.PartitionOptions{
PartitionSizeBytes: opt.PartitionBytes,
MaxPartitions: opt.MaxPartitions,
}
}
// PartitionRead returns a list of Partitions that can be used to read rows from
// the database. These partitions can be executed across multiple processes,
// even across different machines. The partition size and count hints can be
// configured using PartitionOptions.
func (t *BatchReadOnlyTransaction) PartitionRead(ctx context.Context, table string, keys KeySet, columns []string, opt PartitionOptions) ([]*Partition, error) {
return t.PartitionReadUsingIndex(ctx, table, "", keys, columns, opt)
}
// PartitionReadUsingIndex returns a list of Partitions that can be used to read
// rows from the database using an index.
func (t *BatchReadOnlyTransaction) PartitionReadUsingIndex(ctx context.Context, table, index string, keys KeySet, columns []string, opt PartitionOptions) ([]*Partition, error) {
sh, ts, err := t.acquire(ctx)
if err != nil {
return nil, err
}
sid, client := sh.getID(), sh.getClient()
var (
kset *sppb.KeySet
resp *sppb.PartitionResponse
partitions []*Partition
)
kset, err = keys.keySetProto()
// request Partitions
if err != nil {
return nil, err
}
resp, err = client.PartitionRead(ctx, &sppb.PartitionReadRequest{
Session: sid,
Transaction: ts,
Table: table,
Index: index,
Columns: columns,
KeySet: kset,
PartitionOptions: opt.toProto(),
})
// prepare ReadRequest
req := &sppb.ReadRequest{
Session: sid,
Transaction: ts,
Table: table,
Index: index,
Columns: columns,
KeySet: kset,
}
// generate Partitions
for _, p := range resp.GetPartitions() {
partitions = append(partitions, &Partition{
pt: p.PartitionToken,
rreq: req,
})
}
return partitions, err
}
// PartitionQuery returns a list of Partitions that can be used to execute a query against the database.
func (t *BatchReadOnlyTransaction) PartitionQuery(ctx context.Context, statement Statement, opt PartitionOptions) ([]*Partition, error) {
sh, ts, err := t.acquire(ctx)
if err != nil {
return nil, err
}
sid, client := sh.getID(), sh.getClient()
var (
resp *sppb.PartitionResponse
partitions []*Partition
)
// request Partitions
req := &sppb.PartitionQueryRequest{
Session: sid,
Transaction: ts,
Sql: statement.SQL,
PartitionOptions: opt.toProto(),
}
if err := statement.bindParams(req); err != nil {
return nil, err
}
resp, err = client.PartitionQuery(ctx, req)
// prepare ExecuteSqlRequest
r := &sppb.ExecuteSqlRequest{
Session: sid,
Transaction: ts,
Sql: statement.SQL,
}
if err := statement.bindParams(r); err != nil {
return nil, err
}
// generate Partitions
for _, p := range resp.GetPartitions() {
partitions = append(partitions, &Partition{
pt: p.PartitionToken,
qreq: r,
})
}
return partitions, err
}
// release implements txReadEnv.release, noop.
func (t *BatchReadOnlyTransaction) release(err error) {
}
// setTimestamp implements txReadEnv.setTimestamp, noop.
// read timestamp is ready on txn initialization, avoid contending writing to it with future partitions.
func (t *BatchReadOnlyTransaction) setTimestamp(ts time.Time) {
}
// Close marks the txn as closed.
func (t *BatchReadOnlyTransaction) Close() {
t.mu.Lock()
defer t.mu.Unlock()
t.state = txClosed
}
// Cleanup cleans up all the resources used by this transaction and makes
// it unusable. Once this method is invoked, the transaction is no longer
// usable anywhere, including other clients/processes with which this
// transaction was shared.
//
// Calling Cleanup is optional, but recommended. If Cleanup is not called, the
// transaction's resources will be freed when the session expires on the backend and
// is deleted. For more information about recycled sessions, see
// https://cloud.google.com/spanner/docs/sessions.
func (t *BatchReadOnlyTransaction) Cleanup(ctx context.Context) {
t.Close()
t.mu.Lock()
defer t.mu.Unlock()
sh := t.sh
if sh == nil {
return
}
t.sh = nil
sid, client := sh.getID(), sh.getClient()
err := runRetryable(ctx, func(ctx context.Context) error {
_, e := client.DeleteSession(ctx, &sppb.DeleteSessionRequest{Name: sid})
return e
})
if err != nil {
log.Printf("Failed to delete session %v. Error: %v", sid, err)
}
}
// Execute runs a single Partition obtained from PartitionRead or PartitionQuery.
func (t *BatchReadOnlyTransaction) Execute(ctx context.Context, p *Partition) *RowIterator {
var (
sh *sessionHandle
err error
rpc func(ct context.Context, resumeToken []byte) (streamingReceiver, error)
)
if sh, _, err = t.acquire(ctx); err != nil {
return &RowIterator{err: err}
}
client := sh.getClient()
if client == nil {
// Might happen if transaction is closed in the middle of a API call.
return &RowIterator{err: errSessionClosed(sh)}
}
// read or query partition
if p.rreq != nil {
p.rreq.PartitionToken = p.pt
rpc = func(ctx context.Context, resumeToken []byte) (streamingReceiver, error) {
p.rreq.ResumeToken = resumeToken
return client.StreamingRead(ctx, p.rreq)
}
} else {
p.qreq.PartitionToken = p.pt
rpc = func(ctx context.Context, resumeToken []byte) (streamingReceiver, error) {
p.qreq.ResumeToken = resumeToken
return client.ExecuteStreamingSql(ctx, p.qreq)
}
}
return stream(
contextWithOutgoingMetadata(ctx, sh.getMetadata()),
rpc,
t.setTimestamp,
t.release)
}
// MarshalBinary implements BinaryMarshaler.
func (tid BatchReadOnlyTransactionID) MarshalBinary() (data []byte, err error) {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
if err := enc.Encode(tid.tid); err != nil {
return nil, err
}
if err := enc.Encode(tid.sid); err != nil {
return nil, err
}
if err := enc.Encode(tid.rts); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// UnmarshalBinary implements BinaryUnmarshaler.
func (tid *BatchReadOnlyTransactionID) UnmarshalBinary(data []byte) error {
dec := gob.NewDecoder(bytes.NewReader(data))
if err := dec.Decode(&tid.tid); err != nil {
return err
}
if err := dec.Decode(&tid.sid); err != nil {
return err
}
return dec.Decode(&tid.rts)
}
// MarshalBinary implements BinaryMarshaler.
func (p Partition) MarshalBinary() (data []byte, err error) {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
if err := enc.Encode(p.pt); err != nil {
return nil, err
}
var isReadPartition bool
var req proto.Message
if p.rreq != nil {
isReadPartition = true
req = p.rreq
} else {
isReadPartition = false
req = p.qreq
}
if err := enc.Encode(isReadPartition); err != nil {
return nil, err
}
if data, err = proto.Marshal(req); err != nil {
return nil, err
}
if err := enc.Encode(data); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// UnmarshalBinary implements BinaryUnmarshaler.
func (p *Partition) UnmarshalBinary(data []byte) error {
var (
isReadPartition bool
d []byte
err error
)
dec := gob.NewDecoder(bytes.NewReader(data))
if err := dec.Decode(&p.pt); err != nil {
return err
}
if err := dec.Decode(&isReadPartition); err != nil {
return err
}
if err := dec.Decode(&d); err != nil {
return err
}
if isReadPartition {
p.rreq = &sppb.ReadRequest{}
err = proto.Unmarshal(d, p.rreq)
} else {
p.qreq = &sppb.ExecuteSqlRequest{}
err = proto.Unmarshal(d, p.qreq)
}
return err
}
+73
View File
@@ -0,0 +1,73 @@
/*
Copyright 2018 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spanner
import (
"testing"
"time"
sppb "google.golang.org/genproto/googleapis/spanner/v1"
)
func TestPartitionRoundTrip(t *testing.T) {
t.Parallel()
for i, want := range []Partition{
{rreq: &sppb.ReadRequest{Table: "t"}},
{qreq: &sppb.ExecuteSqlRequest{Sql: "sql"}},
} {
got := serdesPartition(t, i, &want)
if !testEqual(got, want) {
t.Errorf("got: %#v\nwant:%#v", got, want)
}
}
}
func TestBROTIDRoundTrip(t *testing.T) {
t.Parallel()
tm := time.Now()
want := BatchReadOnlyTransactionID{
tid: []byte("tid"),
sid: "sid",
rts: tm,
}
data, err := want.MarshalBinary()
if err != nil {
t.Fatal(err)
}
var got BatchReadOnlyTransactionID
if err := got.UnmarshalBinary(data); err != nil {
t.Fatal(err)
}
if !testEqual(got, want) {
t.Errorf("got: %#v\nwant:%#v", got, want)
}
}
// serdesPartition is a helper that serialize a Partition then deserialize it
func serdesPartition(t *testing.T, i int, p1 *Partition) (p2 Partition) {
var (
data []byte
err error
)
if data, err = p1.MarshalBinary(); err != nil {
t.Fatalf("#%d: encoding failed %v", i, err)
}
if err = p2.UnmarshalBinary(data); err != nil {
t.Fatalf("#%d: decoding failed %v", i, err)
}
return p2
}
+109 -3
View File
@@ -18,6 +18,7 @@ package spanner
import (
"fmt"
"log"
"regexp"
"sync/atomic"
"time"
@@ -33,7 +34,7 @@ import (
)
const (
prodAddr = "spanner.googleapis.com:443"
endpoint = "spanner.googleapis.com:443"
// resourcePrefixHeader is the name of the metadata header used to indicate
// the resource being operated on.
@@ -126,7 +127,7 @@ func NewClientWithConfig(ctx context.Context, database string, config ClientConf
xGoogHeaderKey, xGoogHeaderVal),
}
allOpts := []option.ClientOption{
option.WithEndpoint(prodAddr),
option.WithEndpoint(endpoint),
option.WithScopes(Scope),
option.WithGRPCDialOption(
grpc.WithDefaultCallOptions(
@@ -135,7 +136,6 @@ func NewClientWithConfig(ctx context.Context, database string, config ClientConf
),
),
}
allOpts = append(allOpts, openCensusOptions()...)
allOpts = append(allOpts, opts...)
// Prepare gRPC channels.
if config.NumChannels == 0 {
@@ -219,6 +219,112 @@ func (c *Client) ReadOnlyTransaction() *ReadOnlyTransaction {
return t
}
// BatchReadOnlyTransaction returns a BatchReadOnlyTransaction that can be used
// for partitioned reads or queries from a snapshot of the database. This is
// useful in batch processing pipelines where one wants to divide the work of
// reading from the database across multiple machines.
//
// Note: This transaction does not use the underlying session pool but creates a
// new session each time, and the session is reused across clients.
//
// You should call Close() after the txn is no longer needed on local
// client, and call Cleanup() when the txn is finished for all clients, to free
// the session.
func (c *Client) BatchReadOnlyTransaction(ctx context.Context, tb TimestampBound) (*BatchReadOnlyTransaction, error) {
var (
tx transactionID
rts time.Time
s *session
sh *sessionHandle
err error
)
defer func() {
if err != nil && sh != nil {
e := runRetryable(ctx, func(ctx context.Context) error {
_, e := s.client.DeleteSession(ctx, &sppb.DeleteSessionRequest{Name: s.getID()})
return e
})
if e != nil {
log.Printf("Failed to delete session %v. Error: %v", s.getID(), e)
}
}
}()
// create session
sc := c.rrNext()
err = runRetryable(ctx, func(ctx context.Context) error {
sid, e := sc.CreateSession(ctx, &sppb.CreateSessionRequest{Database: c.database})
if e != nil {
return e
}
// If no error, construct the new session.
s = &session{valid: true, client: sc, id: sid.Name, createTime: time.Now(), md: c.md}
return nil
})
if err != nil {
return nil, err
}
sh = &sessionHandle{session: s}
// begin transaction
err = runRetryable(contextWithOutgoingMetadata(ctx, sh.getMetadata()), func(ctx context.Context) error {
res, e := sh.getClient().BeginTransaction(ctx, &sppb.BeginTransactionRequest{
Session: sh.getID(),
Options: &sppb.TransactionOptions{
Mode: &sppb.TransactionOptions_ReadOnly_{
ReadOnly: buildTransactionOptionsReadOnly(tb, true),
},
},
})
if e != nil {
return e
}
tx = res.Id
if res.ReadTimestamp != nil {
rts = time.Unix(res.ReadTimestamp.Seconds, int64(res.ReadTimestamp.Nanos))
}
return nil
})
if err != nil {
return nil, err
}
t := &BatchReadOnlyTransaction{
ReadOnlyTransaction: ReadOnlyTransaction{
tx: tx,
txReadyOrClosed: make(chan struct{}),
state: txActive,
sh: sh,
rts: rts,
},
ID: BatchReadOnlyTransactionID{
tid: tx,
sid: sh.getID(),
rts: rts,
},
}
t.txReadOnly.txReadEnv = t
return t, nil
}
// BatchReadOnlyTransactionFromID reconstruct a BatchReadOnlyTransaction from BatchReadOnlyTransactionID
func (c *Client) BatchReadOnlyTransactionFromID(tid BatchReadOnlyTransactionID) *BatchReadOnlyTransaction {
sc := c.rrNext()
s := &session{valid: true, client: sc, id: tid.sid, createTime: time.Now(), md: c.md}
sh := &sessionHandle{session: s}
t := &BatchReadOnlyTransaction{
ReadOnlyTransaction: ReadOnlyTransaction{
tx: tid.tid,
txReadyOrClosed: make(chan struct{}),
state: txActive,
sh: sh,
rts: tid.rts,
},
ID: tid,
}
t.txReadOnly.txReadEnv = t
return t
}
type transactionInProgressKey struct{}
func checkNestedTxn(ctx context.Context) error {
+1 -1
View File
@@ -192,7 +192,7 @@ For Cloud Spanner columns that may contain NULL, use one of the NullXXX types,
like NullString:
var ns spanner.NullString
if err =: row.Column(0, &ns); err != nil {
if err := row.Column(0, &ns); err != nil {
// TODO: Handle error.
}
if ns.Valid {
+2 -2
View File
@@ -21,8 +21,8 @@ import (
"testing"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func TestToSpannerError(t *testing.T) {
@@ -33,7 +33,7 @@ func TestToSpannerError(t *testing.T) {
{errors.New("wha?"), codes.Unknown},
{context.Canceled, codes.Canceled},
{context.DeadlineExceeded, codes.DeadlineExceeded},
{grpc.Errorf(codes.ResourceExhausted, "so tired"), codes.ResourceExhausted},
{status.Errorf(codes.ResourceExhausted, "so tired"), codes.ResourceExhausted},
{spannerErrorf(codes.InvalidArgument, "bad"), codes.InvalidArgument},
} {
err := toSpannerError(test.err)
+55
View File
@@ -19,6 +19,7 @@ package spanner_test
import (
"errors"
"fmt"
"sync"
"time"
"cloud.google.com/go/spanner"
@@ -550,3 +551,57 @@ func ExampleGenericColumnValue_Decode() {
// int 42
// string my-text
}
func ExampleClient_BatchReadOnlyTransaction() {
ctx := context.Background()
var (
client *spanner.Client
txn *spanner.BatchReadOnlyTransaction
err error
)
if client, err = spanner.NewClient(ctx, myDB); err != nil {
// TODO: Handle error.
}
defer client.Close()
if txn, err = client.BatchReadOnlyTransaction(ctx, spanner.StrongRead()); err != nil {
// TODO: Handle error.
}
defer txn.Close()
// Singer represents the elements in a row from the Singers table.
type Singer struct {
SingerID int64
FirstName string
LastName string
SingerInfo []byte
}
stmt := spanner.Statement{SQL: "SELECT * FROM Singers;"}
partitions, err := txn.PartitionQuery(ctx, stmt, spanner.PartitionOptions{})
if err != nil {
// TODO: Handle error.
}
// Note: here we use multiple goroutines, but you should use separate processes/machines.
wg := sync.WaitGroup{}
for i, p := range partitions {
wg.Add(1)
go func(i int, p *spanner.Partition) {
defer wg.Done()
iter := txn.Execute(ctx, p)
defer iter.Stop()
for {
row, err := iter.Next()
if err == iterator.Done {
break
} else if err != nil {
// TODO: Handle error.
}
var s Singer
if err := row.ToStruct(&s); err != nil {
// TODO: Handle error.
}
_ = s // TODO: Process the row.
}
}(i, p)
}
wg.Wait()
}
+9 -16
View File
@@ -19,29 +19,22 @@ package spanner
import (
"fmt"
ocgrpc "go.opencensus.io/plugin/grpc"
"go.opencensus.io/trace"
"golang.org/x/net/context"
"google.golang.org/api/option"
"google.golang.org/grpc"
)
func openCensusOptions() []option.ClientOption {
return []option.ClientOption{
option.WithGRPCDialOption(grpc.WithStatsHandler(ocgrpc.NewClientStatsHandler())),
}
}
func traceStartSpan(ctx context.Context, name string) context.Context {
return trace.StartSpan(ctx, name)
ctx, _ = trace.StartSpan(ctx, name)
return ctx
}
func traceEndSpan(ctx context.Context, err error) {
span := trace.FromContext(ctx)
if err != nil {
// TODO(jba): Add error code to the status.
span.SetStatus(trace.Status{Message: err.Error()})
}
trace.EndSpan(ctx)
span.End()
}
func tracePrintf(ctx context.Context, attrMap map[string]interface{}, format string, args ...interface{}) {
@@ -50,15 +43,15 @@ func tracePrintf(ctx context.Context, attrMap map[string]interface{}, format str
var a trace.Attribute
switch v := v.(type) {
case string:
a = trace.StringAttribute{k, v}
a = trace.StringAttribute(k, v)
case bool:
a = trace.BoolAttribute{k, v}
a = trace.BoolAttribute(k, v)
case int:
a = trace.Int64Attribute{k, int64(v)}
a = trace.Int64Attribute(k, int64(v))
case int64:
a = trace.Int64Attribute{k, v}
a = trace.Int64Attribute(k, v)
default:
a = trace.StringAttribute{k, fmt.Sprintf("%#v", v)}
a = trace.StringAttribute(k, fmt.Sprintf("%#v", v))
}
attrs = append(attrs, a)
}
+16 -3
View File
@@ -24,6 +24,7 @@ import (
"time"
"golang.org/x/net/context"
"google.golang.org/grpc/status"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes/empty"
@@ -132,7 +133,7 @@ func (m *MockCloudSpannerClient) CreateSession(c context.Context, r *sppb.Create
s := &sppb.Session{}
if r.Database != "mockdb" {
// Reject other databases
return s, grpc.Errorf(codes.NotFound, fmt.Sprintf("database not found: %v", r.Database))
return s, status.Errorf(codes.NotFound, fmt.Sprintf("database not found: %v", r.Database))
}
// Generate & record session name.
s.Name = fmt.Sprintf("mockdb-%v", time.Now().UnixNano())
@@ -150,7 +151,7 @@ func (m *MockCloudSpannerClient) GetSession(c context.Context, r *sppb.GetSessio
}
m.pings = append(m.pings, r.Name)
if _, ok := m.sessions[r.Name]; !ok {
return nil, grpc.Errorf(codes.NotFound, fmt.Sprintf("Session not found: %v", r.Name))
return nil, status.Errorf(codes.NotFound, fmt.Sprintf("Session not found: %v", r.Name))
}
return &sppb.Session{Name: r.Name}, nil
}
@@ -165,7 +166,7 @@ func (m *MockCloudSpannerClient) DeleteSession(c context.Context, r *sppb.Delete
}
if _, ok := m.sessions[r.Name]; !ok {
// Session not found.
return &empty.Empty{}, grpc.Errorf(codes.NotFound, fmt.Sprintf("Session not found: %v", r.Name))
return &empty.Empty{}, status.Errorf(codes.NotFound, fmt.Sprintf("Session not found: %v", r.Name))
}
// Delete session from in-memory table.
delete(m.sessions, r.Name)
@@ -318,6 +319,18 @@ func (m *MockCloudSpannerClient) Rollback(c context.Context, r *sppb.RollbackReq
return nil, nil
}
// PartitionQuery is a placeholder for SpannerServer.PartitionQuery.
func (m *MockCloudSpannerClient) PartitionQuery(ctx context.Context, r *sppb.PartitionQueryRequest, opts ...grpc.CallOption) (*sppb.PartitionResponse, error) {
m.ready()
return nil, errors.New("Unimplemented")
}
// PartitionRead is a placeholder for SpannerServer.PartitionRead.
func (m *MockCloudSpannerClient) PartitionRead(ctx context.Context, r *sppb.PartitionReadRequest, opts ...grpc.CallOption) (*sppb.PartitionResponse, error) {
m.ready()
return nil, errors.New("Unimplemented")
}
func (m *MockCloudSpannerClient) expectAction(methods ...string) (Action, error) {
for _, me := range methods {
if err := m.injErr[me]; err != nil {
+29 -43
View File
@@ -18,10 +18,10 @@ package testutil
import (
"encoding/binary"
"errors"
"fmt"
"io"
"net"
"sync"
"testing"
"time"
@@ -34,6 +34,7 @@ import (
sppb "google.golang.org/genproto/googleapis/spanner/v1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var (
@@ -75,6 +76,10 @@ type MockCloudSpanner struct {
msgs chan MockCtlMsg
readTs time.Time
next int
mu sync.Mutex
nextSession int
sessions map[string]*sppb.Session
}
// Addr returns the listening address of mock server.
@@ -102,26 +107,31 @@ func (m *MockCloudSpanner) Done() {
// CreateSession is a placeholder for SpannerServer.CreateSession.
func (m *MockCloudSpanner) CreateSession(c context.Context, r *sppb.CreateSessionRequest) (*sppb.Session, error) {
m.t.Fatalf("CreateSession is unimplemented")
return nil, errors.New("Unimplemented")
m.mu.Lock()
defer m.mu.Unlock()
name := fmt.Sprintf("session-%d", m.nextSession)
m.nextSession++
s := &sppb.Session{Name: name}
m.sessions[name] = s
return s, nil
}
// GetSession is a placeholder for SpannerServer.GetSession.
func (m *MockCloudSpanner) GetSession(c context.Context, r *sppb.GetSessionRequest) (*sppb.Session, error) {
m.t.Fatalf("GetSession is unimplemented")
return nil, errors.New("Unimplemented")
m.mu.Lock()
defer m.mu.Unlock()
if s, ok := m.sessions[r.Name]; ok {
return s, nil
}
return nil, status.Errorf(codes.NotFound, "not found")
}
// DeleteSession is a placeholder for SpannerServer.DeleteSession.
func (m *MockCloudSpanner) DeleteSession(c context.Context, r *sppb.DeleteSessionRequest) (*empty.Empty, error) {
m.t.Fatalf("DeleteSession is unimplemented")
return nil, errors.New("Unimplemented")
}
// ExecuteSql is a placeholder for SpannerServer.ExecuteSql.
func (m *MockCloudSpanner) ExecuteSql(c context.Context, r *sppb.ExecuteSqlRequest) (*sppb.ResultSet, error) {
m.t.Fatalf("ExecuteSql is unimplemented")
return nil, errors.New("Unimplemented")
m.mu.Lock()
defer m.mu.Unlock()
delete(m.sessions, r.Name)
return &empty.Empty{}, nil
}
// EncodeResumeToken return mock resume token encoding for an uint64 integer.
@@ -144,7 +154,7 @@ func DecodeResumeToken(t []byte) (uint64, error) {
func (m *MockCloudSpanner) ExecuteStreamingSql(r *sppb.ExecuteSqlRequest, s sppb.Spanner_ExecuteStreamingSqlServer) error {
switch r.Sql {
case "SELECT * from t_unavailable":
return grpc.Errorf(codes.Unavailable, "mock table unavailable")
return status.Errorf(codes.Unavailable, "mock table unavailable")
case "SELECT t.key key, t.value value FROM t_mock t":
if r.ResumeToken != nil {
s, err := DecodeResumeToken(r.ResumeToken)
@@ -192,34 +202,9 @@ func (m *MockCloudSpanner) ExecuteStreamingSql(r *sppb.ExecuteSqlRequest, s sppb
}
}
// Read is a placeholder for SpannerServer.Read.
func (m *MockCloudSpanner) Read(c context.Context, r *sppb.ReadRequest) (*sppb.ResultSet, error) {
m.t.Fatalf("Read is unimplemented")
return nil, errors.New("Unimplemented")
}
// StreamingRead is a placeholder for SpannerServer.StreamingRead.
func (m *MockCloudSpanner) StreamingRead(r *sppb.ReadRequest, s sppb.Spanner_StreamingReadServer) error {
m.t.Fatalf("StreamingRead is unimplemented")
return errors.New("Unimplemented")
}
// BeginTransaction is a placeholder for SpannerServer.BeginTransaction.
func (m *MockCloudSpanner) BeginTransaction(c context.Context, r *sppb.BeginTransactionRequest) (*sppb.Transaction, error) {
m.t.Fatalf("BeginTransaction is unimplemented")
return nil, errors.New("Unimplemented")
}
// Commit is a placeholder for SpannerServer.Commit.
func (m *MockCloudSpanner) Commit(c context.Context, r *sppb.CommitRequest) (*sppb.CommitResponse, error) {
m.t.Fatalf("Commit is unimplemented")
return nil, errors.New("Unimplemented")
}
// Rollback is a placeholder for SpannerServer.Rollback.
func (m *MockCloudSpanner) Rollback(c context.Context, r *sppb.RollbackRequest) (*empty.Empty, error) {
m.t.Fatalf("Rollback is unimplemented")
return nil, errors.New("Unimplemented")
return s.Send(&sppb.PartialResultSet{})
}
// Serve runs a MockCloudSpanner listening on a random localhost address.
@@ -249,9 +234,10 @@ func (m *MockCloudSpanner) Stop() {
// NewMockCloudSpanner creates a new MockCloudSpanner instance.
func NewMockCloudSpanner(t *testing.T, ts time.Time) *MockCloudSpanner {
mcs := &MockCloudSpanner{
t: t,
msgs: make(chan MockCtlMsg, 1000),
readTs: ts,
t: t,
msgs: make(chan MockCtlMsg, 1000),
readTs: ts,
sessions: map[string]*sppb.Session{},
}
return mcs
}
+4 -6
View File
@@ -30,13 +30,11 @@ import (
// A Key can be either a Cloud Spanner row's primary key or a secondary index key.
// It is essentially an interface{} array, which represents a set of Cloud Spanner
// columns. A Key type has the following usages:
// columns. A Key can be used as:
//
// - Used as primary key which uniquely identifies a Cloud Spanner row.
// - Used as secondary index key which maps to a set of Cloud Spanner rows
// indexed under it.
// - Used as endpoints of primary key/secondary index ranges,
// see also the KeyRange type.
// - A primary key which uniquely identifies a Cloud Spanner row.
// - A secondary index key which maps to a set of Cloud Spanner rows indexed under it.
// - An endpoint of primary key/secondary index ranges; see the KeyRange type.
//
// Rows that are identified by the Key type are outputs of read operation or targets of
// delete operation in a mutation. Note that for Insert/Update/InsertOrUpdate/Update
+11 -11
View File
@@ -53,23 +53,23 @@ const (
//
// Many mutations can be applied in a single atomic commit. For purposes of
// constraint checking (such as foreign key constraints), the operations can be
// viewed as applying in same order as the mutations are supplied in (so that
// e.g., a row and its logical "child" can be inserted in the same commit).
// viewed as applying in the same order as the mutations are provided (so that, e.g.,
// a row and its logical "child" can be inserted in the same commit).
//
// - The Apply function applies series of mutations.
// - A ReadWriteTransaction applies a series of mutations as part of an
// atomic read-modify-write operation.
// Example:
// The Apply function applies series of mutations. For example,
//
// m := spanner.Insert("User",
// []string{"user_id", "profile"},
// []interface{}{UserID, profile})
// _, err := client.Apply(ctx, []*spanner.Mutation{m})
// m := spanner.Insert("User",
// []string{"user_id", "profile"},
// []interface{}{UserID, profile})
// _, err := client.Apply(ctx, []*spanner.Mutation{m})
//
// In this example, we insert a new row into the User table. The primary key
// inserts a new row into the User table. The primary key
// for the new row is UserID (presuming that "user_id" has been declared as the
// primary key of the "User" table).
//
// To apply a series of mutations as part of an atomic read-modify-write operation,
// use ReadWriteTransaction.
//
// Updating a row
//
// Changing the values of columns in an existing row is very similar to
+1 -6
View File
@@ -16,15 +16,10 @@
package spanner
import (
"golang.org/x/net/context"
"google.golang.org/api/option"
)
import "golang.org/x/net/context"
// OpenCensus only supports go 1.8 and higher.
func openCensusOptions() []option.ClientOption { return nil }
func traceStartSpan(ctx context.Context, _ string) context.Context {
return ctx
}
+72
View File
@@ -0,0 +1,72 @@
// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.8
package spanner
import (
"testing"
"time"
"cloud.google.com/go/spanner/internal/testutil"
"go.opencensus.io/plugin/ocgrpc"
statsview "go.opencensus.io/stats/view"
"golang.org/x/net/context"
"google.golang.org/api/option"
"google.golang.org/grpc"
)
func TestOCStats(t *testing.T) {
// Check that stats are being exported.
te := &testExporter{c: make(chan *statsview.Data)}
statsview.RegisterExporter(te)
defer statsview.UnregisterExporter(te)
statsview.SetReportingPeriod(time.Millisecond)
if err := ocgrpc.ClientRequestCountView.Subscribe(); err != nil {
t.Fatal(err)
}
ms := testutil.NewMockCloudSpanner(t, trxTs)
ms.Serve()
ctx := context.Background()
c, err := NewClient(ctx, "projects/P/instances/I/databases/D",
option.WithEndpoint(ms.Addr()),
option.WithGRPCDialOption(grpc.WithInsecure()),
option.WithoutAuthentication())
if err != nil {
t.Fatal(err)
}
defer c.Close()
c.Single().ReadRow(ctx, "Users", Key{"alice"}, []string{"email"})
// Wait until we see data from the view.
select {
case <-te.c:
case <-time.After(1 * time.Second):
t.Fatal("no stats were exported before timeout")
}
}
type testExporter struct {
c chan *statsview.Data
}
func (e *testExporter) ExportView(vd *statsview.Data) {
if len(vd.Rows) > 0 {
select {
case e.c <- vd:
default:
}
}
}
+9 -8
View File
@@ -25,8 +25,9 @@ import (
"time"
"golang.org/x/net/context"
"google.golang.org/grpc/status"
proto "github.com/golang/protobuf/proto"
"github.com/golang/protobuf/proto"
proto3 "github.com/golang/protobuf/ptypes/struct"
"cloud.google.com/go/spanner/internal/testutil"
@@ -717,7 +718,7 @@ func TestRsdNonblockingStates(t *testing.T) {
queueingRetryable, // got foo-02
aborted, // got error
},
wantErr: grpc.Errorf(codes.Unknown, "I quit"),
wantErr: status.Errorf(codes.Unknown, "I quit"),
},
{
// unConnected->queueingRetryable->queueingUnretryable->queueingUnretryable
@@ -785,7 +786,7 @@ func TestRsdNonblockingStates(t *testing.T) {
s = append(s, aborted) // Error happens
return s
}(),
wantErr: grpc.Errorf(codes.Unknown, "Just Abort It"),
wantErr: status.Errorf(codes.Unknown, "Just Abort It"),
},
}
nextTest:
@@ -901,11 +902,11 @@ func TestRsdBlockingStates(t *testing.T) {
// unConnected -> unConnected
name: "unConnected -> unConnected",
rpc: func(ct context.Context, resumeToken []byte) (streamingReceiver, error) {
return nil, grpc.Errorf(codes.Unavailable, "trust me: server is unavailable")
return nil, status.Errorf(codes.Unavailable, "trust me: server is unavailable")
},
sql: "SELECT * from t_whatever",
stateHistory: []resumableStreamDecoderState{unConnected, unConnected, unConnected},
wantErr: grpc.Errorf(codes.Unavailable, "trust me: server is unavailable"),
wantErr: status.Errorf(codes.Unavailable, "trust me: server is unavailable"),
},
{
// unConnected -> queueingRetryable
@@ -1344,7 +1345,7 @@ func TestResumeToken(t *testing.T) {
}
// Inject resumable failure.
ms.AddMsg(
grpc.Errorf(codes.Unavailable, "mock server unavailable"),
status.Errorf(codes.Unavailable, "mock server unavailable"),
false,
)
// Test if client detects the resumable failure and retries.
@@ -1401,7 +1402,7 @@ func TestResumeToken(t *testing.T) {
// Inject resumable error, but since resumableStreamDecoder is already at queueingUnretryable
// state, query will just fail.
ms.AddMsg(
grpc.Errorf(codes.Unavailable, "mock server wants some sleep"),
status.Errorf(codes.Unavailable, "mock server wants some sleep"),
false,
)
var gotErr error
@@ -1410,7 +1411,7 @@ func TestResumeToken(t *testing.T) {
case <-time.After(10 * time.Second):
t.Fatalf("timeout in waiting for failed query to return.")
}
if wantErr := toSpannerError(grpc.Errorf(codes.Unavailable, "mock server wants some sleep")); !testEqual(gotErr, wantErr) {
if wantErr := toSpannerError(status.Errorf(codes.Unavailable, "mock server wants some sleep")); !testEqual(gotErr, wantErr) {
t.Fatalf("stream() returns error: %v, but want error: %v", gotErr, wantErr)
}
+6 -6
View File
@@ -26,9 +26,9 @@ import (
"github.com/golang/protobuf/ptypes"
"golang.org/x/net/context"
edpb "google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
// Test if runRetryable loop deals with various errors correctly.
@@ -37,10 +37,10 @@ func TestRetry(t *testing.T) {
t.SkipNow()
}
responses := []error{
grpc.Errorf(codes.Internal, "transport is closing"),
grpc.Errorf(codes.Unknown, "unexpected EOF"),
grpc.Errorf(codes.Internal, "stream terminated by RST_STREAM with error code: 2"),
grpc.Errorf(codes.Unavailable, "service is currently unavailable"),
status.Errorf(codes.Internal, "transport is closing"),
status.Errorf(codes.Unknown, "unexpected EOF"),
status.Errorf(codes.Internal, "stream terminated by RST_STREAM with error code: 2"),
status.Errorf(codes.Unavailable, "service is currently unavailable"),
errRetry(fmt.Errorf("just retry it")),
}
err := runRetryable(context.Background(), func(ct context.Context) error {
@@ -100,7 +100,7 @@ func TestRetryInfo(t *testing.T) {
trailers := map[string]string{
retryInfoKey: string(b),
}
gotDelay, ok := extractRetryDelay(errRetry(toSpannerErrorWithMetadata(grpc.Errorf(codes.Aborted, ""), metadata.New(trailers))))
gotDelay, ok := extractRetryDelay(errRetry(toSpannerErrorWithMetadata(status.Errorf(codes.Aborted, ""), metadata.New(trailers))))
if !ok || !testEqual(time.Second, gotDelay) {
t.Errorf("<ok, retryDelay> = <%t, %v>, want <true, %v>", ok, gotDelay, time.Second)
}
+7 -5
View File
@@ -274,11 +274,13 @@ func errToStructArgType(p interface{}) error {
// ToStruct fetches the columns in a row into the fields of a struct.
// The rules for mapping a row's columns into a struct's exported fields
// are as the following:
// 1. If a field has a `spanner: "column_name"` tag, then decode column
// 'column_name' into the field. A special case is the `spanner: "-"`
// tag, which instructs ToStruct to ignore the field during decoding.
// 2. Otherwise, if the name of a field matches the name of a column (ignoring case),
// decode the column into the field.
//
// 1. If a field has a `spanner: "column_name"` tag, then decode column
// 'column_name' into the field. A special case is the `spanner: "-"`
// tag, which instructs ToStruct to ignore the field during decoding.
//
// 2. Otherwise, if the name of a field matches the name of a column (ignoring case),
// decode the column into the field.
//
// The fields of the destination struct can be of any type that is acceptable
// to spanner.Row.Column.
+5 -5
View File
@@ -25,10 +25,10 @@ import (
"time"
"golang.org/x/net/context"
"google.golang.org/grpc/status"
"cloud.google.com/go/spanner/internal/testutil"
sppb "google.golang.org/genproto/googleapis/spanner/v1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
)
@@ -222,7 +222,7 @@ func TestTakeFromIdleListChecked(t *testing.T) {
}
// Inject session error to mockclient, and take the session from the session pool, the old session should be destroyed and
// the session pool will create a new session.
sc.InjectError("GetSession", grpc.Errorf(codes.NotFound, "Session not found:"))
sc.InjectError("GetSession", status.Errorf(codes.NotFound, "Session not found:"))
// Delay to trigger sessionPool.Take to ping the session.
<-time.After(time.Second)
sh, err = sp.take(context.Background())
@@ -279,7 +279,7 @@ func TestTakeFromIdleWriteListChecked(t *testing.T) {
}
// Inject session error to mockclient, and take the session from the session pool, the old session should be destroyed and
// the session pool will create a new session.
sc.InjectError("GetSession", grpc.Errorf(codes.NotFound, "Session not found:"))
sc.InjectError("GetSession", status.Errorf(codes.NotFound, "Session not found:"))
// Delay to trigger sessionPool.Take to ping the session.
<-time.After(time.Second)
sh, err = sp.takeWriteSession(context.Background())
@@ -369,7 +369,7 @@ func TestMaxBurst(t *testing.T) {
sp, sc, cancel := setup(t, SessionPoolConfig{MaxBurst: 1})
defer cancel()
// Will cause session creation RPC to be retried forever.
sc.InjectError("CreateSession", grpc.Errorf(codes.Unavailable, "try later"))
sc.InjectError("CreateSession", status.Errorf(codes.Unavailable, "try later"))
// This session request will never finish until the injected error is cleared.
go sp.take(context.Background())
// Poll for the execution of the first session request.
@@ -619,7 +619,7 @@ func TestSessionHealthCheck(t *testing.T) {
if err != nil {
t.Errorf("cannot get session from session pool: %v", err)
}
sc.InjectError("GetSession", grpc.Errorf(codes.NotFound, "Session not found:"))
sc.InjectError("GetSession", status.Errorf(codes.NotFound, "Session not found:"))
// Wait for healthcheck workers to find the broken session and tear it down.
<-time.After(1 * time.Second)
s := sh.session
+250 -7
View File
@@ -93,6 +93,19 @@ var (
`CREATE INDEX TestTableByValue ON TestTable(StringValue)`,
`CREATE INDEX TestTableByValueDesc ON TestTable(StringValue DESC)`,
}
simpleDBStatements = []string{
`CREATE TABLE test (
a STRING(1024),
b STRING(1024),
) PRIMARY KEY (a)`,
}
simpleDBTableColumns = []string{"a", "b"}
)
const (
str1 = "alice"
str2 = "a@example.com"
)
type testTableRow struct{ Key, StringValue string }
@@ -119,8 +132,7 @@ func initIntegrationTest() {
}
var err error
// Create Admin client and Data client.
// TODO: Remove the EndPoint option once this is the default.
admin, err = database.NewDatabaseAdminClient(ctx, option.WithTokenSource(ts), option.WithEndpoint("spanner.googleapis.com:443"))
admin, err = database.NewDatabaseAdminClient(ctx, option.WithTokenSource(ts), option.WithEndpoint(endpoint))
if err != nil {
log.Fatalf("cannot create admin client: %v", err)
}
@@ -158,16 +170,16 @@ func prepare(ctx context.Context, t *testing.T, statements []string) (client *Cl
}
client, err = NewClientWithConfig(ctx, dbPath, ClientConfig{
SessionPoolConfig: SessionPoolConfig{WriteSessions: 0.2},
}, option.WithTokenSource(testutil.TokenSource(ctx, Scope)))
}, option.WithTokenSource(testutil.TokenSource(ctx, Scope)), option.WithEndpoint(endpoint))
if err != nil {
t.Fatalf("cannot create data client on DB %v: %v", dbPath, err)
}
return client, dbPath, func() {
client.Close()
if err := admin.DropDatabase(ctx, &adminpb.DropDatabaseRequest{dbPath}); err != nil {
t.Logf("failed to drop database %s (error %v), might need a manual removal",
dbPath, err)
}
client.Close()
}
}
@@ -959,8 +971,11 @@ func TestDbRemovalRecovery(t *testing.T) {
) PRIMARY KEY (SingerId)`,
},
})
if err != nil {
t.Fatalf("cannot recreate testing DB %v: %v", dbPath, err)
}
if _, err := op.Wait(ctx); err != nil {
t.Errorf("cannot recreate testing DB %v: %v", dbPath, err)
t.Fatalf("cannot recreate testing DB %v: %v", dbPath, err)
}
// Now, send the query again.
@@ -968,7 +983,7 @@ func TestDbRemovalRecovery(t *testing.T) {
defer iter.Stop()
_, err = iter.Next()
if err != nil && err != iterator.Done {
t.Fatalf("failed to send query to database %v: %v", dbPath, err)
t.Errorf("failed to send query to database %v: %v", dbPath, err)
}
}
@@ -1434,7 +1449,7 @@ func readAllTestTable(iter *RowIterator) ([]testTableRow, error) {
// Test TransactionRunner. Test that transactions are aborted and retried as expected.
func TestTransactionRunner(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
client, _, tearDown := prepare(ctx, t, singerDBStatements)
defer tearDown()
@@ -1559,3 +1574,231 @@ func TestTransactionRunner(t *testing.T) {
}
}
}
// createClient creates Cloud Spanner data client.
func createClient(ctx context.Context, dbPath string) (client *Client, err error) {
client, err = NewClientWithConfig(ctx, dbPath, ClientConfig{
SessionPoolConfig: SessionPoolConfig{WriteSessions: 0.2},
}, option.WithTokenSource(testutil.TokenSource(ctx, Scope)), option.WithEndpoint(endpoint))
if err != nil {
return nil, fmt.Errorf("cannot create data client on DB %v: %v", dbPath, err)
}
return client, nil
}
// populate prepares the database with some data.
func populate(ctx context.Context, client *Client) error {
// Populate data
var err error
m := InsertMap("test", map[string]interface{}{
"a": str1,
"b": str2,
})
_, err = client.Apply(ctx, []*Mutation{m})
return err
}
// Test PartitionQuery of BatchReadOnlyTransaction, create partitions then
// serialize and deserialize both transaction and partition to be used in
// execution on another client, and compare results.
func TestBatchQuery(t *testing.T) {
t.Parallel()
// Set up testing environment.
var (
client2 *Client
err error
)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
client, dbPath, tearDown := prepare(ctx, t, simpleDBStatements)
defer tearDown()
if err = populate(ctx, client); err != nil {
t.Fatal(err)
}
if client2, err = createClient(ctx, dbPath); err != nil {
t.Fatal(err)
}
defer client2.Close()
// PartitionQuery
var (
txn *BatchReadOnlyTransaction
partitions []*Partition
stmt = Statement{SQL: "SELECT * FROM test;"}
)
if txn, err = client.BatchReadOnlyTransaction(ctx, StrongRead()); err != nil {
t.Fatal(err)
}
defer txn.Cleanup(ctx)
if partitions, err = txn.PartitionQuery(ctx, stmt, PartitionOptions{0, 3}); err != nil {
t.Fatal(err)
}
// Reconstruct BatchReadOnlyTransactionID and execute partitions
var (
tid2 BatchReadOnlyTransactionID
data []byte
gotResult bool // if we get matching result from two separate txns
)
if data, err = txn.ID.MarshalBinary(); err != nil {
t.Fatalf("encoding failed %v", err)
}
if err = tid2.UnmarshalBinary(data); err != nil {
t.Fatalf("decoding failed %v", err)
}
txn2 := client2.BatchReadOnlyTransactionFromID(tid2)
// Execute Partitions and compare results
for i, p := range partitions {
iter := txn.Execute(ctx, p)
defer iter.Stop()
p2 := serdesPartition(t, i, p)
iter2 := txn2.Execute(ctx, &p2)
defer iter2.Stop()
row1, err1 := iter.Next()
row2, err2 := iter2.Next()
if err1 != err2 {
t.Fatalf("execution failed for different reasons: %v, %v", err1, err2)
continue
}
if !testEqual(row1, row2) {
t.Fatalf("execution returned different values: %v, %v", row1, row2)
continue
}
if row1 == nil {
continue
}
var a, b string
if err = row1.Columns(&a, &b); err != nil {
t.Fatalf("failed to parse row %v", err)
continue
}
if a == str1 && b == str2 {
gotResult = true
}
}
if !gotResult {
t.Fatalf("execution didn't return expected values")
}
}
// Test PartitionRead of BatchReadOnlyTransaction, similar to TestBatchQuery
func TestBatchRead(t *testing.T) {
t.Parallel()
// Set up testing environment.
var (
client2 *Client
err error
)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
client, dbPath, tearDown := prepare(ctx, t, simpleDBStatements)
defer tearDown()
if err = populate(ctx, client); err != nil {
t.Fatal(err)
}
if client2, err = createClient(ctx, dbPath); err != nil {
t.Fatal(err)
}
defer client2.Close()
// PartitionRead
var (
txn *BatchReadOnlyTransaction
partitions []*Partition
)
if txn, err = client.BatchReadOnlyTransaction(ctx, StrongRead()); err != nil {
t.Fatal(err)
}
defer txn.Cleanup(ctx)
if partitions, err = txn.PartitionRead(ctx, "test", AllKeys(), simpleDBTableColumns, PartitionOptions{0, 3}); err != nil {
t.Fatal(err)
}
// Reconstruct BatchReadOnlyTransactionID and execute partitions
var (
tid2 BatchReadOnlyTransactionID
data []byte
gotResult bool // if we get matching result from two separate txns
)
if data, err = txn.ID.MarshalBinary(); err != nil {
t.Fatalf("encoding failed %v", err)
}
if err = tid2.UnmarshalBinary(data); err != nil {
t.Fatalf("decoding failed %v", err)
}
txn2 := client2.BatchReadOnlyTransactionFromID(tid2)
// Execute Partitions and compare results
for i, p := range partitions {
iter := txn.Execute(ctx, p)
defer iter.Stop()
p2 := serdesPartition(t, i, p)
iter2 := txn2.Execute(ctx, &p2)
defer iter2.Stop()
row1, err1 := iter.Next()
row2, err2 := iter2.Next()
if err1 != err2 {
t.Fatalf("execution failed for different reasons: %v, %v", err1, err2)
continue
}
if !testEqual(row1, row2) {
t.Fatalf("execution returned different values: %v, %v", row1, row2)
continue
}
if row1 == nil {
continue
}
var a, b string
if err = row1.Columns(&a, &b); err != nil {
t.Fatalf("failed to parse row %v", err)
continue
}
if a == str1 && b == str2 {
gotResult = true
}
}
if !gotResult {
t.Fatalf("execution didn't return expected values")
}
}
// Test normal txReadEnv method on BatchReadOnlyTransaction.
func TestBROTNormal(t *testing.T) {
t.Parallel()
// Set up testing environment and create txn.
var (
txn *BatchReadOnlyTransaction
err error
row *Row
i int64
)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
client, _, tearDown := prepare(ctx, t, simpleDBStatements)
defer tearDown()
if txn, err = client.BatchReadOnlyTransaction(ctx, StrongRead()); err != nil {
t.Fatal(err)
}
defer txn.Cleanup(ctx)
if _, err := txn.PartitionRead(ctx, "test", AllKeys(), simpleDBTableColumns, PartitionOptions{0, 3}); err != nil {
t.Fatal(err)
}
// Normal query should work with BatchReadOnlyTransaction
stmt2 := Statement{SQL: "SELECT 1"}
iter := txn.Query(ctx, stmt2)
defer iter.Stop()
row, err = iter.Next()
if err != nil {
t.Errorf("query failed with %v", err)
}
if err = row.Columns(&i); err != nil {
t.Errorf("failed to parse row %v", err)
}
}
+17 -6
View File
@@ -66,12 +66,12 @@ var (
errNoType = errors.New("no type information")
)
// bindParams binds parameters in a Statement to a sppb.ExecuteSqlRequest.
func (s *Statement) bindParams(r *sppb.ExecuteSqlRequest) error {
r.Params = &proto3.Struct{
// bindParams binds parameters in a Statement to a sppb.ExecuteSqlRequest or sppb.PartitionQueryRequest.
func (s *Statement) bindParams(i interface{}) error {
params := &proto3.Struct{
Fields: map[string]*proto3.Value{},
}
r.ParamTypes = map[string]*sppb.Type{}
paramTypes := map[string]*sppb.Type{}
for k, v := range s.Params {
if v == nil {
return errBindParam(k, v, errNilParam)
@@ -83,8 +83,19 @@ func (s *Statement) bindParams(r *sppb.ExecuteSqlRequest) error {
if t == nil { // should not happen, because of nil check above
return errBindParam(k, v, errNoType)
}
r.Params.Fields[k] = val
r.ParamTypes[k] = t
params.Fields[k] = val
paramTypes[k] = t
}
switch r := i.(type) {
default:
return fmt.Errorf("failed to bind query parameter, unexpected request type: %v", r)
case *sppb.ExecuteSqlRequest:
r.Params = params
r.ParamTypes = paramTypes
case *sppb.PartitionQueryRequest:
r.Params = params
r.ParamTypes = paramTypes
}
return nil
}
+17 -22
View File
@@ -39,11 +39,8 @@ const (
// TimestampBound defines how Cloud Spanner will choose a timestamp for a single
// read/query or read-only transaction.
//
// The types of timestamp bound are:
//
// - Strong (the default).
// - Bounded staleness.
// - Exact staleness.
// There are three types of timestamp bound: strong, bounded staleness and exact
// staleness. Strong is the default.
//
// If the Cloud Spanner database to be read is geographically distributed, stale
// read-only transactions can execute more quickly than strong or read-write
@@ -57,7 +54,7 @@ const (
//
// Strong reads are guaranteed to see the effects of all transactions that have
// committed before the start of the read. Furthermore, all rows yielded by a
// single read are consistent with each other - if any part of the read
// single read are consistent with each other: if any part of the read
// observes a transaction, all parts of the read see the transaction.
//
// Strong reads are not repeatable: two consecutive strong read-only
@@ -65,18 +62,17 @@ const (
// writes. If consistency across reads is required, the reads should be
// executed within a transaction or at an exact read timestamp.
//
// Use StrongRead() to create a bound of this type.
// Use StrongRead to create a bound of this type.
//
// Exact staleness
//
// These timestamp bounds execute reads at a user-specified timestamp. Reads at
// a timestamp are guaranteed to see a consistent prefix of the global
// transaction history: they observe modifications done by all transactions
// with a commit timestamp less than or equal to the read timestamp, and
// observe none of the modifications done by transactions with a larger commit
// timestamp. They will block until all conflicting transactions that may be
// assigned commit timestamps less than or equal to the read timestamp have
// finished.
// An exact staleness timestamp bound executes reads at a user-specified timestamp.
// Reads at a timestamp are guaranteed to see a consistent prefix of the global
// transaction history: they observe modifications done by all transactions with a
// commit timestamp less than or equal to the read timestamp, and observe none of the
// modifications done by transactions with a larger commit timestamp. They will block
// until all conflicting transactions that may be assigned commit timestamps less
// than or equal to the read timestamp have finished.
//
// The timestamp can either be expressed as an absolute Cloud Spanner commit
// timestamp or a staleness relative to the current time.
@@ -86,7 +82,7 @@ const (
// concurrency modes. On the other hand, boundedly stale reads usually return
// fresher results.
//
// Use ReadTimestamp() and ExactStaleness() to create a bound of this type.
// Use ReadTimestamp and ExactStaleness to create a bound of this type.
//
// Bounded staleness
//
@@ -95,17 +91,17 @@ const (
// the staleness bound that allows execution of the reads at the closest
// available replica without blocking.
//
// All rows yielded are consistent with each other -- if any part of the read
// All rows yielded are consistent with each other: if any part of the read
// observes a transaction, all parts of the read see the transaction. Boundedly
// stale reads are not repeatable: two stale reads, even if they use the same
// staleness bound, can execute at different timestamps and thus return
// inconsistent results.
//
// Boundedly stale reads execute in two phases: the first phase negotiates a
// Boundedly stale reads execute in two phases. The first phase negotiates a
// timestamp among all replicas needed to serve the read. In the second phase,
// reads are executed at the negotiated timestamp.
//
// As a result of the two phase execution, bounded staleness reads are usually
// As a result of this two-phase execution, bounded staleness reads are usually
// a little slower than comparable exact staleness reads. However, they are
// typically able to return fresher results, and are more likely to execute at
// the closest replica.
@@ -114,7 +110,7 @@ const (
// will be read, it can only be used with single-use reads and single-use
// read-only transactions.
//
// Use MinReadTimestamp() and MaxStaleness() to create a bound of this type.
// Use MinReadTimestamp and MaxStaleness to create a bound of this type.
//
// Old read timestamps and garbage collection
//
@@ -123,7 +119,7 @@ const (
// GC". By default, version GC reclaims versions after they are four hours
// old. Because of this, Cloud Spanner cannot perform reads at read timestamps more
// than four hours in the past. This restriction also applies to in-progress
// reads and/or SQL queries whose timestamp become too old while
// reads and/or SQL queries whose timestamps become too old while
// executing. Reads and SQL queries with too-old read timestamps fail with the
// error ErrorCode.FAILED_PRECONDITION.
type TimestampBound struct {
@@ -174,7 +170,6 @@ func ReadTimestamp(t time.Time) TimestampBound {
}
}
// String implements fmt.Stringer.
func (tb TimestampBound) String() string {
switch tb.mode {
case strong:
+1 -1
View File
@@ -268,7 +268,7 @@ func errUnexpectedTxState(ts txState) error {
// applications do not need to worry about this in practice. See the
// documentation of TimestampBound for more details.
//
// A ReadOnlyTransaction consumes resources on the server until Close() is
// A ReadOnlyTransaction consumes resources on the server until Close is
// called.
type ReadOnlyTransaction struct {
// txReadOnly contains methods for performing transactional reads.
+2 -1
View File
@@ -23,5 +23,6 @@ import (
func testEqual(a, b interface{}) bool {
return testutil.Equal(a, b,
cmp.AllowUnexported(TimestampBound{}, Error{}, Mutation{}, Row{}))
cmp.AllowUnexported(TimestampBound{}, Error{}, Mutation{}, Row{},
Partition{}, BatchReadOnlyTransactionID{}))
}