amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-09-20 02:58:30 +00:00
Code Issues Projects Releases Wiki Activity

Update dependencies

Browse Source 
Among others, this updates minio-go, so that the new "eu-west-3" zone
for AWS is supported.
This commit is contained in:
Alexander Neumann
2018-01-23 19:40:42 +01:00
parent b63de7c798
commit 2b39f9f4b2
3435 changed files with 1317989 additions and 315639 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/cloud.google.com/go/pubsub/apiv1/ListTopics_smoke_test.go
Generated Vendored
+5 -3
View File
@@ -1,10 +1,10 @@
// Copyright 2017, Google Inc. All rights reserved.
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -21,6 +21,7 @@ import (
)
import (
"fmt"
"strconv"
"testing"
"time"
@@ -31,6 +32,7 @@ import (
"google.golang.org/api/option"
)
var _ = fmt.Sprintf
var _ = iterator.Done
var _ = strconv.FormatUint
var _ = time.Now
@@ -53,7 +55,7 @@ func TestPublisherSmoke(t *testing.T) {
t.Fatal(err)
}
var formattedProject string = PublisherProjectPath(projectId)
var formattedProject string = fmt.Sprintf("projects/%s", projectId)
var request = &pubsubpb.ListTopicsRequest{
Project: formattedProject,
}
vendor/cloud.google.com/go/pubsub/apiv1/doc.go
Generated Vendored
+3 -3
View File
@@ -1,10 +1,10 @@
// Copyright 2017, Google Inc. All rights reserved.
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,7 +17,7 @@
// Package pubsub is an auto-generated package for the
// Google Cloud Pub/Sub API.
//
// NOTE: This package is in alpha. It is not stable, and is likely to be subject to changes.
// NOTE: This package is in alpha. It is not stable, and is likely to change.
//
// Provides reliable, many-to-many, asynchronous messaging between
// applications.
vendor/cloud.google.com/go/pubsub/apiv1/mock_test.go
Generated Vendored
+89 -54
View File
@@ -1,10 +1,10 @@
// Copyright 2017, Google Inc. All rights reserved.
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -18,6 +18,7 @@ package pubsub
import (
emptypb "github.com/golang/protobuf/ptypes/empty"
timestamppb "github.com/golang/protobuf/ptypes/timestamp"
iampb "google.golang.org/genproto/googleapis/iam/v1"
pubsubpb "google.golang.org/genproto/googleapis/pubsub/v1"
field_maskpb "google.golang.org/genproto/protobuf/field_mask"
@@ -450,7 +451,7 @@ func TestPublisherCreateTopic(t *testing.T) {
mockPublisher.resps = append(mockPublisher.resps[:0], expectedResponse)
var formattedName string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedName string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.Topic{
Name: formattedName,
}
@@ -479,7 +480,7 @@ func TestPublisherCreateTopicError(t *testing.T) {
errCode := codes.PermissionDenied
mockPublisher.err = gstatus.Error(errCode, "test error")
var formattedName string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedName string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.Topic{
Name: formattedName,
}
@@ -573,7 +574,7 @@ func TestPublisherPublish(t *testing.T) {
mockPublisher.resps = append(mockPublisher.resps[:0], expectedResponse)
var formattedTopic string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var data []byte = []byte("-86")
var messagesElement = &pubsubpb.PubsubMessage{
Data: data,
@@ -608,7 +609,7 @@ func TestPublisherPublishError(t *testing.T) {
errCode := codes.PermissionDenied
mockPublisher.err = gstatus.Error(errCode, "test error")
var formattedTopic string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var data []byte = []byte("-86")
var messagesElement = &pubsubpb.PubsubMessage{
Data: data,
@@ -644,7 +645,7 @@ func TestPublisherGetTopic(t *testing.T) {
mockPublisher.resps = append(mockPublisher.resps[:0], expectedResponse)
var formattedTopic string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.GetTopicRequest{
Topic: formattedTopic,
}
@@ -673,7 +674,7 @@ func TestPublisherGetTopicError(t *testing.T) {
errCode := codes.PermissionDenied
mockPublisher.err = gstatus.Error(errCode, "test error")
var formattedTopic string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.GetTopicRequest{
Topic: formattedTopic,
}
@@ -706,7 +707,7 @@ func TestPublisherListTopics(t *testing.T) {
mockPublisher.resps = append(mockPublisher.resps[:0], expectedResponse)
var formattedProject string = PublisherProjectPath("[PROJECT]")
var formattedProject string = fmt.Sprintf("projects/%s", "[PROJECT]")
var request = &pubsubpb.ListTopicsRequest{
Project: formattedProject,
}
@@ -745,7 +746,7 @@ func TestPublisherListTopicsError(t *testing.T) {
errCode := codes.PermissionDenied
mockPublisher.err = gstatus.Error(errCode, "test error")
var formattedProject string = PublisherProjectPath("[PROJECT]")
var formattedProject string = fmt.Sprintf("projects/%s", "[PROJECT]")
var request = &pubsubpb.ListTopicsRequest{
Project: formattedProject,
}
@@ -778,7 +779,7 @@ func TestPublisherListTopicSubscriptions(t *testing.T) {
mockPublisher.resps = append(mockPublisher.resps[:0], expectedResponse)
var formattedTopic string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.ListTopicSubscriptionsRequest{
Topic: formattedTopic,
}
@@ -817,7 +818,7 @@ func TestPublisherListTopicSubscriptionsError(t *testing.T) {
errCode := codes.PermissionDenied
mockPublisher.err = gstatus.Error(errCode, "test error")
var formattedTopic string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.ListTopicSubscriptionsRequest{
Topic: formattedTopic,
}
@@ -844,7 +845,7 @@ func TestPublisherDeleteTopic(t *testing.T) {
mockPublisher.resps = append(mockPublisher.resps[:0], expectedResponse)
var formattedTopic string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.DeleteTopicRequest{
Topic: formattedTopic,
}
@@ -870,7 +871,7 @@ func TestPublisherDeleteTopicError(t *testing.T) {
errCode := codes.PermissionDenied
mockPublisher.err = gstatus.Error(errCode, "test error")
var formattedTopic string = PublisherTopicPath("[PROJECT]", "[TOPIC]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.DeleteTopicRequest{
Topic: formattedTopic,
}
@@ -905,8 +906,8 @@ func TestSubscriberCreateSubscription(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedName string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedTopic string = SubscriberTopicPath("[PROJECT]", "[TOPIC]")
var formattedName string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.Subscription{
Name: formattedName,
Topic: formattedTopic,
@@ -936,8 +937,8 @@ func TestSubscriberCreateSubscriptionError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedName string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedTopic string = SubscriberTopicPath("[PROJECT]", "[TOPIC]")
var formattedName string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var formattedTopic string = fmt.Sprintf("projects/%s/topics/%s", "[PROJECT]", "[TOPIC]")
var request = &pubsubpb.Subscription{
Name: formattedName,
Topic: formattedTopic,
@@ -974,7 +975,7 @@ func TestSubscriberGetSubscription(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var request = &pubsubpb.GetSubscriptionRequest{
Subscription: formattedSubscription,
}
@@ -1003,7 +1004,7 @@ func TestSubscriberGetSubscriptionError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var request = &pubsubpb.GetSubscriptionRequest{
Subscription: formattedSubscription,
}
@@ -1025,12 +1026,12 @@ func TestSubscriberGetSubscriptionError(t *testing.T) {
func TestSubscriberUpdateSubscription(t *testing.T) {
var name string = "name3373707"
var topic string = "topic110546223"
var ackDeadlineSeconds int32 = 2135351438
var ackDeadlineSeconds2 int32 = 921632575
var retainAckedMessages bool = false
var expectedResponse = &pubsubpb.Subscription{
Name: name,
Topic: topic,
AckDeadlineSeconds: ackDeadlineSeconds,
AckDeadlineSeconds: ackDeadlineSeconds2,
RetainAckedMessages: retainAckedMessages,
}
@@ -1039,8 +1040,15 @@ func TestSubscriberUpdateSubscription(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var subscription *pubsubpb.Subscription = &pubsubpb.Subscription{}
var updateMask *field_maskpb.FieldMask = &field_maskpb.FieldMask{}
var ackDeadlineSeconds int32 = 42
var subscription = &pubsubpb.Subscription{
AckDeadlineSeconds: ackDeadlineSeconds,
}
var pathsElement string = "ack_deadline_seconds"
var paths = []string{pathsElement}
var updateMask = &field_maskpb.FieldMask{
Paths: paths,
}
var request = &pubsubpb.UpdateSubscriptionRequest{
Subscription: subscription,
UpdateMask: updateMask,
@@ -1070,8 +1078,15 @@ func TestSubscriberUpdateSubscriptionError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var subscription *pubsubpb.Subscription = &pubsubpb.Subscription{}
var updateMask *field_maskpb.FieldMask = &field_maskpb.FieldMask{}
var ackDeadlineSeconds int32 = 42
var subscription = &pubsubpb.Subscription{
AckDeadlineSeconds: ackDeadlineSeconds,
}
var pathsElement string = "ack_deadline_seconds"
var paths = []string{pathsElement}
var updateMask = &field_maskpb.FieldMask{
Paths: paths,
}
var request = &pubsubpb.UpdateSubscriptionRequest{
Subscription: subscription,
UpdateMask: updateMask,
@@ -1105,7 +1120,7 @@ func TestSubscriberListSubscriptions(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedProject string = SubscriberProjectPath("[PROJECT]")
var formattedProject string = fmt.Sprintf("projects/%s", "[PROJECT]")
var request = &pubsubpb.ListSubscriptionsRequest{
Project: formattedProject,
}
@@ -1144,7 +1159,7 @@ func TestSubscriberListSubscriptionsError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedProject string = SubscriberProjectPath("[PROJECT]")
var formattedProject string = fmt.Sprintf("projects/%s", "[PROJECT]")
var request = &pubsubpb.ListSubscriptionsRequest{
Project: formattedProject,
}
@@ -1171,7 +1186,7 @@ func TestSubscriberDeleteSubscription(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var request = &pubsubpb.DeleteSubscriptionRequest{
Subscription: formattedSubscription,
}
@@ -1197,7 +1212,7 @@ func TestSubscriberDeleteSubscriptionError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var request = &pubsubpb.DeleteSubscriptionRequest{
Subscription: formattedSubscription,
}
@@ -1223,7 +1238,7 @@ func TestSubscriberModifyAckDeadline(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var ackIds []string = nil
var ackDeadlineSeconds int32 = 2135351438
var request = &pubsubpb.ModifyAckDeadlineRequest{
@@ -1253,7 +1268,7 @@ func TestSubscriberModifyAckDeadlineError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var ackIds []string = nil
var ackDeadlineSeconds int32 = 2135351438
var request = &pubsubpb.ModifyAckDeadlineRequest{
@@ -1283,7 +1298,7 @@ func TestSubscriberAcknowledge(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var ackIds []string = nil
var request = &pubsubpb.AcknowledgeRequest{
Subscription: formattedSubscription,
@@ -1311,7 +1326,7 @@ func TestSubscriberAcknowledgeError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var ackIds []string = nil
var request = &pubsubpb.AcknowledgeRequest{
Subscription: formattedSubscription,
@@ -1339,7 +1354,7 @@ func TestSubscriberPull(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var maxMessages int32 = 496131527
var request = &pubsubpb.PullRequest{
Subscription: formattedSubscription,
@@ -1370,7 +1385,7 @@ func TestSubscriberPullError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var maxMessages int32 = 496131527
var request = &pubsubpb.PullRequest{
Subscription: formattedSubscription,
@@ -1403,7 +1418,7 @@ func TestSubscriberStreamingPull(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var streamAckDeadlineSeconds int32 = 1875467245
var request = &pubsubpb.StreamingPullRequest{
Subscription: formattedSubscription,
@@ -1444,7 +1459,7 @@ func TestSubscriberStreamingPullError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var streamAckDeadlineSeconds int32 = 1875467245
var request = &pubsubpb.StreamingPullRequest{
Subscription: formattedSubscription,
@@ -1483,7 +1498,7 @@ func TestSubscriberModifyPushConfig(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var pushConfig *pubsubpb.PushConfig = &pubsubpb.PushConfig{}
var request = &pubsubpb.ModifyPushConfigRequest{
Subscription: formattedSubscription,
@@ -1511,7 +1526,7 @@ func TestSubscriberModifyPushConfigError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var pushConfig *pubsubpb.PushConfig = &pubsubpb.PushConfig{}
var request = &pubsubpb.ModifyPushConfigRequest{
Subscription: formattedSubscription,
@@ -1545,7 +1560,7 @@ func TestSubscriberListSnapshots(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedProject string = SubscriberProjectPath("[PROJECT]")
var formattedProject string = fmt.Sprintf("projects/%s", "[PROJECT]")
var request = &pubsubpb.ListSnapshotsRequest{
Project: formattedProject,
}
@@ -1584,7 +1599,7 @@ func TestSubscriberListSnapshotsError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedProject string = SubscriberProjectPath("[PROJECT]")
var formattedProject string = fmt.Sprintf("projects/%s", "[PROJECT]")
var request = &pubsubpb.ListSnapshotsRequest{
Project: formattedProject,
}
@@ -1616,8 +1631,8 @@ func TestSubscriberCreateSnapshot(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedName string = SubscriberSnapshotPath("[PROJECT]", "[SNAPSHOT]")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedName string = fmt.Sprintf("projects/%s/snapshots/%s", "[PROJECT]", "[SNAPSHOT]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var request = &pubsubpb.CreateSnapshotRequest{
Name: formattedName,
Subscription: formattedSubscription,
@@ -1647,8 +1662,8 @@ func TestSubscriberCreateSnapshotError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedName string = SubscriberSnapshotPath("[PROJECT]", "[SNAPSHOT]")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedName string = fmt.Sprintf("projects/%s/snapshots/%s", "[PROJECT]", "[SNAPSHOT]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var request = &pubsubpb.CreateSnapshotRequest{
Name: formattedName,
Subscription: formattedSubscription,
@@ -1681,8 +1696,18 @@ func TestSubscriberUpdateSnapshot(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var snapshot *pubsubpb.Snapshot = &pubsubpb.Snapshot{}
var updateMask *field_maskpb.FieldMask = &field_maskpb.FieldMask{}
var seconds int64 = 123456
var expireTime = &timestamppb.Timestamp{
Seconds: seconds,
}
var snapshot = &pubsubpb.Snapshot{
ExpireTime: expireTime,
}
var pathsElement string = "expire_time"
var paths = []string{pathsElement}
var updateMask = &field_maskpb.FieldMask{
Paths: paths,
}
var request = &pubsubpb.UpdateSnapshotRequest{
Snapshot: snapshot,
UpdateMask: updateMask,
@@ -1712,8 +1737,18 @@ func TestSubscriberUpdateSnapshotError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var snapshot *pubsubpb.Snapshot = &pubsubpb.Snapshot{}
var updateMask *field_maskpb.FieldMask = &field_maskpb.FieldMask{}
var seconds int64 = 123456
var expireTime = &timestamppb.Timestamp{
Seconds: seconds,
}
var snapshot = &pubsubpb.Snapshot{
ExpireTime: expireTime,
}
var pathsElement string = "expire_time"
var paths = []string{pathsElement}
var updateMask = &field_maskpb.FieldMask{
Paths: paths,
}
var request = &pubsubpb.UpdateSnapshotRequest{
Snapshot: snapshot,
UpdateMask: updateMask,
@@ -1741,7 +1776,7 @@ func TestSubscriberDeleteSnapshot(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSnapshot string = SubscriberSnapshotPath("[PROJECT]", "[SNAPSHOT]")
var formattedSnapshot string = fmt.Sprintf("projects/%s/snapshots/%s", "[PROJECT]", "[SNAPSHOT]")
var request = &pubsubpb.DeleteSnapshotRequest{
Snapshot: formattedSnapshot,
}
@@ -1767,7 +1802,7 @@ func TestSubscriberDeleteSnapshotError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSnapshot string = SubscriberSnapshotPath("[PROJECT]", "[SNAPSHOT]")
var formattedSnapshot string = fmt.Sprintf("projects/%s/snapshots/%s", "[PROJECT]", "[SNAPSHOT]")
var request = &pubsubpb.DeleteSnapshotRequest{
Snapshot: formattedSnapshot,
}
@@ -1793,7 +1828,7 @@ func TestSubscriberSeek(t *testing.T) {
mockSubscriber.resps = append(mockSubscriber.resps[:0], expectedResponse)
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var request = &pubsubpb.SeekRequest{
Subscription: formattedSubscription,
}
@@ -1822,7 +1857,7 @@ func TestSubscriberSeekError(t *testing.T) {
errCode := codes.PermissionDenied
mockSubscriber.err = gstatus.Error(errCode, "test error")
var formattedSubscription string = SubscriberSubscriptionPath("[PROJECT]", "[SUBSCRIPTION]")
var formattedSubscription string = fmt.Sprintf("projects/%s/subscriptions/%s", "[PROJECT]", "[SUBSCRIPTION]")
var request = &pubsubpb.SeekRequest{
Subscription: formattedSubscription,
}
vendor/cloud.google.com/go/pubsub/apiv1/path_funcs.go
Generated Vendored
+95
View File
@@ -0,0 +1,95 @@
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pubsub
// PublisherProjectPath returns the path for the project resource.
//
// Deprecated: Use
// fmt.Sprintf("projects/%s", project)
// instead.
func PublisherProjectPath(project string) string {
return "" +
"projects/" +
project +
""
}
// PublisherTopicPath returns the path for the topic resource.
//
// Deprecated: Use
// fmt.Sprintf("projects/%s/topics/%s", project, topic)
// instead.
func PublisherTopicPath(project, topic string) string {
return "" +
"projects/" +
project +
"/topics/" +
topic +
""
}
// SubscriberProjectPath returns the path for the project resource.
//
// Deprecated: Use
// fmt.Sprintf("projects/%s", project)
// instead.
func SubscriberProjectPath(project string) string {
return "" +
"projects/" +
project +
""
}
// SubscriberSnapshotPath returns the path for the snapshot resource.
//
// Deprecated: Use
// fmt.Sprintf("projects/%s/snapshots/%s", project, snapshot)
// instead.
func SubscriberSnapshotPath(project, snapshot string) string {
return "" +
"projects/" +
project +
"/snapshots/" +
snapshot +
""
}
// SubscriberSubscriptionPath returns the path for the subscription resource.
//
// Deprecated: Use
// fmt.Sprintf("projects/%s/subscriptions/%s", project, subscription)
// instead.
func SubscriberSubscriptionPath(project, subscription string) string {
return "" +
"projects/" +
project +
"/subscriptions/" +
subscription +
""
}
// SubscriberTopicPath returns the path for the topic resource.
//
// Deprecated: Use
// fmt.Sprintf("projects/%s/topics/%s", project, topic)
// instead.
func SubscriberTopicPath(project, topic string) string {
return "" +
"projects/" +
project +
"/topics/" +
topic +
""
}
vendor/cloud.google.com/go/pubsub/apiv1/publisher_client.go
Generated Vendored
+12 -30
View File
@@ -1,10 +1,10 @@
// Copyright 2017, Google Inc. All rights reserved.
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -105,8 +105,8 @@ type PublisherClient struct {
// The call options for this service.
CallOptions *PublisherCallOptions
// The metadata to be sent with each request.
Metadata metadata.MD
// The x-goog-* metadata to be sent with each request.
xGoogMetadata metadata.MD
}
// NewPublisherClient creates a new publisher client.
@@ -145,25 +145,7 @@ func (c *PublisherClient) Close() error {
func (c *PublisherClient) SetGoogleClientInfo(keyval ...string) {
kv := append([]string{"gl-go", version.Go()}, keyval...)
kv = append(kv, "gapic", version.Repo, "gax", gax.Version, "grpc", grpc.Version)
c.Metadata = metadata.Pairs("x-goog-api-client", gax.XGoogHeader(kv...))
}
// PublisherProjectPath returns the path for the project resource.
func PublisherProjectPath(project string) string {
return "" +
"projects/" +
project +
""
}
// PublisherTopicPath returns the path for the topic resource.
func PublisherTopicPath(project, topic string) string {
return "" +
"projects/" +
project +
"/topics/" +
topic +
""
c.xGoogMetadata = metadata.Pairs("x-goog-api-client", gax.XGoogHeader(kv...))
}
func (c *PublisherClient) SubscriptionIAM(subscription *pubsubpb.Subscription) *iam.Handle {
@@ -176,7 +158,7 @@ func (c *PublisherClient) TopicIAM(topic *pubsubpb.Topic) *iam.Handle {
// CreateTopic creates the given topic with the given name.
func (c *PublisherClient) CreateTopic(ctx context.Context, req *pubsubpb.Topic, opts ...gax.CallOption) (*pubsubpb.Topic, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.CreateTopic[0:len(c.CallOptions.CreateTopic):len(c.CallOptions.CreateTopic)], opts...)
var resp *pubsubpb.Topic
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -197,7 +179,7 @@ func (c *PublisherClient) CreateTopic(ctx context.Context, req *pubsubpb.Topic,
// corrected in V2. See
// https://cloud.google.com/apis/design/standard_methods#update for details.
func (c *PublisherClient) UpdateTopic(ctx context.Context, req *pubsubpb.UpdateTopicRequest, opts ...gax.CallOption) (*pubsubpb.Topic, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.UpdateTopic[0:len(c.CallOptions.UpdateTopic):len(c.CallOptions.UpdateTopic)], opts...)
var resp *pubsubpb.Topic
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -215,7 +197,7 @@ func (c *PublisherClient) UpdateTopic(ctx context.Context, req *pubsubpb.UpdateT
// does not exist. The message payload must not be empty; it must contain
// either a non-empty data field, or at least one attribute.
func (c *PublisherClient) Publish(ctx context.Context, req *pubsubpb.PublishRequest, opts ...gax.CallOption) (*pubsubpb.PublishResponse, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.Publish[0:len(c.CallOptions.Publish):len(c.CallOptions.Publish)], opts...)
var resp *pubsubpb.PublishResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -231,7 +213,7 @@ func (c *PublisherClient) Publish(ctx context.Context, req *pubsubpb.PublishRequ
// GetTopic gets the configuration of a topic.
func (c *PublisherClient) GetTopic(ctx context.Context, req *pubsubpb.GetTopicRequest, opts ...gax.CallOption) (*pubsubpb.Topic, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.GetTopic[0:len(c.CallOptions.GetTopic):len(c.CallOptions.GetTopic)], opts...)
var resp *pubsubpb.Topic
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -247,7 +229,7 @@ func (c *PublisherClient) GetTopic(ctx context.Context, req *pubsubpb.GetTopicRe
// ListTopics lists matching topics.
func (c *PublisherClient) ListTopics(ctx context.Context, req *pubsubpb.ListTopicsRequest, opts ...gax.CallOption) *TopicIterator {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.ListTopics[0:len(c.CallOptions.ListTopics):len(c.CallOptions.ListTopics)], opts...)
it := &TopicIterator{}
it.InternalFetch = func(pageSize int, pageToken string) ([]*pubsubpb.Topic, string, error) {
@@ -282,7 +264,7 @@ func (c *PublisherClient) ListTopics(ctx context.Context, req *pubsubpb.ListTopi
// ListTopicSubscriptions lists the name of the subscriptions for this topic.
func (c *PublisherClient) ListTopicSubscriptions(ctx context.Context, req *pubsubpb.ListTopicSubscriptionsRequest, opts ...gax.CallOption) *StringIterator {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.ListTopicSubscriptions[0:len(c.CallOptions.ListTopicSubscriptions):len(c.CallOptions.ListTopicSubscriptions)], opts...)
it := &StringIterator{}
it.InternalFetch = func(pageSize int, pageToken string) ([]string, string, error) {
@@ -321,7 +303,7 @@ func (c *PublisherClient) ListTopicSubscriptions(ctx context.Context, req *pubsu
// configuration or subscriptions. Existing subscriptions to this topic are
// not deleted, but their topic field is set to _deleted-topic_.
func (c *PublisherClient) DeleteTopic(ctx context.Context, req *pubsubpb.DeleteTopicRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.DeleteTopic[0:len(c.CallOptions.DeleteTopic):len(c.CallOptions.DeleteTopic)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
vendor/cloud.google.com/go/pubsub/apiv1/publisher_client_example_test.go
Generated Vendored
+9 -9
View File
@@ -1,10 +1,10 @@
// Copyright 2017, Google Inc. All rights reserved.
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -75,7 +75,7 @@ func ExamplePublisherClient_CreateTopic() {
}
req := &pubsubpb.Topic{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.CreateTopic(ctx, req)
if err != nil {
@@ -93,7 +93,7 @@ func ExamplePublisherClient_UpdateTopic() {
}
req := &pubsubpb.UpdateTopicRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.UpdateTopic(ctx, req)
if err != nil {
@@ -111,7 +111,7 @@ func ExamplePublisherClient_Publish() {
}
req := &pubsubpb.PublishRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.Publish(ctx, req)
if err != nil {
@@ -129,7 +129,7 @@ func ExamplePublisherClient_GetTopic() {
}
req := &pubsubpb.GetTopicRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.GetTopic(ctx, req)
if err != nil {
@@ -147,7 +147,7 @@ func ExamplePublisherClient_ListTopics() {
}
req := &pubsubpb.ListTopicsRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
it := c.ListTopics(ctx, req)
for {
@@ -171,7 +171,7 @@ func ExamplePublisherClient_ListTopicSubscriptions() {
}
req := &pubsubpb.ListTopicSubscriptionsRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
it := c.ListTopicSubscriptions(ctx, req)
for {
@@ -195,7 +195,7 @@ func ExamplePublisherClient_DeleteTopic() {
}
req := &pubsubpb.DeleteTopicRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
err = c.DeleteTopic(ctx, req)
if err != nil {
vendor/cloud.google.com/go/pubsub/apiv1/subscriber_client.go
Generated Vendored
+20 -58
View File
@@ -1,10 +1,10 @@
// Copyright 2017, Google Inc. All rights reserved.
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -134,8 +134,8 @@ type SubscriberClient struct {
// The call options for this service.
CallOptions *SubscriberCallOptions
// The metadata to be sent with each request.
Metadata metadata.MD
// The x-goog-* metadata to be sent with each request.
xGoogMetadata metadata.MD
}
// NewSubscriberClient creates a new subscriber client.
@@ -174,45 +174,7 @@ func (c *SubscriberClient) Close() error {
func (c *SubscriberClient) SetGoogleClientInfo(keyval ...string) {
kv := append([]string{"gl-go", version.Go()}, keyval...)
kv = append(kv, "gapic", version.Repo, "gax", gax.Version, "grpc", grpc.Version)
c.Metadata = metadata.Pairs("x-goog-api-client", gax.XGoogHeader(kv...))
}
// SubscriberProjectPath returns the path for the project resource.
func SubscriberProjectPath(project string) string {
return "" +
"projects/" +
project +
""
}
// SubscriberSnapshotPath returns the path for the snapshot resource.
func SubscriberSnapshotPath(project, snapshot string) string {
return "" +
"projects/" +
project +
"/snapshots/" +
snapshot +
""
}
// SubscriberSubscriptionPath returns the path for the subscription resource.
func SubscriberSubscriptionPath(project, subscription string) string {
return "" +
"projects/" +
project +
"/subscriptions/" +
subscription +
""
}
// SubscriberTopicPath returns the path for the topic resource.
func SubscriberTopicPath(project, topic string) string {
return "" +
"projects/" +
project +
"/topics/" +
topic +
""
c.xGoogMetadata = metadata.Pairs("x-goog-api-client", gax.XGoogHeader(kv...))
}
func (c *SubscriberClient) SubscriptionIAM(subscription *pubsubpb.Subscription) *iam.Handle {
@@ -234,7 +196,7 @@ func (c *SubscriberClient) TopicIAM(topic *pubsubpb.Topic) *iam.Handle {
// The generated name is populated in the returned Subscription object.
// Note that for REST API requests, you must specify a name in the request.
func (c *SubscriberClient) CreateSubscription(ctx context.Context, req *pubsubpb.Subscription, opts ...gax.CallOption) (*pubsubpb.Subscription, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.CreateSubscription[0:len(c.CallOptions.CreateSubscription):len(c.CallOptions.CreateSubscription)], opts...)
var resp *pubsubpb.Subscription
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -250,7 +212,7 @@ func (c *SubscriberClient) CreateSubscription(ctx context.Context, req *pubsubpb
// GetSubscription gets the configuration details of a subscription.
func (c *SubscriberClient) GetSubscription(ctx context.Context, req *pubsubpb.GetSubscriptionRequest, opts ...gax.CallOption) (*pubsubpb.Subscription, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.GetSubscription[0:len(c.CallOptions.GetSubscription):len(c.CallOptions.GetSubscription)], opts...)
var resp *pubsubpb.Subscription
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -271,7 +233,7 @@ func (c *SubscriberClient) GetSubscription(ctx context.Context, req *pubsubpb.Ge
// corrected in V2. See
// https://cloud.google.com/apis/design/standard_methods#update for details.
func (c *SubscriberClient) UpdateSubscription(ctx context.Context, req *pubsubpb.UpdateSubscriptionRequest, opts ...gax.CallOption) (*pubsubpb.Subscription, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.UpdateSubscription[0:len(c.CallOptions.UpdateSubscription):len(c.CallOptions.UpdateSubscription)], opts...)
var resp *pubsubpb.Subscription
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -287,7 +249,7 @@ func (c *SubscriberClient) UpdateSubscription(ctx context.Context, req *pubsubpb
// ListSubscriptions lists matching subscriptions.
func (c *SubscriberClient) ListSubscriptions(ctx context.Context, req *pubsubpb.ListSubscriptionsRequest, opts ...gax.CallOption) *SubscriptionIterator {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.ListSubscriptions[0:len(c.CallOptions.ListSubscriptions):len(c.CallOptions.ListSubscriptions)], opts...)
it := &SubscriptionIterator{}
it.InternalFetch = func(pageSize int, pageToken string) ([]*pubsubpb.Subscription, string, error) {
@@ -326,7 +288,7 @@ func (c *SubscriberClient) ListSubscriptions(ctx context.Context, req *pubsubpb.
// the same name, but the new one has no association with the old
// subscription or its topic unless the same topic is specified.
func (c *SubscriberClient) DeleteSubscription(ctx context.Context, req *pubsubpb.DeleteSubscriptionRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.DeleteSubscription[0:len(c.CallOptions.DeleteSubscription):len(c.CallOptions.DeleteSubscription)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -342,7 +304,7 @@ func (c *SubscriberClient) DeleteSubscription(ctx context.Context, req *pubsubpb
// processing was interrupted. Note that this does not modify the
// subscription-level ackDeadlineSeconds used for subsequent messages.
func (c *SubscriberClient) ModifyAckDeadline(ctx context.Context, req *pubsubpb.ModifyAckDeadlineRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.ModifyAckDeadline[0:len(c.CallOptions.ModifyAckDeadline):len(c.CallOptions.ModifyAckDeadline)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -360,7 +322,7 @@ func (c *SubscriberClient) ModifyAckDeadline(ctx context.Context, req *pubsubpb.
// but such a message may be redelivered later. Acknowledging a message more
// than once will not result in an error.
func (c *SubscriberClient) Acknowledge(ctx context.Context, req *pubsubpb.AcknowledgeRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.Acknowledge[0:len(c.CallOptions.Acknowledge):len(c.CallOptions.Acknowledge)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -375,7 +337,7 @@ func (c *SubscriberClient) Acknowledge(ctx context.Context, req *pubsubpb.Acknow
// there are too many concurrent pull requests pending for the given
// subscription.
func (c *SubscriberClient) Pull(ctx context.Context, req *pubsubpb.PullRequest, opts ...gax.CallOption) (*pubsubpb.PullResponse, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.Pull[0:len(c.CallOptions.Pull):len(c.CallOptions.Pull)], opts...)
var resp *pubsubpb.PullResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -402,7 +364,7 @@ func (c *SubscriberClient) Pull(ctx context.Context, req *pubsubpb.PullRequest,
// (e.g., a server restart). These should also be retried by the client. Flow
// control can be achieved by configuring the underlying RPC channel.
func (c *SubscriberClient) StreamingPull(ctx context.Context, opts ...gax.CallOption) (pubsubpb.Subscriber_StreamingPullClient, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.StreamingPull[0:len(c.CallOptions.StreamingPull):len(c.CallOptions.StreamingPull)], opts...)
var resp pubsubpb.Subscriber_StreamingPullClient
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -423,7 +385,7 @@ func (c *SubscriberClient) StreamingPull(ctx context.Context, opts ...gax.CallOp
// attributes of a push subscription. Messages will accumulate for delivery
// continuously through the call regardless of changes to the PushConfig.
func (c *SubscriberClient) ModifyPushConfig(ctx context.Context, req *pubsubpb.ModifyPushConfigRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.ModifyPushConfig[0:len(c.CallOptions.ModifyPushConfig):len(c.CallOptions.ModifyPushConfig)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -435,7 +397,7 @@ func (c *SubscriberClient) ModifyPushConfig(ctx context.Context, req *pubsubpb.M
// ListSnapshots lists the existing snapshots.
func (c *SubscriberClient) ListSnapshots(ctx context.Context, req *pubsubpb.ListSnapshotsRequest, opts ...gax.CallOption) *SnapshotIterator {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.ListSnapshots[0:len(c.CallOptions.ListSnapshots):len(c.CallOptions.ListSnapshots)], opts...)
it := &SnapshotIterator{}
it.InternalFetch = func(pageSize int, pageToken string) ([]*pubsubpb.Snapshot, string, error) {
@@ -479,7 +441,7 @@ func (c *SubscriberClient) ListSnapshots(ctx context.Context, req *pubsubpb.List
// The generated name is populated in the returned Snapshot object.
// Note that for REST API requests, you must specify a name in the request.
func (c *SubscriberClient) CreateSnapshot(ctx context.Context, req *pubsubpb.CreateSnapshotRequest, opts ...gax.CallOption) (*pubsubpb.Snapshot, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.CreateSnapshot[0:len(c.CallOptions.CreateSnapshot):len(c.CallOptions.CreateSnapshot)], opts...)
var resp *pubsubpb.Snapshot
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -500,7 +462,7 @@ func (c *SubscriberClient) CreateSnapshot(ctx context.Context, req *pubsubpb.Cre
// corrected in V2. See
// https://cloud.google.com/apis/design/standard_methods#update for details.
func (c *SubscriberClient) UpdateSnapshot(ctx context.Context, req *pubsubpb.UpdateSnapshotRequest, opts ...gax.CallOption) (*pubsubpb.Snapshot, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.UpdateSnapshot[0:len(c.CallOptions.UpdateSnapshot):len(c.CallOptions.UpdateSnapshot)], opts...)
var resp *pubsubpb.Snapshot
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
@@ -519,7 +481,7 @@ func (c *SubscriberClient) UpdateSnapshot(ctx context.Context, req *pubsubpb.Upd
// created with the same name, but the new one has no association with the old
// snapshot or its subscription, unless the same subscription is specified.
func (c *SubscriberClient) DeleteSnapshot(ctx context.Context, req *pubsubpb.DeleteSnapshotRequest, opts ...gax.CallOption) error {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.DeleteSnapshot[0:len(c.CallOptions.DeleteSnapshot):len(c.CallOptions.DeleteSnapshot)], opts...)
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
@@ -532,7 +494,7 @@ func (c *SubscriberClient) DeleteSnapshot(ctx context.Context, req *pubsubpb.Del
// Seek seeks an existing subscription to a point in time or to a given snapshot,
// whichever is provided in the request.
func (c *SubscriberClient) Seek(ctx context.Context, req *pubsubpb.SeekRequest, opts ...gax.CallOption) (*pubsubpb.SeekResponse, error) {
ctx = insertMetadata(ctx, c.Metadata)
ctx = insertMetadata(ctx, c.xGoogMetadata)
opts = append(c.CallOptions.Seek[0:len(c.CallOptions.Seek):len(c.CallOptions.Seek)], opts...)
var resp *pubsubpb.SeekResponse
err := gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
vendor/cloud.google.com/go/pubsub/apiv1/subscriber_client_example_test.go
Generated Vendored
+17 -17
View File
@@ -1,10 +1,10 @@
// Copyright 2017, Google Inc. All rights reserved.
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -77,7 +77,7 @@ func ExampleSubscriberClient_CreateSubscription() {
}
req := &pubsubpb.Subscription{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.CreateSubscription(ctx, req)
if err != nil {
@@ -95,7 +95,7 @@ func ExampleSubscriberClient_GetSubscription() {
}
req := &pubsubpb.GetSubscriptionRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.GetSubscription(ctx, req)
if err != nil {
@@ -113,7 +113,7 @@ func ExampleSubscriberClient_UpdateSubscription() {
}
req := &pubsubpb.UpdateSubscriptionRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.UpdateSubscription(ctx, req)
if err != nil {
@@ -131,7 +131,7 @@ func ExampleSubscriberClient_ListSubscriptions() {
}
req := &pubsubpb.ListSubscriptionsRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
it := c.ListSubscriptions(ctx, req)
for {
@@ -155,7 +155,7 @@ func ExampleSubscriberClient_DeleteSubscription() {
}
req := &pubsubpb.DeleteSubscriptionRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
err = c.DeleteSubscription(ctx, req)
if err != nil {
@@ -171,7 +171,7 @@ func ExampleSubscriberClient_ModifyAckDeadline() {
}
req := &pubsubpb.ModifyAckDeadlineRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
err = c.ModifyAckDeadline(ctx, req)
if err != nil {
@@ -187,7 +187,7 @@ func ExampleSubscriberClient_Acknowledge() {
}
req := &pubsubpb.AcknowledgeRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
err = c.Acknowledge(ctx, req)
if err != nil {
@@ -203,7 +203,7 @@ func ExampleSubscriberClient_Pull() {
}
req := &pubsubpb.PullRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.Pull(ctx, req)
if err != nil {
@@ -225,7 +225,7 @@ func ExampleSubscriberClient_StreamingPull() {
}
go func() {
reqs := []*pubsubpb.StreamingPullRequest{
// TODO: Create requests.
// TODO: Create requests.
}
for _, req := range reqs {
if err := stream.Send(req); err != nil {
@@ -255,7 +255,7 @@ func ExampleSubscriberClient_ModifyPushConfig() {
}
req := &pubsubpb.ModifyPushConfigRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
err = c.ModifyPushConfig(ctx, req)
if err != nil {
@@ -271,7 +271,7 @@ func ExampleSubscriberClient_ListSnapshots() {
}
req := &pubsubpb.ListSnapshotsRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
it := c.ListSnapshots(ctx, req)
for {
@@ -295,7 +295,7 @@ func ExampleSubscriberClient_CreateSnapshot() {
}
req := &pubsubpb.CreateSnapshotRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.CreateSnapshot(ctx, req)
if err != nil {
@@ -313,7 +313,7 @@ func ExampleSubscriberClient_UpdateSnapshot() {
}
req := &pubsubpb.UpdateSnapshotRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.UpdateSnapshot(ctx, req)
if err != nil {
@@ -331,7 +331,7 @@ func ExampleSubscriberClient_DeleteSnapshot() {
}
req := &pubsubpb.DeleteSnapshotRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
err = c.DeleteSnapshot(ctx, req)
if err != nil {
@@ -347,7 +347,7 @@ func ExampleSubscriberClient_Seek() {
}
req := &pubsubpb.SeekRequest{
// TODO: Fill request struct fields.
// TODO: Fill request struct fields.
}
resp, err := c.Seek(ctx, req)
if err != nil {
vendor/cloud.google.com/go/pubsub/endtoend_test.go
Generated Vendored
+1 -2
View File
@@ -20,7 +20,6 @@ import (
"log"
"math/rand"
"os"
"reflect"
"sync"
"testing"
"time"
@@ -147,7 +146,7 @@ loop:
wg.Wait()
ok := true
for i, con := range consumers {
if got, want := con.counts, wantCounts; !reflect.DeepEqual(got, want) {
if got, want := con.counts, wantCounts; !testutil.Equal(got, want) {
t.Errorf("%d: message counts: %v\n", i, diff(got, want))
ok = false
}
vendor/cloud.google.com/go/pubsub/example_test.go
Generated Vendored
+88
View File
@@ -279,3 +279,91 @@ func ExampleSubscription_Update() {
}
_ = subConfig // TODO: Use SubscriptionConfig.
}
func ExampleSubscription_CreateSnapshot() {
ctx := context.Background()
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
// TODO: Handle error.
}
sub := client.Subscription("subName")
snapConfig, err := sub.CreateSnapshot(ctx, "snapshotName")
if err != nil {
// TODO: Handle error.
}
_ = snapConfig // TODO: Use SnapshotConfig.
}
func ExampleSubscription_SeekToSnapshot() {
ctx := context.Background()
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
// TODO: Handle error.
}
sub := client.Subscription("subName")
snap := client.Snapshot("snapshotName")
if err := sub.SeekToSnapshot(ctx, snap); err != nil {
// TODO: Handle error.
}
}
func ExampleSubscription_SeekToTime() {
ctx := context.Background()
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
// TODO: Handle error.
}
sub := client.Subscription("subName")
if err := sub.SeekToTime(ctx, time.Now().Add(-time.Hour)); err != nil {
// TODO: Handle error.
}
}
func ExampleSnapshot_Delete() {
ctx := context.Background()
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
// TODO: Handle error.
}
snap := client.Snapshot("snapshotName")
if err := snap.Delete(ctx); err != nil {
// TODO: Handle error.
}
}
func ExampleClient_Snapshots() {
ctx := context.Background()
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
// TODO: Handle error.
}
// List all snapshots for the project.
iter := client.Snapshots(ctx)
_ = iter // TODO: iterate using Next.
}
func ExampleSnapshotConfigIterator_Next() {
ctx := context.Background()
client, err := pubsub.NewClient(ctx, "project-id")
if err != nil {
// TODO: Handle error.
}
// List all snapshots for the project.
iter := client.Snapshots(ctx)
for {
snapConfig, err := iter.Next()
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
_ = snapConfig // TODO: use the SnapshotConfig.
}
}
// TODO(jba): write an example for PublishResult.Ready
// TODO(jba): write an example for Subscription.IAM
// TODO(jba): write an example for Topic.IAM
// TODO(jba): write an example for Topic.Stop
vendor/cloud.google.com/go/pubsub/fake_test.go
Generated Vendored
+181 -6
View File
@@ -15,17 +15,23 @@
package pubsub
// This file provides a fake/mock in-memory pubsub server.
// (Really just a mock at the moment, but we hope to turn it into
// more of a fake.)
import (
"io"
"sort"
"strings"
"sync"
"time"
"cloud.google.com/go/internal/testutil"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
durpb "github.com/golang/protobuf/ptypes/duration"
emptypb "github.com/golang/protobuf/ptypes/empty"
"golang.org/x/net/context"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
)
type fakeServer struct {
@@ -39,6 +45,8 @@ type fakeServer struct {
Deadlines map[string]int32 // deadlines by message ID
pullResponses []*pullResponse
wg sync.WaitGroup
subs map[string]*pb.Subscription
topics map[string]*pb.Topic
}
type pullResponse struct {
@@ -55,6 +63,8 @@ func newFakeServer() (*fakeServer, error) {
Addr: srv.Addr,
Acked: map[string]bool{},
Deadlines: map[string]int32{},
subs: map[string]*pb.Subscription{},
topics: map[string]*pb.Topic{},
}
pb.RegisterPublisherServer(srv.Gsrv, fake)
pb.RegisterSubscriberServer(srv.Gsrv, fake)
@@ -138,10 +148,175 @@ func (s *fakeServer) StreamingPull(stream pb.Subscriber_StreamingPullServer) err
}
}
const (
minMessageRetentionDuration = 10 * time.Minute
maxMessageRetentionDuration = 168 * time.Hour
)
var defaultMessageRetentionDuration = ptypes.DurationProto(maxMessageRetentionDuration)
func checkMRD(pmrd *durpb.Duration) error {
mrd, err := ptypes.Duration(pmrd)
if err != nil || mrd < minMessageRetentionDuration || mrd > maxMessageRetentionDuration {
return grpc.Errorf(codes.InvalidArgument, "bad message_retention_duration %+v", pmrd)
}
return nil
}
func checkAckDeadline(ads int32) error {
if ads < 10 || ads > 600 {
// PubSub service returns Unknown.
return grpc.Errorf(codes.Unknown, "bad ack_deadline_seconds: %d", ads)
}
return nil
}
func (s *fakeServer) CreateSubscription(ctx context.Context, sub *pb.Subscription) (*pb.Subscription, error) {
if s.subs[sub.Name] != nil {
return nil, grpc.Errorf(codes.AlreadyExists, "subscription %q", sub.Name)
}
sub2 := proto.Clone(sub).(*pb.Subscription)
if err := checkAckDeadline(sub.AckDeadlineSeconds); err != nil {
return nil, err
}
if sub.MessageRetentionDuration == nil {
sub2.MessageRetentionDuration = defaultMessageRetentionDuration
}
if err := checkMRD(sub2.MessageRetentionDuration); err != nil {
return nil, err
}
if sub.PushConfig == nil {
sub2.PushConfig = &pb.PushConfig{}
}
s.subs[sub.Name] = sub2
return sub2, nil
}
func (s *fakeServer) GetSubscription(ctx context.Context, req *pb.GetSubscriptionRequest) (*pb.Subscription, error) {
return &pb.Subscription{
Name: req.Subscription,
AckDeadlineSeconds: 10,
PushConfig: &pb.PushConfig{},
if sub := s.subs[req.Subscription]; sub != nil {
return sub, nil
}
return nil, grpc.Errorf(codes.NotFound, "subscription %q", req.Subscription)
}
func (s *fakeServer) UpdateSubscription(ctx context.Context, req *pb.UpdateSubscriptionRequest) (*pb.Subscription, error) {
sub := s.subs[req.Subscription.Name]
if sub == nil {
return nil, grpc.Errorf(codes.NotFound, "subscription %q", req.Subscription.Name)
}
for _, path := range req.UpdateMask.Paths {
switch path {
case "push_config":
sub.PushConfig = req.Subscription.PushConfig
case "ack_deadline_seconds":
a := req.Subscription.AckDeadlineSeconds
if err := checkAckDeadline(a); err != nil {
return nil, err
}
sub.AckDeadlineSeconds = a
case "retain_acked_messages":
sub.RetainAckedMessages = req.Subscription.RetainAckedMessages
case "message_retention_duration":
if err := checkMRD(req.Subscription.MessageRetentionDuration); err != nil {
return nil, err
}
sub.MessageRetentionDuration = req.Subscription.MessageRetentionDuration
// TODO(jba): labels
default:
return nil, grpc.Errorf(codes.InvalidArgument, "unknown field name %q", path)
}
}
return sub, nil
}
func (s *fakeServer) DeleteSubscription(_ context.Context, req *pb.DeleteSubscriptionRequest) (*emptypb.Empty, error) {
if s.subs[req.Subscription] == nil {
return nil, grpc.Errorf(codes.NotFound, "subscription %q", req.Subscription)
}
delete(s.subs, req.Subscription)
return &emptypb.Empty{}, nil
}
func (s *fakeServer) CreateTopic(_ context.Context, t *pb.Topic) (*pb.Topic, error) {
if s.topics[t.Name] != nil {
return nil, grpc.Errorf(codes.AlreadyExists, "topic %q", t.Name)
}
t2 := proto.Clone(t).(*pb.Topic)
s.topics[t.Name] = t2
return t2, nil
}
func (s *fakeServer) GetTopic(_ context.Context, req *pb.GetTopicRequest) (*pb.Topic, error) {
if t := s.topics[req.Topic]; t != nil {
return t, nil
}
return nil, grpc.Errorf(codes.NotFound, "topic %q", req.Topic)
}
func (s *fakeServer) DeleteTopic(_ context.Context, req *pb.DeleteTopicRequest) (*emptypb.Empty, error) {
if s.topics[req.Topic] == nil {
return nil, grpc.Errorf(codes.NotFound, "topic %q", req.Topic)
}
delete(s.topics, req.Topic)
return &emptypb.Empty{}, nil
}
func (s *fakeServer) ListTopics(_ context.Context, req *pb.ListTopicsRequest) (*pb.ListTopicsResponse, error) {
var names []string
for n := range s.topics {
if strings.HasPrefix(n, req.Project) {
names = append(names, n)
}
}
sort.Strings(names)
from, to, nextToken, err := testutil.PageBounds(int(req.PageSize), req.PageToken, len(names))
if err != nil {
return nil, err
}
res := &pb.ListTopicsResponse{NextPageToken: nextToken}
for i := from; i < to; i++ {
res.Topics = append(res.Topics, s.topics[names[i]])
}
return res, nil
}
func (s *fakeServer) ListSubscriptions(_ context.Context, req *pb.ListSubscriptionsRequest) (*pb.ListSubscriptionsResponse, error) {
var names []string
for _, sub := range s.subs {
if strings.HasPrefix(sub.Name, req.Project) {
names = append(names, sub.Name)
}
}
sort.Strings(names)
from, to, nextToken, err := testutil.PageBounds(int(req.PageSize), req.PageToken, len(names))
if err != nil {
return nil, err
}
res := &pb.ListSubscriptionsResponse{NextPageToken: nextToken}
for i := from; i < to; i++ {
res.Subscriptions = append(res.Subscriptions, s.subs[names[i]])
}
return res, nil
}
func (s *fakeServer) ListTopicSubscriptions(_ context.Context, req *pb.ListTopicSubscriptionsRequest) (*pb.ListTopicSubscriptionsResponse, error) {
var names []string
for _, sub := range s.subs {
if sub.Topic == req.Topic {
names = append(names, sub.Name)
}
}
sort.Strings(names)
from, to, nextToken, err := testutil.PageBounds(int(req.PageSize), req.PageToken, len(names))
if err != nil {
return nil, err
}
return &pb.ListTopicSubscriptionsResponse{
Subscriptions: names[from:to],
NextPageToken: nextToken,
}, nil
}
vendor/cloud.google.com/go/pubsub/integration_test.go
Generated Vendored
+110 -25
View File
@@ -16,7 +16,6 @@ package pubsub
import (
"fmt"
"reflect"
"testing"
"time"
@@ -29,6 +28,8 @@ import (
"cloud.google.com/go/internal/testutil"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
)
var (
@@ -149,7 +150,7 @@ func TestAll(t *testing.T) {
md := extractMessageData(m)
got[md.ID] = md
}
if !reflect.DeepEqual(got, want) {
if !testutil.Equal(got, want) {
t.Errorf("messages: got: %v ; want: %v", got, want)
}
@@ -160,14 +161,14 @@ func TestAll(t *testing.T) {
t.Errorf("sub IAM: %s", msg)
}
snap, err := sub.createSnapshot(ctx, "")
snap, err := sub.CreateSnapshot(ctx, "")
if err != nil {
t.Fatalf("CreateSnapshot error: %v", err)
}
timeoutCtx, _ = context.WithTimeout(ctx, time.Minute)
err = internal.Retry(timeoutCtx, gax.Backoff{}, func() (bool, error) {
snapIt := client.snapshots(timeoutCtx)
snapIt := client.Snapshots(timeoutCtx)
for {
s, err := snapIt.Next()
if err == nil && s.name == snap.name {
@@ -186,7 +187,7 @@ func TestAll(t *testing.T) {
}
err = internal.Retry(timeoutCtx, gax.Backoff{}, func() (bool, error) {
err := sub.seekToSnapshot(timeoutCtx, snap.snapshot)
err := sub.SeekToSnapshot(timeoutCtx, snap.Snapshot)
return err == nil, err
})
if err != nil {
@@ -194,7 +195,7 @@ func TestAll(t *testing.T) {
}
err = internal.Retry(timeoutCtx, gax.Backoff{}, func() (bool, error) {
err := sub.seekToTime(timeoutCtx, time.Now())
err := sub.SeekToTime(timeoutCtx, time.Now())
return err == nil, err
})
if err != nil {
@@ -202,8 +203,8 @@ func TestAll(t *testing.T) {
}
err = internal.Retry(timeoutCtx, gax.Backoff{}, func() (bool, error) {
snapHandle := client.snapshot(snap.ID())
err := snapHandle.delete(timeoutCtx)
snapHandle := client.Snapshot(snap.ID())
err := snapHandle.Delete(timeoutCtx)
return err == nil, err
})
if err != nil {
@@ -247,7 +248,7 @@ func testIAM(ctx context.Context, h *iam.Handle, permission string) (msg string,
if policy, err = h.Policy(ctx); err != nil {
return fmt.Sprintf("Policy: %v", err), false
}
if got, want := policy.Members(iam.Viewer), []string{member}; !reflect.DeepEqual(got, want) {
if got, want := policy.Members(iam.Viewer), []string{member}; !testutil.Equal(got, want) {
return fmt.Sprintf("after Add: got %v, want %v", got, want), false
}
// Now remove that member, set the policy, and check that it's empty again.
@@ -271,7 +272,7 @@ func testIAM(ctx context.Context, h *iam.Handle, permission string) (msg string,
if err != nil {
return fmt.Sprintf("TestPermissions: %v", err), false
}
if !reflect.DeepEqual(gotPerms, wantPerms) {
if !testutil.Equal(gotPerms, wantPerms) {
return fmt.Sprintf("TestPermissions: got %v, want %v", gotPerms, wantPerms), false
}
return "", true
@@ -296,40 +297,62 @@ func TestSubscriptionUpdate(t *testing.T) {
}
defer sub.Delete(ctx)
sc, err := sub.Config(ctx)
got, err := sub.Config(ctx)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(sc.PushConfig, PushConfig{}) {
t.Fatalf("got %+v, want empty PushConfig")
want := SubscriptionConfig{
Topic: topic,
AckDeadline: 10 * time.Second,
RetainAckedMessages: false,
RetentionDuration: defaultRetentionDuration,
}
// Add a PushConfig.
if !testutil.Equal(got, want) {
t.Fatalf("\ngot %+v\nwant %+v", got, want)
}
// Add a PushConfig and change other fields.
projID := testutil.ProjID()
pc := PushConfig{
Endpoint: "https://" + projID + ".appspot.com/_ah/push-handlers/push",
Attributes: map[string]string{"x-goog-version": "v1"},
}
sc, err = sub.Update(ctx, SubscriptionConfigToUpdate{PushConfig: &pc})
got, err = sub.Update(ctx, SubscriptionConfigToUpdate{
PushConfig: &pc,
AckDeadline: 2 * time.Minute,
RetainAckedMessages: true,
RetentionDuration: 2 * time.Hour,
})
if err != nil {
t.Fatal(err)
}
// Despite the docs which say that Get always returns a valid "x-goog-version"
// attribute, none is returned. See
// https://cloud.google.com/pubsub/docs/reference/rpc/google.pubsub.v1#google.pubsub.v1.PushConfig
pc.Attributes = nil
if got, want := sc.PushConfig, pc; !reflect.DeepEqual(got, want) {
t.Fatalf("setting push config: got\n%+v\nwant\n%+v", got, want)
want = SubscriptionConfig{
Topic: topic,
PushConfig: pc,
AckDeadline: 2 * time.Minute,
RetainAckedMessages: true,
RetentionDuration: 2 * time.Hour,
}
if !testutil.Equal(got, want) {
t.Fatalf("\ngot %+v\nwant %+v", got, want)
}
// Remove the PushConfig, turning the subscription back into pull mode.
// Change AckDeadline, but nothing else.
pc = PushConfig{}
sc, err = sub.Update(ctx, SubscriptionConfigToUpdate{PushConfig: &pc})
got, err = sub.Update(ctx, SubscriptionConfigToUpdate{
PushConfig: &pc,
AckDeadline: 30 * time.Second,
})
if err != nil {
t.Fatal(err)
}
if got, want := sc.PushConfig, pc; !reflect.DeepEqual(got, want) {
t.Fatalf("removing push config: got\n%+v\nwant %+v", got, want)
want.PushConfig = pc
want.AckDeadline = 30 * time.Second
// service issue: PushConfig attributes are not removed.
// TODO(jba): remove when issue resolved.
want.PushConfig.Attributes = map[string]string{"x-goog-version": "v1"}
if !testutil.Equal(got, want) {
t.Fatalf("\ngot %+v\nwant %+v", got, want)
}
// If nothing changes, our client returns an error.
_, err = sub.Update(ctx, SubscriptionConfigToUpdate{})
if err == nil {
@@ -361,3 +384,65 @@ func TestPublicTopic(t *testing.T) {
t.Fatal(err)
}
}
func TestIntegration_Errors(t *testing.T) {
// Test various edge conditions.
t.Parallel()
ctx := context.Background()
client := integrationTestClient(t, ctx)
defer client.Close()
topic, err := client.CreateTopic(ctx, topicIDs.New())
if err != nil {
t.Fatalf("CreateTopic error: %v", err)
}
defer topic.Stop()
defer topic.Delete(ctx)
// Out-of-range retention duration.
sub, err := client.CreateSubscription(ctx, subIDs.New(), SubscriptionConfig{
Topic: topic,
RetentionDuration: 1 * time.Second,
})
if want := codes.InvalidArgument; grpc.Code(err) != want {
t.Errorf("got <%v>, want %s", err, want)
}
if err == nil {
sub.Delete(ctx)
}
// Ack deadline less than minimum.
sub, err = client.CreateSubscription(ctx, subIDs.New(), SubscriptionConfig{
Topic: topic,
AckDeadline: 5 * time.Second,
})
if want := codes.Unknown; grpc.Code(err) != want {
t.Errorf("got <%v>, want %s", err, want)
}
if err == nil {
sub.Delete(ctx)
}
// Updating a non-existent subscription.
sub = client.Subscription(subIDs.New())
_, err = sub.Update(ctx, SubscriptionConfigToUpdate{AckDeadline: 20 * time.Second})
if want := codes.NotFound; grpc.Code(err) != want {
t.Errorf("got <%v>, want %s", err, want)
}
// Deleting a non-existent subscription.
err = sub.Delete(ctx)
if want := codes.NotFound; grpc.Code(err) != want {
t.Errorf("got <%v>, want %s", err, want)
}
// Updating out-of-range retention duration.
sub, err = client.CreateSubscription(ctx, subIDs.New(), SubscriptionConfig{Topic: topic})
if err != nil {
t.Fatal(err)
}
defer sub.Delete(ctx)
_, err = sub.Update(ctx, SubscriptionConfigToUpdate{RetentionDuration: 1000 * time.Hour})
if want := codes.InvalidArgument; grpc.Code(err) != want {
t.Errorf("got <%v>, want %s", err, want)
}
}
vendor/cloud.google.com/go/pubsub/iterator.go
Generated Vendored
+68 -51
View File
@@ -18,6 +18,7 @@ import (
"sync"
"time"
vkit "cloud.google.com/go/pubsub/apiv1"
"golang.org/x/net/context"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
)
@@ -26,16 +27,15 @@ import (
// when it is no longer needed.
// subName is the full name of the subscription to pull messages from.
// ctx is the context to use for acking messages and extending message deadlines.
func newMessageIterator(ctx context.Context, s service, subName string, po *pullOptions) *streamingMessageIterator {
sp := s.newStreamingPuller(ctx, subName, int32(po.ackDeadline.Seconds()))
_ = sp.open() // error stored in sp
return newStreamingMessageIterator(ctx, sp, po)
func newMessageIterator(ctx context.Context, subc *vkit.SubscriberClient, subName string, po *pullOptions) *streamingMessageIterator {
ps := newPullStream(ctx, subc, subName, int32(po.ackDeadline.Seconds()))
return newStreamingMessageIterator(ctx, ps, po)
}
type streamingMessageIterator struct {
ctx context.Context
po *pullOptions
sp *streamingPuller
ps *pullStream
kaTicker *time.Ticker // keep-alive (deadline extensions)
ackTicker *time.Ticker // message acks
nackTicker *time.Ticker // message nacks (more frequent than acks)
@@ -47,10 +47,11 @@ type streamingMessageIterator struct {
mu sync.Mutex
keepAliveDeadlines map[string]time.Time
pendingReq *pb.StreamingPullRequest
err error // error from stream failure
pendingModAcks map[string]int32 // ack IDs whose ack deadline is to be modified
err error // error from stream failure
}
func newStreamingMessageIterator(ctx context.Context, sp *streamingPuller, po *pullOptions) *streamingMessageIterator {
func newStreamingMessageIterator(ctx context.Context, ps *pullStream, po *pullOptions) *streamingMessageIterator {
// TODO: make kaTicker frequency more configurable. (ackDeadline - 5s) is a
// reasonable default for now, because the minimum ack period is 10s. This
// gives us 5s grace.
@@ -62,7 +63,7 @@ func newStreamingMessageIterator(ctx context.Context, sp *streamingPuller, po *p
nackTicker := time.NewTicker(100 * time.Millisecond)
it := &streamingMessageIterator{
ctx: ctx,
sp: sp,
ps: ps,
po: po,
kaTicker: kaTicker,
ackTicker: ackTicker,
@@ -72,6 +73,7 @@ func newStreamingMessageIterator(ctx context.Context, sp *streamingPuller, po *p
drained: make(chan struct{}),
keepAliveDeadlines: map[string]time.Time{},
pendingReq: &pb.StreamingPullRequest{},
pendingModAcks: map[string]int32{},
}
it.wg.Add(1)
go it.sender()
@@ -121,20 +123,11 @@ func (it *streamingMessageIterator) done(ackID string, ack bool) {
if ack {
it.pendingReq.AckIds = append(it.pendingReq.AckIds, ackID)
} else {
it.addDeadlineMod(ackID, 0) // Nack indicated by modifying the deadline to zero.
it.pendingModAcks[ackID] = 0 // Nack indicated by modifying the deadline to zero.
}
it.checkDrained()
}
// addDeadlineMod adds the ack ID to the pending request with the given deadline.
//
// Called with the lock held.
func (it *streamingMessageIterator) addDeadlineMod(ackID string, deadlineSecs int32) {
pr := it.pendingReq
pr.ModifyDeadlineAckIds = append(pr.ModifyDeadlineAckIds, ackID)
pr.ModifyDeadlineSeconds = append(pr.ModifyDeadlineSeconds, deadlineSecs)
}
// fail is called when a stream method returns a permanent error.
func (it *streamingMessageIterator) fail(err error) {
it.mu.Lock()
@@ -162,20 +155,33 @@ func (it *streamingMessageIterator) receive() ([]*Message, error) {
return nil, err
}
// Receive messages from stream. This may block indefinitely.
msgs, err := it.sp.fetchMessages()
// The streamingPuller handles retries, so any error here
// is fatal.
res, err := it.ps.Recv()
// The pullStream handles retries, so any error here is fatal.
if err != nil {
it.fail(err)
return nil, err
}
// We received some messages. Remember them so we can
// keep them alive.
deadline := time.Now().Add(it.po.maxExtension)
msgs, err := convertMessages(res.ReceivedMessages)
if err != nil {
it.fail(err)
return nil, err
}
// We received some messages. Remember them so we can keep them alive. Also,
// arrange for a receipt mod-ack (which will occur at the next firing of
// nackTicker).
maxExt := time.Now().Add(it.po.maxExtension)
deadline := trunc32(int64(it.po.ackDeadline.Seconds()))
it.mu.Lock()
for _, m := range msgs {
m.doneFunc = it.done
it.keepAliveDeadlines[m.ackID] = deadline
it.keepAliveDeadlines[m.ackID] = maxExt
// The receipt mod-ack uses the subscription's configured ack deadline. Don't
// change the mod-ack if one is already pending. This is possible if there
// are retries.
if _, ok := it.pendingModAcks[m.ackID]; !ok {
it.pendingModAcks[m.ackID] = deadline
}
}
it.mu.Unlock()
return msgs, nil
@@ -187,7 +193,7 @@ func (it *streamingMessageIterator) sender() {
defer it.kaTicker.Stop()
defer it.ackTicker.Stop()
defer it.nackTicker.Stop()
defer it.sp.closeSend()
defer it.ps.CloseSend()
done := false
for !done {
@@ -206,28 +212,34 @@ func (it *streamingMessageIterator) sender() {
// All outstanding messages have been marked done:
// nothing left to do except send the final request.
it.mu.Lock()
send = (len(it.pendingReq.AckIds) > 0 || len(it.pendingReq.ModifyDeadlineAckIds) > 0)
send = (len(it.pendingReq.AckIds) > 0 || len(it.pendingModAcks) > 0)
done = true
case <-it.kaTicker.C:
it.mu.Lock()
send = it.handleKeepAlives()
it.handleKeepAlives()
send = (len(it.pendingModAcks) > 0)
case <-it.nackTicker.C:
it.mu.Lock()
send = (len(it.pendingReq.ModifyDeadlineAckIds) > 0)
send = (len(it.pendingModAcks) > 0)
case <-it.ackTicker.C:
it.mu.Lock()
send = (len(it.pendingReq.AckIds) > 0)
}
// Lock is held here.
if send {
req := it.pendingReq
it.pendingReq = &pb.StreamingPullRequest{}
modAcks := it.pendingModAcks
it.pendingModAcks = map[string]int32{}
it.mu.Unlock()
err := it.sp.send(req)
for id, s := range modAcks {
req.ModifyDeadlineAckIds = append(req.ModifyDeadlineAckIds, id)
req.ModifyDeadlineSeconds = append(req.ModifyDeadlineSeconds, s)
}
err := it.send(req)
if err != nil {
// The streamingPuller handles retries, so any error here
// is fatal to the iterator.
@@ -240,32 +252,37 @@ func (it *streamingMessageIterator) sender() {
}
}
// handleKeepAlives modifies the pending request to include deadline extensions
// for live messages. It also purges expired messages. It reports whether
// there were any live messages.
//
// Called with the lock held.
func (it *streamingMessageIterator) handleKeepAlives() bool {
live, expired := getKeepAliveAckIDs(it.keepAliveDeadlines)
for _, e := range expired {
delete(it.keepAliveDeadlines, e)
func (it *streamingMessageIterator) send(req *pb.StreamingPullRequest) error {
// Note: len(modAckIDs) == len(modSecs)
var rest *pb.StreamingPullRequest
for len(req.AckIds) > 0 || len(req.ModifyDeadlineAckIds) > 0 {
req, rest = splitRequest(req, maxPayload)
if err := it.ps.Send(req); err != nil {
return err
}
req = rest
}
dl := trunc32(int64(it.po.ackDeadline.Seconds()))
for _, m := range live {
it.addDeadlineMod(m, dl)
}
it.checkDrained()
return len(live) > 0
return nil
}
func getKeepAliveAckIDs(items map[string]time.Time) (live, expired []string) {
// handleKeepAlives modifies the pending request to include deadline extensions
// for live messages. It also purges expired messages.
//
// Called with the lock held.
func (it *streamingMessageIterator) handleKeepAlives() {
now := time.Now()
for id, expiry := range items {
dl := trunc32(int64(it.po.ackDeadline.Seconds()))
for id, expiry := range it.keepAliveDeadlines {
if expiry.Before(now) {
expired = append(expired, id)
// This delete will not result in skipping any map items, as implied by
// the spec at https://golang.org/ref/spec#For_statements, "For
// statements with range clause", note 3, and stated explicitly at
// https://groups.google.com/forum/#!msg/golang-nuts/UciASUb03Js/pzSq5iVFAQAJ.
delete(it.keepAliveDeadlines, id)
} else {
live = append(live, id)
// This will not overwrite a nack, because nacking removes the ID from keepAliveDeadlines.
it.pendingModAcks[id] = dl
}
}
return live, expired
it.checkDrained()
}
vendor/cloud.google.com/go/pubsub/pstest/examples_test.go
Generated Vendored
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pstest_test
import (
"cloud.google.com/go/pubsub"
"cloud.google.com/go/pubsub/pstest"
"golang.org/x/net/context"
"google.golang.org/api/option"
"google.golang.org/grpc"
)
func ExampleNewServer() {
ctx := context.Background()
// Start a fake server running locally.
srv := pstest.NewServer()
// Connect to the server without using TLS.
conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
if err != nil {
// TODO: Handle error.
}
// Use the connection when creating a pubsub client.
client, err := pubsub.NewClient(ctx, "project", option.WithGRPCConn(conn))
if err != nil {
// TODO: Handle error.
}
defer client.Close()
_ = client // TODO: Use the client.
}
vendor/cloud.google.com/go/pubsub/pstest/fake.go
Generated Vendored
+751
View File
@@ -0,0 +1,751 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package pstest provides a fake Cloud PubSub service for testing. It implements a
// simplified form of the service, suitable for unit tests. It may behave
// differently from the actual service in ways in which the service is
// non-deterministic or unspecified: timing, delivery order, etc.
//
// This package is EXPERIMENTAL and is subject to change without notice.
//
// See the example for usage.
package pstest
import (
"fmt"
"io"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"cloud.google.com/go/internal/testutil"
"github.com/golang/protobuf/ptypes"
durpb "github.com/golang/protobuf/ptypes/duration"
emptypb "github.com/golang/protobuf/ptypes/empty"
"golang.org/x/net/context"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
)
// For testing. Note that even though changes to the now variable are atomic, a call
// to the stored function can race with a change to that function. This could be a
// problem if tests are run in parallel, or even if concurrent parts of the same test
// change the value of the variable.
var now atomic.Value
func init() {
now.Store(time.Now)
}
func timeNow() time.Time {
return now.Load().(func() time.Time)()
}
type Server struct {
Addr string // The address that the server is listening on.
gServer gServer
}
type gServer struct {
pb.PublisherServer
pb.SubscriberServer
mu sync.Mutex
topics map[string]*topic
subs map[string]*subscription
msgs []*Message // all messages ever published
msgsByID map[string]*Message
wg sync.WaitGroup
nextID int
streamTimeout time.Duration
}
// NewServer creates a new fake server running in the current process.
func NewServer() *Server {
srv, err := testutil.NewServer()
if err != nil {
panic(fmt.Sprintf("pstest.NewServer: %v", err))
}
s := &Server{
Addr: srv.Addr,
gServer: gServer{
topics: map[string]*topic{},
subs: map[string]*subscription{},
msgsByID: map[string]*Message{},
},
}
pb.RegisterPublisherServer(srv.Gsrv, &s.gServer)
pb.RegisterSubscriberServer(srv.Gsrv, &s.gServer)
srv.Start()
return s
}
// Publish behaves as if the Publish RPC was called with a message with the given
// data and attrs. It returns the ID of the message.
// The topic will be created if it doesn't exist.
//
// Publish panics if there is an error, which is appropriate for testing.
func (s *Server) Publish(topic string, data []byte, attrs map[string]string) string {
_, _ = s.gServer.CreateTopic(nil, &pb.Topic{Name: topic})
req := &pb.PublishRequest{
Topic: topic,
Messages: []*pb.PubsubMessage{{Data: data, Attributes: attrs}},
}
res, err := s.gServer.Publish(nil, req)
if err != nil {
panic(fmt.Sprintf("pstest.Server.Publish: %v", err))
}
return res.MessageIds[0]
}
// SetStreamTimeout sets the amount of time a stream will be active before it shuts
// itself down. This mimics the real service's behavior of closing streams after 30
// minutes. If SetStreamTimeout is never called or is passed zero, streams never shut
// down.
func (s *Server) SetStreamTimeout(d time.Duration) {
s.gServer.mu.Lock()
defer s.gServer.mu.Unlock()
s.gServer.streamTimeout = d
}
// A Message is a message that was published to the server.
type Message struct {
ID string
Data []byte
Attributes map[string]string
PublishTime time.Time
Deliveries int // number of times delivery of the message was attempted
Acks int // number of acks received from clients
// protected by server mutex
deliveries int
acks int
}
// Messages returns information about all messages ever published.
func (s *Server) Messages() []*Message {
s.gServer.mu.Lock()
defer s.gServer.mu.Unlock()
var msgs []*Message
for _, m := range s.gServer.msgs {
m.Deliveries = m.deliveries
m.Acks = m.acks
msgs = append(msgs, m)
}
return msgs
}
// Message returns the message with the given ID, or nil if no message
// with that ID was published.
func (s *Server) Message(id string) *Message {
s.gServer.mu.Lock()
defer s.gServer.mu.Unlock()
m := s.gServer.msgsByID[id]
if m != nil {
m.Deliveries = m.deliveries
m.Acks = m.acks
}
return m
}
// Wait blocks until all server activity has completed.
func (s *Server) Wait() {
s.gServer.wg.Wait()
}
func (s *gServer) CreateTopic(_ context.Context, t *pb.Topic) (*pb.Topic, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.topics[t.Name] != nil {
return nil, grpc.Errorf(codes.AlreadyExists, "topic %q", t.Name)
}
top := newTopic(t)
s.topics[t.Name] = top
return top.proto, nil
}
func (s *gServer) GetTopic(_ context.Context, req *pb.GetTopicRequest) (*pb.Topic, error) {
s.mu.Lock()
defer s.mu.Unlock()
if t := s.topics[req.Topic]; t != nil {
return t.proto, nil
}
return nil, grpc.Errorf(codes.NotFound, "topic %q", req.Topic)
}
func (s *gServer) UpdateTopic(_ context.Context, req *pb.UpdateTopicRequest) (*pb.Topic, error) {
return nil, grpc.Errorf(codes.Unimplemented, "unimplemented")
}
func (s *gServer) ListTopics(_ context.Context, req *pb.ListTopicsRequest) (*pb.ListTopicsResponse, error) {
s.mu.Lock()
defer s.mu.Unlock()
var names []string
for n := range s.topics {
if strings.HasPrefix(n, req.Project) {
names = append(names, n)
}
}
sort.Strings(names)
from, to, nextToken, err := testutil.PageBounds(int(req.PageSize), req.PageToken, len(names))
if err != nil {
return nil, err
}
res := &pb.ListTopicsResponse{NextPageToken: nextToken}
for i := from; i < to; i++ {
res.Topics = append(res.Topics, s.topics[names[i]].proto)
}
return res, nil
}
func (s *gServer) ListTopicSubscriptions(_ context.Context, req *pb.ListTopicSubscriptionsRequest) (*pb.ListTopicSubscriptionsResponse, error) {
s.mu.Lock()
defer s.mu.Unlock()
var names []string
for name, sub := range s.subs {
if sub.topic.proto.Name == req.Topic {
names = append(names, name)
}
}
sort.Strings(names)
from, to, nextToken, err := testutil.PageBounds(int(req.PageSize), req.PageToken, len(names))
if err != nil {
return nil, err
}
return &pb.ListTopicSubscriptionsResponse{
Subscriptions: names[from:to],
NextPageToken: nextToken,
}, nil
}
func (s *gServer) DeleteTopic(_ context.Context, req *pb.DeleteTopicRequest) (*emptypb.Empty, error) {
s.mu.Lock()
defer s.mu.Unlock()
t := s.topics[req.Topic]
if t == nil {
return nil, grpc.Errorf(codes.NotFound, "topic %q", req.Topic)
}
t.stop()
delete(s.topics, req.Topic)
return &emptypb.Empty{}, nil
}
func (s *gServer) CreateSubscription(_ context.Context, ps *pb.Subscription) (*pb.Subscription, error) {
s.mu.Lock()
defer s.mu.Unlock()
if ps.Name == "" {
return nil, grpc.Errorf(codes.InvalidArgument, "missing name")
}
if s.subs[ps.Name] != nil {
return nil, grpc.Errorf(codes.AlreadyExists, "subscription %q", ps.Name)
}
if ps.Topic == "" {
return nil, grpc.Errorf(codes.InvalidArgument, "missing topic")
}
top := s.topics[ps.Topic]
if top == nil {
return nil, grpc.Errorf(codes.NotFound, "topic %q", ps.Topic)
}
if err := checkAckDeadline(ps.AckDeadlineSeconds); err != nil {
return nil, err
}
if ps.MessageRetentionDuration == nil {
ps.MessageRetentionDuration = defaultMessageRetentionDuration
}
if err := checkMRD(ps.MessageRetentionDuration); err != nil {
return nil, err
}
if ps.PushConfig == nil {
ps.PushConfig = &pb.PushConfig{}
}
sub := newSubscription(top, &s.mu, ps)
top.subs[ps.Name] = sub
s.subs[ps.Name] = sub
sub.start(&s.wg)
return ps, nil
}
func checkAckDeadline(ads int32) error {
if ads < 10 || ads > 600 {
// PubSub service returns Unknown.
return grpc.Errorf(codes.Unknown, "bad ack_deadline_seconds: %d", ads)
}
return nil
}
const (
minMessageRetentionDuration = 10 * time.Minute
maxMessageRetentionDuration = 168 * time.Hour
)
var defaultMessageRetentionDuration = ptypes.DurationProto(maxMessageRetentionDuration)
func checkMRD(pmrd *durpb.Duration) error {
mrd, err := ptypes.Duration(pmrd)
if err != nil || mrd < minMessageRetentionDuration || mrd > maxMessageRetentionDuration {
return grpc.Errorf(codes.InvalidArgument, "bad message_retention_duration %+v", pmrd)
}
return nil
}
func (s *gServer) GetSubscription(_ context.Context, req *pb.GetSubscriptionRequest) (*pb.Subscription, error) {
s.mu.Lock()
defer s.mu.Unlock()
if sub := s.subs[req.Subscription]; sub != nil {
return sub.proto, nil
}
return nil, grpc.Errorf(codes.NotFound, "subscription %q", req.Subscription)
}
func (s *gServer) UpdateSubscription(_ context.Context, req *pb.UpdateSubscriptionRequest) (*pb.Subscription, error) {
s.mu.Lock()
defer s.mu.Unlock()
sub := s.subs[req.Subscription.Name]
if sub == nil {
return nil, grpc.Errorf(codes.NotFound, "subscription %q", req.Subscription.Name)
}
for _, path := range req.UpdateMask.Paths {
switch path {
case "push_config":
sub.proto.PushConfig = req.Subscription.PushConfig
case "ack_deadline_seconds":
a := req.Subscription.AckDeadlineSeconds
if err := checkAckDeadline(a); err != nil {
return nil, err
}
sub.proto.AckDeadlineSeconds = a
case "retain_acked_messages":
sub.proto.RetainAckedMessages = req.Subscription.RetainAckedMessages
case "message_retention_duration":
if err := checkMRD(req.Subscription.MessageRetentionDuration); err != nil {
return nil, err
}
sub.proto.MessageRetentionDuration = req.Subscription.MessageRetentionDuration
// TODO(jba): labels
default:
return nil, grpc.Errorf(codes.InvalidArgument, "unknown field name %q", path)
}
}
return sub.proto, nil
}
func (s *gServer) ListSubscriptions(_ context.Context, req *pb.ListSubscriptionsRequest) (*pb.ListSubscriptionsResponse, error) {
s.mu.Lock()
defer s.mu.Unlock()
var names []string
for name := range s.subs {
if strings.HasPrefix(name, req.Project) {
names = append(names, name)
}
}
sort.Strings(names)
from, to, nextToken, err := testutil.PageBounds(int(req.PageSize), req.PageToken, len(names))
if err != nil {
return nil, err
}
res := &pb.ListSubscriptionsResponse{NextPageToken: nextToken}
for i := from; i < to; i++ {
res.Subscriptions = append(res.Subscriptions, s.subs[names[i]].proto)
}
return res, nil
}
func (s *gServer) DeleteSubscription(_ context.Context, req *pb.DeleteSubscriptionRequest) (*emptypb.Empty, error) {
s.mu.Lock()
defer s.mu.Unlock()
sub := s.subs[req.Subscription]
if sub == nil {
return nil, grpc.Errorf(codes.NotFound, "subscription %q", req.Subscription)
}
sub.stop()
delete(s.subs, req.Subscription)
sub.topic.deleteSub(sub)
return &emptypb.Empty{}, nil
}
func (s *gServer) Publish(_ context.Context, req *pb.PublishRequest) (*pb.PublishResponse, error) {
s.mu.Lock()
defer s.mu.Unlock()
if req.Topic == "" {
return nil, grpc.Errorf(codes.InvalidArgument, "missing topic")
}
top := s.topics[req.Topic]
if top == nil {
return nil, grpc.Errorf(codes.NotFound, "topic %q", req.Topic)
}
var ids []string
for _, pm := range req.Messages {
id := fmt.Sprintf("m%d", s.nextID)
s.nextID++
pm.MessageId = id
pubTime := timeNow()
tsPubTime, err := ptypes.TimestampProto(pubTime)
if err != nil {
return nil, grpc.Errorf(codes.Internal, err.Error())
}
pm.PublishTime = tsPubTime
m := &Message{
ID: id,
Data: pm.Data,
Attributes: pm.Attributes,
PublishTime: pubTime,
}
top.publish(pm, m)
ids = append(ids, id)
s.msgs = append(s.msgs, m)
s.msgsByID[id] = m
}
return &pb.PublishResponse{MessageIds: ids}, nil
}
type topic struct {
proto *pb.Topic
subs map[string]*subscription
}
func newTopic(pt *pb.Topic) *topic {
return &topic{
proto: pt,
subs: map[string]*subscription{},
}
}
func (t *topic) stop() {
for _, sub := range t.subs {
sub.proto.Topic = "_deleted-topic_"
sub.stop()
}
}
func (t *topic) deleteSub(sub *subscription) {
delete(t.subs, sub.proto.Name)
}
func (t *topic) publish(pm *pb.PubsubMessage, m *Message) {
for _, s := range t.subs {
s.msgs[pm.MessageId] = &message{
publishTime: m.PublishTime,
proto: &pb.ReceivedMessage{
AckId: pm.MessageId,
Message: pm,
},
deliveries: &m.deliveries,
acks: &m.acks,
streamIndex: -1,
}
}
}
type subscription struct {
topic *topic
mu *sync.Mutex
proto *pb.Subscription
ackTimeout time.Duration
msgs map[string]*message // unacked messages by message ID
streams []*stream
done chan struct{}
}
func newSubscription(t *topic, mu *sync.Mutex, ps *pb.Subscription) *subscription {
return &subscription{
topic: t,
mu: mu,
proto: ps,
ackTimeout: 10 * time.Second,
msgs: map[string]*message{},
done: make(chan struct{}),
}
}
func (s *subscription) start(wg *sync.WaitGroup) {
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-s.done:
return
case <-time.After(1 * time.Second):
s.deliver()
}
}
}()
}
func (s *subscription) stop() {
close(s.done)
}
func (s *gServer) StreamingPull(sps pb.Subscriber_StreamingPullServer) error {
// Receive initial message configuring the pull.
req, err := sps.Recv()
if err != nil {
return err
}
if req.Subscription == "" {
return grpc.Errorf(codes.InvalidArgument, "missing subscription")
}
s.mu.Lock()
sub := s.subs[req.Subscription]
s.mu.Unlock()
if sub == nil {
return grpc.Errorf(codes.NotFound, "subscription %s", req.Subscription)
}
// Create a new stream to handle the pull.
st := sub.newStream(sps, s.streamTimeout)
err = st.pull(&s.wg)
sub.deleteStream(st)
return err
}
var retentionDuration = 10 * time.Minute
func (s *subscription) deliver() {
s.mu.Lock()
defer s.mu.Unlock()
tNow := timeNow()
for id, m := range s.msgs {
// Mark a message as re-deliverable if its ack deadline has expired.
if m.outstanding() && tNow.After(m.ackDeadline) {
m.makeAvailable()
}
pubTime, err := ptypes.Timestamp(m.proto.Message.PublishTime)
if err != nil {
panic(err)
}
// Remove messages that have been undelivered for a long time.
if !m.outstanding() && tNow.Sub(pubTime) > retentionDuration {
delete(s.msgs, id)
}
}
// Try to deliver each remaining message.
curIndex := 0
for _, m := range s.msgs {
// If the message was never delivered before, start with the stream at
// curIndex. If it was delivered before, start with the stream after the one
// that owned it.
if m.streamIndex < 0 {
delIndex, ok := s.deliverMessage(m, curIndex, tNow)
if !ok {
break
}
curIndex = delIndex + 1
m.streamIndex = curIndex
} else {
delIndex, ok := s.deliverMessage(m, m.streamIndex, tNow)
if !ok {
break
}
m.streamIndex = delIndex
}
}
}
// deliverMessage attempts to deliver m to the stream at index i. If it can't, it
// tries streams i+1, i+2, ..., wrapping around. It returns the index of the stream
// it delivered the message to, or 0, false if it didn't deliver the message because
// there are no active streams.
func (s *subscription) deliverMessage(m *message, i int, tNow time.Time) (int, bool) {
for len(s.streams) > 0 {
if i >= len(s.streams) {
i = 0
}
st := s.streams[i]
select {
case <-st.done:
s.streams = deleteStreamAt(s.streams, i)
case st.msgc <- m.proto:
(*m.deliveries)++
m.ackDeadline = tNow.Add(st.ackTimeout)
return i, true
}
}
return 0, false
}
func (s *subscription) newStream(gs pb.Subscriber_StreamingPullServer, timeout time.Duration) *stream {
st := &stream{
sub: s,
done: make(chan struct{}),
msgc: make(chan *pb.ReceivedMessage),
gstream: gs,
ackTimeout: s.ackTimeout,
timeout: timeout,
}
s.mu.Lock()
s.streams = append(s.streams, st)
s.mu.Unlock()
return st
}
func (s *subscription) deleteStream(st *stream) {
s.mu.Lock()
defer s.mu.Unlock()
var i int
for i = 0; i < len(s.streams); i++ {
if s.streams[i] == st {
break
}
}
if i < len(s.streams) {
s.streams = deleteStreamAt(s.streams, i)
}
}
func deleteStreamAt(s []*stream, i int) []*stream {
// Preserve order for round-robin delivery.
return append(s[:i], s[i+1:]...)
}
type message struct {
proto *pb.ReceivedMessage
publishTime time.Time
ackDeadline time.Time
deliveries *int
acks *int
streamIndex int // index of stream that currently owns msg, for round-robin delivery
}
func (m *message) outstanding() bool {
return !m.ackDeadline.IsZero()
}
func (m *message) makeAvailable() {
m.ackDeadline = time.Time{}
}
type stream struct {
sub *subscription
done chan struct{} // closed when the stream is finished
msgc chan *pb.ReceivedMessage
gstream pb.Subscriber_StreamingPullServer
ackTimeout time.Duration
timeout time.Duration
}
// pull manages the StreamingPull interaction for the life of the stream.
func (st *stream) pull(wg *sync.WaitGroup) error {
errc := make(chan error, 2)
wg.Add(2)
go func() {
defer wg.Done()
errc <- st.sendLoop()
}()
go func() {
defer wg.Done()
errc <- st.recvLoop()
}()
var tchan <-chan time.Time
if st.timeout > 0 {
tchan = time.After(st.timeout)
}
// Wait until one of the goroutines returns an error, or we time out.
var err error
select {
case err = <-errc:
if err == io.EOF {
err = nil
}
case <-tchan:
}
close(st.done) // stop the other goroutine
return err
}
func (st *stream) sendLoop() error {
for {
select {
case <-st.done:
return nil
case rm := <-st.msgc:
res := &pb.StreamingPullResponse{ReceivedMessages: []*pb.ReceivedMessage{rm}}
if err := st.gstream.Send(res); err != nil {
return err
}
}
}
}
func (st *stream) recvLoop() error {
for {
req, err := st.gstream.Recv()
if err != nil {
return err
}
st.sub.handleStreamingPullRequest(st, req)
}
}
func (s *subscription) handleStreamingPullRequest(st *stream, req *pb.StreamingPullRequest) {
// Lock the entire server.
s.mu.Lock()
defer s.mu.Unlock()
for _, ackID := range req.AckIds {
s.ack(ackID)
}
for i, id := range req.ModifyDeadlineAckIds {
s.modifyAckDeadline(id, secsToDur(req.ModifyDeadlineSeconds[i]))
}
if req.StreamAckDeadlineSeconds > 0 {
st.ackTimeout = secsToDur(req.StreamAckDeadlineSeconds)
}
}
func (s *subscription) ack(id string) {
m := s.msgs[id]
if m != nil {
(*m.acks)++
delete(s.msgs, id)
}
}
func (s *subscription) modifyAckDeadline(id string, d time.Duration) {
m := s.msgs[id]
if m == nil { // already acked: ignore.
return
}
if d == 0 { // nack
m.makeAvailable()
} else { // extend the deadline by d
m.ackDeadline = timeNow().Add(d)
}
}
func secsToDur(secs int32) time.Duration {
return time.Duration(secs) * time.Second
}
vendor/cloud.google.com/go/pubsub/pstest/fake_test.go
Generated Vendored
+342
View File
@@ -0,0 +1,342 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pstest
import (
"fmt"
"io"
"testing"
"time"
"github.com/golang/protobuf/ptypes"
"cloud.google.com/go/internal/testutil"
"golang.org/x/net/context"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
"google.golang.org/grpc"
)
func TestTopics(t *testing.T) {
pclient, _, server := newFake(t)
ctx := context.Background()
var topics []*pb.Topic
for i := 1; i < 3; i++ {
topics = append(topics, mustCreateTopic(t, pclient, &pb.Topic{
Name: fmt.Sprintf("projects/P/topics/T%d", i),
Labels: map[string]string{"num": fmt.Sprintf("%d", i)},
}))
}
if got, want := len(server.gServer.topics), len(topics); got != want {
t.Fatalf("got %d topics, want %d", got, want)
}
for _, top := range topics {
got, err := pclient.GetTopic(ctx, &pb.GetTopicRequest{Topic: top.Name})
if err != nil {
t.Fatal(err)
}
if !testutil.Equal(got, top) {
t.Errorf("\ngot %+v\nwant %+v", got, top)
}
}
res, err := pclient.ListTopics(ctx, &pb.ListTopicsRequest{Project: "projects/P"})
if err != nil {
t.Fatal(err)
}
if got, want := res.Topics, topics; !testutil.Equal(got, want) {
t.Errorf("\ngot %+v\nwant %+v", got, want)
}
for _, top := range topics {
if _, err := pclient.DeleteTopic(ctx, &pb.DeleteTopicRequest{Topic: top.Name}); err != nil {
t.Fatal(err)
}
}
if got, want := len(server.gServer.topics), 0; got != want {
t.Fatalf("got %d topics, want %d", got, want)
}
}
func TestSubscriptions(t *testing.T) {
pclient, sclient, server := newFake(t)
ctx := context.Background()
topic := mustCreateTopic(t, pclient, &pb.Topic{Name: "projects/P/topics/T"})
var subs []*pb.Subscription
for i := 0; i < 3; i++ {
subs = append(subs, mustCreateSubscription(t, sclient, &pb.Subscription{
Name: fmt.Sprintf("projects/P/subscriptions/S%d", i),
Topic: topic.Name,
AckDeadlineSeconds: int32(10 * (i + 1)),
}))
}
if got, want := len(server.gServer.subs), len(subs); got != want {
t.Fatalf("got %d subscriptions, want %d", got, want)
}
for _, s := range subs {
got, err := sclient.GetSubscription(ctx, &pb.GetSubscriptionRequest{Subscription: s.Name})
if err != nil {
t.Fatal(err)
}
if !testutil.Equal(got, s) {
t.Errorf("\ngot %+v\nwant %+v", got, s)
}
}
res, err := sclient.ListSubscriptions(ctx, &pb.ListSubscriptionsRequest{Project: "projects/P"})
if err != nil {
t.Fatal(err)
}
if got, want := res.Subscriptions, subs; !testutil.Equal(got, want) {
t.Errorf("\ngot %+v\nwant %+v", got, want)
}
res2, err := pclient.ListTopicSubscriptions(ctx, &pb.ListTopicSubscriptionsRequest{Topic: topic.Name})
if err != nil {
t.Fatal(err)
}
if got, want := len(res2.Subscriptions), len(subs); got != want {
t.Fatalf("got %d subs, want %d", got, want)
}
for i, got := range res2.Subscriptions {
want := subs[i].Name
if !testutil.Equal(got, want) {
t.Errorf("\ngot %+v\nwant %+v", got, want)
}
}
for _, s := range subs {
if _, err := sclient.DeleteSubscription(ctx, &pb.DeleteSubscriptionRequest{Subscription: s.Name}); err != nil {
t.Fatal(err)
}
}
if got, want := len(server.gServer.subs), 0; got != want {
t.Fatalf("got %d subscriptions, want %d", got, want)
}
}
func TestPublish(t *testing.T) {
s := NewServer()
var ids []string
for i := 0; i < 3; i++ {
ids = append(ids, s.Publish("t", []byte("hello"), nil))
}
s.Wait()
ms := s.Messages()
if got, want := len(ms), len(ids); got != want {
t.Errorf("got %d messages, want %d", got, want)
}
for i, id := range ids {
if got, want := ms[i].ID, id; got != want {
t.Errorf("got %s, want %s", got, want)
}
}
m := s.Message(ids[1])
if m == nil {
t.Error("got nil, want a message")
}
}
// Note: this sets the fake's "now" time, so it is senstive to concurrent changes to "now".
func publish(t *testing.T, pclient pb.PublisherClient, topic *pb.Topic, messages []*pb.PubsubMessage) map[string]*pb.PubsubMessage {
pubTime := time.Now()
now.Store(func() time.Time { return pubTime })
defer func() { now.Store(time.Now) }()
res, err := pclient.Publish(context.Background(), &pb.PublishRequest{
Topic: topic.Name,
Messages: messages,
})
if err != nil {
t.Fatal(err)
}
tsPubTime, err := ptypes.TimestampProto(pubTime)
if err != nil {
t.Fatal(err)
}
want := map[string]*pb.PubsubMessage{}
for i, id := range res.MessageIds {
want[id] = &pb.PubsubMessage{
Data: messages[i].Data,
Attributes: messages[i].Attributes,
MessageId: id,
PublishTime: tsPubTime,
}
}
return want
}
func TestStreamingPull(t *testing.T) {
// A simple test of streaming pull.
pclient, sclient, _ := newFake(t)
top := mustCreateTopic(t, pclient, &pb.Topic{Name: "projects/P/topics/T"})
sub := mustCreateSubscription(t, sclient, &pb.Subscription{
Name: "projects/P/subscriptions/S",
Topic: top.Name,
AckDeadlineSeconds: 10,
})
want := publish(t, pclient, top, []*pb.PubsubMessage{
{Data: []byte("d1")},
{Data: []byte("d2")},
{Data: []byte("d3")},
})
got := pullN(t, len(want), sclient, sub)
if diff := testutil.Diff(got, want); diff != "" {
t.Error(diff)
}
}
func TestMultiSubs(t *testing.T) {
// Each subscription gets every message.
pclient, sclient, _ := newFake(t)
top := mustCreateTopic(t, pclient, &pb.Topic{Name: "projects/P/topics/T"})
sub1 := mustCreateSubscription(t, sclient, &pb.Subscription{
Name: "projects/P/subscriptions/S1",
Topic: top.Name,
AckDeadlineSeconds: 10,
})
sub2 := mustCreateSubscription(t, sclient, &pb.Subscription{
Name: "projects/P/subscriptions/S2",
Topic: top.Name,
AckDeadlineSeconds: 10,
})
want := publish(t, pclient, top, []*pb.PubsubMessage{
{Data: []byte("d1")},
{Data: []byte("d2")},
{Data: []byte("d3")},
})
got1 := pullN(t, len(want), sclient, sub1)
got2 := pullN(t, len(want), sclient, sub2)
if diff := testutil.Diff(got1, want); diff != "" {
t.Error(diff)
}
if diff := testutil.Diff(got2, want); diff != "" {
t.Error(diff)
}
}
func TestMultiStreams(t *testing.T) {
// Messages are handed out to the streams of a subscription in round-robin order.
pclient, sclient, _ := newFake(t)
top := mustCreateTopic(t, pclient, &pb.Topic{Name: "projects/P/topics/T"})
sub := mustCreateSubscription(t, sclient, &pb.Subscription{
Name: "projects/P/subscriptions/S",
Topic: top.Name,
AckDeadlineSeconds: 10,
})
want := publish(t, pclient, top, []*pb.PubsubMessage{
{Data: []byte("d1")},
{Data: []byte("d2")},
{Data: []byte("d3")},
{Data: []byte("d4")},
})
streams := []pb.Subscriber_StreamingPullClient{
mustStartPull(t, sclient, sub),
mustStartPull(t, sclient, sub),
}
got := map[string]*pb.PubsubMessage{}
for i := 0; i < 2; i++ {
for _, st := range streams {
res, err := st.Recv()
if err != nil {
t.Fatal(err)
}
m := res.ReceivedMessages[0]
got[m.Message.MessageId] = m.Message
}
}
if diff := testutil.Diff(got, want); diff != "" {
t.Error(diff)
}
}
func TestStreamingPullTimeout(t *testing.T) {
pclient, sclient, srv := newFake(t)
timeout := 200 * time.Millisecond
srv.SetStreamTimeout(timeout)
top := mustCreateTopic(t, pclient, &pb.Topic{Name: "projects/P/topics/T"})
sub := mustCreateSubscription(t, sclient, &pb.Subscription{
Name: "projects/P/subscriptions/S",
Topic: top.Name,
AckDeadlineSeconds: 10,
})
stream := mustStartPull(t, sclient, sub)
time.Sleep(2 * timeout)
_, err := stream.Recv()
if err != io.EOF {
t.Errorf("got %v, want io.EOF", err)
}
}
func mustStartPull(t *testing.T, sc pb.SubscriberClient, sub *pb.Subscription) pb.Subscriber_StreamingPullClient {
spc, err := sc.StreamingPull(context.Background())
if err != nil {
t.Fatal(err)
}
if err := spc.Send(&pb.StreamingPullRequest{Subscription: sub.Name}); err != nil {
t.Fatal(err)
}
return spc
}
func pullN(t *testing.T, n int, sc pb.SubscriberClient, sub *pb.Subscription) map[string]*pb.PubsubMessage {
spc := mustStartPull(t, sc, sub)
got := map[string]*pb.PubsubMessage{}
for i := 0; i < n; i++ {
res, err := spc.Recv()
if err != nil {
t.Fatal(err)
}
for _, m := range res.ReceivedMessages {
got[m.Message.MessageId] = m.Message
}
}
if err := spc.CloseSend(); err != nil {
t.Fatal(err)
}
_, err := spc.Recv()
if err != io.EOF {
t.Fatal(err)
}
return got
}
func mustCreateTopic(t *testing.T, pc pb.PublisherClient, topic *pb.Topic) *pb.Topic {
top, err := pc.CreateTopic(context.Background(), topic)
if err != nil {
t.Fatal(err)
}
return top
}
func mustCreateSubscription(t *testing.T, sc pb.SubscriberClient, sub *pb.Subscription) *pb.Subscription {
sub, err := sc.CreateSubscription(context.Background(), sub)
if err != nil {
t.Fatal(err)
}
return sub
}
func newFake(t *testing.T) (pb.PublisherClient, pb.SubscriberClient, *Server) {
srv := NewServer()
conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
if err != nil {
t.Fatal(err)
}
return pb.NewPublisherClient(conn), pb.NewSubscriberClient(conn), srv
}
vendor/cloud.google.com/go/pubsub/pubsub.go
Generated Vendored
+28 -69
View File
@@ -20,12 +20,12 @@ import (
"runtime"
"time"
"google.golang.org/api/iterator"
"cloud.google.com/go/internal/version"
vkit "cloud.google.com/go/pubsub/apiv1"
"golang.org/x/net/context"
"google.golang.org/api/option"
"google.golang.org/grpc"
"google.golang.org/grpc/keepalive"
"golang.org/x/net/context"
)
const (
@@ -46,7 +46,8 @@ const prodAddr = "https://pubsub.googleapis.com/"
// A Client may be shared by multiple goroutines.
type Client struct {
projectID string
s service
pubc *vkit.PublisherClient
subc *vkit.SubscriberClient
}
// NewClient creates a new PubSub client.
@@ -64,86 +65,44 @@ func NewClient(ctx context.Context, projectID string, opts ...option.ClientOptio
o = []option.ClientOption{
// Create multiple connections to increase throughput.
option.WithGRPCConnectionPool(runtime.GOMAXPROCS(0)),
// TODO(grpc/grpc-go#1388) using connection pool without WithBlock
// can cause RPCs to fail randomly. We can delete this after the issue is fixed.
option.WithGRPCDialOption(grpc.WithBlock()),
option.WithGRPCDialOption(grpc.WithKeepaliveParams(keepalive.ClientParameters{
Time: 5 * time.Minute,
})),
}
}
o = append(o, opts...)
s, err := newPubSubService(ctx, o)
pubc, err := vkit.NewPublisherClient(ctx, o...)
if err != nil {
return nil, fmt.Errorf("constructing pubsub client: %v", err)
return nil, fmt.Errorf("pubsub: %v", err)
}
c := &Client{
subc, err := vkit.NewSubscriberClient(ctx, option.WithGRPCConn(pubc.Connection()))
if err != nil {
// Should never happen, since we are passing in the connection.
// If it does, we cannot close, because the user may have passed in their
// own connection originally.
return nil, fmt.Errorf("pubsub: %v", err)
}
pubc.SetGoogleClientInfo("gccl", version.Repo)
subc.SetGoogleClientInfo("gccl", version.Repo)
return &Client{
projectID: projectID,
s: s,
}
return c, nil
pubc: pubc,
subc: subc,
}, nil
}
// Close closes any resources held by the client.
// Close releases any resources held by the client,
// such as memory and goroutines.
//
// Close need not be called at program exit.
// If the client is available for the lifetime of the program, then Close need not be
// called at exit.
func (c *Client) Close() error {
return c.s.close()
// Return the first error, because the first call closes the connection.
err := c.pubc.Close()
_ = c.subc.Close()
return err
}
func (c *Client) fullyQualifiedProjectName() string {
return fmt.Sprintf("projects/%s", c.projectID)
}
// pageToken stores the next page token for a server response which is split over multiple pages.
type pageToken struct {
tok string
explicit bool
}
func (pt *pageToken) set(tok string) {
pt.tok = tok
pt.explicit = true
}
func (pt *pageToken) get() string {
return pt.tok
}
// more returns whether further pages should be fetched from the server.
func (pt *pageToken) more() bool {
return pt.tok != "" || !pt.explicit
}
// stringsIterator provides an iterator API for a sequence of API page fetches that return lists of strings.
type stringsIterator struct {
ctx context.Context
strings []string
token pageToken
fetch func(ctx context.Context, tok string) (*stringsPage, error)
}
// Next returns the next string. If there are no more strings, iterator.Done will be returned.
func (si *stringsIterator) Next() (string, error) {
for len(si.strings) == 0 && si.token.more() {
page, err := si.fetch(si.ctx, si.token.get())
if err != nil {
return "", err
}
si.token.set(page.tok)
si.strings = page.strings
}
if len(si.strings) == 0 {
return "", iterator.Done
}
s := si.strings[0]
si.strings = si.strings[1:]
return s, nil
}
vendor/cloud.google.com/go/pubsub/pullstream.go
Generated Vendored
+147
View File
@@ -0,0 +1,147 @@
// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pubsub
import (
"io"
"sync"
vkit "cloud.google.com/go/pubsub/apiv1"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
"google.golang.org/grpc"
)
// A pullStream supports the methods of a StreamingPullClient, but re-opens
// the stream on a retryable error.
type pullStream struct {
ctx context.Context
open func() (pb.Subscriber_StreamingPullClient, error)
mu sync.Mutex
spc *pb.Subscriber_StreamingPullClient
err error // permanent error
}
func newPullStream(ctx context.Context, subc *vkit.SubscriberClient, subName string, ackDeadlineSecs int32) *pullStream {
return &pullStream{
ctx: ctx,
open: func() (pb.Subscriber_StreamingPullClient, error) {
spc, err := subc.StreamingPull(ctx, gax.WithGRPCOptions(grpc.MaxCallRecvMsgSize(maxSendRecvBytes)))
if err == nil {
err = spc.Send(&pb.StreamingPullRequest{
Subscription: subName,
StreamAckDeadlineSeconds: ackDeadlineSecs,
})
}
if err != nil {
return nil, err
}
return spc, nil
},
}
}
// get returns either a valid *StreamingPullClient (SPC), or a permanent error.
// If the argument is nil, this is the first call for an RPC, and the current
// SPC will be returned (or a new one will be opened). Otherwise, this call is a
// request to re-open the stream because of a retryable error, and the argument
// is a pointer to the SPC that returned the error.
func (s *pullStream) get(spc *pb.Subscriber_StreamingPullClient) (*pb.Subscriber_StreamingPullClient, error) {
s.mu.Lock()
defer s.mu.Unlock()
// A stored error is permanent.
if s.err != nil {
return nil, s.err
}
// If the context is done, so are we.
select {
case <-s.ctx.Done():
s.err = s.ctx.Err()
return nil, s.err
default:
}
// TODO(jba): We can use the following instead of the above after we drop support for 1.8:
// s.err = s.ctx.Err()
// if s.err != nil {
// return nil, s.err
// }
// If the current and argument SPCs differ, return the current one. This subsumes two cases:
// 1. We have an SPC and the caller is getting the stream for the first time.
// 2. The caller wants to retry, but they have an older SPC; we've already retried.
if spc != s.spc {
return s.spc, nil
}
// Either this is the very first call on this stream (s.spc == nil), or we have a valid
// retry request. Either way, open a new stream.
// The lock is held here for a long time, but it doesn't matter because no callers could get
// anything done anyway.
s.spc = new(pb.Subscriber_StreamingPullClient)
*s.spc, s.err = s.open() // Setting s.err means any error from open is permanent. Reconsider.
return s.spc, s.err
}
func (s *pullStream) call(f func(pb.Subscriber_StreamingPullClient) error) error {
var (
spc *pb.Subscriber_StreamingPullClient
err error
bo gax.Backoff
)
for {
spc, err = s.get(spc)
if err != nil {
// Preserve the existing behavior of not retrying on open. Is that a bug?
// (If we do decide to retry, don't retry after we're closed.)
return err
}
err = f(*spc)
if err != nil {
if isRetryable(err) {
gax.Sleep(s.ctx, bo.Pause())
continue
}
s.err = err
}
return err
}
}
func (s *pullStream) Send(req *pb.StreamingPullRequest) error {
return s.call(func(spc pb.Subscriber_StreamingPullClient) error {
return spc.Send(req)
})
}
func (s *pullStream) Recv() (*pb.StreamingPullResponse, error) {
var res *pb.StreamingPullResponse
err := s.call(func(spc pb.Subscriber_StreamingPullClient) error {
var err error
res, err = spc.Recv()
return err
})
return res, err
}
func (s *pullStream) CloseSend() error {
err := s.call(func(spc pb.Subscriber_StreamingPullClient) error {
return spc.CloseSend()
})
s.mu.Lock()
s.err = io.EOF // should not be retried
s.mu.Unlock()
return err
}
vendor/cloud.google.com/go/pubsub/service.go
Generated Vendored
+126 -469
View File
@@ -18,230 +18,12 @@ import (
"fmt"
"math"
"strings"
"sync"
"time"
"github.com/golang/protobuf/ptypes"
"cloud.google.com/go/iam"
"cloud.google.com/go/internal/version"
vkit "cloud.google.com/go/pubsub/apiv1"
durpb "github.com/golang/protobuf/ptypes/duration"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
"google.golang.org/api/option"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
type nextStringFunc func() (string, error)
type nextSnapshotFunc func() (*snapshotConfig, error)
// service provides an internal abstraction to isolate the generated
// PubSub API; most of this package uses this interface instead.
// The single implementation, *apiService, contains all the knowledge
// of the generated PubSub API (except for that present in legacy code).
type service interface {
createSubscription(ctx context.Context, subName string, cfg SubscriptionConfig) error
getSubscriptionConfig(ctx context.Context, subName string) (SubscriptionConfig, string, error)
listProjectSubscriptions(ctx context.Context, projName string) nextStringFunc
deleteSubscription(ctx context.Context, name string) error
subscriptionExists(ctx context.Context, name string) (bool, error)
modifyPushConfig(ctx context.Context, subName string, conf PushConfig) error
createTopic(ctx context.Context, name string) error
deleteTopic(ctx context.Context, name string) error
topicExists(ctx context.Context, name string) (bool, error)
listProjectTopics(ctx context.Context, projName string) nextStringFunc
listTopicSubscriptions(ctx context.Context, topicName string) nextStringFunc
modifyAckDeadline(ctx context.Context, subName string, deadline time.Duration, ackIDs []string) error
fetchMessages(ctx context.Context, subName string, maxMessages int32) ([]*Message, error)
publishMessages(ctx context.Context, topicName string, msgs []*Message) ([]string, error)
// splitAckIDs divides ackIDs into
// * a batch of a size which is suitable for passing to acknowledge or
// modifyAckDeadline, and
// * the rest.
splitAckIDs(ackIDs []string) ([]string, []string)
// acknowledge ACKs the IDs in ackIDs.
acknowledge(ctx context.Context, subName string, ackIDs []string) error
iamHandle(resourceName string) *iam.Handle
newStreamingPuller(ctx context.Context, subName string, ackDeadline int32) *streamingPuller
createSnapshot(ctx context.Context, snapName, subName string) (*snapshotConfig, error)
deleteSnapshot(ctx context.Context, snapName string) error
listProjectSnapshots(ctx context.Context, projName string) nextSnapshotFunc
// TODO(pongad): Raw proto returns an empty SeekResponse; figure out if we want to return it before GA.
seekToTime(ctx context.Context, subName string, t time.Time) error
seekToSnapshot(ctx context.Context, subName, snapName string) error
close() error
}
type apiService struct {
pubc *vkit.PublisherClient
subc *vkit.SubscriberClient
}
func newPubSubService(ctx context.Context, opts []option.ClientOption) (*apiService, error) {
pubc, err := vkit.NewPublisherClient(ctx, opts...)
if err != nil {
return nil, err
}
subc, err := vkit.NewSubscriberClient(ctx, option.WithGRPCConn(pubc.Connection()))
if err != nil {
_ = pubc.Close() // ignore error
return nil, err
}
pubc.SetGoogleClientInfo("gccl", version.Repo)
subc.SetGoogleClientInfo("gccl", version.Repo)
return &apiService{pubc: pubc, subc: subc}, nil
}
func (s *apiService) close() error {
// Return the first error, because the first call closes the connection.
err := s.pubc.Close()
_ = s.subc.Close()
return err
}
func (s *apiService) createSubscription(ctx context.Context, subName string, cfg SubscriptionConfig) error {
var rawPushConfig *pb.PushConfig
if cfg.PushConfig.Endpoint != "" || len(cfg.PushConfig.Attributes) != 0 {
rawPushConfig = &pb.PushConfig{
Attributes: cfg.PushConfig.Attributes,
PushEndpoint: cfg.PushConfig.Endpoint,
}
}
var retentionDuration *durpb.Duration
if cfg.retentionDuration != 0 {
retentionDuration = ptypes.DurationProto(cfg.retentionDuration)
}
_, err := s.subc.CreateSubscription(ctx, &pb.Subscription{
Name: subName,
Topic: cfg.Topic.name,
PushConfig: rawPushConfig,
AckDeadlineSeconds: trunc32(int64(cfg.AckDeadline.Seconds())),
RetainAckedMessages: cfg.retainAckedMessages,
MessageRetentionDuration: retentionDuration,
})
return err
}
func (s *apiService) getSubscriptionConfig(ctx context.Context, subName string) (SubscriptionConfig, string, error) {
rawSub, err := s.subc.GetSubscription(ctx, &pb.GetSubscriptionRequest{Subscription: subName})
if err != nil {
return SubscriptionConfig{}, "", err
}
var rd time.Duration
// TODO(pongad): Remove nil-check after white list is removed.
if rawSub.MessageRetentionDuration != nil {
if rd, err = ptypes.Duration(rawSub.MessageRetentionDuration); err != nil {
return SubscriptionConfig{}, "", err
}
}
sub := SubscriptionConfig{
AckDeadline: time.Second * time.Duration(rawSub.AckDeadlineSeconds),
PushConfig: PushConfig{
Endpoint: rawSub.PushConfig.PushEndpoint,
Attributes: rawSub.PushConfig.Attributes,
},
retainAckedMessages: rawSub.RetainAckedMessages,
retentionDuration: rd,
}
return sub, rawSub.Topic, nil
}
// stringsPage contains a list of strings and a token for fetching the next page.
type stringsPage struct {
strings []string
tok string
}
func (s *apiService) listProjectSubscriptions(ctx context.Context, projName string) nextStringFunc {
it := s.subc.ListSubscriptions(ctx, &pb.ListSubscriptionsRequest{
Project: projName,
})
return func() (string, error) {
sub, err := it.Next()
if err != nil {
return "", err
}
return sub.Name, nil
}
}
func (s *apiService) deleteSubscription(ctx context.Context, name string) error {
return s.subc.DeleteSubscription(ctx, &pb.DeleteSubscriptionRequest{Subscription: name})
}
func (s *apiService) subscriptionExists(ctx context.Context, name string) (bool, error) {
_, err := s.subc.GetSubscription(ctx, &pb.GetSubscriptionRequest{Subscription: name})
if err == nil {
return true, nil
}
if grpc.Code(err) == codes.NotFound {
return false, nil
}
return false, err
}
func (s *apiService) createTopic(ctx context.Context, name string) error {
_, err := s.pubc.CreateTopic(ctx, &pb.Topic{Name: name})
return err
}
func (s *apiService) listProjectTopics(ctx context.Context, projName string) nextStringFunc {
it := s.pubc.ListTopics(ctx, &pb.ListTopicsRequest{
Project: projName,
})
return func() (string, error) {
topic, err := it.Next()
if err != nil {
return "", err
}
return topic.Name, nil
}
}
func (s *apiService) deleteTopic(ctx context.Context, name string) error {
return s.pubc.DeleteTopic(ctx, &pb.DeleteTopicRequest{Topic: name})
}
func (s *apiService) topicExists(ctx context.Context, name string) (bool, error) {
_, err := s.pubc.GetTopic(ctx, &pb.GetTopicRequest{Topic: name})
if err == nil {
return true, nil
}
if grpc.Code(err) == codes.NotFound {
return false, nil
}
return false, err
}
func (s *apiService) listTopicSubscriptions(ctx context.Context, topicName string) nextStringFunc {
it := s.pubc.ListTopicSubscriptions(ctx, &pb.ListTopicSubscriptionsRequest{
Topic: topicName,
})
return it.Next
}
func (s *apiService) modifyAckDeadline(ctx context.Context, subName string, deadline time.Duration, ackIDs []string) error {
return s.subc.ModifyAckDeadline(ctx, &pb.ModifyAckDeadlineRequest{
Subscription: subName,
AckIds: ackIDs,
AckDeadlineSeconds: trunc32(int64(deadline.Seconds())),
})
}
// maxPayload is the maximum number of bytes to devote to actual ids in
// acknowledgement or modifyAckDeadline requests. A serialized
// AcknowledgeRequest proto has a small constant overhead, plus the size of the
@@ -260,36 +42,6 @@ const (
maxSendRecvBytes = 20 * 1024 * 1024 // 20M
)
// splitAckIDs splits ids into two slices, the first of which contains at most maxPayload bytes of ackID data.
func (s *apiService) splitAckIDs(ids []string) ([]string, []string) {
total := reqFixedOverhead
for i, id := range ids {
total += len(id) + overheadPerID
if total > maxPayload {
return ids[:i], ids[i:]
}
}
return ids, nil
}
func (s *apiService) acknowledge(ctx context.Context, subName string, ackIDs []string) error {
return s.subc.Acknowledge(ctx, &pb.AcknowledgeRequest{
Subscription: subName,
AckIds: ackIDs,
})
}
func (s *apiService) fetchMessages(ctx context.Context, subName string, maxMessages int32) ([]*Message, error) {
resp, err := s.subc.Pull(ctx, &pb.PullRequest{
Subscription: subName,
MaxMessages: maxMessages,
}, gax.WithGRPCOptions(grpc.MaxCallRecvMsgSize(maxSendRecvBytes)))
if err != nil {
return nil, err
}
return convertMessages(resp.ReceivedMessages)
}
func convertMessages(rms []*pb.ReceivedMessage) ([]*Message, error) {
msgs := make([]*Message, 0, len(rms))
for i, m := range rms {
@@ -302,38 +54,6 @@ func convertMessages(rms []*pb.ReceivedMessage) ([]*Message, error) {
return msgs, nil
}
func (s *apiService) publishMessages(ctx context.Context, topicName string, msgs []*Message) ([]string, error) {
rawMsgs := make([]*pb.PubsubMessage, len(msgs))
for i, msg := range msgs {
rawMsgs[i] = &pb.PubsubMessage{
Data: msg.Data,
Attributes: msg.Attributes,
}
}
resp, err := s.pubc.Publish(ctx, &pb.PublishRequest{
Topic: topicName,
Messages: rawMsgs,
}, gax.WithGRPCOptions(grpc.MaxCallSendMsgSize(maxSendRecvBytes)))
if err != nil {
return nil, err
}
return resp.MessageIds, nil
}
func (s *apiService) modifyPushConfig(ctx context.Context, subName string, conf PushConfig) error {
return s.subc.ModifyPushConfig(ctx, &pb.ModifyPushConfigRequest{
Subscription: subName,
PushConfig: &pb.PushConfig{
Attributes: conf.Attributes,
PushEndpoint: conf.Endpoint,
},
})
}
func (s *apiService) iamHandle(resourceName string) *iam.Handle {
return iam.InternalNewHandle(s.pubc.Connection(), resourceName)
}
func trunc32(i int64) int32 {
if i > math.MaxInt32 {
i = math.MaxInt32
@@ -341,103 +61,103 @@ func trunc32(i int64) int32 {
return int32(i)
}
func (s *apiService) newStreamingPuller(ctx context.Context, subName string, ackDeadlineSecs int32) *streamingPuller {
p := &streamingPuller{
ctx: ctx,
subName: subName,
ackDeadlineSecs: ackDeadlineSecs,
subc: s.subc,
}
p.c = sync.NewCond(&p.mu)
return p
}
// func newStreamingPuller(ctx context.Context, subc *vkit.SubscriberClient, subName string, ackDeadlineSecs int32) *streamingPuller {
// p := &streamingPuller{
// ctx: ctx,
// subName: subName,
// ackDeadlineSecs: ackDeadlineSecs,
// subc: subc,
// }
// p.c = sync.NewCond(&p.mu)
// return p
// }
type streamingPuller struct {
ctx context.Context
subName string
ackDeadlineSecs int32
subc *vkit.SubscriberClient
// type streamingPuller struct {
// ctx context.Context
// subName string
// ackDeadlineSecs int32
// subc *vkit.SubscriberClient
mu sync.Mutex
c *sync.Cond
inFlight bool
closed bool // set after CloseSend called
spc pb.Subscriber_StreamingPullClient
err error
}
// mu sync.Mutex
// c *sync.Cond
// inFlight bool
// closed bool // set after CloseSend called
// spc pb.Subscriber_StreamingPullClient
// err error
// }
// open establishes (or re-establishes) a stream for pulling messages.
// It takes care that only one RPC is in flight at a time.
func (p *streamingPuller) open() error {
p.c.L.Lock()
defer p.c.L.Unlock()
p.openLocked()
return p.err
}
// // open establishes (or re-establishes) a stream for pulling messages.
// // It takes care that only one RPC is in flight at a time.
// func (p *streamingPuller) open() error {
// p.c.L.Lock()
// defer p.c.L.Unlock()
// p.openLocked()
// return p.err
// }
func (p *streamingPuller) openLocked() {
if p.inFlight {
// Another goroutine is opening; wait for it.
for p.inFlight {
p.c.Wait()
}
return
}
// No opens in flight; start one.
// Keep the lock held, to avoid a race where we
// close the old stream while opening a new one.
p.inFlight = true
spc, err := p.subc.StreamingPull(p.ctx, gax.WithGRPCOptions(grpc.MaxCallRecvMsgSize(maxSendRecvBytes)))
if err == nil {
err = spc.Send(&pb.StreamingPullRequest{
Subscription: p.subName,
StreamAckDeadlineSeconds: p.ackDeadlineSecs,
})
}
p.spc = spc
p.err = err
p.inFlight = false
p.c.Broadcast()
}
// func (p *streamingPuller) openLocked() {
// if p.inFlight {
// // Another goroutine is opening; wait for it.
// for p.inFlight {
// p.c.Wait()
// }
// return
// }
// // No opens in flight; start one.
// // Keep the lock held, to avoid a race where we
// // close the old stream while opening a new one.
// p.inFlight = true
// spc, err := p.subc.StreamingPull(p.ctx, gax.WithGRPCOptions(grpc.MaxCallRecvMsgSize(maxSendRecvBytes)))
// if err == nil {
// err = spc.Send(&pb.StreamingPullRequest{
// Subscription: p.subName,
// StreamAckDeadlineSeconds: p.ackDeadlineSecs,
// })
// }
// p.spc = spc
// p.err = err
// p.inFlight = false
// p.c.Broadcast()
// }
func (p *streamingPuller) call(f func(pb.Subscriber_StreamingPullClient) error) error {
p.c.L.Lock()
defer p.c.L.Unlock()
// Wait for an open in flight.
for p.inFlight {
p.c.Wait()
}
var err error
var bo gax.Backoff
for {
select {
case <-p.ctx.Done():
p.err = p.ctx.Err()
default:
}
if p.err != nil {
return p.err
}
spc := p.spc
// Do not call f with the lock held. Only one goroutine calls Send
// (streamingMessageIterator.sender) and only one calls Recv
// (streamingMessageIterator.receiver). If we locked, then a
// blocked Recv would prevent a Send from happening.
p.c.L.Unlock()
err = f(spc)
p.c.L.Lock()
if !p.closed && err != nil && isRetryable(err) {
// Sleep with exponential backoff. Normally we wouldn't hold the lock while sleeping,
// but here it can't do any harm, since the stream is broken anyway.
gax.Sleep(p.ctx, bo.Pause())
p.openLocked()
continue
}
// Not an error, or not a retryable error; stop retrying.
p.err = err
return err
}
}
// func (p *streamingPuller) call(f func(pb.Subscriber_StreamingPullClient) error) error {
// p.c.L.Lock()
// defer p.c.L.Unlock()
// // Wait for an open in flight.
// for p.inFlight {
// p.c.Wait()
// }
// var err error
// var bo gax.Backoff
// for {
// select {
// case <-p.ctx.Done():
// p.err = p.ctx.Err()
// default:
// }
// if p.err != nil {
// return p.err
// }
// spc := p.spc
// // Do not call f with the lock held. Only one goroutine calls Send
// // (streamingMessageIterator.sender) and only one calls Recv
// // (streamingMessageIterator.receiver). If we locked, then a
// // blocked Recv would prevent a Send from happening.
// p.c.L.Unlock()
// err = f(spc)
// p.c.L.Lock()
// if !p.closed && err != nil && isRetryable(err) {
// // Sleep with exponential backoff. Normally we wouldn't hold the lock while sleeping,
// // but here it can't do any harm, since the stream is broken anyway.
// gax.Sleep(p.ctx, bo.Pause())
// p.openLocked()
// continue
// }
// // Not an error, or not a retryable error; stop retrying.
// p.err = err
// return err
// }
// }
// Logic from https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/StatusUtil.java.
func isRetryable(err error) bool {
@@ -455,42 +175,42 @@ func isRetryable(err error) bool {
}
}
func (p *streamingPuller) fetchMessages() ([]*Message, error) {
var res *pb.StreamingPullResponse
err := p.call(func(spc pb.Subscriber_StreamingPullClient) error {
var err error
res, err = spc.Recv()
return err
})
if err != nil {
return nil, err
}
return convertMessages(res.ReceivedMessages)
}
// func (p *streamingPuller) fetchMessages() ([]*Message, error) {
// var res *pb.StreamingPullResponse
// err := p.call(func(spc pb.Subscriber_StreamingPullClient) error {
// var err error
// res, err = spc.Recv()
// return err
// })
// if err != nil {
// return nil, err
// }
// return convertMessages(res.ReceivedMessages)
// }
func (p *streamingPuller) send(req *pb.StreamingPullRequest) error {
// Note: len(modAckIDs) == len(modSecs)
var rest *pb.StreamingPullRequest
for len(req.AckIds) > 0 || len(req.ModifyDeadlineAckIds) > 0 {
req, rest = splitRequest(req, maxPayload)
err := p.call(func(spc pb.Subscriber_StreamingPullClient) error {
x := spc.Send(req)
return x
})
if err != nil {
return err
}
req = rest
}
return nil
}
// func (p *streamingPuller) send(req *pb.StreamingPullRequest) error {
// // Note: len(modAckIDs) == len(modSecs)
// var rest *pb.StreamingPullRequest
// for len(req.AckIds) > 0 || len(req.ModifyDeadlineAckIds) > 0 {
// req, rest = splitRequest(req, maxPayload)
// err := p.call(func(spc pb.Subscriber_StreamingPullClient) error {
// x := spc.Send(req)
// return x
// })
// if err != nil {
// return err
// }
// req = rest
// }
// return nil
// }
func (p *streamingPuller) closeSend() {
p.mu.Lock()
p.closed = true
p.spc.CloseSend()
p.mu.Unlock()
}
// func (p *streamingPuller) closeSend() {
// p.mu.Lock()
// p.closed = true
// p.spc.CloseSend()
// p.mu.Unlock()
// }
// Split req into a prefix that is smaller than maxSize, and a remainder.
func splitRequest(req *pb.StreamingPullRequest, maxSize int) (prefix, remainder *pb.StreamingPullRequest) {
@@ -533,66 +253,3 @@ func splitRequest(req *pb.StreamingPullRequest, maxSize int) (prefix, remainder
req.ModifyDeadlineSeconds = req.ModifyDeadlineSeconds[:k]
return req, remainder
}
func (s *apiService) createSnapshot(ctx context.Context, snapName, subName string) (*snapshotConfig, error) {
snap, err := s.subc.CreateSnapshot(ctx, &pb.CreateSnapshotRequest{
Name: snapName,
Subscription: subName,
})
if err != nil {
return nil, err
}
return s.toSnapshotConfig(snap)
}
func (s *apiService) deleteSnapshot(ctx context.Context, snapName string) error {
return s.subc.DeleteSnapshot(ctx, &pb.DeleteSnapshotRequest{Snapshot: snapName})
}
func (s *apiService) listProjectSnapshots(ctx context.Context, projName string) nextSnapshotFunc {
it := s.subc.ListSnapshots(ctx, &pb.ListSnapshotsRequest{
Project: projName,
})
return func() (*snapshotConfig, error) {
snap, err := it.Next()
if err != nil {
return nil, err
}
return s.toSnapshotConfig(snap)
}
}
func (s *apiService) toSnapshotConfig(snap *pb.Snapshot) (*snapshotConfig, error) {
exp, err := ptypes.Timestamp(snap.ExpireTime)
if err != nil {
return nil, err
}
return &snapshotConfig{
snapshot: &snapshot{
s: s,
name: snap.Name,
},
Topic: newTopic(s, snap.Topic),
Expiration: exp,
}, nil
}
func (s *apiService) seekToTime(ctx context.Context, subName string, t time.Time) error {
ts, err := ptypes.TimestampProto(t)
if err != nil {
return err
}
_, err = s.subc.Seek(ctx, &pb.SeekRequest{
Subscription: subName,
Target: &pb.SeekRequest_Time{ts},
})
return err
}
func (s *apiService) seekToSnapshot(ctx context.Context, subName, snapName string) error {
_, err := s.subc.Seek(ctx, &pb.SeekRequest{
Subscription: subName,
Target: &pb.SeekRequest_Snapshot{snapName},
})
return err
}
vendor/cloud.google.com/go/pubsub/service_test.go
Generated Vendored
+4 -3
View File
@@ -15,9 +15,10 @@
package pubsub
import (
"reflect"
"testing"
"cloud.google.com/go/internal/testutil"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
)
@@ -58,10 +59,10 @@ func TestSplitRequest(t *testing.T) {
ModifyDeadlineSeconds: modDeadlines[len(m1) : len(m1)+len(m2)],
}
got1, got2 := splitRequest(req, reqFixedOverhead+40)
if !reflect.DeepEqual(got1, want1) {
if !testutil.Equal(got1, want1) {
t.Errorf("#%d: first:\ngot %+v\nwant %+v", i, got1, want1)
}
if !reflect.DeepEqual(got2, want2) {
if !testutil.Equal(got2, want2) {
t.Errorf("#%d: second:\ngot %+v\nwant %+v", i, got2, want2)
}
}
vendor/cloud.google.com/go/pubsub/snapshot.go
Generated Vendored
+71 -30
View File
@@ -15,23 +15,25 @@
package pubsub
import (
"fmt"
"strings"
"time"
vkit "cloud.google.com/go/pubsub/apiv1"
"github.com/golang/protobuf/ptypes"
"golang.org/x/net/context"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
)
// Snapshot is a reference to a PubSub snapshot.
type snapshot struct {
s service
type Snapshot struct {
c *Client
// The fully qualified identifier for the snapshot, in the format "projects/<projid>/snapshots/<snap>"
name string
}
// ID returns the unique identifier of the snapshot within its project.
func (s *snapshot) ID() string {
func (s *Snapshot) ID() string {
slash := strings.LastIndex(s.name, "/")
if slash == -1 {
// name is not a fully-qualified name.
@@ -41,44 +43,52 @@ func (s *snapshot) ID() string {
}
// SnapshotConfig contains the details of a Snapshot.
type snapshotConfig struct {
*snapshot
type SnapshotConfig struct {
*Snapshot
Topic *Topic
Expiration time.Time
}
// Snapshot creates a reference to a snapshot.
func (c *Client) snapshot(id string) *snapshot {
return &snapshot{
s: c.s,
name: vkit.SubscriberSnapshotPath(c.projectID, id),
func (c *Client) Snapshot(id string) *Snapshot {
return &Snapshot{
c: c,
name: fmt.Sprintf("projects/%s/snapshots/%s", c.projectID, id),
}
}
// Snapshots returns an iterator which returns snapshots for this project.
func (c *Client) snapshots(ctx context.Context) *snapshotConfigIterator {
return &snapshotConfigIterator{
next: c.s.listProjectSnapshots(ctx, c.fullyQualifiedProjectName()),
func (c *Client) Snapshots(ctx context.Context) *SnapshotConfigIterator {
it := c.subc.ListSnapshots(ctx, &pb.ListSnapshotsRequest{
Project: c.fullyQualifiedProjectName(),
})
next := func() (*SnapshotConfig, error) {
snap, err := it.Next()
if err != nil {
return nil, err
}
return toSnapshotConfig(snap, c)
}
return &SnapshotConfigIterator{next: next}
}
// SnapshotConfigIterator is an iterator that returns a series of snapshots.
type snapshotConfigIterator struct {
next nextSnapshotFunc
type SnapshotConfigIterator struct {
next func() (*SnapshotConfig, error)
}
// Next returns the next SnapshotConfig. Its second return value is iterator.Done if there are no more results.
// Once Next returns iterator.Done, all subsequent calls will return iterator.Done.
func (snaps *snapshotConfigIterator) Next() (*snapshotConfig, error) {
func (snaps *SnapshotConfigIterator) Next() (*SnapshotConfig, error) {
return snaps.next()
}
// Delete deletes a snapshot.
func (snap *snapshot) delete(ctx context.Context) error {
return snap.s.deleteSnapshot(ctx, snap.name)
func (snap *Snapshot) Delete(ctx context.Context) error {
return snap.c.subc.DeleteSnapshot(ctx, &pb.DeleteSnapshotRequest{Snapshot: snap.name})
}
// SeekTime seeks the subscription to a point in time.
// SeekToTime seeks the subscription to a point in time.
//
// Messages retained in the subscription that were published before this
// time are marked as acknowledged, and messages retained in the
@@ -89,11 +99,19 @@ func (snap *snapshot) delete(ctx context.Context) error {
// window (or to a point before the system's notion of the subscription
// creation time), only retained messages will be marked as unacknowledged,
// and already-expunged messages will not be restored.
func (s *Subscription) seekToTime(ctx context.Context, t time.Time) error {
return s.s.seekToTime(ctx, s.name, t)
func (s *Subscription) SeekToTime(ctx context.Context, t time.Time) error {
ts, err := ptypes.TimestampProto(t)
if err != nil {
return err
}
_, err = s.c.subc.Seek(ctx, &pb.SeekRequest{
Subscription: s.name,
Target: &pb.SeekRequest_Time{ts},
})
return err
}
// Snapshot creates a new snapshot from this subscription.
// CreateSnapshot creates a new snapshot from this subscription.
// The snapshot will be for the topic this subscription is subscribed to.
// If the name is empty string, a unique name is assigned.
//
@@ -103,17 +121,40 @@ func (s *Subscription) seekToTime(ctx context.Context, t time.Time) error {
// unacknowledged when Snapshot returns without error.
// (b) Any messages published to the subscription's topic following
// Snapshot returning without error.
func (s *Subscription) createSnapshot(ctx context.Context, name string) (*snapshotConfig, error) {
func (s *Subscription) CreateSnapshot(ctx context.Context, name string) (*SnapshotConfig, error) {
if name != "" {
name = vkit.SubscriberSnapshotPath(strings.Split(s.name, "/")[1], name)
name = fmt.Sprintf("projects/%s/snapshots/%s", strings.Split(s.name, "/")[1], name)
}
return s.s.createSnapshot(ctx, name, s.name)
snap, err := s.c.subc.CreateSnapshot(ctx, &pb.CreateSnapshotRequest{
Name: name,
Subscription: s.name,
})
if err != nil {
return nil, err
}
return toSnapshotConfig(snap, s.c)
}
// SeekSnapshot seeks the subscription to a snapshot.
// SeekToSnapshot seeks the subscription to a snapshot.
//
// The snapshot needs not be created from this subscription,
// but the snapshot must be for the topic this subscription is subscribed to.
func (s *Subscription) seekToSnapshot(ctx context.Context, snap *snapshot) error {
return s.s.seekToSnapshot(ctx, s.name, snap.name)
// The snapshot need not be created from this subscription,
// but it must be for the topic this subscription is subscribed to.
func (s *Subscription) SeekToSnapshot(ctx context.Context, snap *Snapshot) error {
_, err := s.c.subc.Seek(ctx, &pb.SeekRequest{
Subscription: s.name,
Target: &pb.SeekRequest_Snapshot{snap.name},
})
return err
}
func toSnapshotConfig(snap *pb.Snapshot, c *Client) (*SnapshotConfig, error) {
exp, err := ptypes.Timestamp(snap.ExpireTime)
if err != nil {
return nil, err
}
return &SnapshotConfig{
Snapshot: &Snapshot{c: c, name: snap.Name},
Topic: newTopic(c, snap.Topic),
Expiration: exp,
}, nil
}
vendor/cloud.google.com/go/pubsub/streaming_pull_test.go
Generated Vendored
+21 -9
View File
@@ -20,14 +20,17 @@ package pubsub
import (
"io"
"reflect"
"strconv"
"sync"
"sync/atomic"
"testing"
"time"
"cloud.google.com/go/internal/testutil"
tspb "github.com/golang/protobuf/ptypes/timestamp"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/net/context"
"google.golang.org/api/option"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
@@ -57,8 +60,18 @@ func TestStreamingPullMultipleFetches(t *testing.T) {
testStreamingPullIteration(t, client, server, testMessages)
}
func newTestSubscription(t *testing.T, client *Client, name string) *Subscription {
topic := client.Topic("t")
sub, err := client.CreateSubscription(context.Background(), name,
SubscriptionConfig{Topic: topic})
if err != nil {
t.Fatalf("CreateSubscription: %v", err)
}
return sub
}
func testStreamingPullIteration(t *testing.T, client *Client, server *fakeServer, msgs []*pb.ReceivedMessage) {
sub := client.Subscription("s")
sub := newTestSubscription(t, client, "s")
gotMsgs, err := pullN(context.Background(), sub, len(msgs), func(_ context.Context, m *Message) {
id, err := strconv.Atoi(m.ackID)
if err != nil {
@@ -89,8 +102,7 @@ func testStreamingPullIteration(t *testing.T, client *Client, server *fakeServer
t.Errorf("%d: no message for ackID %q", i, want.ackID)
continue
}
got.doneFunc = nil // Don't compare done; it's a function.
if !reflect.DeepEqual(got, want) {
if !testutil.Equal(got, want, cmp.AllowUnexported(Message{}), cmpopts.IgnoreTypes(func(string, bool) {})) {
t.Errorf("%d: got\n%#v\nwant\n%#v", i, got, want)
}
}
@@ -116,7 +128,7 @@ func TestStreamingPullError(t *testing.T) {
client, server := newFake(t)
server.addStreamingPullMessages(testMessages[:1])
server.addStreamingPullError(grpc.Errorf(codes.Unknown, ""))
sub := client.Subscription("s")
sub := newTestSubscription(t, client, "s")
// Use only one goroutine, since the fake server is configured to
// return only one error.
sub.ReceiveSettings.NumGoroutines = 1
@@ -144,7 +156,7 @@ func TestStreamingPullCancel(t *testing.T) {
// return and all messages have been acked.
client, server := newFake(t)
server.addStreamingPullMessages(testMessages)
sub := client.Subscription("s")
sub := newTestSubscription(t, client, "s")
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
var n int32
err := sub.Receive(ctx, func(ctx2 context.Context, m *Message) {
@@ -179,7 +191,7 @@ func TestStreamingPullOneActive(t *testing.T) {
// Only one call to Pull can be active at a time.
client, srv := newFake(t)
srv.addStreamingPullMessages(testMessages[:1])
sub := client.Subscription("s")
sub := newTestSubscription(t, client, "s")
ctx, cancel := context.WithCancel(context.Background())
err := sub.Receive(ctx, func(ctx context.Context, m *Message) {
m.Ack()
@@ -209,7 +221,7 @@ func TestStreamingPullConcurrent(t *testing.T) {
for i := 0; i < nMessages; i += 2 {
server.addStreamingPullMessages([]*pb.ReceivedMessage{newMsg(i), newMsg(i + 1)})
}
sub := client.Subscription("s")
sub := newTestSubscription(t, client, "s")
ctx, _ := context.WithTimeout(context.Background(), time.Second)
gotMsgs, err := pullN(ctx, sub, nMessages, func(ctx context.Context, m *Message) {
m.Ack()
@@ -233,7 +245,7 @@ func TestStreamingPullFlowControl(t *testing.T) {
// Callback invocations should not occur if flow control limits are exceeded.
client, server := newFake(t)
server.addStreamingPullMessages(testMessages)
sub := client.Subscription("s")
sub := newTestSubscription(t, client, "s")
sub.ReceiveSettings.MaxOutstandingMessages = 2
ctx, cancel := context.WithCancel(context.Background())
activec := make(chan int)
vendor/cloud.google.com/go/pubsub/subscription.go
Generated Vendored
+133 -31
View File
@@ -23,15 +23,20 @@ import (
"time"
"cloud.google.com/go/iam"
"cloud.google.com/go/internal/optional"
"github.com/golang/protobuf/ptypes"
durpb "github.com/golang/protobuf/ptypes/duration"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
fmpb "google.golang.org/genproto/protobuf/field_mask"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
)
// Subscription is a reference to a PubSub subscription.
type Subscription struct {
s service
c *Client
// The fully qualified identifier for the subscription, in the format "projects/<projid>/subscriptions/<name>"
name string
@@ -45,13 +50,9 @@ type Subscription struct {
// Subscription creates a reference to a subscription.
func (c *Client) Subscription(id string) *Subscription {
return newSubscription(c.s, fmt.Sprintf("projects/%s/subscriptions/%s", c.projectID, id))
}
func newSubscription(s service, name string) *Subscription {
return &Subscription{
s: s,
name: name,
c: c,
name: fmt.Sprintf("projects/%s/subscriptions/%s", c.projectID, id),
}
}
@@ -72,16 +73,25 @@ func (s *Subscription) ID() string {
// Subscriptions returns an iterator which returns all of the subscriptions for the client's project.
func (c *Client) Subscriptions(ctx context.Context) *SubscriptionIterator {
it := c.subc.ListSubscriptions(ctx, &pb.ListSubscriptionsRequest{
Project: c.fullyQualifiedProjectName(),
})
return &SubscriptionIterator{
s: c.s,
next: c.s.listProjectSubscriptions(ctx, c.fullyQualifiedProjectName()),
c: c,
next: func() (string, error) {
sub, err := it.Next()
if err != nil {
return "", err
}
return sub.Name, nil
},
}
}
// SubscriptionIterator is an iterator that returns a series of subscriptions.
type SubscriptionIterator struct {
s service
next nextStringFunc
c *Client
next func() (string, error)
}
// Next returns the next subscription. If there are no more subscriptions, iterator.Done will be returned.
@@ -90,7 +100,7 @@ func (subs *SubscriptionIterator) Next() (*Subscription, error) {
if err != nil {
return nil, err
}
return newSubscription(subs.s, subName), nil
return &Subscription{c: subs.c, name: subName}, nil
}
// PushConfig contains configuration for subscriptions that operate in push mode.
@@ -102,6 +112,13 @@ type PushConfig struct {
Attributes map[string]string
}
func (pc *PushConfig) toProto() *pb.PushConfig {
return &pb.PushConfig{
Attributes: pc.Attributes,
PushEndpoint: pc.Endpoint,
}
}
// Subscription config contains the configuration of a subscription.
type SubscriptionConfig struct {
Topic *Topic
@@ -115,13 +132,52 @@ type SubscriptionConfig struct {
// Whether to retain acknowledged messages. If true, acknowledged messages
// will not be expunged until they fall out of the RetentionDuration window.
retainAckedMessages bool
RetainAckedMessages bool
// How long to retain messages in backlog, from the time of publish. If RetainAckedMessages is true,
// this duration affects the retention of acknowledged messages,
// otherwise only unacknowledged messages are retained.
// How long to retain messages in backlog, from the time of publish. If
// RetainAckedMessages is true, this duration affects the retention of
// acknowledged messages, otherwise only unacknowledged messages are retained.
// Defaults to 7 days. Cannot be longer than 7 days or shorter than 10 minutes.
retentionDuration time.Duration
RetentionDuration time.Duration
}
func (cfg *SubscriptionConfig) toProto(name string) *pb.Subscription {
var pbPushConfig *pb.PushConfig
if cfg.PushConfig.Endpoint != "" || len(cfg.PushConfig.Attributes) != 0 {
pbPushConfig = &pb.PushConfig{
Attributes: cfg.PushConfig.Attributes,
PushEndpoint: cfg.PushConfig.Endpoint,
}
}
var retentionDuration *durpb.Duration
if cfg.RetentionDuration != 0 {
retentionDuration = ptypes.DurationProto(cfg.RetentionDuration)
}
return &pb.Subscription{
Name: name,
Topic: cfg.Topic.name,
PushConfig: pbPushConfig,
AckDeadlineSeconds: trunc32(int64(cfg.AckDeadline.Seconds())),
RetainAckedMessages: cfg.RetainAckedMessages,
MessageRetentionDuration: retentionDuration,
}
}
func protoToSubscriptionConfig(pbSub *pb.Subscription, c *Client) (SubscriptionConfig, error) {
rd, err := ptypes.Duration(pbSub.MessageRetentionDuration)
if err != nil {
return SubscriptionConfig{}, err
}
return SubscriptionConfig{
Topic: newTopic(c, pbSub.Topic),
AckDeadline: time.Second * time.Duration(pbSub.AckDeadlineSeconds),
PushConfig: PushConfig{
Endpoint: pbSub.PushConfig.PushEndpoint,
Attributes: pbSub.PushConfig.Attributes,
},
RetainAckedMessages: pbSub.RetainAckedMessages,
RetentionDuration: rd,
}, nil
}
// ReceiveSettings configure the Receive method.
@@ -171,31 +227,47 @@ var DefaultReceiveSettings = ReceiveSettings{
// Delete deletes the subscription.
func (s *Subscription) Delete(ctx context.Context) error {
return s.s.deleteSubscription(ctx, s.name)
return s.c.subc.DeleteSubscription(ctx, &pb.DeleteSubscriptionRequest{Subscription: s.name})
}
// Exists reports whether the subscription exists on the server.
func (s *Subscription) Exists(ctx context.Context) (bool, error) {
return s.s.subscriptionExists(ctx, s.name)
_, err := s.c.subc.GetSubscription(ctx, &pb.GetSubscriptionRequest{Subscription: s.name})
if err == nil {
return true, nil
}
if grpc.Code(err) == codes.NotFound {
return false, nil
}
return false, err
}
// Config fetches the current configuration for the subscription.
func (s *Subscription) Config(ctx context.Context) (SubscriptionConfig, error) {
conf, topicName, err := s.s.getSubscriptionConfig(ctx, s.name)
pbSub, err := s.c.subc.GetSubscription(ctx, &pb.GetSubscriptionRequest{Subscription: s.name})
if err != nil {
return SubscriptionConfig{}, err
}
conf.Topic = &Topic{
s: s.s,
name: topicName,
cfg, err := protoToSubscriptionConfig(pbSub, s.c)
if err != nil {
return SubscriptionConfig{}, err
}
return conf, nil
return cfg, nil
}
// SubscriptionConfigToUpdate describes how to update a subscription.
type SubscriptionConfigToUpdate struct {
// If non-nil, the push config is changed.
PushConfig *PushConfig
// If non-zero, the ack deadline is changed.
AckDeadline time.Duration
// If set, RetainAckedMessages is changed.
RetainAckedMessages optional.Bool
// If non-zero, RetentionDuration is changed.
RetentionDuration time.Duration
}
// Update changes an existing subscription according to the fields set in cfg.
@@ -203,17 +275,44 @@ type SubscriptionConfigToUpdate struct {
//
// Update returns an error if no fields were modified.
func (s *Subscription) Update(ctx context.Context, cfg SubscriptionConfigToUpdate) (SubscriptionConfig, error) {
if cfg.PushConfig == nil {
req := s.updateRequest(&cfg)
if len(req.UpdateMask.Paths) == 0 {
return SubscriptionConfig{}, errors.New("pubsub: UpdateSubscription call with nothing to update")
}
if err := s.s.modifyPushConfig(ctx, s.name, *cfg.PushConfig); err != nil {
rpsub, err := s.c.subc.UpdateSubscription(ctx, req)
if err != nil {
return SubscriptionConfig{}, err
}
return s.Config(ctx)
return protoToSubscriptionConfig(rpsub, s.c)
}
func (s *Subscription) updateRequest(cfg *SubscriptionConfigToUpdate) *pb.UpdateSubscriptionRequest {
psub := &pb.Subscription{Name: s.name}
var paths []string
if cfg.PushConfig != nil {
psub.PushConfig = cfg.PushConfig.toProto()
paths = append(paths, "push_config")
}
if cfg.AckDeadline != 0 {
psub.AckDeadlineSeconds = trunc32(int64(cfg.AckDeadline.Seconds()))
paths = append(paths, "ack_deadline_seconds")
}
if cfg.RetainAckedMessages != nil {
psub.RetainAckedMessages = optional.ToBool(cfg.RetainAckedMessages)
paths = append(paths, "retain_acked_messages")
}
if cfg.RetentionDuration != 0 {
psub.MessageRetentionDuration = ptypes.DurationProto(cfg.RetentionDuration)
paths = append(paths, "message_retention_duration")
}
return &pb.UpdateSubscriptionRequest{
Subscription: psub,
UpdateMask: &fmpb.FieldMask{Paths: paths},
}
}
func (s *Subscription) IAM() *iam.Handle {
return s.s.iamHandle(s.name)
return iam.InternalNewHandle(s.c.subc.Connection(), s.name)
}
// CreateSubscription creates a new subscription on a topic.
@@ -250,8 +349,11 @@ func (c *Client) CreateSubscription(ctx context.Context, id string, cfg Subscrip
}
sub := c.Subscription(id)
err := c.s.createSubscription(ctx, sub.name, cfg)
return sub, err
_, err := c.subc.CreateSubscription(ctx, cfg.toProto(sub.name))
if err != nil {
return nil, err
}
return sub, nil
}
var errReceiveInProgress = errors.New("pubsub: Receive already in progress for this subscription")
@@ -347,7 +449,7 @@ func (s *Subscription) receive(ctx context.Context, po *pullOptions, fc *flowCon
// The iterator does not use the context passed to Receive. If it did, canceling
// that context would immediately stop the iterator without waiting for unacked
// messages.
iter := newMessageIterator(context.Background(), s.s, s.name, po)
iter := newMessageIterator(context.Background(), s.c.subc, s.name, po)
// We cannot use errgroup from Receive here. Receive might already be calling group.Wait,
// and group.Wait cannot be called concurrently with group.Go. We give each receive() its
vendor/cloud.google.com/go/pubsub/subscription_test.go
Generated Vendored
+121 -90
View File
@@ -15,49 +15,17 @@
package pubsub
import (
"reflect"
"fmt"
"testing"
"time"
"cloud.google.com/go/internal/testutil"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
type subListService struct {
service
subs []string
err error
t *testing.T // for error logging.
}
func (s *subListService) newNextStringFunc() nextStringFunc {
return func() (string, error) {
if len(s.subs) == 0 {
return "", iterator.Done
}
sn := s.subs[0]
s.subs = s.subs[1:]
return sn, s.err
}
}
func (s *subListService) listProjectSubscriptions(ctx context.Context, projName string) nextStringFunc {
if projName != "projects/projid" {
s.t.Fatalf("unexpected call: projName: %q", projName)
return nil
}
return s.newNextStringFunc()
}
func (s *subListService) listTopicSubscriptions(ctx context.Context, topicName string) nextStringFunc {
if topicName != "projects/projid/topics/topic" {
s.t.Fatalf("unexpected call: topicName: %q", topicName)
return nil
}
return s.newNextStringFunc()
}
// All returns the remaining subscriptions from this iterator.
func slurpSubs(it *SubscriptionIterator) ([]*Subscription, error) {
var subs []*Subscription
@@ -75,75 +43,138 @@ func slurpSubs(it *SubscriptionIterator) ([]*Subscription, error) {
func TestSubscriptionID(t *testing.T) {
const id = "id"
serv := &subListService{
subs: []string{"projects/projid/subscriptions/s1", "projects/projid/subscriptions/s2"},
t: t,
}
c := &Client{projectID: "projid", s: serv}
c := &Client{projectID: "projid"}
s := c.Subscription(id)
if got, want := s.ID(), id; got != want {
t.Errorf("Subscription.ID() = %q; want %q", got, want)
}
want := []string{"s1", "s2"}
subs, err := slurpSubs(c.Subscriptions(context.Background()))
if err != nil {
t.Errorf("error listing subscriptions: %v", err)
}
func TestListProjectSubscriptions(t *testing.T) {
ctx := context.Background()
c, _ := newFake(t)
topic := mustCreateTopic(t, c, "t")
var want []string
for i := 1; i <= 2; i++ {
id := fmt.Sprintf("s%d", i)
want = append(want, id)
_, err := c.CreateSubscription(ctx, id, SubscriptionConfig{Topic: topic})
if err != nil {
t.Fatal(err)
}
}
for i, s := range subs {
if got, want := s.ID(), want[i]; got != want {
t.Errorf("Subscription.ID() = %q; want %q", got, want)
subs, err := slurpSubs(c.Subscriptions(ctx))
if err != nil {
t.Fatal(err)
}
got := getSubIDs(subs)
if !testutil.Equal(got, want) {
t.Errorf("got %v, want %v", got, want)
}
}
func getSubIDs(subs []*Subscription) []string {
var names []string
for _, sub := range subs {
names = append(names, sub.ID())
}
return names
}
func TestListTopicSubscriptions(t *testing.T) {
ctx := context.Background()
c, _ := newFake(t)
topics := []*Topic{
mustCreateTopic(t, c, "t0"),
mustCreateTopic(t, c, "t1"),
}
wants := make([][]string, 2)
for i := 0; i < 5; i++ {
id := fmt.Sprintf("s%d", i)
sub, err := c.CreateSubscription(ctx, id, SubscriptionConfig{Topic: topics[i%2]})
if err != nil {
t.Fatal(err)
}
wants[i%2] = append(wants[i%2], sub.ID())
}
for i, topic := range topics {
subs, err := slurpSubs(topic.Subscriptions(ctx))
if err != nil {
t.Fatal(err)
}
got := getSubIDs(subs)
if !testutil.Equal(got, wants[i]) {
t.Errorf("#%d: got %v, want %v", i, got, wants[i])
}
}
}
func TestListProjectSubscriptions(t *testing.T) {
snames := []string{"projects/projid/subscriptions/s1", "projects/projid/subscriptions/s2",
"projects/projid/subscriptions/s3"}
s := &subListService{subs: snames, t: t}
c := &Client{projectID: "projid", s: s}
subs, err := slurpSubs(c.Subscriptions(context.Background()))
const defaultRetentionDuration = 168 * time.Hour
func TestUpdateSubscription(t *testing.T) {
ctx := context.Background()
client, _ := newFake(t)
defer client.Close()
topic := client.Topic("t")
sub, err := client.CreateSubscription(ctx, "s", SubscriptionConfig{Topic: topic})
if err != nil {
t.Errorf("error listing subscriptions: %v", err)
t.Fatal(err)
}
got := subNames(subs)
want := []string{
"projects/projid/subscriptions/s1",
"projects/projid/subscriptions/s2",
"projects/projid/subscriptions/s3"}
if !reflect.DeepEqual(got, want) {
t.Errorf("sub list: got: %v, want: %v", got, want)
cfg, err := sub.Config(ctx)
if err != nil {
t.Fatal(err)
}
if len(s.subs) != 0 {
t.Errorf("outstanding subs: %v", s.subs)
want := SubscriptionConfig{
Topic: topic,
AckDeadline: 10 * time.Second,
RetainAckedMessages: false,
RetentionDuration: defaultRetentionDuration,
}
if !testutil.Equal(cfg, want) {
t.Fatalf("\ngot %+v\nwant %+v", cfg, want)
}
got, err := sub.Update(ctx, SubscriptionConfigToUpdate{
AckDeadline: 20 * time.Second,
RetainAckedMessages: true,
})
if err != nil {
t.Fatal(err)
}
want = SubscriptionConfig{
Topic: topic,
AckDeadline: 20 * time.Second,
RetainAckedMessages: true,
RetentionDuration: defaultRetentionDuration,
}
if !testutil.Equal(got, want) {
t.Fatalf("\ngot %+v\nwant %+v", got, want)
}
got, err = sub.Update(ctx, SubscriptionConfigToUpdate{RetentionDuration: 2 * time.Hour})
if err != nil {
t.Fatal(err)
}
want.RetentionDuration = 2 * time.Hour
if !testutil.Equal(got, want) {
t.Fatalf("\ngot %+v\nwant %+v", got, want)
}
_, err = sub.Update(ctx, SubscriptionConfigToUpdate{})
if err == nil {
t.Fatal("got nil, want error")
}
}
func TestListTopicSubscriptions(t *testing.T) {
snames := []string{"projects/projid/subscriptions/s1", "projects/projid/subscriptions/s2",
"projects/projid/subscriptions/s3"}
s := &subListService{subs: snames, t: t}
c := &Client{projectID: "projid", s: s}
subs, err := slurpSubs(c.Topic("topic").Subscriptions(context.Background()))
if err != nil {
t.Errorf("error listing subscriptions: %v", err)
func (t1 *Topic) Equal(t2 *Topic) bool {
if t1 == nil && t2 == nil {
return true
}
got := subNames(subs)
want := []string{
"projects/projid/subscriptions/s1",
"projects/projid/subscriptions/s2",
"projects/projid/subscriptions/s3"}
if !reflect.DeepEqual(got, want) {
t.Errorf("sub list: got: %v, want: %v", got, want)
}
if len(s.subs) != 0 {
t.Errorf("outstanding subs: %v", s.subs)
if t1 == nil || t2 == nil {
return false
}
}
func subNames(subs []*Subscription) []string {
var names []string
for _, sub := range subs {
names = append(names, sub.name)
}
return names
return t1.c == t2.c && t1.name == t2.name
}
vendor/cloud.google.com/go/pubsub/timeout_test.go
Generated Vendored
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pubsub
import (
"log"
"sync/atomic"
"testing"
"time"
"golang.org/x/net/context"
"cloud.google.com/go/pubsub/pstest"
"google.golang.org/api/option"
"google.golang.org/grpc"
)
// Using the fake PubSub server in the pstest package, verify that streaming
// pull resumes if the server stream times out.
func TestStreamTimeout(t *testing.T) {
log.SetFlags(log.Lmicroseconds)
ctx := context.Background()
srv := pstest.NewServer()
srv.SetStreamTimeout(2 * time.Second)
conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
if err != nil {
t.Fatal(err)
}
client, err := NewClient(ctx, "P", option.WithGRPCConn(conn))
if err != nil {
t.Fatal(err)
}
defer client.Close()
topic, err := client.CreateTopic(ctx, "T")
if err != nil {
t.Fatal(err)
}
sub, err := client.CreateSubscription(ctx, "sub", SubscriptionConfig{Topic: topic, AckDeadline: 10 * time.Second})
if err != nil {
t.Fatal(err)
}
const nPublish = 8
rctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
errc := make(chan error)
var nSeen int64
go func() {
errc <- sub.Receive(rctx, func(ctx context.Context, m *Message) {
m.Ack()
n := atomic.AddInt64(&nSeen, 1)
if n >= nPublish {
cancel()
}
})
}()
for i := 0; i < nPublish; i++ {
pr := topic.Publish(ctx, &Message{Data: []byte("msg")})
_, err := pr.Get(ctx)
if err != nil {
t.Fatal(err)
}
time.Sleep(250 * time.Millisecond)
}
err = <-errc
if err := sub.Delete(ctx); err != nil {
t.Fatal(err)
}
n := atomic.LoadInt64(&nSeen)
t.Logf("Receive returned %v after seeing %d messages\n", err, n)
if n < nPublish {
t.Errorf("got %d messages, want %d", n, nPublish)
}
}
vendor/cloud.google.com/go/pubsub/topic.go
Generated Vendored
+52 -23
View File
@@ -24,9 +24,12 @@ import (
"cloud.google.com/go/iam"
"github.com/golang/protobuf/proto"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
"google.golang.org/api/support/bundler"
pb "google.golang.org/genproto/googleapis/pubsub/v1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
)
const (
@@ -47,7 +50,7 @@ var ErrOversizedMessage = bundler.ErrOversizedItem
//
// The methods of Topic are safe for use by multiple goroutines.
type Topic struct {
s service
c *Client
// The fully qualified identifier for the topic, in the format "projects/<projid>/topics/<name>"
name string
@@ -102,8 +105,11 @@ var DefaultPublishSettings = PublishSettings{
// If the topic already exists an error will be returned.
func (c *Client) CreateTopic(ctx context.Context, id string) (*Topic, error) {
t := c.Topic(id)
err := c.s.createTopic(ctx, t.name)
return t, err
_, err := c.pubc.CreateTopic(ctx, &pb.Topic{Name: t.name})
if err != nil {
return nil, err
}
return t, nil
}
// Topic creates a reference to a topic in the client's project.
@@ -123,15 +129,15 @@ func (c *Client) Topic(id string) *Topic {
//
// Avoid creating many Topic instances if you use them to publish.
func (c *Client) TopicInProject(id, projectID string) *Topic {
return newTopic(c.s, fmt.Sprintf("projects/%s/topics/%s", projectID, id))
return newTopic(c, fmt.Sprintf("projects/%s/topics/%s", projectID, id))
}
func newTopic(s service, name string) *Topic {
func newTopic(c *Client, name string) *Topic {
// bundlec is unbuffered. A buffer would occupy memory not
// accounted for by the bundler, so BufferedByteLimit would be a lie:
// the actual memory consumed would be higher.
return &Topic{
s: s,
c: c,
name: name,
PublishSettings: DefaultPublishSettings,
bundlec: make(chan []*bundledMessage),
@@ -140,16 +146,23 @@ func newTopic(s service, name string) *Topic {
// Topics returns an iterator which returns all of the topics for the client's project.
func (c *Client) Topics(ctx context.Context) *TopicIterator {
it := c.pubc.ListTopics(ctx, &pb.ListTopicsRequest{Project: c.fullyQualifiedProjectName()})
return &TopicIterator{
s: c.s,
next: c.s.listProjectTopics(ctx, c.fullyQualifiedProjectName()),
c: c,
next: func() (string, error) {
topic, err := it.Next()
if err != nil {
return "", err
}
return topic.Name, nil
},
}
}
// TopicIterator is an iterator that returns a series of topics.
type TopicIterator struct {
s service
next nextStringFunc
c *Client
next func() (string, error)
}
// Next returns the next topic. If there are no more topics, iterator.Done will be returned.
@@ -158,7 +171,7 @@ func (tps *TopicIterator) Next() (*Topic, error) {
if err != nil {
return nil, err
}
return newTopic(tps.s, topicName), nil
return newTopic(tps.c, topicName), nil
}
// ID returns the unique idenfier of the topic within its project.
@@ -178,7 +191,7 @@ func (t *Topic) String() string {
// Delete deletes the topic.
func (t *Topic) Delete(ctx context.Context) error {
return t.s.deleteTopic(ctx, t.name)
return t.c.pubc.DeleteTopic(ctx, &pb.DeleteTopicRequest{Topic: t.name})
}
// Exists reports whether the topic exists on the server.
@@ -186,21 +199,30 @@ func (t *Topic) Exists(ctx context.Context) (bool, error) {
if t.name == "_deleted-topic_" {
return false, nil
}
return t.s.topicExists(ctx, t.name)
_, err := t.c.pubc.GetTopic(ctx, &pb.GetTopicRequest{Topic: t.name})
if err == nil {
return true, nil
}
if grpc.Code(err) == codes.NotFound {
return false, nil
}
return false, err
}
func (t *Topic) IAM() *iam.Handle {
return t.s.iamHandle(t.name)
return iam.InternalNewHandle(t.c.pubc.Connection(), t.name)
}
// Subscriptions returns an iterator which returns the subscriptions for this topic.
//
// Some of the returned subscriptions may belong to a project other than t.
func (t *Topic) Subscriptions(ctx context.Context) *SubscriptionIterator {
// NOTE: zero or more Subscriptions that are ultimately returned by this
// Subscriptions iterator may belong to a different project to t.
it := t.c.pubc.ListTopicSubscriptions(ctx, &pb.ListTopicSubscriptionsRequest{
Topic: t.name,
})
return &SubscriptionIterator{
s: t.s,
next: t.s.listTopicSubscriptions(ctx, t.name),
c: t.c,
next: it.Next,
}
}
@@ -353,16 +375,23 @@ func (t *Topic) initBundler() {
}
func (t *Topic) publishMessageBundle(ctx context.Context, bms []*bundledMessage) {
msgs := make([]*Message, len(bms))
pbMsgs := make([]*pb.PubsubMessage, len(bms))
for i, bm := range bms {
msgs[i], bm.msg = bm.msg, nil // release bm.msg for GC
pbMsgs[i] = &pb.PubsubMessage{
Data: bm.msg.Data,
Attributes: bm.msg.Attributes,
}
bm.msg = nil // release bm.msg for GC
}
ids, err := t.s.publishMessages(ctx, t.name, msgs)
res, err := t.c.pubc.Publish(ctx, &pb.PublishRequest{
Topic: t.name,
Messages: pbMsgs,
}, gax.WithGRPCOptions(grpc.MaxCallSendMsgSize(maxSendRecvBytes)))
for i, bm := range bms {
if err != nil {
bm.res.set("", err)
} else {
bm.res.set(ids[i], nil)
bm.res.set(res.MessageIds[i], nil)
}
}
}
vendor/cloud.google.com/go/pubsub/topic_test.go
Generated Vendored
+27 -67
View File
@@ -15,11 +15,13 @@
package pubsub
import (
"fmt"
"net"
"reflect"
"testing"
"time"
"cloud.google.com/go/internal/testutil"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
@@ -28,46 +30,18 @@ import (
"google.golang.org/grpc/codes"
)
type topicListService struct {
service
topics []string
err error
t *testing.T // for error logging.
}
func (s *topicListService) newNextStringFunc() nextStringFunc {
return func() (string, error) {
if len(s.topics) == 0 {
return "", iterator.Done
}
tn := s.topics[0]
s.topics = s.topics[1:]
return tn, s.err
}
}
func (s *topicListService) listProjectTopics(ctx context.Context, projName string) nextStringFunc {
if projName != "projects/projid" {
s.t.Fatalf("unexpected call: projName: %q", projName)
return nil
}
return s.newNextStringFunc()
}
func checkTopicListing(t *testing.T, want []string) {
s := &topicListService{topics: want, t: t}
c := &Client{projectID: "projid", s: s}
func checkTopicListing(t *testing.T, c *Client, want []string) {
topics, err := slurpTopics(c.Topics(context.Background()))
if err != nil {
t.Errorf("error listing topics: %v", err)
t.Fatalf("error listing topics: %v", err)
}
got := topicNames(topics)
if !reflect.DeepEqual(got, want) {
var got []string
for _, topic := range topics {
got = append(got, topic.ID())
}
if !testutil.Equal(got, want) {
t.Errorf("topic list: got: %v, want: %v", got, want)
}
if len(s.topics) != 0 {
t.Errorf("outstanding topics: %v", s.topics)
}
}
// All returns the remaining topics from this iterator.
@@ -87,38 +61,27 @@ func slurpTopics(it *TopicIterator) ([]*Topic, error) {
func TestTopicID(t *testing.T) {
const id = "id"
serv := &topicListService{
topics: []string{"projects/projid/topics/t1", "projects/projid/topics/t2"},
t: t,
}
c := &Client{projectID: "projid", s: serv}
c, _ := newFake(t)
s := c.Topic(id)
if got, want := s.ID(), id; got != want {
t.Errorf("Token.ID() = %q; want %q", got, want)
}
want := []string{"t1", "t2"}
topics, err := slurpTopics(c.Topics(context.Background()))
if err != nil {
t.Errorf("error listing topics: %v", err)
}
for i, topic := range topics {
if got, want := topic.ID(), want[i]; got != want {
t.Errorf("Token.ID() = %q; want %q", got, want)
}
}
}
func TestListTopics(t *testing.T) {
checkTopicListing(t, []string{
"projects/projid/topics/t1",
"projects/projid/topics/t2",
"projects/projid/topics/t3",
"projects/projid/topics/t4"})
c, _ := newFake(t)
var ids []string
for i := 1; i <= 4; i++ {
id := fmt.Sprintf("t%d", i)
ids = append(ids, id)
mustCreateTopic(t, c, id)
}
checkTopicListing(t, c, ids)
}
func TestListCompletelyEmptyTopics(t *testing.T) {
var want []string
checkTopicListing(t, want)
c, _ := newFake(t)
checkTopicListing(t, c, nil)
}
func TestStopPublishOrder(t *testing.T) {
@@ -148,11 +111,10 @@ func TestPublishTimeout(t *testing.T) {
if err != nil {
t.Fatal(err)
}
s, err := newPubSubService(context.Background(), []option.ClientOption{option.WithGRPCConn(conn)})
c, err := NewClient(ctx, "projectID", option.WithGRPCConn(conn))
if err != nil {
t.Fatal(err)
}
c := &Client{s: s}
topic := c.Topic("t")
topic.PublishSettings.Timeout = 3 * time.Second
r := topic.Publish(ctx, &Message{})
@@ -176,12 +138,10 @@ func (s *alwaysFailPublish) Publish(ctx context.Context, req *pubsubpb.PublishRe
return nil, grpc.Errorf(codes.Unavailable, "try again")
}
func topicNames(topics []*Topic) []string {
var names []string
for _, topic := range topics {
names = append(names, topic.name)
func mustCreateTopic(t *testing.T, c *Client, id string) *Topic {
topic, err := c.CreateTopic(context.Background(), id)
if err != nil {
t.Fatal(err)
}
return names
return topic
}
vendor/cloud.google.com/go/pubsub/utils_test.go
Generated Vendored
-63
View File
@@ -1,63 +0,0 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pubsub
import (
"time"
"golang.org/x/net/context"
)
type modDeadlineCall struct {
subName string
deadline time.Duration
ackIDs []string
}
type acknowledgeCall struct {
subName string
ackIDs []string
}
type testService struct {
service
// The arguments of each call to modifyAckDealine are written to this channel.
modDeadlineCalled chan modDeadlineCall
// The arguments of each call to acknowledge are written to this channel.
acknowledgeCalled chan acknowledgeCall
}
func (s *testService) modifyAckDeadline(ctx context.Context, subName string, deadline time.Duration, ackIDs []string) error {
s.modDeadlineCalled <- modDeadlineCall{
subName: subName,
deadline: deadline,
ackIDs: ackIDs,
}
return nil
}
func (s *testService) acknowledge(ctx context.Context, subName string, ackIDs []string) error {
s.acknowledgeCalled <- acknowledgeCall{
subName: subName,
ackIDs: ackIDs,
}
return nil
}
func (s *testService) splitAckIDs(ids []string) ([]string, []string) {
return ids, nil
}