amorfo77/restic
1
0 Fork 0
mirror of https://github.com/restic/restic.git synced 2026-10-29 03:25:42 +00:00
Code Issues Projects Releases Wiki Activity

Update vendored dependencies

Browse Source 
This includes github.com/kurin/blazer 0.2.0, which resolves #1291
This commit is contained in:
Alexander Neumann committed 2017-10-01 10:13:39 +02:00
1 parent ba23d24dd1
commit 61cb1cc6f8
1044 files changed
+203017 -97704
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vendor/cloud.google.com/go/bigquery/copy.go
Generated Vendored
+8 -3
View File
@@ -21,9 +21,12 @@ import (
// CopyConfig holds the configuration for a copy job.
type CopyConfig struct {
// JobID is the ID to use for the copy job. If unset, a job ID will be automatically created.
// JobID is the ID to use for the job. If empty, a random job ID will be generated.
JobID string
// If AddJobIDSuffix is true, then a random string will be appended to JobID.
AddJobIDSuffix bool
// Srcs are the tables from which data will be copied.
Srcs []*Table
@@ -68,7 +71,9 @@ func (c *Copier) Run(ctx context.Context) (*Job, error) {
for _, t := range c.Srcs {
conf.SourceTables = append(conf.SourceTables, t.tableRefProto())
}
job := &bq.Job{Configuration: &bq.JobConfiguration{Copy: conf}}
setJobRef(job, c.JobID, c.c.projectID)
job := &bq.Job{
JobReference: createJobRef(c.JobID, c.AddJobIDSuffix, c.c.projectID),
Configuration: &bq.JobConfiguration{Copy: conf},
}
return c.c.insertJob(ctx, &insertJobConf{job: job})
}
vendor/cloud.google.com/go/bigquery/copy_test.go
Generated Vendored
+2 -19
View File
@@ -17,16 +17,13 @@ package bigquery
import (
"testing"
"cloud.google.com/go/internal/testutil"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/net/context"
bq "google.golang.org/api/bigquery/v2"
)
func defaultCopyJob() *bq.Job {
return &bq.Job{
JobReference: &bq.JobReference{ProjectId: "client-project-id"},
JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
Configuration: &bq.JobConfiguration{
Copy: &bq.JobConfigurationTableCopy{
DestinationTable: &bq.TableReference{
@@ -47,6 +44,7 @@ func defaultCopyJob() *bq.Job {
}
func TestCopy(t *testing.T) {
defer fixRandomJobID("RANDOM")()
testCases := []struct {
dst *Table
srcs []*Table
@@ -132,18 +130,3 @@ func TestCopy(t *testing.T) {
checkJob(t, i, s.Job, tc.want)
}
}
func checkJob(t *testing.T, i int, got, want *bq.Job) {
if got.JobReference == nil {
t.Errorf("#%d: empty job reference", i)
return
}
if got.JobReference.JobId == "" {
t.Errorf("#%d: empty job ID", i)
return
}
d := testutil.Diff(got, want, cmpopts.IgnoreFields(bq.JobReference{}, "JobId"))
if d != "" {
t.Errorf("#%d: (got=-, want=+) %s", i, d)
}
}
vendor/cloud.google.com/go/bigquery/create_table_test.go
Generated Vendored
-122
View File
@@ -1,122 +0,0 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bigquery
import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"cloud.google.com/go/internal/testutil"
"golang.org/x/net/context"
bq "google.golang.org/api/bigquery/v2"
)
type createTableRecorder struct {
conf *createTableConf
service
}
func (rec *createTableRecorder) createTable(ctx context.Context, conf *createTableConf) error {
rec.conf = conf
return nil
}
func TestCreateTableOptions(t *testing.T) {
s := &createTableRecorder{}
c := &Client{
projectID: "p",
service: s,
}
ds := c.Dataset("d")
table := ds.Table("t")
exp := time.Now()
q := "query"
if err := table.Create(context.Background(), TableExpiration(exp), ViewQuery(q), UseStandardSQL()); err != nil {
t.Fatalf("err calling Table.Create: %v", err)
}
want := createTableConf{
projectID: "p",
datasetID: "d",
tableID: "t",
expiration: exp,
viewQuery: q,
useStandardSQL: true,
}
equal := func(x, y createTableConf) bool {
return testutil.Equal(x, y, cmp.AllowUnexported(createTableConf{}))
}
if !equal(*s.conf, want) {
t.Errorf("createTableConf: got:\n%v\nwant:\n%v", *s.conf, want)
}
sc := Schema{fieldSchema("desc", "name", "STRING", false, true)}
if err := table.Create(context.Background(), TableExpiration(exp), sc); err != nil {
t.Fatalf("err calling Table.Create: %v", err)
}
want = createTableConf{
projectID: "p",
datasetID: "d",
tableID: "t",
expiration: exp,
// No need for an elaborate schema, that is tested in schema_test.go.
schema: &bq.TableSchema{
Fields: []*bq.TableFieldSchema{
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
},
},
}
if !equal(*s.conf, want) {
t.Errorf("createTableConf: got:\n%v\nwant:\n%v", *s.conf, want)
}
partitionCases := []struct {
timePartitioning TimePartitioning
expectedExpiration time.Duration
}{
{TimePartitioning{}, time.Duration(0)},
{TimePartitioning{time.Second}, time.Second},
}
for _, c := range partitionCases {
if err := table.Create(context.Background(), c.timePartitioning); err != nil {
t.Fatalf("err calling Table.Create: %v", err)
}
want = createTableConf{
projectID: "p",
datasetID: "d",
tableID: "t",
timePartitioning: &TimePartitioning{c.expectedExpiration},
}
if !equal(*s.conf, want) {
t.Errorf("createTableConf: got:\n%v\nwant:\n%v", *s.conf, want)
}
}
}
func TestCreateTableOptionsLegacySQL(t *testing.T) {
c := &Client{
projectID: "p",
service: &bigqueryService{},
}
ds := c.Dataset("d")
table := ds.Table("t")
if err := table.Create(context.Background(), UseStandardSQL(), UseLegacySQL()); err == nil {
t.Fatal("no error using both standard and legacy SQL options")
}
}
vendor/cloud.google.com/go/bigquery/dataset.go
Generated Vendored
+16 -10
View File
@@ -30,22 +30,28 @@ type Dataset struct {
c *Client
}
// DatasetMetadata contains information about a BigQuery dataset.
type DatasetMetadata struct {
CreationTime time.Time
LastModifiedTime time.Time // When the dataset or any of its tables were modified.
DefaultTableExpiration time.Duration // The default expiration time for new tables.
Description string // The user-friendly description of this dataset.
Name string // The user-friendly name for this dataset.
ID string
// These fields can be set when creating a dataset.
Name string // The user-friendly name for this dataset.
Description string // The user-friendly description of this dataset.
Location string // The geo location of the dataset.
DefaultTableExpiration time.Duration // The default expiration time for new tables.
Labels map[string]string // User-provided labels.
// These fields are read-only.
CreationTime time.Time
LastModifiedTime time.Time // When the dataset or any of its tables were modified.
FullID string // The full dataset ID in the form projectID:datasetID.
// ETag is the ETag obtained when reading metadata. Pass it to Dataset.Update to
// ensure that the metadata hasn't changed since it was read.
ETag string
// TODO(jba): access rules
}
// DatasetMetadataToUpdate is used when updating a dataset's metadata.
// Only non-nil fields will be updated.
type DatasetMetadataToUpdate struct {
Description optional.String // The user-friendly description of this table.
Name optional.String // The user-friendly name for this dataset.
@@ -89,10 +95,10 @@ func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset {
}
}
// Create creates a dataset in the BigQuery service. An error will be returned
// if the dataset already exists.
func (d *Dataset) Create(ctx context.Context) error {
return d.c.service.insertDataset(ctx, d.DatasetID, d.ProjectID)
// Create creates a dataset in the BigQuery service. An error will be returned if the
// dataset already exists. Pass in a DatasetMetadata value to configure the dataset.
func (d *Dataset) Create(ctx context.Context, md *DatasetMetadata) error {
return d.c.service.insertDataset(ctx, d.DatasetID, d.ProjectID, md)
}
// Delete deletes the dataset.
vendor/cloud.google.com/go/bigquery/examples_test.go
Generated Vendored
+12 -5
View File
@@ -17,6 +17,7 @@ package bigquery_test
import (
"fmt"
"os"
"time"
"cloud.google.com/go/bigquery"
"golang.org/x/net/context"
@@ -233,7 +234,8 @@ func ExampleDataset_Create() {
if err != nil {
// TODO: Handle error.
}
if err := client.Dataset("my_dataset").Create(ctx); err != nil {
ds := client.Dataset("my_dataset")
if err := ds.Create(ctx, &bigquery.DatasetMetadata{Location: "EU"}); err != nil {
// TODO: Handle error.
}
}
@@ -389,13 +391,13 @@ func ExampleTable_Create() {
// TODO: Handle error.
}
t := client.Dataset("my_dataset").Table("new-table")
if err := t.Create(ctx); err != nil {
if err := t.Create(ctx, nil); err != nil {
// TODO: Handle error.
}
}
// If you know your table's schema initially, pass a Schema to Create.
func ExampleTable_Create_schema() {
// Initialize a new table by passing TableMetadata to Table.Create.
func ExampleTable_Create_initialize() {
ctx := context.Background()
// Infer table schema from a Go type.
schema, err := bigquery.InferSchema(Item{})
@@ -407,7 +409,12 @@ func ExampleTable_Create_schema() {
// TODO: Handle error.
}
t := client.Dataset("my_dataset").Table("new-table")
if err := t.Create(ctx, schema); err != nil {
if err := t.Create(ctx,
&bigquery.TableMetadata{
Name: "My New Table",
Schema: schema,
ExpirationTime: time.Now().Add(24 * time.Hour),
}); err != nil {
// TODO: Handle error.
}
}
vendor/cloud.google.com/go/bigquery/extract.go
Generated Vendored
+19 -15
View File
@@ -21,9 +21,12 @@ import (
// ExtractConfig holds the configuration for an extract job.
type ExtractConfig struct {
// JobID is the ID to use for the extract job. If empty, a job ID will be automatically created.
// JobID is the ID to use for the job. If empty, a random job ID will be generated.
JobID string
// If AddJobIDSuffix is true, then a random string will be appended to JobID.
AddJobIDSuffix bool
// Src is the table from which data will be extracted.
Src *Table
@@ -55,22 +58,23 @@ func (t *Table) ExtractorTo(dst *GCSReference) *Extractor {
// Run initiates an extract job.
func (e *Extractor) Run(ctx context.Context) (*Job, error) {
conf := &bq.JobConfigurationExtract{}
job := &bq.Job{Configuration: &bq.JobConfiguration{Extract: conf}}
setJobRef(job, e.JobID, e.c.projectID)
conf.DestinationUris = append([]string{}, e.Dst.uris...)
conf.Compression = string(e.Dst.Compression)
conf.DestinationFormat = string(e.Dst.DestinationFormat)
conf.FieldDelimiter = e.Dst.FieldDelimiter
conf.SourceTable = e.Src.tableRefProto()
var printHeader *bool
if e.DisableHeader {
f := false
conf.PrintHeader = &f
printHeader = &f
}
job := &bq.Job{
JobReference: createJobRef(e.JobID, e.AddJobIDSuffix, e.c.projectID),
Configuration: &bq.JobConfiguration{
Extract: &bq.JobConfigurationExtract{
DestinationUris: append([]string{}, e.Dst.uris...),
Compression: string(e.Dst.Compression),
DestinationFormat: string(e.Dst.DestinationFormat),
FieldDelimiter: e.Dst.FieldDelimiter,
SourceTable: e.Src.tableRefProto(),
PrintHeader: printHeader,
},
},
}
return e.c.insertJob(ctx, &insertJobConf{job: job})
}
vendor/cloud.google.com/go/bigquery/extract_test.go
Generated Vendored
+2 -1
View File
@@ -24,7 +24,7 @@ import (
func defaultExtractJob() *bq.Job {
return &bq.Job{
JobReference: &bq.JobReference{ProjectId: "client-project-id"},
JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
Configuration: &bq.JobConfiguration{
Extract: &bq.JobConfigurationExtract{
SourceTable: &bq.TableReference{
@@ -39,6 +39,7 @@ func defaultExtractJob() *bq.Job {
}
func TestExtract(t *testing.T) {
defer fixRandomJobID("RANDOM")()
s := &testService{}
c := &Client{
service: s,
vendor/cloud.google.com/go/bigquery/integration_test.go
Generated Vendored
+102 -38
View File
@@ -49,6 +49,7 @@ var (
}},
}
testTableExpiration time.Time
datasetIDs = testutil.NewUIDSpace("dataset")
)
func TestMain(m *testing.M) {
@@ -82,13 +83,13 @@ func initIntegrationTest() {
log.Fatalf("NewClient: %v", err)
}
dataset = client.Dataset("bigquery_integration_test")
if err := dataset.Create(ctx); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
if err := dataset.Create(ctx, nil); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
log.Fatalf("creating dataset: %v", err)
}
testTableExpiration = time.Now().Add(10 * time.Minute).Round(time.Second)
}
func TestIntegration_Create(t *testing.T) {
func TestIntegration_TableCreate(t *testing.T) {
// Check that creating a record field with an empty schema is an error.
if client == nil {
t.Skip("Integration tests skipped")
@@ -97,7 +98,10 @@ func TestIntegration_Create(t *testing.T) {
schema := Schema{
{Name: "rec", Type: RecordFieldType, Schema: Schema{}},
}
err := table.Create(context.Background(), schema, TableExpiration(time.Now().Add(5*time.Minute)))
err := table.Create(context.Background(), &TableMetadata{
Schema: schema,
ExpirationTime: time.Now().Add(5 * time.Minute),
})
if err == nil {
t.Fatal("want error, got nil")
}
@@ -106,7 +110,7 @@ func TestIntegration_Create(t *testing.T) {
}
}
func TestIntegration_CreateView(t *testing.T) {
func TestIntegration_TableCreateView(t *testing.T) {
if client == nil {
t.Skip("Integration tests skipped")
}
@@ -116,8 +120,12 @@ func TestIntegration_CreateView(t *testing.T) {
// Test that standard SQL views work.
view := dataset.Table("t_view_standardsql")
query := ViewQuery(fmt.Sprintf("SELECT APPROX_COUNT_DISTINCT(name) FROM `%s.%s.%s`", dataset.ProjectID, dataset.DatasetID, table.TableID))
err := view.Create(context.Background(), UseStandardSQL(), query)
query := fmt.Sprintf("SELECT APPROX_COUNT_DISTINCT(name) FROM `%s.%s.%s`",
dataset.ProjectID, dataset.DatasetID, table.TableID)
err := view.Create(context.Background(), &TableMetadata{
ViewQuery: query,
UseStandardSQL: true,
})
if err != nil {
t.Fatalf("table.create: Did not expect an error, got: %v", err)
}
@@ -137,8 +145,8 @@ func TestIntegration_TableMetadata(t *testing.T) {
t.Fatal(err)
}
// TODO(jba): check md more thorougly.
if got, want := md.ID, fmt.Sprintf("%s:%s.%s", dataset.ProjectID, dataset.DatasetID, table.TableID); got != want {
t.Errorf("metadata.ID: got %q, want %q", got, want)
if got, want := md.FullID, fmt.Sprintf("%s:%s.%s", dataset.ProjectID, dataset.DatasetID, table.TableID); got != want {
t.Errorf("metadata.FullID: got %q, want %q", got, want)
}
if got, want := md.Type, RegularTable; got != want {
t.Errorf("metadata.Type: got %v, want %v", got, want)
@@ -162,7 +170,11 @@ func TestIntegration_TableMetadata(t *testing.T) {
}
for i, c := range partitionCases {
table := dataset.Table(fmt.Sprintf("t_metadata_partition_%v", i))
err = table.Create(context.Background(), schema, c.timePartitioning, TableExpiration(time.Now().Add(5*time.Minute)))
err = table.Create(context.Background(), &TableMetadata{
Schema: schema,
TimePartitioning: &c.timePartitioning,
ExpirationTime: time.Now().Add(5 * time.Minute),
})
if err != nil {
t.Fatal(err)
}
@@ -180,6 +192,33 @@ func TestIntegration_TableMetadata(t *testing.T) {
}
}
func TestIntegration_DatasetCreate(t *testing.T) {
if client == nil {
t.Skip("Integration tests skipped")
}
ctx := context.Background()
uid := strings.Replace(datasetIDs.New(), "-", "_", -1)
ds := client.Dataset(uid)
wmd := &DatasetMetadata{Name: "name", Location: "EU"}
err := ds.Create(ctx, wmd)
if err != nil {
t.Fatal(err)
}
gmd, err := ds.Metadata(ctx)
if err != nil {
t.Fatal(err)
}
if got, want := gmd.Name, wmd.Name; got != want {
t.Errorf("name: got %q, want %q", got, want)
}
if got, want := gmd.Location, wmd.Location; got != want {
t.Errorf("location: got %q, want %q", got, want)
}
if err := ds.Delete(ctx); err != nil {
t.Fatalf("deleting dataset %s: %v", ds, err)
}
}
func TestIntegration_DatasetMetadata(t *testing.T) {
if client == nil {
t.Skip("Integration tests skipped")
@@ -189,8 +228,8 @@ func TestIntegration_DatasetMetadata(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if got, want := md.ID, fmt.Sprintf("%s:%s", dataset.ProjectID, dataset.DatasetID); got != want {
t.Errorf("ID: got %q, want %q", got, want)
if got, want := md.FullID, fmt.Sprintf("%s:%s", dataset.ProjectID, dataset.DatasetID); got != want {
t.Errorf("FullID: got %q, want %q", got, want)
}
jan2016 := time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC)
if md.CreationTime.Before(jan2016) {
@@ -213,7 +252,7 @@ func TestIntegration_DatasetDelete(t *testing.T) {
}
ctx := context.Background()
ds := client.Dataset("delete_test")
if err := ds.Create(ctx); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
if err := ds.Create(ctx, nil); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
t.Fatalf("creating dataset %s: %v", ds, err)
}
if err := ds.Delete(ctx); err != nil {
@@ -276,8 +315,7 @@ func TestIntegration_DatasetUpdateDefaultExpiration(t *testing.T) {
t.Fatal(err)
}
// Set the default expiration time.
md, err = dataset.Update(ctx,
DatasetMetadataToUpdate{DefaultTableExpiration: time.Hour}, "")
md, err = dataset.Update(ctx, DatasetMetadataToUpdate{DefaultTableExpiration: time.Hour}, "")
if err != nil {
t.Fatal(err)
}
@@ -293,8 +331,7 @@ func TestIntegration_DatasetUpdateDefaultExpiration(t *testing.T) {
t.Fatalf("got %s, want 1h", md.DefaultTableExpiration)
}
// Setting it to 0 deletes it (which looks like a 0 duration).
md, err = dataset.Update(ctx,
DatasetMetadataToUpdate{DefaultTableExpiration: time.Duration(0)}, "")
md, err = dataset.Update(ctx, DatasetMetadataToUpdate{DefaultTableExpiration: time.Duration(0)}, "")
if err != nil {
t.Fatal(err)
}
@@ -746,15 +783,7 @@ func TestIntegration_TableUpdate(t *testing.T) {
schema3[0], schema3[1], schema3[2],
{Name: "rec2", Type: RecordFieldType, Schema: Schema{}}}},
} {
for {
_, err = table.Update(ctx, TableMetadataToUpdate{Schema: Schema(test.fields)}, "")
if !hasStatusCode(err, 403) {
break
}
// We've hit the rate limit for updates. Wait a bit and retry.
t.Logf("%s: retrying after getting %v", test.desc, err)
time.Sleep(4 * time.Second)
}
_, err = table.Update(ctx, TableMetadataToUpdate{Schema: Schema(test.fields)}, "")
if err == nil {
t.Errorf("%s: want error, got nil", test.desc)
} else if !hasStatusCode(err, 400) {
@@ -818,7 +847,9 @@ func TestIntegration_DML(t *testing.T) {
q.UseStandardSQL = true // necessary for DML
job, err := q.Run(ctx)
if err != nil {
fmt.Printf("q.Run: %v\n", err)
if e, ok := err.(*googleapi.Error); ok && e.Code < 500 {
return true, err // fail on 4xx
}
return false, err
}
if err := wait(ctx, job); err != nil {
@@ -961,6 +992,7 @@ func TestIntegration_LegacyQuery(t *testing.T) {
}
for _, c := range testCases {
q := client.Query(c.query)
q.UseLegacySQL = true
it, err := q.Read(ctx)
if err != nil {
t.Fatal(err)
@@ -1070,11 +1102,11 @@ var useLegacySqlTests = []struct {
}{
{t: legacyName, std: false, legacy: true, err: false},
{t: legacyName, std: true, legacy: false, err: true},
{t: legacyName, std: false, legacy: false, err: false}, // legacy SQL is default
{t: legacyName, std: false, legacy: false, err: true}, // standard SQL is default
{t: legacyName, std: true, legacy: true, err: true},
{t: stdName, std: false, legacy: true, err: true},
{t: stdName, std: true, legacy: false, err: false},
{t: stdName, std: false, legacy: false, err: true}, // legacy SQL is default
{t: stdName, std: false, legacy: false, err: false}, // standard SQL is default
{t: stdName, std: true, legacy: true, err: true},
}
@@ -1099,7 +1131,7 @@ func TestIntegration_QueryUseLegacySQL(t *testing.T) {
}
func TestIntegration_TableUseLegacySQL(t *testing.T) {
// Test the UseLegacySQL and UseStandardSQL options for CreateTable.
// Test UseLegacySQL and UseStandardSQL for Table.Create.
if client == nil {
t.Skip("Integration tests skipped")
}
@@ -1108,15 +1140,12 @@ func TestIntegration_TableUseLegacySQL(t *testing.T) {
defer table.Delete(ctx)
for i, test := range useLegacySqlTests {
view := dataset.Table(fmt.Sprintf("t_view_%d", i))
vq := ViewQuery(fmt.Sprintf("SELECT word from %s", test.t))
opts := []CreateTableOption{vq}
if test.std {
opts = append(opts, UseStandardSQL())
tm := &TableMetadata{
ViewQuery: fmt.Sprintf("SELECT word from %s", test.t),
UseStandardSQL: test.std,
UseLegacySQL: test.legacy,
}
if test.legacy {
opts = append(opts, UseLegacySQL())
}
err := view.Create(ctx, opts...)
err := view.Create(ctx, tm)
gotErr := err != nil
if gotErr && !test.err {
t.Errorf("%+v:\nunexpected error: %v", test, err)
@@ -1127,11 +1156,46 @@ func TestIntegration_TableUseLegacySQL(t *testing.T) {
}
}
func TestIntegration_ListJobs(t *testing.T) {
// It's difficult to test the list of jobs, because we can't easily
// control what's in it. Also, there are many jobs in the test project,
// and it takes considerable time to list them all.
if client == nil {
t.Skip("Integration tests skipped")
}
ctx := context.Background()
// About all we can do is list a few jobs.
const max = 20
var jis []JobInfo
it := client.Jobs(ctx)
for {
ji, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
t.Fatal(err)
}
jis = append(jis, ji)
if len(jis) >= max {
break
}
}
// We expect that there is at least one job in the last few months.
if len(jis) == 0 {
t.Fatal("did not get any jobs")
}
}
// Creates a new, temporary table with a unique name and the given schema.
func newTable(t *testing.T, s Schema) *Table {
name := fmt.Sprintf("t%d", time.Now().UnixNano())
table := dataset.Table(name)
err := table.Create(context.Background(), s, TableExpiration(testTableExpiration))
err := table.Create(context.Background(), &TableMetadata{
Schema: s,
ExpirationTime: testTableExpiration,
})
if err != nil {
t.Fatal(err)
}
vendor/cloud.google.com/go/bigquery/job.go
Generated Vendored
+97 -15
View File
@@ -16,6 +16,7 @@ package bigquery
import (
"errors"
"fmt"
"math/rand"
"os"
"sync"
@@ -25,6 +26,7 @@ import (
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
bq "google.golang.org/api/bigquery/v2"
"google.golang.org/api/iterator"
)
// A Job represents an operation which has been submitted to BigQuery for processing.
@@ -58,7 +60,8 @@ func (j *Job) ID() string {
type State int
const (
Pending State = iota
StateUnspecified State = iota // used only as a default in JobIterator
Pending
Running
Done
)
@@ -77,16 +80,17 @@ type JobStatus struct {
Statistics *JobStatistics
}
// setJobRef initializes job's JobReference if given a non-empty jobID.
// createJobRef creates a JobReference.
// projectID must be non-empty.
func setJobRef(job *bq.Job, jobID, projectID string) {
func createJobRef(jobID string, addJobIDSuffix bool, projectID string) *bq.JobReference {
if jobID == "" {
// Generate an ID on the client so that insertJob can be idempotent.
jobID = randomJobID()
jobID = randomJobIDFn()
} else if addJobIDSuffix {
jobID += "-" + randomJobIDFn()
}
// We don't check whether projectID is empty; the server will return an
// error when it encounters the resulting JobReference.
job.JobReference = &bq.JobReference{
return &bq.JobReference{
JobId: jobID,
ProjectId: projectID,
}
@@ -99,8 +103,11 @@ var (
rng = rand.New(rand.NewSource(time.Now().UnixNano() ^ int64(os.Getpid())))
)
// For testing.
var randomJobIDFn = randomJobID
func randomJobID() string {
// As of August 2017, the BigQuery service uses 27 alphanumeric characters.
// As of August 2017, the BigQuery service uses 27 alphanumeric characters for suffixes.
var b [27]byte
rngMu.Lock()
for i := 0; i < len(b); i++ {
@@ -121,20 +128,25 @@ func (s *JobStatus) Err() error {
return s.err
}
// Fill in the client field of Tables in the statistics.
func (s *JobStatus) setClient(c *Client) {
if s.Statistics == nil {
return
}
if qs, ok := s.Statistics.Details.(*QueryStatistics); ok {
for _, t := range qs.ReferencedTables {
t.c = c
}
}
}
// Status returns the current status of the job. It fails if the Status could not be determined.
func (j *Job) Status(ctx context.Context) (*JobStatus, error) {
js, err := j.c.service.jobStatus(ctx, j.projectID, j.jobID)
if err != nil {
return nil, err
}
// Fill in the client field of Tables in the statistics.
if js.Statistics != nil {
if qs, ok := js.Statistics.Details.(*QueryStatistics); ok {
for _, t := range qs.ReferencedTables {
t.c = j.c
}
}
}
js.setClient(j.c)
return js, nil
}
@@ -346,3 +358,73 @@ type ExplainQueryStep struct {
func (*ExtractStatistics) implementsStatistics() {}
func (*LoadStatistics) implementsStatistics() {}
func (*QueryStatistics) implementsStatistics() {}
// Jobs lists jobs within a project.
func (c *Client) Jobs(ctx context.Context) *JobIterator {
it := &JobIterator{
ctx: ctx,
c: c,
ProjectID: c.projectID,
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
return it
}
// A JobInfo consists of a Job and a JobStatus.
type JobInfo struct {
Job *Job
Status *JobStatus
}
// JobIterator iterates over jobs in a project.
type JobIterator struct {
ProjectID string // Project ID of the jobs to list. Default is the client's project.
AllUsers bool // Whether to list jobs owned by all users in the project, or just the current caller.
State State // List only jobs in the given state. Defaults to all states.
ctx context.Context
c *Client
pageInfo *iterator.PageInfo
nextFunc func() error
items []JobInfo
}
func (it *JobIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
func (it *JobIterator) Next() (JobInfo, error) {
if err := it.nextFunc(); err != nil {
return JobInfo{}, err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *JobIterator) fetch(pageSize int, pageToken string) (string, error) {
var st string
switch it.State {
case StateUnspecified:
st = ""
case Pending:
st = "pending"
case Running:
st = "running"
case Done:
st = "done"
default:
return "", fmt.Errorf("bigquery: invalid value for JobIterator.State: %d", it.State)
}
jobInfos, nextPageToken, err := it.c.service.listJobs(it.ctx, it.ProjectID, pageSize, pageToken, it.AllUsers, st)
if err != nil {
return "", err
}
for _, ji := range jobInfos {
ji.Job.c = it.c
ji.Status.setClient(it.c)
it.items = append(it.items, ji)
}
return nextPageToken, nil
}
vendor/cloud.google.com/go/bigquery/job_test.go
Generated Vendored
+95
View File
@@ -0,0 +1,95 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bigquery
import (
"testing"
"cloud.google.com/go/internal/testutil"
"golang.org/x/net/context"
bq "google.golang.org/api/bigquery/v2"
)
func TestCreateJobRef(t *testing.T) {
defer fixRandomJobID("RANDOM")()
for _, test := range []struct {
jobID string
addJobIDSuffix bool
want string
}{
{
jobID: "foo",
addJobIDSuffix: false,
want: "foo",
},
{
jobID: "",
addJobIDSuffix: false,
want: "RANDOM",
},
{
jobID: "",
addJobIDSuffix: true, // irrelevant
want: "RANDOM",
},
{
jobID: "foo",
addJobIDSuffix: true,
want: "foo-RANDOM",
},
} {
jr := createJobRef(test.jobID, test.addJobIDSuffix, "projectID")
got := jr.JobId
if got != test.want {
t.Errorf("%q, %t: got %q, want %q", test.jobID, test.addJobIDSuffix, got, test.want)
}
}
}
func fixRandomJobID(s string) func() {
prev := randomJobIDFn
randomJobIDFn = func() string { return s }
return func() { randomJobIDFn = prev }
}
func checkJob(t *testing.T, i int, got, want *bq.Job) {
if got.JobReference == nil {
t.Errorf("#%d: empty job reference", i)
return
}
if got.JobReference.JobId == "" {
t.Errorf("#%d: empty job ID", i)
return
}
d := testutil.Diff(got, want)
if d != "" {
t.Errorf("#%d: (got=-, want=+) %s", i, d)
}
}
type testService struct {
*bq.Job
service
}
func (s *testService) insertJob(ctx context.Context, projectID string, conf *insertJobConf) (*Job, error) {
s.Job = conf.job
return &Job{}, nil
}
func (s *testService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) {
return &JobStatus{State: Done}, nil
}
vendor/cloud.google.com/go/bigquery/load.go
Generated Vendored
+5 -4
View File
@@ -21,9 +21,12 @@ import (
// LoadConfig holds the configuration for a load job.
type LoadConfig struct {
// JobID is the ID to use for the load job. If unset, a job ID will be automatically created.
// JobID is the ID to use for the job. If empty, a random job ID will be generated.
JobID string
// If AddJobIDSuffix is true, then a random string will be appended to JobID.
AddJobIDSuffix bool
// Src is the source from which data will be loaded.
Src LoadSource
@@ -71,6 +74,7 @@ func (t *Table) LoaderFrom(src LoadSource) *Loader {
// Run initiates a load job.
func (l *Loader) Run(ctx context.Context) (*Job, error) {
job := &bq.Job{
JobReference: createJobRef(l.JobID, l.AddJobIDSuffix, l.c.projectID),
Configuration: &bq.JobConfiguration{
Load: &bq.JobConfigurationLoad{
CreateDisposition: string(l.CreateDisposition),
@@ -80,9 +84,6 @@ func (l *Loader) Run(ctx context.Context) (*Job, error) {
}
conf := &insertJobConf{job: job}
l.Src.populateInsertJobConfForLoad(conf)
setJobRef(job, l.JobID, l.c.projectID)
job.Configuration.Load.DestinationTable = l.Dst.tableRefProto()
return l.c.insertJob(ctx, conf)
}
vendor/cloud.google.com/go/bigquery/load_test.go
Generated Vendored
+2 -1
View File
@@ -25,7 +25,7 @@ import (
func defaultLoadJob() *bq.Job {
return &bq.Job{
JobReference: &bq.JobReference{ProjectId: "client-project-id"},
JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
Configuration: &bq.JobConfiguration{
Load: &bq.JobConfigurationLoad{
DestinationTable: &bq.TableReference{
@@ -67,6 +67,7 @@ func bqNestedFieldSchema() *bq.TableFieldSchema {
}
func TestLoad(t *testing.T) {
defer fixRandomJobID("RANDOM")()
c := &Client{projectID: "client-project-id"}
testCases := []struct {
vendor/cloud.google.com/go/bigquery/query.go
Generated Vendored
+9 -10
View File
@@ -23,10 +23,12 @@ import (
// QueryConfig holds the configuration for a query job.
type QueryConfig struct {
// JobID is the ID to use for the query job. If this field is empty, a job ID
// will be automatically created.
// JobID is the ID to use for the job. If empty, a random job ID will be generated.
JobID string
// If AddJobIDSuffix is true, then a random string will be appended to JobID.
AddJobIDSuffix bool
// Dst is the table into which the results of the query will be written.
// If this field is nil, a temporary table will be created.
Dst *Table
@@ -87,8 +89,7 @@ type QueryConfig struct {
// used.
MaxBytesBilled int64
// UseStandardSQL causes the query to use standard SQL.
// The default is false (using legacy SQL).
// UseStandardSQL causes the query to use standard SQL. The default.
UseStandardSQL bool
// UseLegacySQL causes the query to use legacy SQL.
@@ -128,12 +129,11 @@ func (c *Client) Query(q string) *Query {
// Run initiates a query job.
func (q *Query) Run(ctx context.Context) (*Job, error) {
job := &bq.Job{
JobReference: createJobRef(q.JobID, q.AddJobIDSuffix, q.client.projectID),
Configuration: &bq.JobConfiguration{
Query: &bq.JobConfigurationQuery{},
},
}
setJobRef(job, q.JobID, q.client.projectID)
if err := q.QueryConfig.populateJobQueryConfig(job.Configuration.Query); err != nil {
return nil, err
}
@@ -188,12 +188,11 @@ func (q *QueryConfig) populateJobQueryConfig(conf *bq.JobConfigurationQuery) err
if len(q.Parameters) > 0 && q.UseLegacySQL {
return errors.New("bigquery: cannot provide both Parameters (implying standard SQL) and UseLegacySQL")
}
if q.UseStandardSQL || len(q.Parameters) > 0 {
conf.UseLegacySql = false
conf.ForceSendFields = append(conf.ForceSendFields, "UseLegacySql")
}
if q.UseLegacySQL {
conf.UseLegacySql = true
} else {
conf.UseLegacySql = false
conf.ForceSendFields = append(conf.ForceSendFields, "UseLegacySql")
}
if q.Dst != nil && !q.Dst.implicitTable() {
conf.DestinationTable = q.Dst.tableRefProto()
vendor/cloud.google.com/go/bigquery/query_test.go
Generated Vendored
+34 -8
View File
@@ -15,7 +15,6 @@
package bigquery
import (
"fmt"
"testing"
"cloud.google.com/go/internal/testutil"
@@ -27,7 +26,7 @@ import (
func defaultQueryJob() *bq.Job {
return &bq.Job{
JobReference: &bq.JobReference{ProjectId: "client-project-id"},
JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
Configuration: &bq.JobConfiguration{
Query: &bq.JobConfigurationQuery{
DestinationTable: &bq.TableReference{
@@ -40,12 +39,15 @@ func defaultQueryJob() *bq.Job {
ProjectId: "def-project-id",
DatasetId: "def-dataset-id",
},
UseLegacySql: false,
ForceSendFields: []string{"UseLegacySql"},
},
},
}
}
func TestQuery(t *testing.T) {
defer fixRandomJobID("RANDOM")()
c := &Client{
projectID: "client-project-id",
}
@@ -70,6 +72,20 @@ func TestQuery(t *testing.T) {
return j
}(),
},
{
dst: c.Dataset("dataset-id").Table("table-id"),
src: &QueryConfig{
Q: "query string",
JobID: "jobID",
AddJobIDSuffix: true,
},
want: func() *bq.Job {
j := defaultQueryJob()
j.Configuration.Query.DefaultDataset = nil
j.JobReference.JobId = "jobID-RANDOM"
return j
}(),
},
{
dst: &Table{},
src: defaultQuery,
@@ -246,10 +262,20 @@ func TestQuery(t *testing.T) {
DefaultDatasetID: "def-dataset-id",
UseStandardSQL: true,
},
want: defaultQueryJob(),
},
{
dst: c.Dataset("dataset-id").Table("table-id"),
src: &QueryConfig{
Q: "query string",
DefaultProjectID: "def-project-id",
DefaultDatasetID: "def-dataset-id",
UseLegacySQL: true,
},
want: func() *bq.Job {
j := defaultQueryJob()
j.Configuration.Query.UseLegacySql = false
j.Configuration.Query.ForceSendFields = []string{"UseLegacySql"}
j.Configuration.Query.UseLegacySql = true
j.Configuration.Query.ForceSendFields = nil
return j
}(),
},
@@ -290,6 +316,8 @@ func TestConfiguringQuery(t *testing.T) {
ProjectId: "def-project-id",
DatasetId: "def-dataset-id",
},
UseLegacySql: false,
ForceSendFields: []string{"UseLegacySql"},
},
},
JobReference: &bq.JobReference{
@@ -301,8 +329,8 @@ func TestConfiguringQuery(t *testing.T) {
if _, err := query.Run(context.Background()); err != nil {
t.Fatalf("err calling Query.Run: %v", err)
}
if !testutil.Equal(s.Job, want) {
t.Errorf("querying: got:\n%v\nwant:\n%v", s.Job, want)
if diff := testutil.Diff(s.Job, want); diff != "" {
t.Errorf("querying: -got +want:\n%s", diff)
}
}
@@ -324,7 +352,5 @@ func TestQueryLegacySQL(t *testing.T) {
_, err = q.Run(context.Background())
if err == nil {
t.Error("Parameters and UseLegacySQL: got nil, want error")
} else {
fmt.Println(err)
}
}
vendor/cloud.google.com/go/bigquery/schema.go
Generated Vendored
-5
View File
@@ -77,11 +77,6 @@ func (s Schema) asTableSchema() *bq.TableSchema {
return &bq.TableSchema{Fields: fields}
}
// customizeCreateTable allows a Schema to be used directly as an option to CreateTable.
func (s Schema) customizeCreateTable(conf *createTableConf) {
conf.schema = s.asTableSchema()
}
func convertTableFieldSchema(tfs *bq.TableFieldSchema) *FieldSchema {
fs := &FieldSchema{
Description: tfs.Description,
vendor/cloud.google.com/go/bigquery/service.go
Generated Vendored
+185 -79
View File
@@ -41,9 +41,10 @@ type service interface {
getJob(ctx context.Context, projectId, jobID string) (*Job, error)
jobCancel(ctx context.Context, projectId, jobID string) error
jobStatus(ctx context.Context, projectId, jobID string) (*JobStatus, error)
listJobs(ctx context.Context, projectId string, maxResults int, pageToken string, all bool, state string) ([]JobInfo, string, error)
// Tables
createTable(ctx context.Context, conf *createTableConf) error
createTable(ctx context.Context, projectID, datasetID, tableID string, tm *TableMetadata) error
getTableMetadata(ctx context.Context, projectID, datasetID, tableID string) (*TableMetadata, error)
deleteTable(ctx context.Context, projectID, datasetID, tableID string) error
@@ -56,7 +57,7 @@ type service interface {
insertRows(ctx context.Context, projectID, datasetID, tableID string, rows []*insertionRow, conf *insertRowsConf) error
// Datasets
insertDataset(ctx context.Context, datasetID, projectID string) error
insertDataset(ctx context.Context, datasetID, projectID string, dm *DatasetMetadata) error
deleteDataset(ctx context.Context, datasetID, projectID string) error
getDatasetMetadata(ctx context.Context, projectID, datasetID string) (*DatasetMetadata, error)
patchDataset(ctx context.Context, projectID, datasetID string, dm *DatasetMetadataToUpdate, etag string) (*DatasetMetadata, error)
@@ -313,22 +314,11 @@ func (s *bigqueryService) insertRows(ctx context.Context, projectID, datasetID,
}
func (s *bigqueryService) getJob(ctx context.Context, projectID, jobID string) (*Job, error) {
job, err := s.getJobInternal(ctx, projectID, jobID, "configuration")
bqjob, err := s.getJobInternal(ctx, projectID, jobID, "configuration", "jobReference")
if err != nil {
return nil, err
}
var isQuery bool
var dest *bq.TableReference
if job.Configuration.Query != nil {
isQuery = true
dest = job.Configuration.Query.DestinationTable
}
return &Job{
projectID: projectID,
jobID: jobID,
isQuery: isQuery,
destinationTable: dest,
}, nil
return jobFromProtos(bqjob.JobReference, bqjob.Configuration), nil
}
func (s *bigqueryService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) {
@@ -346,9 +336,10 @@ func (s *bigqueryService) jobStatus(ctx context.Context, projectID, jobID string
func (s *bigqueryService) getJobInternal(ctx context.Context, projectID, jobID string, fields ...googleapi.Field) (*bq.Job, error) {
var job *bq.Job
call := s.s.Jobs.Get(projectID, jobID).
Fields(fields...).
Context(ctx)
call := s.s.Jobs.Get(projectID, jobID).Context(ctx)
if len(fields) > 0 {
call = call.Fields(fields...)
}
setClientHeader(call.Header())
err := runWithRetry(ctx, func() (err error) {
job, err = call.Do()
@@ -376,6 +367,21 @@ func (s *bigqueryService) jobCancel(ctx context.Context, projectID, jobID string
})
}
func jobFromProtos(jr *bq.JobReference, config *bq.JobConfiguration) *Job {
var isQuery bool
var dest *bq.TableReference
if config.Query != nil {
isQuery = true
dest = config.Query.DestinationTable
}
return &Job{
projectID: jr.ProjectId,
jobID: jr.JobId,
isQuery: isQuery,
destinationTable: dest,
}
}
var stateMap = map[string]State{"PENDING": Pending, "RUNNING": Running, "DONE": Done}
func jobStatusFromProto(status *bq.JobStatus) (*JobStatus, error) {
@@ -496,63 +502,88 @@ func (s *bigqueryService) listTables(ctx context.Context, projectID, datasetID s
return tables, res.NextPageToken, nil
}
type createTableConf struct {
projectID, datasetID, tableID string
expiration time.Time
viewQuery string
schema *bq.TableSchema
useStandardSQL bool
useLegacySQL bool
timePartitioning *TimePartitioning
}
// createTable creates a table in the BigQuery service.
// expiration is an optional time after which the table will be deleted and its storage reclaimed.
// If viewQuery is non-empty, the created table will be of type VIEW.
// If tm.ViewQuery is non-empty, the created table will be of type VIEW.
// Note: expiration can only be set during table creation.
// Note: after table creation, a view can be modified only if its table was initially created with a view.
func (s *bigqueryService) createTable(ctx context.Context, conf *createTableConf) error {
if conf.useStandardSQL && conf.useLegacySQL {
return errors.New("bigquery: cannot provide both UseStandardSQL and UseLegacySQL")
func (s *bigqueryService) createTable(ctx context.Context, projectID, datasetID, tableID string, tm *TableMetadata) error {
table, err := bqTableFromMetadata(tm)
if err != nil {
return err
}
table := &bq.Table{
// TODO(jba): retry? Is this always idempotent?
TableReference: &bq.TableReference{
ProjectId: conf.projectID,
DatasetId: conf.datasetID,
TableId: conf.tableID,
},
table.TableReference = &bq.TableReference{
ProjectId: projectID,
DatasetId: datasetID,
TableId: tableID,
}
if !conf.expiration.IsZero() {
table.ExpirationTime = conf.expiration.UnixNano() / 1e6
req := s.s.Tables.Insert(projectID, datasetID, table).Context(ctx)
setClientHeader(req.Header())
_, err = req.Do()
return err
}
func bqTableFromMetadata(tm *TableMetadata) (*bq.Table, error) {
t := &bq.Table{}
if tm == nil {
return t, nil
}
// TODO(jba): make it impossible to provide both a view query and a schema.
if conf.viewQuery != "" {
table.View = &bq.ViewDefinition{
Query: conf.viewQuery,
if tm.Schema != nil && tm.ViewQuery != "" {
return nil, errors.New("bigquery: provide Schema or ViewQuery, not both")
}
t.FriendlyName = tm.Name
t.Description = tm.Description
if tm.Schema != nil {
t.Schema = tm.Schema.asTableSchema()
}
if tm.ViewQuery != "" {
if tm.UseStandardSQL && tm.UseLegacySQL {
return nil, errors.New("bigquery: cannot provide both UseStandardSQL and UseLegacySQL")
}
if conf.useStandardSQL {
table.View.UseLegacySql = false
table.View.ForceSendFields = append(table.View.ForceSendFields, "UseLegacySql")
}
if conf.useLegacySQL {
table.View.UseLegacySql = true
t.View = &bq.ViewDefinition{Query: tm.ViewQuery}
if tm.UseLegacySQL {
t.View.UseLegacySql = true
} else {
t.View.UseLegacySql = false
t.View.ForceSendFields = append(t.View.ForceSendFields, "UseLegacySql")
}
} else if tm.UseLegacySQL || tm.UseStandardSQL {
return nil, errors.New("bigquery: UseLegacy/StandardSQL requires ViewQuery")
}
if conf.schema != nil {
table.Schema = conf.schema
}
if conf.timePartitioning != nil {
table.TimePartitioning = &bq.TimePartitioning{
if tm.TimePartitioning != nil {
t.TimePartitioning = &bq.TimePartitioning{
Type: "DAY",
ExpirationMs: int64(conf.timePartitioning.Expiration.Seconds() * 1000),
ExpirationMs: int64(tm.TimePartitioning.Expiration / time.Millisecond),
}
}
if !tm.ExpirationTime.IsZero() {
t.ExpirationTime = tm.ExpirationTime.UnixNano() / 1e6
}
req := s.s.Tables.Insert(conf.projectID, conf.datasetID, table).Context(ctx)
setClientHeader(req.Header())
_, err := req.Do()
return err
if tm.FullID != "" {
return nil, errors.New("cannot set FullID on create")
}
if tm.Type != "" {
return nil, errors.New("cannot set Type on create")
}
if !tm.CreationTime.IsZero() {
return nil, errors.New("cannot set CreationTime on create")
}
if !tm.LastModifiedTime.IsZero() {
return nil, errors.New("cannot set LastModifiedTime on create")
}
if tm.NumBytes != 0 {
return nil, errors.New("cannot set NumBytes on create")
}
if tm.NumRows != 0 {
return nil, errors.New("cannot set NumRows on create")
}
if tm.StreamingBuffer != nil {
return nil, errors.New("cannot set StreamingBuffer on create")
}
if tm.ETag != "" {
return nil, errors.New("cannot set ETag on create")
}
return t, nil
}
func (s *bigqueryService) getTableMetadata(ctx context.Context, projectID, datasetID, tableID string) (*TableMetadata, error) {
@@ -580,7 +611,7 @@ func bqTableToMetadata(t *bq.Table) *TableMetadata {
Description: t.Description,
Name: t.FriendlyName,
Type: TableType(t.Type),
ID: t.Id,
FullID: t.Id,
NumBytes: t.NumBytes,
NumRows: t.NumRows,
ExpirationTime: unixMillisToTime(t.ExpirationTime),
@@ -592,7 +623,8 @@ func bqTableToMetadata(t *bq.Table) *TableMetadata {
md.Schema = convertTableSchema(t.Schema)
}
if t.View != nil {
md.View = t.View.Query
md.ViewQuery = t.View.Query
md.UseLegacySQL = t.View.UseLegacySql
}
if t.TimePartitioning != nil {
md.TimePartitioning = &TimePartitioning{
@@ -617,7 +649,7 @@ func bqDatasetToMetadata(d *bq.Dataset) *DatasetMetadata {
DefaultTableExpiration: time.Duration(d.DefaultTableExpirationMs) * time.Millisecond,
Description: d.Description,
Name: d.FriendlyName,
ID: d.Id,
FullID: d.Id,
Location: d.Location,
Labels: d.Labels,
ETag: d.Etag,
@@ -678,39 +710,72 @@ func (s *bigqueryService) patchTable(ctx context.Context, projectID, datasetID,
if etag != "" {
call.Header().Set("If-Match", etag)
}
table, err := call.Do()
if err != nil {
var table *bq.Table
if err := runWithRetry(ctx, func() (err error) {
table, err = call.Do()
return err
}); err != nil {
return nil, err
}
return bqTableToMetadata(table), nil
}
func (s *bigqueryService) insertDataset(ctx context.Context, datasetID, projectID string) error {
func (s *bigqueryService) insertDataset(ctx context.Context, datasetID, projectID string, dm *DatasetMetadata) error {
// TODO(jba): retry?
ds := &bq.Dataset{
DatasetReference: &bq.DatasetReference{DatasetId: datasetID},
ds, err := bqDatasetFromMetadata(dm)
if err != nil {
return err
}
ds.DatasetReference = &bq.DatasetReference{DatasetId: datasetID}
req := s.s.Datasets.Insert(projectID, ds).Context(ctx)
setClientHeader(req.Header())
_, err := req.Do()
_, err = req.Do()
return err
}
func (s *bigqueryService) patchDataset(ctx context.Context, projectID, datasetID string, dm *DatasetMetadataToUpdate, etag string) (*DatasetMetadata, error) {
ds := bqDatasetFromMetadata(dm)
ds := bqDatasetFromUpdateMetadata(dm)
call := s.s.Datasets.Patch(projectID, datasetID, ds).Context(ctx)
setClientHeader(call.Header())
if etag != "" {
call.Header().Set("If-Match", etag)
}
ds2, err := call.Do()
if err != nil {
var ds2 *bq.Dataset
if err := runWithRetry(ctx, func() (err error) {
ds2, err = call.Do()
return err
}); err != nil {
return nil, err
}
return bqDatasetToMetadata(ds2), nil
}
func bqDatasetFromMetadata(dm *DatasetMetadataToUpdate) *bq.Dataset {
func bqDatasetFromMetadata(dm *DatasetMetadata) (*bq.Dataset, error) {
ds := &bq.Dataset{}
if dm == nil {
return ds, nil
}
ds.FriendlyName = dm.Name
ds.Description = dm.Description
ds.Location = dm.Location
ds.DefaultTableExpirationMs = int64(dm.DefaultTableExpiration / time.Millisecond)
ds.Labels = dm.Labels
if !dm.CreationTime.IsZero() {
return nil, errors.New("bigquery: Dataset.CreationTime is not writable")
}
if !dm.LastModifiedTime.IsZero() {
return nil, errors.New("bigquery: Dataset.LastModifiedTime is not writable")
}
if dm.FullID != "" {
return nil, errors.New("bigquery: Dataset.FullID is not writable")
}
if dm.ETag != "" {
return nil, errors.New("bigquery: Dataset.ETag is not writable")
}
return ds, nil
}
func bqDatasetFromUpdateMetadata(dm *DatasetMetadataToUpdate) *bq.Dataset {
ds := &bq.Dataset{}
forceSend := func(field string) {
ds.ForceSendFields = append(ds.ForceSendFields, field)
@@ -730,7 +795,7 @@ func bqDatasetFromMetadata(dm *DatasetMetadataToUpdate) *bq.Dataset {
// Send a null to delete the field.
ds.NullFields = append(ds.NullFields, "DefaultTableExpirationMs")
} else {
ds.DefaultTableExpirationMs = int64(dur.Seconds() * 1000)
ds.DefaultTableExpirationMs = int64(dur / time.Millisecond)
}
}
if dm.setLabels != nil || dm.deleteLabels != nil {
@@ -801,13 +866,54 @@ func (s *bigqueryService) convertListedDataset(d *bq.DatasetListDatasets) *Datas
}
}
func (s *bigqueryService) listJobs(ctx context.Context, projectID string, maxResults int, pageToken string, all bool, state string) ([]JobInfo, string, error) {
req := s.s.Jobs.List(projectID).
Context(ctx).
PageToken(pageToken).
Projection("full").
AllUsers(all)
if state != "" {
req.StateFilter(state)
}
setClientHeader(req.Header())
if maxResults > 0 {
req.MaxResults(int64(maxResults))
}
res, err := req.Do()
if err != nil {
return nil, "", err
}
var jobInfos []JobInfo
for _, j := range res.Jobs {
ji, err := s.convertListedJob(j)
if err != nil {
return nil, "", err
}
jobInfos = append(jobInfos, ji)
}
return jobInfos, res.NextPageToken, nil
}
func (s *bigqueryService) convertListedJob(j *bq.JobListJobs) (JobInfo, error) {
st, err := jobStatusFromProto(j.Status)
if err != nil {
return JobInfo{}, err
}
st.Statistics = jobStatisticsFromProto(j.Statistics)
return JobInfo{
Job: jobFromProtos(j.JobReference, j.Configuration),
Status: st,
}, nil
}
// runWithRetry calls the function until it returns nil or a non-retryable error, or
// the context is done.
// See the similar function in ../storage/invoke.go. The main difference is the
// reason for retrying.
func runWithRetry(ctx context.Context, call func() error) error {
// These parameters match the suggestions in https://cloud.google.com/bigquery/sla.
backoff := gax.Backoff{
Initial: 2 * time.Second,
Initial: 1 * time.Second,
Max: 32 * time.Second,
Multiplier: 2,
}
@@ -820,7 +926,7 @@ func runWithRetry(ctx context.Context, call func() error) error {
})
}
// Use the criteria in https://cloud.google.com/bigquery/troubleshooting-errors.
// This is the correct definition of retryable according to the BigQuery team.
func retryableError(err error) bool {
e, ok := err.(*googleapi.Error)
if !ok {
@@ -830,5 +936,5 @@ func retryableError(err error) bool {
if len(e.Errors) > 0 {
reason = e.Errors[0].Reason
}
return reason == "backendError" && (e.Code == 500 || e.Code == 503)
return reason == "backendError" || reason == "rateLimitExceeded"
}
vendor/cloud.google.com/go/bigquery/service_test.go
Generated Vendored
+149 -6
View File
@@ -59,15 +59,15 @@ func TestBQTableToMetadata(t *testing.T) {
&TableMetadata{
Description: "desc",
Name: "fname",
View: "view-query",
ID: "id",
ViewQuery: "view-query",
FullID: "id",
Type: ExternalTable,
ExpirationTime: aTime.Truncate(time.Millisecond),
CreationTime: aTime.Truncate(time.Millisecond),
LastModifiedTime: aTime.Truncate(time.Millisecond),
NumBytes: 123,
NumRows: 7,
TimePartitioning: &TimePartitioning{Expiration: time.Duration(7890) * time.Millisecond},
TimePartitioning: &TimePartitioning{Expiration: 7890 * time.Millisecond},
StreamingBuffer: &StreamingBuffer{
EstimatedBytes: 11,
EstimatedRows: 3,
@@ -78,13 +78,156 @@ func TestBQTableToMetadata(t *testing.T) {
},
} {
got := bqTableToMetadata(test.in)
if !testutil.Equal(got, test.want) {
t.Errorf("%v:\ngot %+v\nwant %+v", test.in, got, test.want)
if diff := testutil.Diff(got, test.want); diff != "" {
t.Errorf("%+v:\n, -got, +want:\n%s", test.in, diff)
}
}
}
func TestBQTableFromMetadata(t *testing.T) {
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
aTimeMillis := aTime.UnixNano() / 1e6
sc := Schema{fieldSchema("desc", "name", "STRING", false, true)}
for _, test := range []struct {
in *TableMetadata
want *bq.Table
}{
{nil, &bq.Table{}},
{&TableMetadata{}, &bq.Table{}},
{
&TableMetadata{
Name: "n",
Description: "d",
Schema: sc,
ExpirationTime: aTime,
},
&bq.Table{
FriendlyName: "n",
Description: "d",
Schema: &bq.TableSchema{
Fields: []*bq.TableFieldSchema{
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
},
},
ExpirationTime: aTimeMillis,
},
},
{
&TableMetadata{ViewQuery: "q"},
&bq.Table{
View: &bq.ViewDefinition{
Query: "q",
UseLegacySql: false,
ForceSendFields: []string{"UseLegacySql"},
},
},
},
{
&TableMetadata{
ViewQuery: "q",
UseLegacySQL: true,
TimePartitioning: &TimePartitioning{},
},
&bq.Table{
View: &bq.ViewDefinition{
Query: "q",
UseLegacySql: true,
},
TimePartitioning: &bq.TimePartitioning{
Type: "DAY",
ExpirationMs: 0,
},
},
},
{
&TableMetadata{
ViewQuery: "q",
UseStandardSQL: true,
TimePartitioning: &TimePartitioning{time.Second},
},
&bq.Table{
View: &bq.ViewDefinition{
Query: "q",
UseLegacySql: false,
ForceSendFields: []string{"UseLegacySql"},
},
TimePartitioning: &bq.TimePartitioning{
Type: "DAY",
ExpirationMs: 1000,
},
},
},
} {
got, err := bqTableFromMetadata(test.in)
if err != nil {
t.Fatalf("%+v: %v", test.in, err)
}
if diff := testutil.Diff(got, test.want); diff != "" {
t.Errorf("%+v:\n-got, +want:\n%s", test.in, diff)
}
}
// Errors
for _, in := range []*TableMetadata{
{Schema: sc, ViewQuery: "q"}, // can't have both schema and query
{UseLegacySQL: true}, // UseLegacySQL without query
{UseStandardSQL: true}, // UseStandardSQL without query
// read-only fields
{FullID: "x"},
{Type: "x"},
{CreationTime: aTime},
{LastModifiedTime: aTime},
{NumBytes: 1},
{NumRows: 1},
{StreamingBuffer: &StreamingBuffer{}},
{ETag: "x"},
} {
_, err := bqTableFromMetadata(in)
if err == nil {
t.Errorf("%+v: got nil, want error", in)
}
}
}
func TestBQDatasetFromMetadata(t *testing.T) {
for _, test := range []struct {
in *DatasetMetadata
want *bq.Dataset
}{
{nil, &bq.Dataset{}},
{&DatasetMetadata{Name: "name"}, &bq.Dataset{FriendlyName: "name"}},
{&DatasetMetadata{
Name: "name",
Description: "desc",
DefaultTableExpiration: time.Hour,
Location: "EU",
Labels: map[string]string{"x": "y"},
}, &bq.Dataset{
FriendlyName: "name",
Description: "desc",
DefaultTableExpirationMs: 60 * 60 * 1000,
Location: "EU",
Labels: map[string]string{"x": "y"},
}},
} {
got, err := bqDatasetFromMetadata(test.in)
if err != nil {
t.Fatal(err)
}
if !testutil.Equal(got, test.want) {
t.Errorf("%v:\ngot %+v\nwant %+v", test.in, got, test.want)
}
}
// Check that non-writeable fields are unset.
_, err := bqDatasetFromMetadata(&DatasetMetadata{FullID: "x"})
if err == nil {
t.Error("got nil, want error")
}
}
func TestBQDatasetFromUpdateMetadata(t *testing.T) {
dm := DatasetMetadataToUpdate{
Description: "desc",
Name: "name",
@@ -93,7 +236,7 @@ func TestBQDatasetFromMetadata(t *testing.T) {
dm.SetLabel("label", "value")
dm.DeleteLabel("del")
got := bqDatasetFromMetadata(&dm)
got := bqDatasetFromUpdateMetadata(&dm)
want := &bq.Dataset{
Description: "desc",
FriendlyName: "name",
vendor/cloud.google.com/go/bigquery/table.go
Generated Vendored
+38 -78
View File
@@ -39,18 +39,39 @@ type Table struct {
// TableMetadata contains information about a BigQuery table.
type TableMetadata struct {
Description string // The user-friendly description of this table.
Name string // The user-friendly name for this table.
Schema Schema
View string
// The following fields can be set when creating a table.
ID string // An opaque ID uniquely identifying the table.
Type TableType
// The user-friendly name for the table.
Name string
// The user-friendly description of the table.
Description string
// The table schema. If provided on create, ViewQuery must be empty.
Schema Schema
// The query to use for a view. If provided on create, Schema must be nil.
ViewQuery string
// Use Legacy SQL for the view query.
// At most one of UseLegacySQL and UseStandardSQL can be true.
UseLegacySQL bool
// Use Legacy SQL for the view query. The default.
// At most one of UseLegacySQL and UseStandardSQL can be true.
UseStandardSQL bool
// If non-nil, the table is partitioned by time.
TimePartitioning *TimePartitioning
// The time when this table expires. If not set, the table will persist
// indefinitely. Expired tables will be deleted and their storage reclaimed.
ExpirationTime time.Time
// All the fields below are read-only.
FullID string // An opaque ID uniquely identifying the table.
Type TableType
CreationTime time.Time
LastModifiedTime time.Time
@@ -62,9 +83,6 @@ type TableMetadata struct {
// This does not include data that is being buffered during a streaming insert.
NumRows uint64
// The time-based partitioning settings for this table.
TimePartitioning *TimePartitioning
// Contains information regarding this table's streaming buffer, if one is
// present. This field will be nil if the table is not being streamed to or if
// there is no data in the streaming buffer.
@@ -115,6 +133,14 @@ const (
ExternalTable TableType = "EXTERNAL"
)
// TimePartitioning describes the time-based date partitioning on a table.
// For more information see: https://cloud.google.com/bigquery/docs/creating-partitioned-tables.
type TimePartitioning struct {
// The amount of time to keep the storage for a partition.
// If the duration is empty (0), the data in the partitions do not expire.
Expiration time.Duration
}
// StreamingBuffer holds information about the streaming buffer.
type StreamingBuffer struct {
// A lower-bound estimate of the number of bytes currently in the streaming
@@ -148,18 +174,9 @@ func (t *Table) implicitTable() bool {
}
// Create creates a table in the BigQuery service.
// To create a table with a schema, pass in a Schema to Create;
// Schema is a valid CreateTableOption.
func (t *Table) Create(ctx context.Context, options ...CreateTableOption) error {
conf := &createTableConf{
projectID: t.ProjectID,
datasetID: t.DatasetID,
tableID: t.TableID,
}
for _, o := range options {
o.customizeCreateTable(conf)
}
return t.c.service.createTable(ctx, conf)
// Pass in a TableMetadata value to configure the dataset.
func (t *Table) Create(ctx context.Context, tm *TableMetadata) error {
return t.c.service.createTable(ctx, t.ProjectID, t.DatasetID, t.TableID, tm)
}
// Metadata fetches the metadata for the table.
@@ -172,63 +189,6 @@ func (t *Table) Delete(ctx context.Context) error {
return t.c.service.deleteTable(ctx, t.ProjectID, t.DatasetID, t.TableID)
}
// A CreateTableOption is an optional argument to CreateTable.
type CreateTableOption interface {
customizeCreateTable(*createTableConf)
}
type tableExpiration time.Time
// TableExpiration returns a CreateTableOption that will cause the created table to be deleted after the expiration time.
func TableExpiration(exp time.Time) CreateTableOption { return tableExpiration(exp) }
func (opt tableExpiration) customizeCreateTable(conf *createTableConf) {
conf.expiration = time.Time(opt)
}
type viewQuery string
// ViewQuery returns a CreateTableOption that causes the created table to be a virtual table defined by the supplied query.
// For more information see: https://cloud.google.com/bigquery/querying-data#views
func ViewQuery(query string) CreateTableOption { return viewQuery(query) }
func (opt viewQuery) customizeCreateTable(conf *createTableConf) {
conf.viewQuery = string(opt)
}
type useStandardSQL struct{}
// UseStandardSQL returns a CreateTableOption to set the table to use standard SQL.
// The default setting is false (using legacy SQL).
func UseStandardSQL() CreateTableOption { return useStandardSQL{} }
func (opt useStandardSQL) customizeCreateTable(conf *createTableConf) {
conf.useStandardSQL = true
}
type useLegacySQL struct{}
// UseLegacySQL returns a CreateTableOption to set the table to use legacy SQL.
// This is currently the default.
func UseLegacySQL() CreateTableOption { return useLegacySQL{} }
func (opt useLegacySQL) customizeCreateTable(conf *createTableConf) {
conf.useLegacySQL = true
}
// TimePartitioning is a CreateTableOption that can be used to set time-based
// date partitioning on a table.
// For more information see: https://cloud.google.com/bigquery/docs/creating-partitioned-tables
type TimePartitioning struct {
// (Optional) The amount of time to keep the storage for a partition.
// If the duration is empty (0), the data in the partitions do not expire.
Expiration time.Duration
}
func (opt TimePartitioning) customizeCreateTable(conf *createTableConf) {
conf.timePartitioning = &opt
}
// Read fetches the contents of the table.
func (t *Table) Read(ctx context.Context) *RowIterator {
return newRowIterator(ctx, t.c.service, &readTableConf{
vendor/cloud.google.com/go/bigquery/utils_test.go
Generated Vendored
-20
View File
@@ -14,11 +14,6 @@
package bigquery
import (
"golang.org/x/net/context"
bq "google.golang.org/api/bigquery/v2"
)
func defaultGCS() *GCSReference {
return &GCSReference{
uris: []string{"uri"},
@@ -30,18 +25,3 @@ var defaultQuery = &QueryConfig{
DefaultProjectID: "def-project-id",
DefaultDatasetID: "def-dataset-id",
}
type testService struct {
*bq.Job
service
}
func (s *testService) insertJob(ctx context.Context, projectID string, conf *insertJobConf) (*Job, error) {
s.Job = conf.job
return &Job{}, nil
}
func (s *testService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) {
return &JobStatus{State: Done}, nil
}