com.google.api.services.bigquery.model.Table

Here are the examples of the java api class com.google.api.services.bigquery.model.Table taken from open source projects.

1. TableInfo#toPb()

Project: gcloud-java
Source File: TableInfo.java
View license
Table toPb() {
    Table tablePb = definition.toPb();
    tablePb.setTableReference(tableId.toPb());
    if (lastModifiedTime != null) {
        tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime));
    }
    tablePb.setCreationTime(creationTime);
    tablePb.setDescription(description);
    tablePb.setEtag(etag);
    tablePb.setExpirationTime(expirationTime);
    tablePb.setFriendlyName(friendlyName);
    tablePb.setId(generatedId);
    tablePb.setSelfLink(selfLink);
    return tablePb;
}

2. StandardTableDefinition#toPb()

View license
@Override
Table toPb() {
    Table tablePb = super.toPb();
    if (numRows != null) {
        tablePb.setNumRows(BigInteger.valueOf(numRows));
    }
    tablePb.setNumBytes(numBytes);
    tablePb.setLocation(location);
    if (streamingBuffer != null) {
        tablePb.setStreamingBuffer(streamingBuffer.toPb());
    }
    return tablePb;
}

3. BigQueryServicesImplTest#testExecuteWithRetries()

View license
@Test
public void testExecuteWithRetries() throws IOException, InterruptedException {
    Table testTable = new Table();
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(testTable));
    Table table = BigQueryServicesImpl.executeWithRetries(bigquery.tables().get("projectId", "datasetId", "tableId"), "Failed to get table.", Sleeper.DEFAULT, BackOff.STOP_BACKOFF);
    assertEquals(testTable, table);
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
}

4. BigQueryTableInserterTest#testCreateTableSucceeds()

View license
/**
   * Tests that {@link BigQueryTableInserter} succeeds on the first try.
   */
@Test
public void testCreateTableSucceeds() throws IOException {
    Table testTable = new Table().setDescription("a table");
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(testTable));
    BigQueryTableInserter inserter = new BigQueryTableInserter(bigquery, options);
    Table ret = inserter.tryCreateTable(new Table(), "project", "dataset", new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
    assertEquals(testTable, ret);
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
}

5. BigQueryServicesImplTest#testExecuteWithRetries()

View license
@Test
public void testExecuteWithRetries() throws IOException, InterruptedException {
    Table testTable = new Table();
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(testTable));
    Table table = BigQueryServicesImpl.executeWithRetries(bigquery.tables().get("projectId", "datasetId", "tableId"), "Failed to get table.", Sleeper.DEFAULT, BackOff.STOP_BACKOFF);
    assertEquals(testTable, table);
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
}

6. BigQueryTableInserterTest#testCreateTableSucceeds()

View license
/**
   * Tests that {@link BigQueryTableInserter} succeeds on the first try.
   */
@Test
public void testCreateTableSucceeds() throws IOException {
    Table testTable = new Table().setDescription("a table");
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(testTable));
    BigQueryTableInserter inserter = new BigQueryTableInserter(bigquery);
    Table ret = inserter.tryCreateTable(new Table(), "project", "dataset", new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
    assertEquals(testTable, ret);
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
}

7. ViewDefinition#toPb()

Project: gcloud-java
Source File: ViewDefinition.java
View license
@Override
Table toPb() {
    Table tablePb = super.toPb();
    com.google.api.services.bigquery.model.ViewDefinition viewDefinition = new com.google.api.services.bigquery.model.ViewDefinition().setQuery(query);
    if (userDefinedFunctions != null) {
        viewDefinition.setUserDefinedFunctionResources(Lists.transform(userDefinedFunctions, UserDefinedFunction.TO_PB_FUNCTION));
    }
    tablePb.setView(viewDefinition);
    return tablePb;
}

8. DataflowExampleUtils#setupBigQueryTable()

View license
private void setupBigQueryTable(String projectId, String datasetId, String tableId, TableSchema schema) throws IOException {
    if (bigQueryClient == null) {
        bigQueryClient = Transport.newBigQueryClient(options.as(BigQueryOptions.class)).build();
    }
    Datasets datasetService = bigQueryClient.datasets();
    if (executeNullIfNotFound(datasetService.get(projectId, datasetId)) == null) {
        Dataset newDataset = new Dataset().setDatasetReference(new DatasetReference().setProjectId(projectId).setDatasetId(datasetId));
        datasetService.insert(projectId, newDataset).execute();
    }
    Tables tableService = bigQueryClient.tables();
    Table table = executeNullIfNotFound(tableService.get(projectId, datasetId, tableId));
    if (table == null) {
        Table newTable = new Table().setSchema(schema).setTableReference(new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId));
        tableService.insert(projectId, datasetId, newTable).execute();
    } else if (!table.getSchema().equals(schema)) {
        throw new RuntimeException("Table exists and schemas do not match, expecting: " + schema.toPrettyString() + ", actual: " + table.getSchema().toPrettyString());
    }
}

9. BigQueryTableRowIterator#open()

View license
/**
   * Opens the table for read.
   * @throws IOException on failure
   */
public void open() throws IOException, InterruptedException {
    if (query != null) {
        ref = executeQueryAndWaitForCompletion();
    }
    // Get table schema.
    Bigquery.Tables.Get get = client.tables().get(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
    Table table = executeWithBackOff(get, "Error opening BigQuery table  %s of dataset %s  : {}", ref.getTableId(), ref.getDatasetId());
    schema = table.getSchema();
}

10. DataflowExampleUtils#setupBigQueryTable()

View license
private void setupBigQueryTable(String projectId, String datasetId, String tableId, TableSchema schema) throws IOException {
    if (bigQueryClient == null) {
        bigQueryClient = Transport.newBigQueryClient(options.as(BigQueryOptions.class)).build();
    }
    Datasets datasetService = bigQueryClient.datasets();
    if (executeNullIfNotFound(datasetService.get(projectId, datasetId)) == null) {
        Dataset newDataset = new Dataset().setDatasetReference(new DatasetReference().setProjectId(projectId).setDatasetId(datasetId));
        datasetService.insert(projectId, newDataset).execute();
    }
    Tables tableService = bigQueryClient.tables();
    Table table = executeNullIfNotFound(tableService.get(projectId, datasetId, tableId));
    if (table == null) {
        Table newTable = new Table().setSchema(schema).setTableReference(new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId));
        tableService.insert(projectId, datasetId, newTable).execute();
    } else if (!table.getSchema().equals(schema)) {
        throw new RuntimeException("Table exists and schemas do not match, expecting: " + schema.toPrettyString() + ", actual: " + table.getSchema().toPrettyString());
    }
}

11. BigQueryTableInserterTest#testCreateTableSucceedsAlreadyExists()

View license
/**
   * Tests that {@link BigQueryTableInserter} succeeds when the table already exists.
   */
@Test
public void testCreateTableSucceedsAlreadyExists() throws IOException {
    // 409 means already exists
    when(response.getStatusCode()).thenReturn(409);
    BigQueryTableInserter inserter = new BigQueryTableInserter(bigquery, options);
    Table ret = inserter.tryCreateTable(new Table(), "project", "dataset", new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
    assertNull(ret);
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
}

12. BigQueryTableInserterTest#testCreateTableSucceedsAlreadyExists()

View license
/**
   * Tests that {@link BigQueryTableInserter} succeeds when the table already exists.
   */
@Test
public void testCreateTableSucceedsAlreadyExists() throws IOException {
    // 409 means already exists
    when(response.getStatusCode()).thenReturn(409);
    BigQueryTableInserter inserter = new BigQueryTableInserter(bigquery);
    Table ret = inserter.tryCreateTable(new Table(), "project", "dataset", new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
    assertNull(ret);
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
}

13. BigQueryTableRowIterator#open()

View license
/**
   * Opens the table for read.
   * @throws IOException on failure
   */
public void open() throws IOException, InterruptedException {
    if (query != null) {
        ref = executeQueryAndWaitForCompletion();
    }
    // Get table schema.
    Bigquery.Tables.Get get = client.tables().get(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
    Table table = executeWithBackOff(get, "Error opening BigQuery table  %s of dataset %s  : {}", ref.getTableId(), ref.getDatasetId());
    schema = table.getSchema();
}

14. ExternalTableDefinition#toPb()

View license
@Override
com.google.api.services.bigquery.model.Table toPb() {
    Table tablePb = super.toPb();
    tablePb.setExternalDataConfiguration(toExternalDataConfigurationPb());
    return tablePb;
}

15. TableDefinition#toPb()

Project: gcloud-java
Source File: TableDefinition.java
View license
Table toPb() {
    Table tablePb = new Table();
    if (schema != null) {
        tablePb.setSchema(schema.toPb());
    }
    tablePb.setType(type.name());
    return tablePb;
}

16. BigQueryTableInserterTest#testCreateTableRetry()

View license
/**
   * Tests that {@link BigQueryTableInserter} retries quota rate limited attempts.
   */
@Test
public void testCreateTableRetry() throws IOException {
    TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    Table testTable = new Table().setTableReference(ref);
    // First response is 403 rate limited, second response has valid payload.
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(403).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403))).thenReturn(toStream(testTable));
    BigQueryTableInserter inserter = new BigQueryTableInserter(bigquery);
    Table ret = inserter.tryCreateTable(testTable, "project", "dataset", new RetryBoundedBackOff(3, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
    assertEquals(testTable, ret);
    verify(response, times(2)).getStatusCode();
    verify(response, times(2)).getContent();
    verify(response, times(2)).getContentType();
    verifyNotNull(ret.getTableReference());
    expectedLogs.verifyInfo("Quota limit reached when creating table project:dataset.table, " + "retrying up to 5.0 minutes");
}

17. BigQueryTableInserterTest#testCreateTableRetry()

View license
/**
   * Tests that {@link BigQueryTableInserter} retries quota rate limited attempts.
   */
@Test
public void testCreateTableRetry() throws IOException {
    TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
    Table testTable = new Table().setTableReference(ref);
    // First response is 403 rate limited, second response has valid payload.
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(403).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403))).thenReturn(toStream(testTable));
    BigQueryTableInserter inserter = new BigQueryTableInserter(bigquery, options);
    Table ret = inserter.tryCreateTable(testTable, "project", "dataset", new RetryBoundedBackOff(3, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
    assertEquals(testTable, ret);
    verify(response, times(2)).getStatusCode();
    verify(response, times(2)).getContent();
    verify(response, times(2)).getContentType();
    verifyNotNull(ret.getTableReference());
    expectedLogs.verifyInfo("Quota limit reached when creating table project:dataset.table, " + "retrying up to 5.0 minutes");
}

18. BigQueryTableInserterTest#testCreateTableDoesNotRetry()

View license
/**
   * Tests that {@link BigQueryTableInserter} does not retry non-rate-limited attempts.
   */
@Test
public void testCreateTableDoesNotRetry() throws IOException {
    Table testTable = new Table().setDescription("a table");
    // First response is 403 not-rate-limited, second response has valid payload but should not
    // be invoked.
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(403).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(errorWithReasonAndStatus("actually forbidden", 403))).thenReturn(toStream(testTable));
    thrown.expect(GoogleJsonResponseException.class);
    thrown.expectMessage("actually forbidden");
    BigQueryTableInserter inserter = new BigQueryTableInserter(bigquery, options);
    try {
        inserter.tryCreateTable(new Table(), "project", "dataset", new RetryBoundedBackOff(3, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
        fail();
    } catch (IOException e) {
        verify(response, times(1)).getStatusCode();
        verify(response, times(1)).getContent();
        verify(response, times(1)).getContentType();
        throw e;
    }
}

19. BigQueryTableInserter#tryCreateTable()

View license
/**
   * Tries to create the BigQuery table.
   * If a table with the same name already exists in the dataset, the table
   * creation fails, and the function returns null.  In such a case,
   * the existing table doesn't necessarily have the same schema as specified
   * by the parameter.
   *
   * @param schema Schema of the new BigQuery table.
   * @return The newly created BigQuery table information, or null if the table
   *     with the same name already exists.
   * @throws IOException if other error than already existing table occurs.
   */
@Nullable
public Table tryCreateTable(TableReference ref, TableSchema schema) throws IOException {
    LOG.info("Trying to create BigQuery table: {}", BigQueryIO.toTableSpec(ref));
    BackOff backoff = new ExponentialBackOff.Builder().setMaxElapsedTimeMillis(RETRY_CREATE_TABLE_DURATION_MILLIS).build();
    Table table = new Table().setTableReference(ref).setSchema(schema);
    return tryCreateTable(table, ref.getProjectId(), ref.getDatasetId(), backoff, Sleeper.DEFAULT);
}

20. BigQueryTableInserter#getOrCreateTable()

View license
/**
   * Retrieves or creates the table.
   *
   * <p>The table is checked to conform to insertion requirements as specified
   * by WriteDisposition and CreateDisposition.
   *
   * <p>If table truncation is requested (WriteDisposition.WRITE_TRUNCATE), then
   * this will re-create the table if necessary to ensure it is empty.
   *
   * <p>If an empty table is required (WriteDisposition.WRITE_EMPTY), then this
   * will fail if the table exists and is not empty.
   *
   * <p>When constructing a table, a {@code TableSchema} must be available.  If a
   * schema is provided, then it will be used.  If no schema is provided, but
   * an existing table is being cleared (WRITE_TRUNCATE option above), then
   * the existing schema will be re-used.  If no schema is available, then an
   * {@code IOException} is thrown.
   */
public Table getOrCreateTable(TableReference ref, WriteDisposition writeDisposition, CreateDisposition createDisposition, @Nullable TableSchema schema) throws IOException {
    // Check if table already exists.
    Bigquery.Tables.Get get = client.tables().get(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
    Table table = null;
    try {
        table = get.execute();
    } catch (IOException e) {
        ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
        if (!errorExtractor.itemNotFound(e) || createDisposition != CreateDisposition.CREATE_IF_NEEDED) {
            throw e;
        }
    }
    // If we want an empty table, and it isn't, then delete it first.
    if (table != null) {
        if (writeDisposition == WriteDisposition.WRITE_APPEND) {
            return table;
        }
        boolean empty = isEmpty(ref);
        if (empty) {
            if (writeDisposition == WriteDisposition.WRITE_TRUNCATE) {
                LOG.info("Empty table found, not removing {}", BigQueryIO.toTableSpec(ref));
            }
            return table;
        } else if (writeDisposition == WriteDisposition.WRITE_EMPTY) {
            throw new IOException("WriteDisposition is WRITE_EMPTY, " + "but table is not empty");
        }
        // Reuse the existing schema if none was provided.
        if (schema == null) {
            schema = table.getSchema();
        }
        // Delete table and fall through to re-creating it below.
        LOG.info("Deleting table {}", BigQueryIO.toTableSpec(ref));
        Bigquery.Tables.Delete delete = client.tables().delete(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
        delete.execute();
    }
    if (schema == null) {
        throw new IllegalArgumentException("Table schema required for new table.");
    }
    // Create the table.
    return tryCreateTable(ref, schema);
}

21. BigQueryTableInserterTest#testCreateTableDoesNotRetry()

View license
/**
   * Tests that {@link BigQueryTableInserter} does not retry non-rate-limited attempts.
   */
@Test
public void testCreateTableDoesNotRetry() throws IOException {
    Table testTable = new Table().setDescription("a table");
    // First response is 403 not-rate-limited, second response has valid payload but should not
    // be invoked.
    when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
    when(response.getStatusCode()).thenReturn(403).thenReturn(200);
    when(response.getContent()).thenReturn(toStream(errorWithReasonAndStatus("actually forbidden", 403))).thenReturn(toStream(testTable));
    thrown.expect(GoogleJsonResponseException.class);
    thrown.expectMessage("actually forbidden");
    BigQueryTableInserter inserter = new BigQueryTableInserter(bigquery);
    try {
        inserter.tryCreateTable(new Table(), "project", "dataset", new RetryBoundedBackOff(3, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
        fail();
    } catch (IOException e) {
        verify(response, times(1)).getStatusCode();
        verify(response, times(1)).getContent();
        verify(response, times(1)).getContentType();
        throw e;
    }
}

22. BigQueryTableInserter#tryCreateTable()

View license
/**
   * Tries to create the BigQuery table.
   * If a table with the same name already exists in the dataset, the table
   * creation fails, and the function returns null.  In such a case,
   * the existing table doesn't necessarily have the same schema as specified
   * by the parameter.
   *
   * @param schema Schema of the new BigQuery table.
   * @return The newly created BigQuery table information, or null if the table
   *     with the same name already exists.
   * @throws IOException if other error than already existing table occurs.
   */
@Nullable
public Table tryCreateTable(TableReference ref, TableSchema schema) throws IOException {
    LOG.info("Trying to create BigQuery table: {}", BigQueryIO.toTableSpec(ref));
    BackOff backoff = new ExponentialBackOff.Builder().setMaxElapsedTimeMillis(RETRY_CREATE_TABLE_DURATION_MILLIS).build();
    Table table = new Table().setTableReference(ref).setSchema(schema);
    return tryCreateTable(table, ref.getProjectId(), ref.getDatasetId(), backoff, Sleeper.DEFAULT);
}

23. BigQueryTableInserter#getOrCreateTable()

View license
/**
   * Retrieves or creates the table.
   *
   * <p>The table is checked to conform to insertion requirements as specified
   * by WriteDisposition and CreateDisposition.
   *
   * <p>If table truncation is requested (WriteDisposition.WRITE_TRUNCATE), then
   * this will re-create the table if necessary to ensure it is empty.
   *
   * <p>If an empty table is required (WriteDisposition.WRITE_EMPTY), then this
   * will fail if the table exists and is not empty.
   *
   * <p>When constructing a table, a {@code TableSchema} must be available.  If a
   * schema is provided, then it will be used.  If no schema is provided, but
   * an existing table is being cleared (WRITE_TRUNCATE option above), then
   * the existing schema will be re-used.  If no schema is available, then an
   * {@code IOException} is thrown.
   */
public Table getOrCreateTable(TableReference ref, WriteDisposition writeDisposition, CreateDisposition createDisposition, @Nullable TableSchema schema) throws IOException {
    // Check if table already exists.
    Bigquery.Tables.Get get = client.tables().get(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
    Table table = null;
    try {
        table = get.execute();
    } catch (IOException e) {
        ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
        if (!errorExtractor.itemNotFound(e) || createDisposition != CreateDisposition.CREATE_IF_NEEDED) {
            throw e;
        }
    }
    // If we want an empty table, and it isn't, then delete it first.
    if (table != null) {
        if (writeDisposition == WriteDisposition.WRITE_APPEND) {
            return table;
        }
        boolean empty = isEmpty(ref);
        if (empty) {
            if (writeDisposition == WriteDisposition.WRITE_TRUNCATE) {
                LOG.info("Empty table found, not removing {}", BigQueryIO.toTableSpec(ref));
            }
            return table;
        } else if (writeDisposition == WriteDisposition.WRITE_EMPTY) {
            throw new IOException("WriteDisposition is WRITE_EMPTY, " + "but table is not empty");
        }
        // Reuse the existing schema if none was provided.
        if (schema == null) {
            schema = table.getSchema();
        }
        // Delete table and fall through to re-creating it below.
        LOG.info("Deleting table {}", BigQueryIO.toTableSpec(ref));
        Bigquery.Tables.Delete delete = client.tables().delete(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
        delete.execute();
    }
    if (schema == null) {
        throw new IllegalArgumentException("Table schema required for new table.");
    }
    // Create the table.
    return tryCreateTable(ref, schema);
}