easy_sql.sql_processor.backend.bigquery

Module Contents

Classes

BigQueryBackend

table_partitions_table_name;

BigQueryRow

TimeLog

Functions

_exec_sql(conn, sql, *args, **kwargs)

class easy_sql.sql_processor.backend.bigquery.BigQueryBackend(url, credentials=None, sql_expr=None, partitions_table_name='dataplat.__table_partitions__', engine=None)

Bases: easy_sql.sql_processor.backend.base.Backend

table_partitions_table_name; means the table name which save the static partition info for all partition tables in data warehouse, for now need support backend type: [clickhouse] others backend has another method to manage static partition info or just support static partition

Parameters
broadcast_table(self, table, name)
Parameters
  • table (RdbTable) –

  • name (str) –

clean(self)
clear_cache(self)
clear_temp_tables(self, exclude=None)
Parameters

exclude (Optional[List[str]]) –

create_cache_table(self, table, name)
Parameters
  • table (RdbTable) –

  • name (str) –

create_empty_table(self)
create_table_with_data(self, full_table_name, values, schema, partitions)
Parameters
create_temp_table(self, table, name)
Parameters
  • table (RdbTable) –

  • name (str) –

create_temp_table_with_data(self, table_name, values, schema)
Parameters
  • table_name (str) –

  • values (List[List[Any]]) –

  • schema (List[easy_sql.sql_processor.backend.base.Col]) –

db_exists(self, db_name)
Parameters

db_name (str) –

Return type

bool

exec_native_sql(self, sql)
Parameters

sql (Union[str, List[str]]) –

Return type

Any

exec_sql(self, sql)
Parameters

sql (str) –

Return type

RdbTable

get_column_names(self, table_name, schema=None, **kw)
Return type

List[str]

get_columns(self, table_name, schema=None, raw=False, **kw)
Return type

List[Dict]

init_udfs(self, *args, **kwargs)
property inspector(self)
Return type

sqlalchemy.engine.reflection.Inspector

refresh_table_partitions(self, table)
Parameters

table (easy_sql.sql_processor.backend.base.TableMeta) –

register_udfs(self, funcs)
Parameters

funcs (Dict[str, Callable[[], Union[str, List[str]]]]) –

reset(self)
save_table(self, source_table, target_table, save_mode, create_target_table)
Parameters
save_table_sql(self, source_table, source_table_sql, target_table)
Parameters
Return type

str

table_exists(self, table)
Parameters

table (easy_sql.sql_processor.backend.base.TableMeta) –

temp_tables(self)
Return type

List[str]

class easy_sql.sql_processor.backend.bigquery.BigQueryRow(cols, values)

Bases: easy_sql.sql_processor.backend.base.Row

Parameters
  • cols (List[str]) –

  • values (Tuple) –

as_dict(self)
as_tuple(self)
class easy_sql.sql_processor.backend.bigquery.TimeLog(start_log, end_log_tpl)
Parameters
  • start_log (str) –

  • end_log_tpl (str) –

time_took_tpl = time took: {time_took:.3f}s