`easy_sql.sql_processor.backend`

Subpackages

easy_sql.sql_processor.backend.sql_dialect

Submodules

Package Contents

Classes

`Backend`
`FlinkBackend`
`FlinkRow`
`FlinkTable`
`FlinkTablesConfig`
`Partition`
`Row`
`SaveMode`	Generic enumeration.
`SparkBackend`
`SparkRow`
`SparkTable`
`Table`
`TableMeta`

class easy_sql.sql_processor.backend.Backend

abstract broadcast_table(self, table, name)

Parameters

table (Table) –
name (str) –

abstract clean(self)

abstract clear_cache(self)

abstract clear_temp_tables(self, exclude=None)

Parameters: exclude (Optional[List[str]]) –

abstract create_cache_table(self, table, name)

Parameters

table (Table) –
name (str) –

abstract create_empty_table(self)

abstract create_table_with_data(self, full_table_name, values, schema, partitions)

Parameters

full_table_name (str) –
values (List[List[Any]]) –
schema (Union[pyspark.sql.types.StructType, List[Col]]) –
partitions (List[Partition]) –

abstract create_temp_table(self, table, name)

Parameters

table (Table) –
name (str) –

abstract create_temp_table_with_data(self, table_name, values, schema)

Parameters

table_name (str) –
values (List[List[Any]]) –
schema (Union[pyspark.sql.types.StructType, List[Col]]) –

abstract exec_native_sql(self, sql)

Parameters: sql (str) –
Return type: Any

abstract exec_sql(self, sql)

Parameters: sql (str) –
Return type: Table

abstract init_udfs(self, *args, **kwargs)

property is_bigquery_backend(self)

property is_clickhouse_backend(self)

property is_flink_backend(self)

property is_postgres_backend(self)

property is_rdb_backend(self)

property is_spark_backend(self)

abstract refresh_table_partitions(self, table)

Parameters: table (TableMeta) –

abstract register_udfs(self, funcs)

Parameters: funcs (Dict[str, Callable]) –

abstract reset(self)

abstract save_table(self, source_table, target_table, save_mode, create_target_table)

Parameters

source_table (TableMeta) –
target_table (TableMeta) –
save_mode (SaveMode) –
create_target_table (bool) –

abstract save_table_sql(self, source_table, source_table_sql, target_table)

Parameters

source_table (TableMeta) –
source_table_sql (str) –
target_table (TableMeta) –

Return type

str

table_exists(self, table)

Parameters: table (TableMeta) –

abstract temp_tables(self)

Return type: List[str]

class easy_sql.sql_processor.backend.FlinkBackend(is_batch=True, flink_tables_config=None)

Bases: easy_sql.sql_processor.backend.base.Backend

digraph inheritanceff1e9190c3 { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "Backend" [URL="base/index.html#easy_sql.sql_processor.backend.base.Backend",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "FlinkBackend" [URL="flink/index.html#easy_sql.sql_processor.backend.flink.FlinkBackend",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "Backend" -> "FlinkBackend" [arrowsize=0.5,style="setlinewidth(0.5)"]; }

Parameters

is_batch (Optional[bool]) –
flink_tables_config (Optional[FlinkTablesConfig]) –

add_jars(self, jars_path)

Parameters: jars_path (List[str]) –

clean(self)

create_cache_table(self, table, name)

Parameters

table (easy_sql.sql_processor.backend.base.Table) –
name (str) –

create_empty_table(self)

create_temp_table(self, table, name)

Parameters

table (easy_sql.sql_processor.backend.base.Table) –
name (str) –

exec_native_sql(self, sql)

Parameters: sql (str) –
Return type: pyflink.table.TableResult

exec_native_sql_query(self, sql)

Parameters: sql (str) –
Return type: pyflink.table.Table

exec_sql(self, sql)

Parameters: sql (str) –
Return type: easy_sql.sql_processor.backend.base.Table

execute_streaming_inserts(self)

init_udfs(self, scala_udf_initializer=None, *args, **kwargs)

Parameters: scala_udf_initializer (Optional[str]) –

refresh_table_partitions(self, table)

Parameters: table (easy_sql.sql_processor.backend.base.TableMeta) –

register_tables(self)

register_udfs(self, funcs)

Parameters: funcs (Dict[str, Callable]) –

save_table(self, source_table_meta, target_table_meta, save_mode, create_target_table=False)

Parameters

source_table_meta (easy_sql.sql_processor.backend.base.TableMeta) –
target_table_meta (easy_sql.sql_processor.backend.base.TableMeta) –
save_mode (easy_sql.sql_processor.backend.base.SaveMode) –
create_target_table (bool) –

save_table_sql(self, source_table, source_table_sql, target_table)

Parameters

source_table (easy_sql.sql_processor.backend.base.TableMeta) –
source_table_sql (str) –
target_table (easy_sql.sql_processor.backend.base.TableMeta) –

Return type

str

set_configurations(self, configs)

Parameters: configs (dict) –

table_exists(self, table)

Parameters: table (easy_sql.sql_processor.backend.base.TableMeta) –

class easy_sql.sql_processor.backend.FlinkRow(row=None, fields=None)

Bases: easy_sql.sql_processor.backend.base.Row

digraph inheritance1e085b47c1 { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "FlinkRow" [URL="flink/index.html#easy_sql.sql_processor.backend.flink.FlinkRow",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "Row" -> "FlinkRow" [arrowsize=0.5,style="setlinewidth(0.5)"]; "Row" [URL="base/index.html#easy_sql.sql_processor.backend.base.Row",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; }

Parameters: fields (Optional[List[str]]) –

as_dict(self)

as_tuple(self)

Return type: Tuple

class easy_sql.sql_processor.backend.FlinkTable(table)

Bases: easy_sql.sql_processor.backend.base.Table

digraph inheritancea151cfaa79 { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "FlinkTable" [URL="flink/index.html#easy_sql.sql_processor.backend.flink.FlinkTable",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "Table" -> "FlinkTable" [arrowsize=0.5,style="setlinewidth(0.5)"]; "Table" [URL="base/index.html#easy_sql.sql_processor.backend.base.Table",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; }

collect(self)

Return type: List[easy_sql.sql_processor.backend.base.Row]

count(self)

Return type: int

field_names(self)

Return type: List[str]

first(self)

Return type: easy_sql.sql_processor.backend.base.Row

is_empty(self)

Return type: bool

limit(self, count)

Parameters: count (int) –
Return type: FlinkTable

show(self, count=20)

Parameters: count (int) –

with_column(self, name, value)

Parameters

name (str) –
value (Any) –

Return type

FlinkTable

class easy_sql.sql_processor.backend.FlinkTablesConfig

class Catalog

databases :Dict[str, FlinkTablesConfig]

options :str

temporary_tables :Dict[str, FlinkTablesConfig]

static from_dict(data)

Parameters: data (dict) –
Return type: FlinkTablesConfig

class Connector

options :str

static from_dict(data)

Parameters: data (dict) –
Return type: FlinkTablesConfig

class Database

tables :Dict[str, FlinkTablesConfig]

static from_dict(data)

Parameters: data (dict) –
Return type: FlinkTablesConfig

class Table

connector :str | None

options :str | None

partition_by :str | None

schema :str

static from_dict(data)

Parameters: data (dict) –
Return type: FlinkTablesConfig

catalogs :Dict[str, Catalog]

connectors :Dict[str, Connector]

static from_dict(data)

Parameters: data (dict) –
Return type: FlinkTablesConfig

static from_yml(file_path)

Parameters: file_path (str | None) –
Return type: FlinkTablesConfig

generate_catalog_ddl(self)

Return type: Iterable[tuple[str, str]]

generate_db_ddl(self)

Return type: Iterable[str]

generate_table_ddl(self)

Return type: Iterable[str]

get_connector_options(self, name)

Parameters: name (str) –
Return type: Dict[str, str]

static parse_options(option_str)

Parameters: option_str (str) –
Return type: Dict[str, str]

class easy_sql.sql_processor.backend.Partition(field, value=None)

Parameters: field (str) –

class easy_sql.sql_processor.backend.Row

abstract as_dict(self)

Return type: Dict[str, Any]

abstract as_tuple(self)

Return type: Tuple

class easy_sql.sql_processor.backend.SaveMode

Bases: enum.Enum

digraph inheritanceb2edf6a3a6 { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "Enum" [fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",tooltip="Generic enumeration."]; "SaveMode" [URL="base/index.html#easy_sql.sql_processor.backend.base.SaveMode",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "Enum" -> "SaveMode" [arrowsize=0.5,style="setlinewidth(0.5)"]; }

Generic enumeration.

Derive from this class to define new enumerations.

append = 1

overwrite = [0]

class easy_sql.sql_processor.backend.SparkBackend(spark, scala_udf_initializer=None)

Bases: easy_sql.sql_processor.backend.base.Backend

digraph inheritance64f15a669e { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "Backend" [URL="base/index.html#easy_sql.sql_processor.backend.base.Backend",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "SparkBackend" [URL="spark/index.html#easy_sql.sql_processor.backend.spark.SparkBackend",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "Backend" -> "SparkBackend" [arrowsize=0.5,style="setlinewidth(0.5)"]; }

Parameters: scala_udf_initializer (Optional[str]) –

broadcast_table(self, table, name)

Parameters

table (SparkTable) –
name (str) –

clean(self)

clear_cache(self)

clear_temp_tables(self, exclude=None)

Parameters: exclude (Optional[List[str]]) –

create_cache_table(self, table, name)

Parameters

table (SparkTable) –
name (str) –

create_empty_table(self)

create_table_with_data(self, full_table_name, values, schema, partitions)

Parameters

full_table_name (str) –
values (List[List[Any]]) –
schema (Union[pyspark.sql.types.StructType, List[easy_sql.sql_processor.backend.base.Col]]) –
partitions (List[easy_sql.sql_processor.backend.base.Partition]) –

create_temp_table(self, table, name)

Parameters

table (SparkTable) –
name (str) –

create_temp_table_with_data(self, table_name, values, schema)

Parameters

table_name (str) –
values (List[List[Any]]) –
schema (pyspark.sql.types.StructType) –

exec_native_sql(self, sql)

Parameters: sql (str) –
Return type: pyspark.sql.DataFrame

exec_sql(self, sql)

Parameters: sql (str) –
Return type: easy_sql.sql_processor.backend.base.Table

init_udfs(self, scala_udf_initializer=None, *args, **kwargs)

Parameters: scala_udf_initializer (Optional[str]) –

refresh_table_partitions(self, table)

Parameters: table (easy_sql.sql_processor.backend.base.TableMeta) –

register_udfs(self, funcs)

Parameters: funcs (Dict[str, Callable]) –

reset(self)

save_table(self, source_table_meta, target_table_meta, save_mode, create_target_table)

Parameters

source_table_meta (easy_sql.sql_processor.backend.base.TableMeta) –
target_table_meta (easy_sql.sql_processor.backend.base.TableMeta) –
save_mode (easy_sql.sql_processor.backend.base.SaveMode) –
create_target_table (bool) –

save_table_sql(self, source_table, source_table_sql, target_table)

Parameters

source_table (easy_sql.sql_processor.backend.base.TableMeta) –
source_table_sql (str) –
target_table (easy_sql.sql_processor.backend.base.TableMeta) –

Return type

str

set_spark_configs(self, configs)

Parameters: configs (Dict[str, str]) –

table_exists(self, table)

Parameters: table (easy_sql.sql_processor.backend.base.TableMeta) –

temp_tables(self)

Return type: List[str]

class easy_sql.sql_processor.backend.SparkRow(row)

Bases: easy_sql.sql_processor.backend.base.Row

digraph inheritanceca34bc58c4 { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "Row" [URL="base/index.html#easy_sql.sql_processor.backend.base.Row",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "SparkRow" [URL="spark/index.html#easy_sql.sql_processor.backend.spark.SparkRow",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "Row" -> "SparkRow" [arrowsize=0.5,style="setlinewidth(0.5)"]; }

as_dict(self)

as_tuple(self)

Return type: Tuple

class easy_sql.sql_processor.backend.SparkTable(df)

Bases: easy_sql.sql_processor.backend.base.Table

digraph inheritance670259b8b5 { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "SparkTable" [URL="spark/index.html#easy_sql.sql_processor.backend.spark.SparkTable",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "Table" -> "SparkTable" [arrowsize=0.5,style="setlinewidth(0.5)"]; "Table" [URL="base/index.html#easy_sql.sql_processor.backend.base.Table",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; }

collect(self)

Return type: List[easy_sql.sql_processor.backend.base.Row]

count(self)

Return type: int

field_names(self)

Return type: List[str]

first(self)

Return type: easy_sql.sql_processor.backend.base.Row

is_empty(self)

Return type: bool

limit(self, count)

Parameters: count (int) –
Return type: SparkTable

show(self, count=20)

Parameters: count (int) –

with_column(self, name, value)

Parameters

name (str) –
value (Any) –

Return type

SparkTable

class easy_sql.sql_processor.backend.Table

abstract collect(self)

Return type: List[Row]

abstract count(self)

Return type: int

abstract field_names(self)

Return type: List[str]

abstract first(self)

Return type: Row

abstract is_empty(self)

Return type: bool

abstract limit(self, count)

Parameters: count (int) –
Return type: Table

abstract show(self, count)

Parameters: count (int) –

abstract with_column(self, name, value)

Parameters

name (str) –
value (Any) –

Return type

Table

class easy_sql.sql_processor.backend.TableMeta(table_name, partitions=None)

Parameters

table_name (str) –
partitions (Optional[List[Partition]]) –

clone_with_name(self, table_name)

Parameters: table_name (str) –
Return type: TableMeta

clone_with_partitions(self, partitions)

Parameters: partitions (List[Partition]) –
Return type: TableMeta

get_full_table_name(self, temp_db=None)

Parameters: temp_db (Optional[str]) –

has_dynamic_partition(self)

has_partitions(self)

update_partitions(self, partitions)

Parameters: partitions (List[Partition]) –
Return type: TableMeta

easy_sql.sql_processor.backend

Subpackages

Submodules

Package Contents

Classes

`easy_sql.sql_processor.backend`