easy_sql.sql_processor

Subpackages

• easy_sql.sql_processor.backend
  • easy_sql.sql_processor.backend.sql_dialect
    • easy_sql.sql_processor.backend.sql_dialect.bigquery
    • easy_sql.sql_processor.backend.sql_dialect.clickhouse
    • easy_sql.sql_processor.backend.sql_dialect.postgres
  • easy_sql.sql_processor.backend.base
  • easy_sql.sql_processor.backend.bigquery
  • easy_sql.sql_processor.backend.clickhouse
  • easy_sql.sql_processor.backend.flink
  • easy_sql.sql_processor.backend.maxcompute
  • easy_sql.sql_processor.backend.postgres
  • easy_sql.sql_processor.backend.rdb
  • easy_sql.sql_processor.backend.spark

Submodules

• easy_sql.sql_processor.common
• easy_sql.sql_processor.context
• easy_sql.sql_processor.funcs
• easy_sql.sql_processor.funcs_common
• easy_sql.sql_processor.funcs_flink
• easy_sql.sql_processor.funcs_rdb
• easy_sql.sql_processor.funcs_spark
• easy_sql.sql_processor.report
• easy_sql.sql_processor.sql_processor
• easy_sql.sql_processor.step

Package Contents

Classes

Column
FuncRunner
SqlProcessor
SqlProcessorReporter
Step
StepConfig
StepReport
StepStatus
StepType
VarsContext

exception easy_sql.sql_processor.SqlProcessorException(message)

Bases: Exception

digraph inheritance89bd82674e { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "SqlProcessorException" [URL="common/index.html#easy_sql.sql_processor.common.SqlProcessorException",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; }

Common base class for all non-exit exceptions.

Parameters

message (str) –

class easy_sql.sql_processor.Column(name, value)

Parameters

• name (str) –

• value (Any) –

class easy_sql.sql_processor.FuncRunner(funcs=None)

Parameters

funcs (Optional[Dict[str, Callable]]) –

static create(backend)

Parameters

backend (easy_sql.sql_processor.backend.Backend) –

Return type

FuncRunner

static easysql_funcs()

Return type

Dict[str, Callable]

register_funcs(self, funcs)

Parameters

funcs (Dict[str, Callable]) –

run_func(self, func_def, vars_replacer)

Parameters

• func_def (str) –

• vars_replacer (easy_sql.sql_processor.common.VarsReplacer) –

Return type

Optional[str]

static system_funcs()

Return type

Dict[str, Callable]

class easy_sql.sql_processor.SqlProcessor(backend, sql, extra_cols=None, variables=None, report_hdfs_path=None, report_task_id=None, report_es_url=None, report_es_index_prefix=None, scala_udf_initializer=None, templates=None, includes=None, config=None, executed_sql_transformer=None)

Parameters

• backend (Union[pyspark.sql.SparkSession, easy_sql.sql_processor.backend.Backend]) –

• sql (str) –

• extra_cols (Optional[List[easy_sql.sql_processor.common.Column]]) –

• variables (Optional[dict]) –

• report_hdfs_path (Optional[str]) –

• report_task_id (Optional[str]) –

• report_es_url (Optional[str]) –

• report_es_index_prefix (Optional[str]) –

• scala_udf_initializer (Optional[str]) –

• templates (Optional[dict]) –

• includes (Optional[Dict[str, str]]) –

• config (Any) –

• executed_sql_transformer (Optional[easy_sql.sql_processor.step.ExecutedSqlTransformer]) –

add_vars(self, vars)

Parameters

vars (Dict[str, Any]) –

property extra_cols(self)

Return type

List[easy_sql.sql_processor.common.Column]

register_funcs(self, funcs)

Parameters

funcs (Dict[str, Callable]) –

register_funcs_from_pyfile(self, funcs_py_file)

Parameters

funcs_py_file (str) –

register_udfs(self, funcs)

Parameters

funcs (Dict[str, Callable]) –

register_udfs_from_pyfile(self, funcs_py_file)

Parameters

funcs_py_file (str) –

run(self, dry_run=False)

Parameters

dry_run (bool) –

run_step(self, step, dry_run)

Parameters

• step (easy_sql.sql_processor.step.Step) –

• dry_run (bool) –

set_spark_configs(self, configs)

Parameters

configs (Dict[str, str]) –

set_vars(self, vars)

Parameters

vars (Dict[str, Any]) –

property templates(self)

Return type

Dict[str, str]

property variables(self)

Return type

Dict[str, Any]

class easy_sql.sql_processor.SqlProcessorReporter(report_task_id, report_hdfs_path=None, report_es_url=None, report_es_index_prefix=None)

Bases: easy_sql.sql_processor.step.ReportCollector

digraph inheritanced393cd2836 { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "ReportCollector" [URL="step/index.html#easy_sql.sql_processor.step.ReportCollector",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "SqlProcessorReporter" [URL="report/index.html#easy_sql.sql_processor.report.SqlProcessorReporter",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "ReportCollector" -> "SqlProcessorReporter" [arrowsize=0.5,style="setlinewidth(0.5)"]; }

Parameters

• report_task_id (str) –

• report_hdfs_path (Optional[str]) –

• report_es_url (Optional[str]) –

• report_es_index_prefix (Optional[str]) –

collect_report(self, step, status=None, message=None)

Parameters

• step (easy_sql.sql_processor.step.Step) –

• status (Optional[str]) –

• message (Optional[str]) –

get_report(self, verbose=False)

Parameters

verbose (bool) –

Return type

str

init(self, steps)

Parameters

steps (List[easy_sql.sql_processor.step.Step]) –

print_report(self, verbose=False)

Parameters

verbose (bool) –

class easy_sql.sql_processor.Step(id, reporter_collector, func_runner, target_config=None, select_sql=None, debug_var_tmpl_replace=False, *, executed_sql_transformer=None)

Parameters

• id (str) –

• reporter_collector (ReportCollector) –

• func_runner (easy_sql.sql_processor.funcs.FuncRunner) –

• target_config (Optional[StepConfig]) –

• select_sql (Optional[str]) –

• debug_var_tmpl_replace (bool) –

• executed_sql_transformer (Optional[ExecutedSqlTransformer]) –

collect_report(self, status=None, message=None)

get_executed_sql(self)

Return type

str

is_template_statement(self)

preprocess_select_sql(self, context)

Parameters

context (easy_sql.sql_processor.context.ProcessorContext) –

read(self, backend, context)

Parameters

• backend (easy_sql.sql_processor.backend.Backend) –

• context (easy_sql.sql_processor.context.ProcessorContext) –

Return type

Optional[easy_sql.sql_processor.backend.Table]

should_run(self, context)

Parameters

context (easy_sql.sql_processor.context.ProcessorContext) –

write(self, backend, table, context, dry_run=False)

Parameters

• backend (easy_sql.sql_processor.backend.Backend) –

• table (Optional[easy_sql.sql_processor.backend.Table]) –

• context (easy_sql.sql_processor.context.ProcessorContext) –

• dry_run (bool) –

class easy_sql.sql_processor.StepConfig(step_type, step_name, condition, line_no, step_config_str='')

Parameters

• step_type (str) –

• step_name (Optional[str]) –

• condition (Optional[str]) –

• line_no (int) –

• step_config_str (str) –

STEP_CONFIG_PATTERN = ^-- target\s*=\s*(\w+)(.*)$

static from_config_line(config_line, line_no)

Parameters

• config_line (str) –

• line_no (int) –

Return type

StepConfig

has_condition(self)

is_target_name_a_func(self)

class easy_sql.sql_processor.StepReport(step)

Parameters

step (easy_sql.sql_processor.step.Step) –

report_as_text(self, total_execution_time, verbose=False)

Parameters

• total_execution_time (float) –

• verbose (bool) –

update(self, status=None, message=None)

Parameters

• status (Optional[str]) –

• message (Optional[str]) –

Return type

StepReport

class easy_sql.sql_processor.StepStatus

FAILED = FAILED

NOT_STARTED = NOT_STARTED

RUNNING = RUNNING

SKIPPED = SKIPPED

SUCCEEDED = SUCCEEDED

class easy_sql.sql_processor.StepType

ACTION = action

BROADCAST = broadcast

CACHE = cache

CHECK = check

FUNC = func

HIVE = hive

LIST_VARIABLES = list_variables

LOG = log

OUTPUT = output

TEMP = temp

TEMPLATE = template

VARIABLES = variables

static all()

Return type

List[str]

class easy_sql.sql_processor.VarsContext(vars=None, list_vars=None, debug_log=False)

Bases: easy_sql.sql_processor.common.VarsReplacer

digraph inheritanced87d1daf3d { bgcolor=transparent; rankdir=LR; size="8.0, 12.0"; "VarsContext" [URL="context/index.html#easy_sql.sql_processor.context.VarsContext",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; "VarsReplacer" -> "VarsContext" [arrowsize=0.5,style="setlinewidth(0.5)"]; "VarsReplacer" [URL="common/index.html#easy_sql.sql_processor.common.VarsReplacer",fillcolor=white,fontname="Vera Sans, DejaVu Sans, Liberation Sans, Arial, Helvetica, sans",fontsize=10,height=0.25,shape=box,style="setlinewidth(0.5),filled",target="_top"]; }

Parameters

• vars (Optional[Dict[str, Any]]) –

• list_vars (Optional[Dict[str, List]]) –

• debug_log (bool) –

add_list_vars(self, vars)

Parameters

vars (Dict[str, List]) –

add_vars(self, vars)

Parameters

vars (Dict[str, str]) –

init(self, func_runner)

Parameters

func_runner (easy_sql.sql_processor.FuncRunner) –

replace_variables(self, text, include_funcs=True)

Parameters

• text (str) –

• include_funcs (bool) –

Return type

str