easy_sql.sql_processor.sql_processor

Module Contents

Classes

SqlCollector
SqlProcessor

Functions

extract_funcs_from_pyfile(funcs_py_file)

class easy_sql.sql_processor.sql_processor.SqlCollector

collect_sql(self, sql_lines)

Parameters

sql_lines (str) –

collected_sql(self)

Return type

str

class easy_sql.sql_processor.sql_processor.SqlProcessor(backend, sql, extra_cols=None, variables=None, report_hdfs_path=None, report_task_id=None, report_es_url=None, report_es_index_prefix=None, scala_udf_initializer=None, templates=None, includes=None, config=None, executed_sql_transformer=None)

Parameters

• backend (Union[pyspark.sql.SparkSession, easy_sql.sql_processor.backend.Backend]) –

• sql (str) –

• extra_cols (Optional[List[easy_sql.sql_processor.common.Column]]) –

• variables (Optional[dict]) –

• report_hdfs_path (Optional[str]) –

• report_task_id (Optional[str]) –

• report_es_url (Optional[str]) –

• report_es_index_prefix (Optional[str]) –

• scala_udf_initializer (Optional[str]) –

• templates (Optional[dict]) –

• includes (Optional[Dict[str, str]]) –

• config (Any) –

• executed_sql_transformer (Optional[easy_sql.sql_processor.step.ExecutedSqlTransformer]) –

add_vars(self, vars)

Parameters

vars (Dict[str, Any]) –

property extra_cols(self)

Return type

List[easy_sql.sql_processor.common.Column]

register_funcs(self, funcs)

Parameters

funcs (Dict[str, Callable]) –

register_funcs_from_pyfile(self, funcs_py_file)

Parameters

funcs_py_file (str) –

register_udfs(self, funcs)

Parameters

funcs (Dict[str, Callable]) –

register_udfs_from_pyfile(self, funcs_py_file)

Parameters

funcs_py_file (str) –

run(self, dry_run=False)

Parameters

dry_run (bool) –

run_step(self, step, dry_run)

Parameters

• step (easy_sql.sql_processor.step.Step) –

• dry_run (bool) –

set_spark_configs(self, configs)

Parameters

configs (Dict[str, str]) –

set_vars(self, vars)

Parameters

vars (Dict[str, Any]) –

property templates(self)

Return type

Dict[str, str]

property variables(self)

Return type

Dict[str, Any]

easy_sql.sql_processor.sql_processor.extract_funcs_from_pyfile(funcs_py_file)