How to use diff_queries method in dbt-osmosis

Best Python code snippet using dbt-osmosis_python

diff.py

Source:diff.py

...5from dbt.adapters.base.relation import BaseRelation6from git import Repo7from dbt_osmosis.core.log_controller import logger8from dbt_osmosis.core.osmosis import DbtProject9def build_diff_queries(model: str, runner: DbtProject) -> Tuple[str, str]:10    """Leverage git to build two temporary tables for diffing the results of a query throughout a change"""11    # Resolve git node12    node = runner.get_ref_node(model)13    dbt_path = Path(node.root_path)14    repo = Repo(dbt_path, search_parent_directories=True)15    t = next(Path(repo.working_dir).rglob(node.original_file_path)).relative_to(repo.working_dir)16    sha = repo.head.object.hexsha17    target = repo.head.object.tree[str(t)]18    # Create original node19    git_node_name = "z_" + sha[-7:]20    original_node = runner.get_server_node(target.data_stream.read().decode("utf-8"), git_node_name)21    # Alias changed node22    changed_node = node23    # Compile models24    original_node = runner.compile_node(original_node)25    changed_node = runner.compile_node(changed_node)26    return original_node.compiled_sql, changed_node.compiled_sql27def build_diff_tables(model: str, runner: DbtProject) -> Tuple[BaseRelation, BaseRelation]:28    """Leverage git to build two temporary tables for diffing the results of a query throughout a change"""29    # Resolve git node30    node = runner.get_ref_node(model)31    dbt_path = Path(node.root_path)32    repo = Repo(dbt_path, search_parent_directories=True)33    t = next(Path(repo.working_dir).rglob(node.original_file_path)).relative_to(repo.working_dir)34    sha = repo.head.object.hexsha35    target = repo.head.object.tree[str(t)]36    # Create original node37    git_node_name = "z_" + sha[-7:]38    original_node = runner.get_server_node(target.data_stream.read().decode("utf-8"), git_node_name)39    # Alias changed node40    changed_node = node41    # Compile models42    original_node = runner.compile_node(original_node).node43    changed_node = runner.compile_node(changed_node).node44    # Lookup and resolve original ref based on git sha45    git_node_parts = original_node.database, "dbt_diff", git_node_name46    ref_A, did_exist = runner.get_or_create_relation(*git_node_parts)47    if not did_exist:48        logger().info("Creating new relation for %s", ref_A)49        with runner.adapter.connection_named("dbt-osmosis"):50            runner.execute_macro(51                "create_schema",52                kwargs={"relation": ref_A},53            )54            runner.execute_macro(55                "create_table_as",56                kwargs={57                    "sql": original_node.compiled_sql,58                    "relation": ref_A,59                    "temporary": True,60                },61                run_compiled_sql=True,62            )63    # Resolve modified fake ref based on hash of it compiled SQL64    temp_node_name = "z_" + hashlib.md5(changed_node.compiled_sql.encode("utf-8")).hexdigest()[-7:]65    git_node_parts = original_node.database, "dbt_diff", temp_node_name66    ref_B, did_exist = runner.get_or_create_relation(*git_node_parts)67    if not did_exist:68        ref_B = runner.adapter.Relation.create(*git_node_parts)69        logger().info("Creating new relation for %s", ref_B)70        with runner.adapter.connection_named("dbt-osmosis"):71            runner.execute_macro(72                "create_schema",73                kwargs={"relation": ref_B},74            )75            runner.execute_macro(76                "create_table_as",77                kwargs={78                    "sql": original_node.compiled_sql,79                    "relation": ref_B,80                    "temporary": True,81                },82                run_compiled_sql=True,83            )84    return ref_A, ref_B85def diff_tables(86    ref_A: BaseRelation,87    ref_B: BaseRelation,88    pk: str,89    runner: DbtProject,90    aggregate: bool = True,91) -> agate.Table:92    logger().info("Running diff")93    _, table = runner.adapter_execute(94        runner.execute_macro(95            "_dbt_osmosis_compare_relations_agg" if aggregate else "_dbt_osmosis_compare_relations",96            kwargs={97                "a_relation": ref_A,98                "b_relation": ref_B,99                "primary_key": pk,100            },101        ),102        auto_begin=True,103        fetch=True,104    )105    return table106def diff_queries(107    sql_A: str, sql_B: str, pk: str, runner: DbtProject, aggregate: bool = True108) -> agate.Table:109    logger().info("Running diff")110    _, table = runner.adapter_execute(111        runner.execute_macro(112            "_dbt_osmosis_compare_queries_agg" if aggregate else "_dbt_osmosis_compare_queries",113            kwargs={114                "a_query": sql_A,115                "b_query": sql_B,116                "primary_key": pk,117            },118        ),119        auto_begin=True,120        fetch=True,121    )122    return table123def diff_and_print_to_console(124    model: str,125    pk: str,126    runner: DbtProject,127    make_temp_tables: bool = False,128    agg: bool = True,129    output: str = "table",130) -> None:131    """132    Compare two tables and print the results to the console133    """134    if make_temp_tables:135        table = diff_tables(*build_diff_tables(model, runner), pk, runner, agg)136    else:137        table = diff_queries(*build_diff_queries(model, runner), pk, runner, agg)138    print("")139    output = output.lower()140    if output == "table":141        table.print_table()142    elif output in ("chart", "bar"):143        if not agg:144            logger().warn(145                "Cannot render output format chart with --no-agg option, defaulting to table"146            )147            table.print_table()148        else:149            _table = table.compute(150                [151                    (...

search.py

Source:search.py

1import path  # noqa: F4012from sys import argv3from elasticsearch import Elasticsearch4from math import log5import pandas as pd6# Best baseline thusfar7# from .rerank_simple_slop_search import \8#    rerank_slop_search_remaining_lines_max_snippet_at_59from vmware.search.compound_search import with_best_compounds_at_5_only_phrase_search10def damage(results1, results2, at=10):11    """ How "damaging" could the change from results1 -> results2 be?12        (results1, results2 are an array of document identifiers)13        For each result in result1,14            Is the result in result2[:at]15                If so, how far has it moved?16            If not,17                Consider it a move of at+118            damage += discount(idx) * moveDist19                """20    def discount(idx):21        return 1.0 / log(idx + 2)22    idx = 023    dmg = 0.024    if len(results1) < at:25        at = len(results1)26    for result in results1[:at]:27        movedToIdx = at + 1  # out of the window28        if result in results2:29            movedToIdx = results2.index(result)30        moveDist = abs(movedToIdx - idx)31        dmg += discount(idx) * moveDist32        idx += 133    return dmg34def search(query,35           strategy=with_best_compounds_at_5_only_phrase_search):36    print(query)37    es = Elasticsearch('http://localhost:9200', timeout=30, max_retries=10,38                       retry_on_status=True, retry_on_timeout=True)39    hits = strategy(es, query=query)40    for hit in hits:41        print("**********************************")42        print(hit['_source']['title'] if 'title' in hit['_source'] else '',43              '||',44              hit['_source']['first_line'])45        print(f"MAX SIM {hit['_source']['max_sim']} | SCORE {hit['_score']}")46        print("----------------------------------")47def submission(baseline=with_best_compounds_at_5_only_phrase_search,48               test=with_best_compounds_at_5_only_phrase_search,49               verbose=False):50    """Search all test queries to generate a submission."""51    queries = pd.read_csv('data/test.csv')52    all_results = []53    es = Elasticsearch('http://localhost:9200', timeout=30, max_retries=10,54                       retry_on_status=True, retry_on_timeout=True)55    for query in queries.to_dict(orient='records'):56        results_control = baseline(es, query=query['Query'])57        results_test = test(es, query=query['Query'])58        delta_damage = damage([r['_id'] for r in results_control],59                              [r['_id'] for r in results_test])60        if verbose:61            print(query['Query'])62            print(f"DAMAGE: {delta_damage}")63            print("----------------------------------")64        for rank, (result_control, result_test) in enumerate(zip(results_control, results_test)):65            source_control = result_control['_source']66            source_test = result_test['_source']67            source_test['rank'] = rank68            source_test['score_test'] = result_test['_score']69            source_test['score_control'] = result_control['_score']70            source_test['damage'] = delta_damage71            source_test['DocumentId_test'] = source_test['id']72            source_test['DocumentId'] = source_test['id']73            source_test['DocumentId_control'] = source_control['id']74            source_test['splainer_test'] = source_test['splainer']75            source_test['splainer_control'] = source_control['splainer']76            source_test['first_line_control'] = source_control['first_line']77            source_test['first_line_test'] = source_test['first_line']78            source_test['raw_text_control'] = source_control['raw_text']79            source_test['QueryId'] = query['QueryId']80            all_results.append(source_test)81    all_results = pd.DataFrame(all_results)82    all_results = queries.merge(all_results, how='left', on='QueryId')\83        .sort_values(['QueryId', 'rank'])84    write_submission(all_results, test.__name__)85    return all_results86def write_submission(all_results, name):87    from time import time88    timestamp = str(time()).replace('.', '')89    fname = f'data/{name}_turnbull_{timestamp}.csv'90    print("Writing To: ", fname)91    all_results[['QueryId', 'DocumentId']].to_csv(fname, index=False)92def diff_results(all_results):93    diff_queries = all_results[all_results['damage'] > 0][['Query', 'damage',94                                                           'first_line_control', 'first_line',95                                                           'splainer_test', 'splainer_control']]96    last_query = None97    for result in diff_queries.sort_values(['damage', 'Query']).to_dict(orient='records'):98        if result['Query'] != last_query:99            last_query = result['Query']100            print(f"-------- {result['Query']} -- {result['damage']}-------")101        print(result['first_line_control'], "|||", result['first_line'])102    print("----------------------------------")103    print(f"Changed Queries - {len(diff_queries['Query'].unique())} different queries")104if __name__ == "__main__":...

time_render_middle.py

Source:time_render_middle.py

1from django.db import connection2from time import time3#------------------------------------------------------------------------------4#------------------------------------------------------------------------------5class TimeRenderMiddleware(object):6#------------------------------------------------------------------------------7    def process_request(self, request):8        # edit this stuff9        request.session['time_start'] = time()10        request.session['num_queries_old'] = len(connection.queries)11#------------------------------------------------------------------------------12    def process_response(self, request, response):13        14        queries = connection.queries15        query_time = 016        query_count = 017        18        for query in queries:19            query_time += float(query['time'])20            query_count += 121        end_time = time()22        num_queries_new =  len(connection.queries)23        24        content = response.content25        index = content.upper().find('</BODY>')26        if index == -1:27            return response28        29        before = content[:index]30        after = content[index:]31        try:32            diff_time = end_time - request.session['time_start']33            diff_queries = num_queries_new - request.session['num_queries_old']34        except KeyError:35            diff_time = 036            diff_queries = 037        38        message = """<div id="query_time"></br>\n<small><center>Time to render page is: %.4s</br>\n39        Time to sql Query: %s</br>40        Count of queries: %s</center></small></div>""" % (diff_time, query_time, diff_queries)41        content = before + message + after42        response.content =  content43        return response...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.