#!/usr/bin/env qore
# -*- mode: qore; indent-tabs-mode: nil -*-

# @file sqlutil example program for the SqlUtil module

/*  Copyright 2013 - 2023 Qore Technologies, s.r.o.

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/

%new-style
%enable-all-warnings
%require-types
%strict-args

%requires SqlUtil
%requires BulkSqlUtil
%requires Util
%try-module yaml
%define NoYaml
%endtry

%requires DatasourceProvider

const Opts = {
    "align_all":         "A,align-all=s",
    "align_data":        "D,align-data=s",
    "align_data_strict": "s,align-data-strict",
    "align_table":       "a,align-table=s",
    "dump_table":        "t,dump-table=s@",
    "export_table":      "export=s",
    "import_table":      "import=s",
    "dump_sequence":     "S,dump-sequence=s@",
    "dump_object":       "dump=s@",
    "omit":              "O,omit=s@",
    "list_tables":       "L,list-tables",
    "list_objects":      "list=s@",
    "select":            "select=s",

    "data_tablespace":   "d,data-tablespace=s",
    "index_tablespace":  "i,index-tablespace=s",
    "target_ds":         "T,target-ds=s",
    "verbose":           "v,verbose:i+",
    "help":              "h,help",
};

# list of options that require a target datasource
const RequiresTarget = {
    "align_table": True,
};

# map of the keyword : Database feature name
const MAP_OBJECT_FEATURE = {
    "tables"        : SqlUtil::DB_TABLES,
    "views"         : SqlUtil::DB_VIEWS,
    "mviews"        : SqlUtil::DB_MVIEWS,
    "sequences"     : SqlUtil::DB_SEQUENCES,
    "functions"     : SqlUtil::DB_FUNCTIONS,
    "procedures"    : SqlUtil::DB_PROCEDURES,
    "packages"      : SqlUtil::DB_PACKAGES,
    "types"         : SqlUtil::DB_TYPES,
    "synonyms"      : SqlUtil::DB_SYNONYMS,
};

# map of the keyword : Database iterator method name
const LIST_OBJECTS = {
    "tables"        : "tableIterator",
    "views"         : "viewIterator",
    "mviews"        : "materializedViewIterator",
    "sequences"     : "sequenceIterator",
    "functions"     : "functionIterator",
    "procedures"    : "procedureIterator",
    "packages"      : "packageIterator",
    "types"         : "typeIterator",
    "synonyms"      : "synonymIterator",
};

# 'meta' keys for list objects listings
const META_LIST_OBJECTS = {
    "all"           : LIST_OBJECTS.keys(),
    "selectable"    : ( "tables", "views", "mviews", ),
    "code"          : ( "packages", "functions", "procedures", "types", ),
};

const DUMP_OBJECTS = {
    "tables"        : ( "getobj" : "getTable", "sql" : "getCreateSqlString"),
    "views"         : ( "getobj" : "getView", "sql" : "getCreateSql"),
    "mviews"        : ( "getobj" : "getMaterializedView", "sql" : "getCreateSql"),
    "sequences"     : ( "getobj" : "getSequence", "sql" : "getCreateSql"),
    "functions"     : ( "getobj" : "getFunction", "sql" : "getCreateSql"),
    "procedures"    : ( "getobj" : "getProcedure", "sql" : "getCreateSql"),
    "packages"      : ( "getobj" : "getPackage", "sql" : "getCreateSql"),
    "types"         : ( "getobj" : "getType", "sql" : "getCreateSql"),
    "synonyms"      : ( "getobj" : "getSynonym", "sql" : "getCreateSql"),
};

main();

sub usage() {
    printf("usage: %s [options] <datasource>
<datasource> format: driver:user/pass@db[%host[:port]]
  -A,--align-all=ARG          ARG=table to align structure and data in a
                              target database; requires -T
  -D,--align-data=ARG         ARG=table to align data in a target database;
                              requires -T
  -s,--align-data-strict      deletes extra rows in the target table;
                              requires -D
  -a,--align-table=ARG        ARG=table to align structure in a target
                              database; requires -T
  -L,--list-tables            list tables in source
     --list=ARG               list objects of given type. Objects available:
                               %y
                              or some special categories:
                               %y
  -O,--omit=ARG               omit options: \"triggers\", \"foreign_constraints\",
                              \"indexes\"
  -d,--data-tablespace=ARG    use the given data tablespace name for output
  -i,--index-tablespace=ARG   use the given index tablespace name for output
  -t,--dump-table=ARG         dump the DDL for the given table
  -S,--dump-sequence=ARG      dump the DDL for the given sequence
     --dump=ARG1:ARG2         dump the DDL for the given object type ARG1:
                               %y
                              or 'all' for all objects of this type.
                              With name ARG2.
     --select=ARG             use ARG as the select hash for exporting data
                              to be parsed with Util::parse_to_qore_value()
  -T,--target-ds=ARG          target datasource
  -v,--verbose=ARG            increase verbosity level (more v's = more info)
     --export=ARG             export table data to standard output in YAML format
     --import=ARG:ARG         import YAML data into table (ARG 1) from file
                              name (ARG 2). Batch commit per 5000 lines.
", get_script_name(), LIST_OBJECTS.keys(), META_LIST_OBJECTS.keys(), DUMP_OBJECTS.keys());
    exit(1);
}
#' just fix syntax highlighting

sub error(string fmt) {
    vprintf(fmt, argv);
    exit(1);
}

class GlobTables inherits Tables {
    constructor(Database db, AbstractDatasource source, string tspec) {
        # add matching tables to table cache
        foreach string t in (db.tableIterator()) {
            if (t.regex(tspec, RE_Caseless))
                get(source, t);
        }

        if (empty())
            printf("%y: no matching tables\n", tspec);
    }
}

string sub get_regex(string str) {
    str =~ s/\./\\./g;
    str =~ s/\?/./g;
    str =~ s/\*/.*/g; #//;
    str = sprintf("^%s\$", str);
    #printf("str: %s\n", str);
    return str;
}

bool sub is_object_supported (SqlUtil::Database db, string type) {
    return inlist (MAP_OBJECT_FEATURE{type}, db.features());
}

sub list_objects(SqlUtil::Database db, softlist types) {
    list processed = list();
    foreach string i in (types) {
        # do not repeat output for already processed key
        if (inlist(i, processed))
            continue;
        push processed, i;
        if (exists LIST_OBJECTS{i}) {
            if (is_object_supported (db, i))
                map printf("%-30s %s\n", $1, i), call_object_method(db, LIST_OBJECTS{i});
            else
                printf("Unsupported object type/key: %s\n", i);
        } else if (exists META_LIST_OBJECTS{i}) {
            foreach string j in (META_LIST_OBJECTS{i})
                if (is_object_supported (db, j))
                    map printf("%-30s %s\n", $1, j), call_object_method(db, LIST_OBJECTS{j});
        } else {
            printf("Unknown object type/key: %s\n", i);
        }
    }
}

class DumpObjects {
    private {
        hash m_copt;
        list m_processed;
        SqlUtil::Database m_db;
    }

    constructor(SqlUtil::Database db, softlist values, hash copt) {
        m_db = db;
        m_copt = copt;
        doObjects(values);
    }

    private bool processed(string val) {
        if (inlist(val, m_processed))
            return True;
        push m_processed, val;
        return False;
    }

    private doObjects(softlist values) {
        foreach string val in (values) {
            list l = val.split(":");
            string type = l[0];
            string name = l[1];

            if (type == "all") {
                foreach string i in (DUMP_OBJECTS.keys())
                    if (is_object_supported (m_db, i))
                        doObjects(sprintf("%s:all", i));
                continue;
            } else if (!DUMP_OBJECTS{type}) {
                printf("Unable to dump object of type: %n\n", type);
                continue;
            } else if (!is_object_supported (m_db, type)) {
                printf("Unable to dump object of type: %n\n", type);
                continue;
            }

            if (name.lwr() == "all") {
                ListIterator it = call_object_method(m_db, LIST_OBJECTS{type});
                while (it.next()) {
                    doObject(type, it.getValue());
                }
            } else {
                doObject(type, name);
            }
        }
    }

    private doObject(string type, string name) {
        if (processed(name))
            return;

        *object o = call_object_method(m_db, DUMP_OBJECTS{type}.getobj, name);
        if (!o) {
            throw sprintf("%s-ERROR", type.upr()), sprintf("%s %s does not exist or is not accessible to this user",
                type, name);
        }
        string singular = type;
        singular =~ s/s$//;
        printf("-- DDL of %s %s\n", singular, name);
        any ret = call_object_method(o, DUMP_OBJECTS{type}.sql, m_copt);
        string str;
        # unfortunately it can be list or string
        if (ret.typeCode() == NT_LIST) {
            foreach string i in (ret)
                str += i;
        } else
            str = ret;
        if (str.substr(-1) != ";")
            str += ";";
        printf("%s\n", str);
    }

 } # class DumpObjects


sub main() {
    GetOpt g(Opts);
    hash opt = g.parse3(\ARGV);

    if (opt.help || !ARGV[0])
        usage();

    if (opt.align_all) {
        opt.align_data = opt.align_all;
        opt.align_table = opt.align_all;
    }

    Datasource target;
    if (opt.target_ds)
        target = new Datasource(opt.target_ds);

    # if we don't have a target datasource, check for an option that requires one
    if (!target) {
        foreach string os in (RequiresTarget.keyIterator()) {
            if (opt{os})
                error("option --%s requires a target datasource (-T)\n", os);
        }
    }

    # source datasource
    Datasource source(get_ds_string(ARGV[0]));

    # info callback closure and embedded variables
    int change_count = 0;
    int dot_count = 0;
    code info_callback = sub (string str, int ac, string type, string name, *string table, *string new_name,
            *string info) {
        # verbosity threshold
        int t = 0;
        if (ac != AbstractDatabase::AC_Unchanged)
            ++change_count;
        else
            t = 1;
        if (opt.verbose > t) {
            if (dot_count) {
                print("\n");
                dot_count = 0;
            }
            printf("%s %s\n", t ? "+++" : "***", str);
        } else {
            ++dot_count;
            print(AbstractDatabase::ActionLetterMap{ac});
            flush();
        }
    };

    code upsert_callback = sub (string table_name, hash row, int result) {
        # verbosity threshold
        int t = 0;
        if (result != AbstractTable::UR_Unchanged)
            ++change_count;
        else
            t = 1;

        if (opt.verbose > t) {
            if (dot_count) {
                print("\n");
                dot_count = 0;
            }
            printf("%s reference data %s: %y: %s\n", t ? "+++" : "***", table_name, row,
                AbstractTable::UpsertResultMap{result});
        } else {
            ++dot_count;
            print(AbstractTable::UpsertResultLetterMap{result});
            flush();
        }
    };

    # sql callback closure
    code sql_callback = sub (string str) {
        if (opt.verbose > 1)
            printf("%s\n", str);
        target.execRaw(str);
    };

    # callback reset closure
    code reset = sub () {
        if (dot_count) {
            print("\n");
            dot_count = 0;
        }
        if (change_count) {
            printf("%d change%s made\n", change_count, change_count == 1 ? "" : "s");
            change_count = 0;
        } else
            print("no changes made\n");
    };

    # creation options
    hash copt = opt.("index_tablespace", "data_tablespace");
    if (opt.omit)
        copt.omit = opt.omit;

    # callback options
    hash cbopt = ("sql_callback": sql_callback, "info_callback": info_callback, "sql_callback_executed": True);

    # alignment options
    hash aopt = copt + cbopt;

    Database db(source);

    bool done = False;

    # backward compatibility
    if (opt.list_tables) {
        opt.list_objects = "tables";
    }
    # real list of objects
    if (opt.list_objects) {
        list_objects(db, opt.list_objects);
        done = True;
    }

    # backward compatibility
    foreach string name in (opt.dump_table) {
        if (opt.dump_object.typeCode() != NT_LIST)
            opt.dump_object = list();
        push opt.dump_object, sprintf("tables:%s", name);
    }
    # backward compatibility
    foreach string name in (opt.dump_sequence) {
        if (opt.dump_object.typeCode() != NT_LIST)
            opt.dump_object = list();
        push opt.dump_object, sprintf("sequences:%s", name);
    }
    # real dump of objects
    if (opt.dump_object) {
        done = True;
        new DumpObjects(db, opt.dump_object, copt);
    }

    if (opt.export_table) {
%ifdef NoYaml
        throw "YAML-ERROR", "the binary yaml module is not available";
%else
        hash sh;
        # issue #2509 allow exported rows to be filtered
        if (opt.select) {
            auto v = parse_to_qore_value(opt.select);
            if (exists v && v.typeCode() != NT_HASH)
                throw "SELECT-ERROR", sprintf("the --select argument must be assigned to a value that parses to a "
                    "hash; %y => type %y", opt.select, v.type());
            if (v)
                sh = v;
        }

        on_success source.commit();
        on_error source.rollback();

        done = True;
        Table table(source, opt.export_table);
        AbstractSQLStatement sql = table.getStatement(sh);
        while (sql.next()) {
            hash row = sql.fetchRow();
            printf("%s", make_yaml(row));
        }
%endif
    }

    if (opt.import_table) {
%ifdef NoYaml
        throw "YAML-ERROR", "the binary yaml module is not available";
%else
        on_success source.commit();
        on_error source.rollback();

        list arg = opt.import_table.split(":");
        if (!exists arg[0])
            throw "TABLE-IMPORT-ERROR", "No argument with table name";
        if (!exists arg[1])
            throw "TABLE-IMPORT-ERROR", "No argument with file name";

        done = True;
        Table table(source, arg[0]);

        hash bulk_opts = {
                "block_size" : 5000,
        };

        BulkSqlUtil::BulkInsertOperation bulk(table, bulk_opts);
        on_success {
            if (opt.verbose) {
                printf("%n: import %s: cache: %d; count: %d\n",
                    now(), arg[0], bulk.size(), bulk.getRowCount()
                );
            }
            bulk.flush();
        }

        on_error bulk.discard();

        FileLineIterator it(arg[1]);
        while (it.next()) {
            hash row = parse_yaml(it.getValue());
            bulk.queueData(row);

            if (it.index() % bulk_opts.block_size == 0) {
                if (opt.verbose) {
                    printf("%n: import %s: cache: %d; count: %d\n",
                        now(), arg[0], bulk.size(), bulk.getRowCount()
                    );
                }
                bulk.commit();
            }
        }
%endif
    }

    code align_table = sub (AbstractTable source_table) {
        done = True;
        Table target_table(target, source_table.getName());
        on_error target_table.rollback();
        on_success target_table.commit();
        target_table.getAlignSql(source_table, aopt);
    };

    code align_data = sub (AbstractTable source_table) {
        done = True;
        Table target_table(target, source_table.getName());
        on_error {
            target_table.rollback();
            source_table.rollback();
        }
        on_success {
            target_table.commit();
            source_table.commit();
        }
        int us = opt.verbose ? AbstractTable::UpsertSelectFirst : AbstractTable::UpsertAuto;
        target_table.upsertFromSelect(source_table, NOTHING, us, {
            "delete_others": opt.align_data_strict,
            "info_callback": upsert_callback,
        });
    };

    if (opt.align_table) {
        on_success target.commit();
        on_error target.rollback();

        if (opt.align_table =~ /[\*\?]/) {
            # transform into regex
            string tspec = get_regex(opt.align_table);
            GlobTables tables(db, source, tspec);

            foreach AbstractTable t in (tables.iterator()) {
                if (dot_count)
                    print("\n");
                printf("%s table structure: ", t.getName());
                align_table(t);
            }
        } else {
            Table table(source, opt.align_table);
            align_table(table.getTable());
        }
        reset();
    }

    if (opt.align_data) {
        if (opt.align_data =~ /[\*\?]/) {
            # transform into regex
            string tspec = get_regex(opt.align_data);
            GlobTables tables(db, source, tspec);

            foreach AbstractTable t in (tables.iterator()) {
                if (dot_count)
                    print("\n");
                printf("%s table data: ", t.getName());
                align_data(t);
            }
        } else {
            Table table(source, opt.align_data);
            align_data(table.getTable());
        }
        reset();
    }

    if (!done) {
        printf("Server:\n%n\n", source.getServerVersion());
        printf("Client:\n%n\n", source.getClientVersion());
        printf("\nUse -h to display options\n\n");
    }
}
