#!/usr/bin/env python
#
# ddr-search
#
#

description = """Tools for managing DDR metadata in ElasticSearch."""

subparser_title = """Subcommands"""
subparser_description = """Additional help is available for each of the following subcommands.
Example:
    $ ddr-index status --help
"""
subparser_help = ''

epilog = """
Example of setting up a new index:
    
    # Initialize index (creates index, adds mappings and facets)
    $ ddr-index init -H localhost:9200 -i documents /var/www/media/ddr/ddr
    
    # Set or remove an alias for the index
    $ ddr-index alias -H localhost:9200 -i documents -a ddrpublic-dev
    $ ddr-index alias -H localhost:9200 -i documents -a ddrpublic-dev --remove

    # Add/update base repository info:
    $ ddr-index repo -H localhost:9200 -i documents /var/www/media/ddr/REPO/repository.json

    # Add/update organization info:
    $ ddr-index org -H localhost:9200 -i documents /var/www/media/ddr/REPO-ORG/organization.json

    # Add/update narrators info:
    $ ddr-index narrators -H localhost:9200 -i documents /etc/ddr/ddr-defs/vocab/narrators.json
    
    # Choose signatures for non-node objects
    ddr-signatures -u USERNAME -m USERMAIL /var/www/media/ddr/ddr-testing-123
    
    # Index a collection for ddr-local
    $ ddr-index index -H localhost:9200 -i documents --recursive /var/www/media/ddr/ddr-testing-123
    
    # Index a whole directory of collections for ddr-local
    $ ddr-index index -H localhost:9200 -i documents --recursive /var/www/media/ddr
    
    # Index a whole directory of collections for ddr-public
    $ ddr-index index -H localhost:9200 -i documents --recursive --newstyle /var/www/media/ddr

Maintenance tasks:
    
    # Check status
    $ ddr-index status -H localhost:9200 -i documents
    
    # Create index (does not add mappings, facets)
    $ ddr-index create -H localhost:9200 -i documents
    
    # Add/update mappings to existing index.
    $ ddr-index mappings -H localhost:9200 -i documents /var/www/media/ddr/ddr
    
    # Add/update facets to existing index.
    $ ddr-index facets -H localhost:9200 -i documents /var/www/media/ddr/ddr

    # Remove organization metadata:
    $ ddr-index org -H localhost:9200 -i documents --remove /var/www/media/ddr/REPO-ORG/organization.json
    
    # Delete existing index
    $ ddr-index remove -H localhost:9200 -i documents
    
    # Reindex
    $ ddr-index reindex -H localhost:9200 source dest
    
    # See if document exists
    $ ddr-index exists -H localhost:9200 -i documents collection ddr-testing-123
    
    # Search
    $ ddr-index search -H localhost:9200 -i documents collection,entity Minidoka

    # Get document
    $ ddr-index get -H localhost:9200 -i documents collection ddr-testing-123

Note: You can set environment variables got HOSTS and INDEX.:

    $ export DDR_ES_HOSTS=localhost:9200
    $ export DDR_ES_INDEX=dev
"""


import argparse
from datetime import datetime
import json
import logging
import os
import sys

import envoy
import git

from DDR.config import DEBUG, LOG_FILE, LOG_LEVEL
from DDR import format_json
from DDR import docstore
from DDR import identifier
from DDR import models

LOGGING_FORMAT = '%(asctime)s %(levelname)s %(message)s'
LOGGING_DATEFMT = '%Y-%m-%d %H:%M:%S'
LOGGING_FILE = LOG_FILE
if LOG_LEVEL == 'debug':
    LOGGING_LEVEL = logging.DEBUG
else:
    LOGGING_LEVEL = logging.ERROR
#logging.basicConfig(format=LOGGING_FORMAT, datefmt=LOGGING_DATEFMT, level=LOGGING_LEVEL, filename=LOGGING_FILE)
logging.basicConfig(format=LOGGING_FORMAT, datefmt=LOGGING_DATEFMT, level=logging.DEBUG, filename=LOGGING_FILE)

def split_docstring(func):
    """Split function docstring into description and epilog, removing params.
    """
    description = ''
    lines = [l.rstrip().replace('    ','',1) for l in func.__doc__.split('\n')]
    if lines:
        description = lines.pop(0)
    e = [l for l in lines if l.strip() and ('@param' not in l) and ('@return' not in l)]
    epilog = '\n'.join(e)
    return description,epilog

def make_hosts( text ):
    hosts = []
    for host in text.split(','):
        h,p = host.split(':')
        hosts.append( {'host':h, 'port':p} )
    return hosts
    
def host_status(d, mappings=False):
    """Gets host and index status and prints.
    
    @param d: Docstore object
    @param mappings: bool
    """
    status = d.status()
    aliases = d.aliases()

    # list indices
    print('Indices on this cluster:')
    indices = status['indices'].keys()
    indices.sort()
    print(format_json(indices))

    # aliases
    print('Aliases')
    als = [' -> '.join([alias[1], alias[0]]) for alias in aliases]
    print(format_json(als))
    
    # selected info on selected index
    if d.indexname and (d.indexname not in indices):
        print('Selected index not present in cluster: %s' % d.indexname)
        sys.exit(1)
    print('Selected index: %s' % d.indexname)
    this_index = {
        key: status['indices'][d.indexname]['total'][key]
        for key in [
            'docs', 'store'
        ]
    }
    print(format_json(this_index))

    # mappings
    if mappings:
        print('Mappings')
        print(format_json(d.get_mappings()))

def search_results(d, doctype, text, must=None, should=None, mustnot=None, raw=False):
    if doctype in ['*', 'all', 'all_', '_all']:
        doctypes = []
    else:
        doctypes = doctype.strip().split(',')
    
    def make_terms(arg):
        # "language:eng,jpn!creators.role:author"
        terms = []
        if arg:
            for term in arg.strip().split('!'):
                fieldname,value = term.strip().split(':')
                values = value.strip().split(',')
                terms.append(
                    {'terms': {fieldname: values}}
                )
        return terms
    
    q = docstore.search_query(
        text=text,
        must=make_terms(must),
        should=make_terms(should),
        mustnot=make_terms(mustnot),
    )
    if raw:
        print(format_json(q, sort_keys=False))
    
    data = d.search(
        doctypes=doctypes,
        query=q,
        fields=['id','title'],
    )
    if raw:
        print(format_json(data, sort_keys=False))
    else:
        try:
            for item in data['hits']['hits']:
                chunks = (
                    item['_id'],
                    item['_type'],
                    item['_source'].get('title', ''),
                )
                print(chunks)
        except:
            print(format_json(data, sort_keys=False))


def main():
    
    # look for HOSTS and INDEX in environment
    if 'DDR_ES_HOSTS' in os.environ:
        hostsarg = { 'default': os.environ['DDR_ES_HOSTS'] }
    else:
        hostsarg = { 'required': True }
    if 'DDR_ES_INDEX' in os.environ:
        indexarg = { 'default': os.environ['DDR_ES_INDEX'] }
    else:
        indexarg = { 'required': True }

    formatter = argparse.RawDescriptionHelpFormatter
    parser = argparse.ArgumentParser(description=description, epilog=epilog,
                                     formatter_class=formatter,)
    
    subparsers = parser.add_subparsers(
        dest='cmd',
        title=subparser_title,
        description=subparser_description,
        help=subparser_help)
    
    #help
    post_descr,post_epilog = split_docstring(docstore.Docstore.post)
    search_descr,search_epilog = split_docstring(docstore.Docstore.search)
    get_descr,get_epilog = split_docstring(docstore.Docstore.get)
    init_descr,init_epilog = split_docstring(docstore.Docstore.index)
    create_descr,create_epilog = split_docstring(docstore.Docstore.index)
    remove_descr,remove_epilog = split_docstring(docstore.Docstore.index)
    index_descr,index_epilog = split_docstring(docstore.Docstore.index)
    reindex_descr,reindex_epilog = split_docstring(docstore.Docstore.reindex)
    alias_descr,alias_epilog = split_docstring(docstore.Docstore.set_alias)
    mappings_descr,mappings_epilog = split_docstring(docstore.Docstore.put_mappings)
    facets_descr,facets_epilog = split_docstring(docstore.Docstore.put_facets)
    exists_descr,exists_epilog = split_docstring(docstore.Docstore.exists)
    status_descr,status_epilog = split_docstring(docstore.Docstore.status)
    repo_descr,repo_epilog = split_docstring(docstore.Docstore.repo)
    org_descr,org_epilog = split_docstring(docstore.Docstore.org)
    narr_descr,narr_epilog = split_docstring(docstore.Docstore.narrators)
    delete_descr,delete_epilog = split_docstring(docstore.Docstore.delete)
    
    help_parser = subparsers.add_parser('help', description=description, epilog=epilog, formatter_class=formatter,)
    post_parser = subparsers.add_parser('post', description=post_descr, epilog=post_epilog, formatter_class=formatter,)
    search_parser = subparsers.add_parser('search', description=search_descr, epilog=search_epilog, formatter_class=formatter,)
    get_parser = subparsers.add_parser('get', description=get_descr, epilog=get_epilog, formatter_class=formatter,)
    init_parser = subparsers.add_parser('init', description=init_descr, epilog=init_epilog, formatter_class=formatter,)
    create_parser = subparsers.add_parser('create', description=create_descr, epilog=create_epilog, formatter_class=formatter,)
    remove_parser = subparsers.add_parser('remove', description=remove_descr, epilog=remove_epilog, formatter_class=formatter,)
    index_parser = subparsers.add_parser('index', description=index_descr, epilog=index_epilog, formatter_class=formatter,)
    reindex_parser = subparsers.add_parser('reindex', description=reindex_descr, epilog=reindex_epilog, formatter_class=formatter,)
    alias_parser = subparsers.add_parser('alias', description=alias_descr, epilog=alias_epilog, formatter_class=formatter,)
    mappings_parser = subparsers.add_parser('mappings', description=mappings_descr, epilog=mappings_epilog, formatter_class=formatter,)
    facets_parser = subparsers.add_parser('facets', description=facets_descr, epilog=facets_epilog, formatter_class=formatter,)
    exists_parser = subparsers.add_parser('exists', description=exists_descr, epilog=exists_epilog, formatter_class=formatter,)
    status_parser = subparsers.add_parser('status', description=status_descr, epilog=status_epilog, formatter_class=formatter,)
    repo_parser = subparsers.add_parser('repo', description=repo_descr, epilog=repo_epilog, formatter_class=formatter,)
    org_parser = subparsers.add_parser('org', description=org_descr, epilog=org_epilog, formatter_class=formatter,)
    narr_parser = subparsers.add_parser('narrators', description=narr_descr, epilog=narr_epilog, formatter_class=formatter,)
    delete_parser = subparsers.add_parser('delete', description=delete_descr, epilog=delete_epilog, formatter_class=formatter,)
    
    post_parser.set_defaults(func=docstore.Docstore.post)
    search_parser.set_defaults(func=docstore.Docstore.search)
    get_parser.set_defaults(func=docstore.Docstore.get)
    init_parser.set_defaults(func=docstore.Docstore.init_index)
    create_parser.set_defaults(func=docstore.Docstore.create_index)
    remove_parser.set_defaults(func=docstore.Docstore.delete_index)
    index_parser.set_defaults(func=docstore.Docstore.index)
    reindex_parser.set_defaults(func=docstore.Docstore.reindex)
    alias_parser.set_defaults(func=docstore.Docstore.set_alias)
    mappings_parser.set_defaults(func=docstore.Docstore.put_mappings)
    facets_parser.set_defaults(func=docstore.Docstore.put_facets)
    exists_parser.set_defaults(func=docstore.Docstore.exists)
    status_parser.set_defaults(func=docstore.Docstore.status)
    repo_parser.set_defaults(func=docstore.Docstore.repo)
    org_parser.set_defaults(func=docstore.Docstore.org)
    narr_parser.set_defaults(func=docstore.Docstore.narrators)
    delete_parser.set_defaults(func=docstore.Docstore.delete)
    
    help_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    #help_parser.add_argument('-l', '--log', help='Log file..')
    
    post_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    post_parser.add_argument('-l', '--log', help='Log file..')
    post_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    post_parser.add_argument('-i', '--index', help='index.', **indexarg)
    post_parser.add_argument('path', help='Absolute path to JSON file.')
    
    search_parser.add_argument('-D', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    search_parser.add_argument('-l', '--log', help='Log file..')
    search_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    search_parser.add_argument('-i', '--index', help='index.', **indexarg)
    search_parser.add_argument('-d', '--doctype', help='One or more doctypes (comma-separated).')
    search_parser.add_argument('-t', '--text', help='Query string, in double or single quotes.')
    search_parser.add_argument('-a', '--AND', help='AND arg(s) (e.g. "language:eng,jpn!creators.role:author").')
    search_parser.add_argument('-o', '--OR',  help ='OR arg(s) (e.g. "language:eng,jpn!creators.role:author").')
    search_parser.add_argument('-n', '--NOT', help='NOT arg(s) (e.g. "language:eng,jpn!creators.role:author").')
    search_parser.add_argument('-r', '--raw', action='store_true', help='Print raw Elasticsearch DSL and output.')
    
    get_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    get_parser.add_argument('-l', '--log', help='Log file..')
    get_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    get_parser.add_argument('-i', '--index', help='index.', **indexarg)
    get_parser.add_argument('model', help='model.')
    get_parser.add_argument('id', help='Document ID.')
    
    init_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    init_parser.add_argument('-l', '--log', help='Log file..')
    init_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    init_parser.add_argument('-i', '--index', help='index.', **indexarg)
    init_parser.add_argument('path', help='Absolute path to "ddr repo".')
    
    create_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    create_parser.add_argument('-l', '--log', help='Log file..')
    create_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    create_parser.add_argument('-i', '--index', help='index.', **indexarg)
    
    remove_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    remove_parser.add_argument('-l', '--log', help='Log file..')
    remove_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    remove_parser.add_argument('-i', '--index', help='index.', **indexarg)
    
    index_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    index_parser.add_argument('-l', '--log', help='Log file..')
    index_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    index_parser.add_argument('-i', '--index', help='index.', **indexarg)
    index_parser.add_argument('-r', '--recursive', action='store_true', help='Recurse into subdirectories.')
    index_parser.add_argument('-P', '--public', action='store_true', help='For publication (fields not marked public will be omitted.')
    index_parser.add_argument('path', help='Absolute path to directory containing metadata file(s).')
    
    reindex_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    reindex_parser.add_argument('-l', '--log', help='Log file..')
    reindex_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    reindex_parser.add_argument('-i', '--index', help='index.', **indexarg)
    reindex_parser.add_argument('source', help='Source index.')
    reindex_parser.add_argument('dest', help='Destination index.')
    
    alias_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    alias_parser.add_argument('-l', '--log', help='Log file..')
    alias_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    alias_parser.add_argument('-i', '--index', help='Name of index.', **indexarg)
    alias_parser.add_argument('-a', '--alias', help='Alias.')
    alias_parser.add_argument('-R', '--remove', action='store_true', help='Remove the alias.')
    
    mappings_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    mappings_parser.add_argument('-l', '--log', help='Log file..')
    mappings_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    mappings_parser.add_argument('-i', '--index', help='index.', **indexarg)
    mappings_parser.add_argument('path', help='Absolute path to "ddr repo".')
    
    facets_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    facets_parser.add_argument('-l', '--log', help='Log file..')
    facets_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    facets_parser.add_argument('-i', '--index', help='index.', **indexarg)
    facets_parser.add_argument('path', help='Absolute path to "ddr repo".')
    
    exists_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    exists_parser.add_argument('-l', '--log', help='Log file..')
    exists_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    exists_parser.add_argument('-i', '--index', help='index.', **indexarg)
    exists_parser.add_argument('-m', '--model', required=True, help='model.')
    exists_parser.add_argument('-I', '--id', required=True, help='Document ID.')
    
    status_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    status_parser.add_argument('-l', '--log', help='Log file..')
    status_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    status_parser.add_argument('-i', '--index', help='index.', **indexarg)
    status_parser.add_argument('-m', '--mappings', action='store_true', help='Include index mappings.')
   
    repo_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    repo_parser.add_argument('-l', '--log', help='Log file..')
    repo_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    repo_parser.add_argument('-i', '--index', help='index.', **indexarg)
    repo_parser.add_argument('path', help='Absolute path to repository.json file.')
    
    org_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    org_parser.add_argument('-l', '--log', help='Log file..')
    org_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    org_parser.add_argument('-i', '--index', help='index.', **indexarg)
    org_parser.add_argument('-R', '--remove', action='store_true', help='Remove the organization.')
    org_parser.add_argument('path', help='Absolute path to organization.json file.')
    
    narr_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    narr_parser.add_argument('-l', '--log', help='Log file..')
    narr_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    narr_parser.add_argument('-i', '--index', help='index.', **indexarg)
    narr_parser.add_argument('path', help='Absolute path to narrators.json file.')
    
    delete_parser.add_argument('-d', '--debug', action='store_true', help='Debug; prints lots of debug info.')
    delete_parser.add_argument('-l', '--log', help='Log file..')
    delete_parser.add_argument('-H', '--host', help='Hostname and port (HOST:PORT).', **hostsarg)
    delete_parser.add_argument('-i', '--index', help='index.', **indexarg)
    delete_parser.add_argument('-I', '--id', required=True, help='Document ID.')
    delete_parser.add_argument('-r', '--recursive', action='store_true', help='Delete children of this document.')
    
    args = parser.parse_args()
    
    if args.cmd == 'help':
        print(description)
        print(epilog)
        sys.exit(0)
    
    if args.debug:
        print(args)

    if hasattr(args, 'path') and args.path and not os.path.exists(args.path):
        print('ERROR: Path does not exist: %s' % args.path)
        sys.exit(1)
    
    hosts = make_hosts( args.host )
    d = docstore.Docstore(args.host, args.index)
    
    if args.log and (os.path.exists(args.log) or os.path.exists(os.path.basename(args.log))):
        logging.basicConfig(format=LOGGING_FORMAT, datefmt=LOGGING_DATEFMT, level=logging.DEBUG, filename=args.log)
    
    # call selected function
    exit = 0
    if args.cmd == 'post':
        document = json.loads(
            identifier.Identifier(args.path).object().dump_json()
        )
        msg = d.post(document)
        print(msg)
    elif args.cmd == 'search':
        if not (args.text or args.AND or args.OR or args.NOT):
            print('ERROR: No search terms. Hint: --help')
            sys.exit(1)
        search_results(
            d,
            args.doctype,
            args.text,
            must=args.AND,
            should=args.OR,
            mustnot=args.NOT,
            raw=args.raw
        )
    elif args.cmd == 'get':
        msg = d.get(args.model, args.id)
        print(format_json(msg, sort_keys=False))
    elif args.cmd == 'init':
        statuses = d.init_index(args.path)
        for key,val in statuses.iteritems():
            print('%s: %s' % (key,val))
    elif args.cmd == 'create':
        results = d.create_index()
    elif args.cmd == 'remove':
        results = d.delete_index()
    
    elif args.cmd == 'index':
        start = datetime.now()
        results = d.index(args.path, recursive=True, public=True)
        end = datetime.now()
        elapsed = end - start
        if results['bad']:
            print('------------------------------------------------------------------------')
            print('The following paths had problems:\n')
            for path,status,response in results['bad']:
                print(' -- '.join([str(status), path, response]))
        print('------------------------------------------------------------------------')
        print('ES host/index:   %s/%s' % (hosts, args.index))
        print('Path:            %s' % args.path)
        print('Recursive:       %s' % args.recursive)
        print('Files processed: %s' % results['total'])
        print('Successful:      %s' % results['successful'])
        print('Errors:          %s' % len(results['bad']))
        print('Time elapsed:    %s' % elapsed)
    elif args.cmd == 'reindex':
        results = d.reindex(args.source, args.dest)
        print(results)
    elif args.cmd == 'alias':
        if args.remove:
            msg = d.rm_alias(args.alias, args.index)
        else:
            msg = d.set_alias(args.alias, args.index)
        print(msg)
    elif args.cmd == 'mappings':
        mpath = d.mappings_path(args.path)
        msg = d.put_mappings(mpath)
        print(msg)
    elif args.cmd == 'facets':
        fpath = d.facets_path(args.path)
        msg = d.put_facets(fpath)
        print(msg)
    elif args.cmd == 'exists':
        msg = d.exists(args.model, args.id)
        print(msg)
    elif args.cmd == 'status':
        host_status(d, args.mappings)
    elif args.cmd == 'repo':
        msg = d.repo(args.path)
        print(msg)
    elif args.cmd == 'org':
        msg = d.org(args.path, args.remove)
        print(msg)
    elif args.cmd == 'narrators':
        msg = d.narrators(args.path)
        print(msg)
    elif args.cmd == 'delete':
        msg = d.delete(args.id, recursive=args.recursive)
        print(msg)
    
    if exit:
        print(msg)
    sys.exit(exit)


if __name__ == '__main__':
    main()
