#!/usr/bin/python
"""
Update cache or -n (--noupdate) and search cache for rec_ids:
 surlatablo.py -q search_pattern

Update cache with new recording and deletions
 surlatablo.py 

Search for recordings and convert them
 surlatablo.py -n -q search_pattern -c

Use option --help for detailed usage.
"""
SLT_GLOBAL = {}
SLT_GLOBAL['PGM_VERSION'] = '0.5'
# Sur la Tablo (on the Tablo)
SLT_GLOBAL['PGM_NAME'] = 'surlatablo'
# License
SLT_GLOBAL['PGM_LICENSE'] = 'GPLv2'
# Configuration file
#  You can override option variables below by putting your into
#  this file.
SLT_GLOBAL['CONF'] = '~/surlatablo.conf'


### PROGRAM LOCATIONS

# Replace with full path location of your programs
FFMPEG = '/usr/bin/ffmpeg'
CCEXTRACTOR = '/usr/local/bin/ccextractor'

### IMPORTANT LOCAL SETTINGS

SURLATABLO_ROOT = '/SurLaTablo'    # Cache db goes here
TABLO_IPS = ['127.0.0.1']       # Should we detect?
                                   #  Use a python list of Tablo IPs.
LOCAL_TIMEZONE = 'US/Central'      # Your current timezone

NOZEROTV = True                    # Substitutes s<lair_date_month>e<lair_date_day> for when
                                   #  season or episode are zero (for meta_type TV)
                                   # Also sets episode title 

import os
if (os.name == 'nt'):    
    GIF_FONTFILE = 'C:/Windows/Fonts/airial.ttf' # Populates dynamic meta var ${gif_fontfile}
else:
    GIF_FONTFILE = '/usr/share/fonts/truetype/DejaVuSans.ttf' # Populates dynamic meta var ${gif_fontfile}

### DIRECTORY AND FILE CREATION PATTERNS

# Patterns for output handling by meta_type (e.g. TV, Sports, Movie, Manual)
#  Patterns can contain meta information made by this program using
#  ${meta-name}. Just like TabloTV, this programs metadata is non-uniform
#  by media type (aka meta_type)
#
BASE_DIRS={'Default':'./${meta_type}',
    'Sports':'./${meta_type}/${sport_type}/Season ${season_number}',
    'TV':'./${meta_type}/${Eseries}/Season ${season_number}',
    'Movie':'./${meta_type}/${Etitle} (${release_year})'
}
# Everything but the final extension of the final transcoded file. See TRANSCODER_OPTS last element.
FILENAME_PATS={'Default':'${Etitle}',
    'TV':'${Eseries} - s${plex_season_number}e${plex_episode_number} - ${Etitle}',
    'Sports':'${sport_type} - s${plex_season_number}e${plex_episode_number} - ${Etitle}',
    'Movie':'${Etitle} (${release_year})'
}





###############################################################################
# BE CAREFUL ABOUT MODS TO OPTIONS BELOW
###############################################################################

# CHANGELOG
# 
# 0.5 -
#   Quick fix to correct Eseries bug.
# 0.4 - 
#   1. You may now have a SLT_GLOBAL['CONF'] file (defaults to homedir
#      surlatablo.conf) which can be used to override the configuration variables.
#      This way you won't have to keep editing the source file for local site
#      changes.
#   2. TV shows with season 0 and episode 0 (e.g. News) will now default
#      to s<lair_date_month>e<lair_date_day> and Unknown titles will default
#      to Episode - <lair_date_year>-<lair_date_month>-<lair_date_day>.  You can
#      disable this by setting NOZERTV = False in your .conf file (see #1).
#      original_air_date cannot be used for this because quite often it is wrong.
#   3. Pathnames for output files are now santized for Windows.
#   4. surlatablo.py now takes arugments beyond switches which are to specify
#      the transcodes to do on the downloaded data.  Default if not present is
#      just Mp4.  You can override the default by setting in you .conf file:
#       TRANSCODER_DEFAULT = [ 'Mp4', 'Gif', etc. ]
#      You can specify on the command like:
#       surlatablo.py .... Mp4 Gif Json
#      or
#       surlatablo.py .... +Gif +Json
#      The '+' means in addition to the TRANSCODER_DEFAULT values.
#      Use '-t' option to see brief description of available transcoders.
#   5. Options --wcrop (-w) and --cropformat (-W) have been removed.  Create and
#      use a new TRANSCODER_OPTS definition for this.
#   6. You can now use --keepdir @ to say you want to save the .ts and .srt file
#      in the output directory with the other files.
#   7. New predefined TRANSCODER_OPTS have been added, created. (see -t option)
#        a. Mp4 - (the norm) to create a passthrough .mp4
#        b. Gif - create short duration animated gif
#        c. Flash - create an .flv
#        d. HardSubs - (assumes use of -C) to embed subitles directly in .mp4
#        e. Shrink - create a more compressed .mp4
#        f. Json - create .json file containing SurLaTablo meta information for program
#        g. Null - used mainly to say "no transcode", for times when you want to use
#            --keepdir @ to output and save just the .ts and (optionally) .srt
#        h. ... (more)
#   8. Replace sanitizeFilename with sltSanitize for dynamic meta type strings
#      like Etitle, ESeries, essentially things that could be used in path and filenames.
#      Special dynamic variable Efriendly_title2 is a two line escaped string for
#      internal ffmpeg option use (see Gif).
#   9. Bug fixes.... etc...
#
#Sat Jan 17 14:24:17 CST 2015
# 0.3 - New metatype, sort_title, which is the title minus leading
#   a/an/the article.  Added a sanitizeFilename function, mainly for
#   Windows.
#
#Thu Jan 15 00:00:26 CST 2015
# 0.2 - Defined some timezone stuff for when there is no pytz.  Doesn't handle
#   everything.  Handle escapes like \n, \t in -Q.  Check for ability to
#   create SURLATABLO_ROOT and exit with a message instead of fail with stack
#   trace.  New meta-type, friendly_title, added to make it easier to use
#   friendly_title~='Series - s01' for easy season selection.


### COMMAND OPTIONS

### TRANSCODER options
#  Allowed functions that can be called as transcoder.
TRANSCODER_FUNCS = [ 'shutil.copyfile', 'dumpJson', 'print' ]
# Default transcode(s) to provide if none provided
TRANSCODER_DEFAULT = [ 'Mp4' ]
# The allowed transcoders
#  All data options here need to be strings.
TRANSCODER_OPTS = { 
                'Mp4': {
                    'help': [ 'Standard pass through conversion of video and audio from Tablo .ts to .mp4.' ],
                    'command': [ FFMPEG ],
                    'inputfile': [ '-threads', '0', '-i', '${ts_filename}' ],
                    'subtitlefile': [ '-f', 'srt', '-i', '${srt_filename}' ],
                    'options': [ '-bsf:a', 'aac_adtstoasc', '-c:v', 'copy',
                         '-c:a', 'copy' ],
                    'subtitleoptions': [ '-c:s', 'mov_text', '-metadata:s:s:0', 'language=${lang3}' ],
                    'audiooptions': [ '-metadata:s:a:0', 'language=${lang3}' ],
                    'metadata': [],
                    'ext': [ '.mp4' ]
                },
                'Mp4W': {
                    'help': [ 'Crop a 4:3 480i channel presentation of a 16:9 widescreen movie.' ],
                    'command': [ FFMPEG ],
                    'inputfile': [ '-threads', '0', '-i', '${ts_filename}' ],
                    'subtitlefile': [ '-f', 'srt', '-i', '${srt_filename}' ],
                    'options': [ '-bsf:a', 'aac_adtstoasc', '-vf', 'crop=720:360:0:60', '-c:a', 'copy' ],
                    'subtitleoptions': [ '-c:s', 'mov_text', '-metadata:s:s:0', 'language=${lang3}' ],
                    'audiooptions': [ '-metadata:s:a:0', 'language=${lang3}' ],
                    'metadata': [],
                    'ext': [ '-w.mp4' ]
                },
                'Gif': {
                    'help': [ 'Create an short animated Gif' ],
                    'command': [ FFMPEG ],
                    'inputfile': [ '-ss', '${gifstart}', '-i', '${ts_filename}',
                        '-t', '${gifduration}' ],
                    'options': [ '-loop', '0', '-r', '${gifrate}', '-s', '${gifsize}', '-vf', 'drawtext=fontfile=${gif_fontfile}:text=\'${Efriendly_title2}\':fontcolor=white@1.0:fontsize=30:x=10:y=40' ],
                    'ext': [ '.gif' ]
                },
                'Flash': {
                    'help': [ 'Create an flash .flv file' ],
                    'command': [ FFMPEG ],
                    'inputfile': [ '-i', '${ts_filename}' ],
                    'options': [ '-ab', '56', '-ar', '44100', '-r', '25', '-qmin', '3', '-qmax', '6', '-s', '320x240' ],
                    'ext': [ '.flv' ]
                },
                'HardSubs': {
                    'help': [ 'For use with -C, burn the subtitles directly into the video of the .mp4' ],
                    'command': [ FFMPEG ],
                    'inputfile': [ '-i', '${ts_filename}' ],
                    'options': [],
                    'subtitleoptions': [ '-vf', 'subtitles=\'${srt_filename}\'' ],
                    'audiooptions': [ '-metadata:s:a:0', 'language=${lang3}' ],
                    'ext': [ '-hs.mp4' ]
                },
                'Shrink': {
                    'help': [ 'Create a smaller .mp4 file.' ],
                    'command': [ FFMPEG ],
                    'inputfile': [ '-threads', '0', '-i', '${ts_filename}' ],
                    'subtitlefile': [ '-f', 'srt', '-i', '${srt_filename}' ],
                    'options': [ '-crf', '25' ],
                    'subtitleoptions': [ '-c:s', 'mov_text', '-metadata:s:s:0', 'language=${lang3}' ],
                    'audiooptions': [ '-metadata:s:a:0', 'language=${lang3}' ],
                    'metadata': [],
                    'ext': [ '-sm.mp4' ]
                },
                # This is here more or less as an example, normally you would do this with -k @
                #  An example of using a python libray module function instead of an executable.
                'Ts': {
                    'help': [ 'Testing, do not use. Use --keepdir @ instead.' ],
                    'command': [ 'shutil.copyfile' ],
                    'inputfile': [ '${ts_filename}' ],
                    'ext': [ '.ts' ]
                },
                # This is so that if we transcode, and then go back an delete off of the Tablo,
                #  we can have a way to preserve the meta data for the program.
                #  This is an example of calling a global procedure defined here.
                'Json': {
                    'help': [ 'Save json meta data into .json file.' ],
                    'command': [ 'dumpJson' ],
                    'inputfile': [ '"${rec_id}": ${json}' ],
                    'ext': [ '.json' ],
                    'skip_ts': [ 'True' ]
                },
                'Null': {
                    'help': [ 'To specify no transcodes.  Useful when combined with --keepdir @.' ],
                    'command': [ 'print' ],
                    'ext': [ '.always' ]
                },
                # Slow
                'Glow': {
                    'help': [ 'Apply the Frei04 Glow affect to video.' ],
                    'command': [ FFMPEG ],
                    'inputfile': [ '-threads', '0', '-i', '${ts_filename}' ],
                    'subtitlefile': [ '-f', 'srt', '-i', '${srt_filename}' ],
                    'options': [ '-vf', 'frei0r=glow:0.5' ],
                    'subtitleoptions': [ '-c:s', 'mov_text', '-metadata:s:s:0', 'language=${lang3}' ],
                    'audiooptions': [ '-metadata:s:a:0', 'language=${lang3}' ],
                    'metadata': [],
                    'ext': [ '-glow.mp4' ]
                },
                # Slow
                'h265': {
                    'help': [ 'Product an HEVC h265 .mp4 file.' ],
                    'command': [ FFMPEG ],
                    'inputfile': [ '-threads', '0', '-i', '${ts_filename}' ],
                    'subtitlefile': [ '-f', 'srt', '-i', '${srt_filename}' ],
                    'options': [ '-bsf:a', 'aac_adtstoasc', '-c:v', 'libx265', '-preset', 'ultrafast', '-x265-params', 'crf=28', '-c:a', 'copy'  ],
                    'subtitleoptions': [ '-c:s', 'mov_text', '-metadata:s:s:0', 'language=${lang3}' ],
                    'audiooptions': [ '-metadata:s:a:0', 'language=${lang3}' ],
                    'metadata': [],
                    'ext': [ '-h265.mp4' ]
                }
}              

### CCEXTRACTOR options
CCEXTRACTOR_OPTS = { 'Default': {
                         'command': [ CCEXTRACTOR ],
                         'inputfile': [ '${ts_filename}' ],
                         'options': [ '--gui_mode_reports' ],
                         'outputfile': [ '-o', '${srt_filename}' ] }
}
              
### OTHER MODIFIERS

# CCOFFSET options
#  Results in setting an itsoffet (ts_itsoffset) parameter to ffmpeg on the
#   video stream (for adjusting subtitle/ccaption) Note: ffmpeg does not
#   allow itsoffset to work on subtitle streams which is why it's done of
#   the Tablo .ts file instead.
CCOFFSET={'Default':'',
    'Sports':'4'
}

# Tablo url templates
TABLO_SEGS='http://${tablo_ip}:18080/pvr/${rec_id}/segs'
TABLO_REC_IDS='http://${tablo_ip}:18080/plex/rec_ids'
TABLO_CHAN_IDS='http://${tablo_ip}:18080/plex/ch_ids'
TABLO_CHAN_INFO='http://${tablo_ip}:18080/plex/ch_info?id={}'
TABLO_REC_INFO='http://${tablo_ip}:18080/plex/rec_info?id={}'

# Some defaults (to be redone)
options = {
    'keepdir': '',
    'noclobber' : True,
    'ccaption' : False,
    'ccaptionopts' : CCEXTRACTOR_OPTS['Default'],
    'srtsubtitle' : '',
    'srtmovtext' : '',
    'ccoffset' : '',
    'gifstart' : '00:01:00',
    'gifduration' : '10',
    'gifrate' : '10',
    'gifsize' : '320x200'
}

### CONF FILE PROCESSING
# You can override and/or augment anything above this line using
#  the contents of the config file.
#try:
if (True):
    if (os.path.isfile(os.path.expanduser(SLT_GLOBAL['CONF']))):
        execfile(os.path.expanduser(SLT_GLOBAL['CONF']))
#except:
#    pass

# Mapping of 2char lang to 3char lang.
#  Assumes that Gracenote/Tribune data even knows about all of these.
lang3 = {
    'ab': 'abk', 'aa': 'aar', 'af': 'afr', 'ak': 'aka', 'sq': 'sqi', 'am': 'amh',
    'ar': 'ara', 'an': 'arg', 'hy': 'hye', 'as': 'asm', 'av': 'ava', 'ae': 'ave',
    'az': 'aze', 'bm': 'bam', 'ba': 'bak', 'eu': 'eus', 'be': 'bel', 'bn': 'ben',
    'bh': 'bih', 'bi': 'bis', 'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'mya',
    'ca': 'cat', 'ch': 'cha', 'ce': 'che', 'ny': 'nya', 'zh': 'zho', 'cv': 'chv',
    'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'hr': 'hrv', 'cs': 'ces', 'da': 'dan',
    'dv': 'div', 'nl': 'nld', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est',
    'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fra', 'ff': 'ful',
    'gl': 'glg', 'ka': 'kat', 'de': 'deu', 'el': 'ell', 'gn': 'grn', 'gu': 'guj',
    'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin', 'ho': 'hmo',
    'hu': 'hun', 'ia': 'ina', 'id': 'ind', 'ie': 'ile', 'ga': 'gle', 'ig': 'ibo',
    'ik': 'ipk', 'io': 'ido', 'is': 'isl', 'it': 'ita', 'iu': 'iku', 'ja': 'jpn',
    'jv': 'jav', 'kl': 'kal', 'kn': 'kan', 'kr': 'kau', 'ks': 'kas', 'kk': 'kaz',
    'km': 'khm', 'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon',
    'ko': 'kor', 'ku': 'kur', 'kj': 'kua', 'la': 'lat', 'lb': 'ltz', 'lg': 'lug',
    'li': 'lim', 'ln': 'lin', 'lo': 'lao', 'lt': 'lit', 'lu': 'lub', 'lv': 'lav',
    'gv': 'glv', 'mk': 'mkd', 'mg': 'mlg', 'ms': 'msa', 'ml': 'mal', 'mt': 'mlt',
    'mi': 'mri', 'mr': 'mar', 'mh': 'mah', 'mn': 'mon', 'na': 'nau', 'nv': 'nav',
    'nd': 'nde', 'ne': 'nep', 'ng': 'ndo', 'nb': 'nob', 'nn': 'nno', 'no': 'nor',
    'ii': 'iii', 'nr': 'nbl', 'oc': 'oci', 'oj': 'oji', 'cu': 'chu', 'om': 'orm',
    'or': 'ori', 'os': 'oss', 'pa': 'pan', 'pi': 'pli', 'fa': 'fas', 'pl': 'pol',
    'ps': 'pus', 'pt': 'por', 'qu': 'que', 'rm': 'roh', 'rn': 'run', 'ro': 'ron',
    'ru': 'rus', 'sa': 'san', 'sc': 'srd', 'sd': 'snd', 'se': 'sme', 'sm': 'smo',
    'sg': 'sag', 'sr': 'srp', 'gd': 'gla', 'sn': 'sna', 'si': 'sin', 'sk': 'slk',
    'sl': 'slv', 'so': 'som', 'st': 'sot', 'es': 'spa', 'su': 'sun', 'sw': 'swa',
    'ss': 'ssw', 'sv': 'swe', 'ta': 'tam', 'te': 'tel', 'tg': 'tgk', 'th': 'tha',
    'ti': 'tir', 'bo': 'bod', 'tk': 'tuk', 'tl': 'tgl', 'tn': 'tsn', 'to': 'ton',
    'tr': 'tur', 'ts': 'tso', 'tt': 'tat', 'tw': 'twi', 'ty': 'tah', 'ug': 'uig',
    'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie', 'vo': 'vol',
    'wa': 'wln', 'cy': 'cym', 'wo': 'wol', 'fy': 'fry', 'xh': 'xho', 'yi': 'yid',
    'yo': 'yor', 'za': 'zha', 'zu': 'zul'
}

def detailedHelp():
    helpstring = """
${PGM_NAME} ${PGM_VERSION} ${PGM_LICENSE}


FIRST THINGS
------------
Edit this program and change the values for (near top of file):

    # Location of your ffmpeg program (optional if you won't convert to mp4)
    FFMPEG = '/usr/bin/ffmpeg' 

    # Location of your ccextractor program (optional)
    CCEXTRACTOR = '/usr/local/bin/ccextractor'

    # Location directory where meta db caches go
    SURLATABLO_ROOT = '/SurLaTablo'

    # List of your Tablo IPs, this program does not auto detect today
    TABLO_IPS = ['192.168.1.73']



UPDATE AND SEARCH
-----------------
Update cache db (need metadata db to do anything)
 (typically you do this first, first run will take a bit)

    surlatablo.py

Update and search db

    surlatablo.py --query Green\ Acres

Do not update cache, just search db
 (note matches Green Acres anywhere in the metadata)

    surlatablo.py --noupdate --query Green\ Acres

Search db by metadata

    # Find all for TV series called Green Acres
    surlatablo.py --noupdate --query series~=Green\ Acres

    # Find all Sports records
    surlatablo.py --noupdate --query meta_type~=Sports

Use a query format string to change output

    # Find all TV shows and user a query format for output
    # (note --queryformat will not do anything combined with --convert)

    surlatablo.py --noupdate --query meta_type~=TV --queryformat '${lair_date}\t${rec_id}\t${series}\t${title}'

Display full TabloTV metadata for a specific recording id.

    # (note --rec_id can be used with --convert)
    surlatablo.py --noupdate --rec_id 28191

Note:  You can easily force a full refresh of local cached data by removing
 the files at SULATABLO_ROOT/Tablo_IP/*



CONVERT OR PREVIEW CONVERT (ADD SUBTITLES/CC, MP4 METADATA)
-----------------------------------------------------------------

Locate a particular recording, convert with close captioning done as a subtitle. Sample interative output also shown.

    surlatablo.py --noupdate --query Fuji\ falls --convert --ccaption --clobber
    <screen output follows>
    Working on:                             [./TV/McHale's Navy/Season 3/McHale's Navy - s03e15 - Fuji's Big Romance.mp4]
     Retrieving Tablo Data (28191):         [####################] 100%
     Extracting CC as Subtitle:             [####################] 100%
     Transcoding:                           [####################] 100%

Locate all sports and use default season of game_year and use supplied episode
 start number by sport_type. Defaulting starting with episode number 1 for
 unspecified types.

    surlatablo.py --noupdate --query meta_type~=Sports -E "NFL Football::3,NBA Basketball::5,1" --convert

Use the --debug (probably misnamed) to show what surlatablo.py would do, but
 don't do it (option to be combined with --convert)

    surlatablo.py --noupdate --query Fuji\ falls --convert --ccaption --clobber --debug
    [ ./TV/McHale's Navy/Season 3/McHale's Navy - s03e15 - Fuji's Big Romance.mp4 ]
    [ http://192.168.1.73:18080/pvr/28191/segs -> /tmp/TabloE1_zkh.ts]
    ['/usr/local/bin/ccextractor', '/tmp/TabloE1_zkh.ts', '--gui_mode_reports', '-o', '/tmp/TabloZdkhvf.srt']
    ['/usr/bin/ffmpeg', '-i', '/tmp/TabloE1_zkh.ts', '-f', 'srt', '-i', '/tmp/TabloZdkhvf.srt', '-bsf:a', 'aac_adtstoasc', '-c:v', 'copy', '-c:a', 'copy', '-c:s', 'mov_text', '-metadata:s:s:0', 'language=eng', '-metadata:s:a:0', 'language=eng', '-metadata', "title=Fuji's Big Romance", '-metadata', 'date=1964-12-25', '-metadata', 'network=Antenna', '-metadata', "album=McHale's Navy", '-metadata', "show=McHale's Navy", '-metadata', 'genre=Sitcom', '-metadata', 'episode_id=s03e15', '-metadata', "synopsis=Fuji falls in love with a native chief's daughter.", '-metadata', "description=Fuji falls in love with a native chief's daughter.", '-metadata', "comment=Fuji falls in love with a native chief's daughter.", '-y', "./TV/McHale's Navy/Season 3/McHale's Navy - s03e15 - Fuji's Big Romance.mp4"]

Note: Better, full output of an actual --convert can be captured simply be
 redirecting output to a file instead of to screen/terminal.  Useful for when
 things aren't working (bugs in the software).

Note: Program uses Python tempfile functions.  To adjust your temporary
 directory place:
    tempfile.tempdir = '/path-to/my-tempdir'
 For other notes on how Python finds your temporary file location, see Python
 documentation.

MORE EXAMPLES
-------------

Find all Mister Ed episodes and convert them using the Default transoding plug a
short animated Gif preview:

    surlatablo.py -n -q series~='Mister Ed' -C -c +Gif


BASIC CONVERT OPTIONS (for use with --convert)
---------------------

--ccaption         = Pull closed captioning off of Tablo recording and convert
                      it to subtitle format for inclusion with final transcode.
--clobber          = By default, this program will not convert if the target
                      end result filename already exists.
--keepdir <dir>    = Instead of deleting intermediate temporary files (e.g. the
                      Tablo .ts file, the subtitle .srt file), keep them here.    
                      If keepdir is @, files will go to the destination directory
                      for the final transcoded files and be named like --filename.



ADVANCED CONVERT OPTIONS
------------------------
As with query formats, you may use metadata keys inside of the paths here.

--sportsepisode    = Format of the form: [<sport_type>::]<start_episode_number>
                      Use episode_number starting with start_number and
                      incrementing by one for every match.  Different
                      start_numbers can be specified for the different
                      sport_types.
--sportsseason     = Format of the form: [<sport_type>::]<season_number>
                      Override using the game_date and use supplied
                      season_number as the season for the sporting event, a
                      different season can be specified by each sport_type.
--basedir          = Format of the form: <meta_type>::<formatted-path-string>,..
                      Base directory location for final converted mp4 file
                      based on meta_type.  Predefines are (BASE_DIRS):
                      Default    = ./${meta_type}
                        (e.g. ./Movie, ./Manual)
                      Sports     = ./${meta_type}/${sport_type}/Season ${season_number}
                        (e.g. ./Sports/NFL Football/Season 2014)
                        (remember, you can override the season_number for Sports on the
                        command line with -S)
                      TV         = ./${meta_type}/${Eseries}/Season ${season_number}
--filename         = Format of the form: <meta_type>::<formatted-path-to-file-string>,... 
                      (minus the file extension)
                      Predefines are (FILENAME_PATS):
                      Default    = ${Etitle}
                      TV         = ${Eseries} - s${plex_season_number}e${plex_episode_number} - ${Etitle}
                      Sports     = ${sport_type} - s${plex_season_number}e${plex_episode_number} - ${Etitle}
                      Movie      = ${Etitle} (${release_year})

Note: The Default applies only in cases where there is no meta_type match.  So
 if we wanted to override the location for a TV show.  We could override with,
  --basedir TV::. --filename TV::MyOwnFilename
 The output then would simply be placed at ./MyOwnFilename.mp4.  If we want
 Movies to stored inside of directory instead of flat inside of the Default
 basedir, we could use,
  --basedir Movie::./${meta_type}/${Etitle} (${release_year})

You can alter the values of BASE_DIRS and FILENAME_PATS in this program to make
 your own "default" settings persistent.


                     
COMMAND LINE SWITCH SUMMARY WITH SHORTCUTS
------------------------------------------

-h, --help             = Print this very long help.
-t, --transcodehelp    = Summary of valid transcode arguments
-v, --version          = Print this program version.
-d, --debug            = Turn on "debug" output, which is really just a show
                          without execution flag for --convert.
-c, --convert          = Attempt to transcode a matching query set or --rec_id
                          value.
-C, --ccaption         = Extract subtitle .srt file from closed captioning and
                          insert subtitle stream into final ouput file.
-y, --clobber          = Overwrite final destination file.  Normally, program
                          exits if final output file exists.
-n, --noupdate         = Do not process new or deletions off Tablo unit, just
                          go off existing cached data.
-i, --rec_id <id>      = Show TabloTV/Tribune full metadata for supplied id.
-k, --keepdir <dir>    = Keep temporary files used in --convert at this
                          directory location.
-q, --query <str>      = Query the meta db, in particular format can be of the
                          form metakey~=Value.
-b, --basedir <str>    = Comma separted list of basedir format patterns,
                          [<meta-type::]path
-f, --filename <str>   = Formatted pattern to produce final name (without
                          extension).
-S, --sportsseason <num> = Format is [<sport_type>::]season-number instead of
                          using
                          game_year.
-E, --sportsepisode <num> = Format is [<sport_type>::]episode-number.
-Q, --queryformat <str> = Format is re string (matches across full record) or
                          re string patterned automatically.
-O, --ccoffset <time>  = Can be just [+-]seconds or [+-]HH:MM,  sets itsoffset
                          on the TabloTV .ts data.


QUERY/BASEDIR/FILENAME FORMAT METAKEYS
--------------------------------------
Just like Tablo's non-uniform metadata, this program's data is also
 non-uniform across media types (meta_type).

This means the keys for Movies, TV, Sports and Manual are different.  Metakeys
 describe metadata for each recording.  Metakeys can be used in queries,
 queryformatting and for determing the BASEDIR and FILENAME creation when
 converting Tablo recording to mp4 format.

Common Metakeys
---------------
rec_id            = Tablo recording identifier
tablo_ip          = IP of Tablo for matching record
meta_type         = Type of media value: TV, Movie, Sports, or Manual
video_size        = Length in bytes of Tablo recording
video_sizeh       = Human readable version of video_size (e.g. size output in
                     MiB, GiB instead of bytes)
video_width       = Resolution width of recording (as contrained by Tablo
                     settings)
video_height      = Resolution height of recording (as contrained by Tablo
                     settings)
video_duration    = Duration in seconds of Tablo recording
video_durationh   = Duration in HH:MM:SS form
air_date          = UTC date and time of recording (YYYY-MM-DDTHH:MMZ)
air_date_year     = UTC year of recording (YYYY)
air_date_month    = UTC month of recording (MM)
air_date_day      = UTC day of recording (DD)
air_date_hour     = UTC hour of recording (HH)
air_date_minute   = UTC minute of recording (MM)
lair_date         = Local timezone date and time of recording
                     (YYYY-MM-DDTHH:MM+TZOFFSET)
                     (note: stored with the cached metadata db, this is the
                     local timezone value at the time of the cache entry)
lair_date_year    = Local timezone year of recording (YYYY)
lair_date_month   = Local timezone month of recording (MM)
lair_date_day     = Local timezone day of recording (DD)
lair_date_hour    = Local timezone hour of recording (HH)
lair_date_minute  = Local timezone minute of recording (MM)
lair_date_tz      = Local timezone
title             = Recording title (TV = episode title, Movie = title, 
                     Sports = sports event, Manual = title)
friendly_title    = For TV, ${series} - sXXeYY - ${title}, for Movie
                     ${title} (${release_year}), otherwise same as ${title}


Almost Common Metakeys
----------------------
lang              = 2 char ISO 639-1 language of recording (probablistic) (not
                     in Manual)
synopsis          = Recording/show short description (not in Manual)
description       = Recording/show long description (not in Manual)
channel_sign      = TV Channel name (e.g. KDAF-DT, Antenna, MeTV, MOVIES!)
                      (not in Manual)
channel_affliate  = Optionally suppied affiliation of channel_sign (e.g. FOX,
                      CW, ION, MNT) (not in Manual)
channel_num       = Traditional tuning channel number (not in Manual)
channel_res_height= Resolution height of channel (not in Manual)
channel_res_width = Resolution width of channel (not in Manual)
channel_res_name  = Friendly resolution name (e.g. 720p, 1080i, 480i) (not in
                     Manual)

Note: Channel determination is somewhat probablistic.

Almost Dynamic Common Metakeys
------------------------------
plex_season_number  = Zero padded 2 digit respesentation of season_number
                       (for Sports and TV)
plex_episode_number = Zero padded 2 digit respesentation of episode_number (for 
i                      Sports and TV)
Etitle              = ${title} with filesystem unsafe chars removed.
Eseries             = ${series} with filesystem unsafe chars removed.
Efriendly_title     = ${friendly_title} with ffmpeg unsafe chars removed.

Note: This is according to Plex naming requirements and in particular for use
 with thetvdb.org agent.

Sports Metakeys (meta_type = Sports)
---------------
game_date         = Date of sporting event (YYYY-MM-DD)
game_date_year    = Year of sporting event (YYYY)
game_date_month   = Month of sporting event (MM)
game_date_day     = Day of sporting event
sport_type        = Name of sporting event (e.g. NFL Football, NBA Baskeball)
teams_            = Semicolon delimited list of teams in sporting event    

Sports Dynamic Metakeys
-----------------------
These keys are not part of the metadata db, but can be assigned automatically
 or through the command line.

season_number     = Defaults to game_date_year, artificial season number for a
                     sporting event
                    (override on command line with -S <season number>)
episode_number    = Defaults to zero, but for multiple search match of same
                     sport_type it is incremeted by one. 
                    (override on command line with -E <episode number>)

TV Show Metakeys (meta_type = TV)
----------------
original_air_date = Date the TV show originally aired (YYYY-MM-DD)
original_air_date_year = Year the TV show originally aired (YYYY)
original_air_date_month = Month the TV show originally aired (MM)
original_air_date_day = Day the TV show originally aired (DD)
cast_             = Semicolon delimited list of cast in TV show
rating            = TV Rating code (e.g. TV-14, TVG)
genres            = Semicolon delimited list of genres for TV show (e.g. Sitcom)
series            = TV show series name (e.g. Green Acres, Arrow, Gotham)
season_number     = TV show series season number (defaults to 0)
episode_number    = TV show series episode number (defaults to 0)

Movie Metakeys
--------------
release_year      = Year movie was released (YYYY)
cast_             = Semicolon delimited list of cast in Movie
rating            = MPAA Rating code (e.g. R, PG, PG-13, G)
genres            = Semicolon delimited list of genres for Movie (e.g. Comedy;Thriller)



"""
    print Template(helpstring).safe_substitute(SLT_GLOBAL)




# Import python modules.  There is an assumption that you have all of these.
#  Biggest risk is probably pytz.
import urllib2,json,sys,time,re,getopt,string,subprocess,tempfile,importlib
from datetime import datetime, tzinfo, timedelta
from string import Template
from copy import deepcopy
try:
    import pytz
    from pytz import reference
    has_pytz = True
except:
    has_pytz = False

def transcodeHelp():
    for transname in TRANSCODER_OPTS:
        try:
            print transname + '\t' + TRANSCODER_OPTS[transname]['help'][0]
        except:
            pass

def printError(*args):
    sys.stderr.write(' '.join(map(str,args)) + '\n')
    sys.stderr.flush()

# Attempt to handle timzone data without pytz
if (not has_pytz):
    ZERO = timedelta(0)
    HOUR = timedelta(hours=1)

    # A UTC class.

    class UTC(tzinfo):
        """UTC"""

        def utcoffset(self, dt):
            return ZERO

        def tzname(self, dt):
            return "UTC"

        def dst(self, dt):
            return ZERO

    utc = UTC()

    # A class building tzinfo objects for fixed-offset time zones.
    # Note that FixedOffset(0, "UTC") is a different way to build a
    # UTC tzinfo object.

    class FixedOffset(tzinfo):
        """Fixed offset in minutes east from UTC."""

        def __init__(self, offset, name):
            self.__offset = timedelta(minutes = offset)
            self.__name = name

        def utcoffset(self, dt):
            return self.__offset

        def tzname(self, dt):
            return self.__name

        def dst(self, dt):
            return ZERO

    # A class capturing the platform's idea of local time.

    import time as _time

    STDOFFSET = timedelta(seconds = -_time.timezone)
    if _time.daylight:
        DSTOFFSET = timedelta(seconds = -_time.altzone)
    else:
        DSTOFFSET = STDOFFSET

    DSTDIFF = DSTOFFSET - STDOFFSET

    class LocalTimezone(tzinfo):

        def utcoffset(self, dt):
            if self._isdst(dt):
                return DSTOFFSET
            else:
                return STDOFFSET

        def dst(self, dt):
            if self._isdst(dt):
                return DSTDIFF
            else:
                return ZERO

        def tzname(self, dt):
            return _time.tzname[self._isdst(dt)]

        def _isdst(self, dt):
            tt = (dt.year, dt.month, dt.day,
                  dt.hour, dt.minute, dt.second,
                  dt.weekday(), 0, 0)
            stamp = _time.mktime(tt)
            tt = _time.localtime(stamp)
            return tt.tm_isdst > 0

    Local = LocalTimezone()

    # A complete implementation of current DST rules for major US time zones.

    def first_sunday_on_or_after(dt):
        days_to_go = 6 - dt.weekday()
        if days_to_go:
            dt += timedelta(days_to_go)
        return dt


    # US DST Rules
    #
    # This is a simplified (i.e., wrong for a few cases) set of rules for US
    # DST start and end times. For a complete and up-to-date set of DST rules
    # and timezone definitions, visit the Olson Database (or try pytz):
    # http://www.twinsun.com/tz/tz-link.htm
    # http://sourceforge.net/projects/pytz/ (might not be up-to-date)
    #
    # In the US, since 2007, DST starts at 2am (standard time) on the second
    # Sunday in March, which is the first Sunday on or after Mar 8.
    DSTSTART_2007 = datetime(1, 3, 8, 2)
    # and ends at 2am (DST time; 1am standard time) on the first Sunday of Nov.
    DSTEND_2007 = datetime(1, 11, 1, 1)
    # From 1987 to 2006, DST used to start at 2am (standard time) on the first
    # Sunday in April and to end at 2am (DST time; 1am standard time) on the last
    # Sunday of October, which is the first Sunday on or after Oct 25.
    DSTSTART_1987_2006 = datetime(1, 4, 1, 2)
    DSTEND_1987_2006 = datetime(1, 10, 25, 1)
    # From 1967 to 1986, DST used to start at 2am (standard time) on the last
    # Sunday in April (the one on or after April 24) and to end at 2am (DST time;
    # 1am standard time) on the last Sunday of October, which is the first Sunday
    # on or after Oct 25.
    DSTSTART_1967_1986 = datetime(1, 4, 24, 2)
    DSTEND_1967_1986 = DSTEND_1987_2006

    class USTimeZone(tzinfo):

        def __init__(self, hours, reprname, stdname, dstname):
            self.stdoffset = timedelta(hours=hours)
            self.reprname = reprname
            self.stdname = stdname
            self.dstname = dstname

        def __repr__(self):
            return self.reprname

        def tzname(self, dt):
            if self.dst(dt):
                return self.dstname
            else:
                return self.stdname

        def utcoffset(self, dt):
            return self.stdoffset + self.dst(dt)
    
        def dst(self, dt):
            if dt is None or dt.tzinfo is None:
                # An exception may be sensible here, in one or both cases.
                # It depends on how you want to treat them.  The default
                # fromutc() implementation (called by the default astimezone()
                # implementation) passes a datetime with dt.tzinfo is self.
                return ZERO
            assert dt.tzinfo is self

            # Find start and end times for US DST. For years before 1967, return
            # ZERO for no DST.
            if 2006 < dt.year:
                dststart, dstend = DSTSTART_2007, DSTEND_2007
            elif 1986 < dt.year < 2007:
                dststart, dstend = DSTSTART_1987_2006, DSTEND_1987_2006
            elif 1966 < dt.year < 1987:
                dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986
            else:
                return ZERO

            start = first_sunday_on_or_after(dststart.replace(year=dt.year))
            end = first_sunday_on_or_after(dstend.replace(year=dt.year))

            # Can't compare naive to aware objects, so strip the timezone from
            # dt first.
            if start <= dt.replace(tzinfo=None) < end:
                return HOUR
            else:
                return ZERO

    # Ok, this just handles hour based offset timezones and not
    #  all are defined here.  Sorry Newfoundland...
    timezone = {}
    timezone['US/Eastern']  = USTimeZone(-5, "Eastern",  "EST", "EDT")
    timezone['US/Central']  = USTimeZone(-6, "Central",  "CST", "CDT")
    timezone['US/Mountain'] = USTimeZone(-7, "Mountain", "MST", "MDT")
    timezone['US/Pacific']  = USTimeZone(-8, "Pacific",  "PST", "PDT")
    timezone['Canada/Atlantic'] = USTimeZone(-4, "Atlantic", "AST", "ADT")
    timezone['Canada/Central'] = timezone['US/Central']
    timezone['Canada/Eastern'] = timezone['US/Eastern']
    timezone['Canada/Mountain'] = timezone['US/Mountain']
    timezone['Canada/Pacific'] = timezone['US/Pacific']
    timezone['America/Halifax'] = timezone['Canada/Atlantic']
    timezone['America/Glace_Bay'] = timezone['Canada/Atlantic']
    timezone['America/Moncton'] = timezone['Canada/Atlantic']
    timezone['America/Goose_Bay'] = timezone['Canada/Atlantic']
    timezone['America/Blanc-Sablon'] = FixedOffset(-4, "AST")
    timezone['America/Montreal'] = timezone['US/Eastern']
    timezone['America/Toronto'] = timezone['US/Eastern']
    timezone['America/Nipigon'] = timezone['US/Eastern']
    timezone['America/Thunder_Bay'] = timezone['US/Eastern']
    timezone['America/Iqaluit'] = timezone['US/Eastern']
    timezone['America/Pangnirtung'] = timezone['US/Eastern']
    timezone['America/Resolute'] = timezone['US/Central']
    timezone['America/Atikokan'] = FixedOffset(-5, "EST")
    timezone['America/Rankin_Inlet'] = timezone['US/Central']
    timezone['America/Winnipeg'] = timezone['US/Central']
    timezone['America/Resolute'] = timezone['US/Central']
    timezone['America/Rainy_River'] = timezone['US/Central']
    timezone['America/Regina'] = FixedOffset(-6, "CST")
    timezone['America/Swift_Current'] = FixedOffset(-6, "CST")
    timezone['America/Edmonton'] = timezone['US/Mountain']
    timezone['America/Cambridge_Bay'] = timezone['US/Mountain']
    timezone['America/Yellowknife'] = timezone['US/Mountain']
    timezone['America/Inuvik'] = timezone['US/Mountain']
    timezone['America/Creston'] = FixedOffset(-7, "MST")
    timezone['America/Dawson_Creek'] = FixedOffset(-7, "MST")
    timezone['America/Vancouver'] = timezone['US/Pacific']
    timezone['America/Whitehorse'] = timezone['US/Pacific']
    timezone['America/Dawson'] = timezone['US/Pacific']

# Dump json in string format.  You can load by reading the string and surrounding
#  the read string with curly braces and using json.loads on it.
def dumpJson(jsonstring, filename):
    with open(filename, 'w') as f:
        f.write(jsonstring)

# Remove single quotes and possibly other unsafe characters.  Used for 'E'
#  strings used in transcoder (ffmpeg) options.  Side effect for 
#  friendly_titles... in that sXXeYY info goes on first line and title
#  goes on 2nd line.  Yes... this is very specific.  Alternatively, could
#  create 'G' names for things like series, title (Gseries, Gtitle) and
#  leave the rest to you to configure your TRANSCODER_OPTS.
def sltEscapeString(s):
    temp = re.sub(r"[':]","",s)
    return re.sub(r'(.* - s[0-9][0-9]+e[0-9][0-9]+) - (.*)',r'\1\n\2',temp)

# Substitute for bad chars (for things that go into files)
#  These are aslo used in 'E' meta types like Etitle, Eseries
def sltSanitize(name):
    if (os.name == 'nt'):    
        # Feel free to add to this.
        return re.sub(r'[:"/?<>\\*|]','_',name)
    else:
        return name

# Return title minus any first word article, see articles
def sortTitle(title):
    # Check first word against articles array
    #  if matched, remove word and return the rest.
    word = title.split(' ',1)[0].lower()
    for article in articles:
       if (article == word):
           return title.split(' ',1)[1]
    return title

# This function takes meta-dataname~=value and turns it into
#  an appropriate re for line searching.
def transformSearch(search_pat):
    nsearch_pat = re.sub(r'(.*)~=(.*)',r'"\1": ["]?\2', search_pat)
    if (nsearch_pat):
        search_pat = nsearch_pat
    return search_pat

# This may get redone.  Simple way to do a progress bar.
def displayProgress(progress_counter, progress_max, bar_length, prefix=''):
    percent = float(progress_counter) / progress_max
    hashes = '#' * int(round(percent * bar_length))
    spaces = ' ' * (bar_length - len(hashes))
    if (sys.stdout.isatty()):
        sys.stdout.write('\r' + prefix + '[{0}] {1}% '.format(hashes + spaces, int(round(percent * 100))))
    else:
        sys.stdout.write(str(int(round(percent * 100))) + ',')
        if (percent > .995):
            print(' <-' + prefix)
    sys.stdout.flush()

# Turn bytes into a human readable string.
def humanizeValue(num, suffix='B'):
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return '%3.1f%s%s' % (num, unit, suffix)
        num /= 1024.0
    return '%.1f%s%s' % (num, 'Yi', suffix)

# Returns the first rating code found (input being Tribune data).  Somewhat probabilistic assumption.
def getRating(ratings):
    for r in ratings:
        return r['code'].encode('utf-8').strip()
    return 'Unknown'

def createTempfile(prefix, ext, keepdir=''):
    # Create temporary file with extesion, optional keepdir
    #  Returns array of open file handle and the fullname of the temporary file.
    #  On error, returns an empty array.
    if (keepdir):
        # This file stays in provided keepdir
        try:
            fd, tempfilename = tempfile.mkstemp(suffix = ext,
                dir = keepdir,
                prefix = prefix)
        except:
            printError('Could not create temporary file in ' + keepdir)
            return 
    else:
        # This file stays in system temp area
        try:
            fd, tempfilename = tempfile.mkstemp(suffix = ext,
                prefix = prefix)
        except:
            printError('Could not create temporary file (' + ext + ').')
            return 
    try:
        if (not debug):
            # Create an open file handle opened for writing
            fh = os.fdopen(fd, 'wb+')
            fh.close()
        if (keepdir):
            print('Preserved temporary (' + ext + ') file location: [' +
                tempfilename + ']')
    except:
        printError('Could not create temporary file (' + ext + ').')
        return 
    return tempfilename


# Does the work of converting a db_rec into a transcoded file(s) (see Steps
#  comments below).
def doConvert(dbrecord, options, basedirs, filename_pats, transcoder_names):

    # If there is no transcoder (assumes FFMPEG right now), nothing to do!
    #if (not os.path.isfile(FFMPEG)):
    #    return

    # Essentials 
    # We will manipulate dbrecord with dynamic metadata, so
    #  to keep away from a sideeffect, we deepcopy it
    db_rec = deepcopy(dbrecord)
    db_rec.update(options)
    tablo_ip = db_rec['tablo_ip']
    rec_id = db_rec['rec_id']
    air_date = db_rec['air_date']
    meta_type = db_rec['meta_type']
    try:
        db_rec['lang3'] = lang3[db_rec['lang']]
    except:
        db_rec['lang3'] = 'eng'
        db_rec['lang'] = 'en'
    db_rec['json'] = json.dumps(dbrecord)

    # I really don't like having to do this, but if we include friendly_title as
    #  embedded, for example, ffmpeg option, have to replace/escape the single quotes.
    # Break Efriendly_title2 into two lines by inserting ^L
    db_rec['Efriendly_title2'] = sltEscapeString(db_rec['friendly_title'])
    
    # Create the 'E' sanitized strings for things usually found in filenames and paths.
    db_rec['Etitle'] = sltSanitize(db_rec['title'])
    try:
        db_rec['Eseries'] = sltSanitize(db_rec['series'])
    except:
        pass
    
    # Populate gif_fontfile
    db_rec['gif_fontfile'] = GIF_FONTFILE

    # Only TV and Sports could have season, episode
    try:
        season_number = db_rec['season_number']
    except:
        season_number = 0

    try:
        episode_number = db_rec['episode_number']
    except:
        episode_number = 0

    # We will export these back into db_rec anyway for all types
    #  and to handle situations where we did not have values for TV or Sports
    db_rec['season_number'] = season_number
    db_rec['episode_number'] = episode_number
    # Plex uses thetvdb.com for TV lookups, but seems to prefer padded 2 digit entries
    db_rec['plex_season_number'] = '{0:02d}'.format(season_number)
    db_rec['plex_episode_number'] = '{0:02d}'.format(episode_number)

    # A flag to tell us if we need to pull ts data from the Tablo, or
    #  has it already been done.
    have_tsfile = False
    # A flag to tell us if we need to pull subtite data from the ts file, or
    #  has it already been done.
    have_srtfile = False

    # After all db_rec meta data updates and augments
    #  Replace variables inside of basedir and filename using augmented db_rec
    try:
        basedir = basedirs[meta_type]
    except:
        basedir = basedirs['Default']
    basedir_t = Template(basedir)
    basedir = basedir_t.safe_substitute(db_rec)
    try:
        filename = filename_pats[meta_type]
    except:
        filename = filename_pats['Default']
    filename_t=Template(filename)
    filename = filename_t.safe_substitute(db_rec)


    # The fully qualified filename is the basedir + filename
    #local_encoding = sys.getfilesystemencoding()
    fq_filename = basedir + '/' + filename
    # Expand filename in case it refers to user's home by tilde or other.
    fq_filename = os.path.expanduser(fq_filename)
    # This may not seemed required, but it is possible for the filename_pat to
    #  have directories paths in it.  Just trying to be flexible.
    final_dir = os.path.dirname(fq_filename)
    final_filename = os.path.basename(fq_filename)
    fq_filename = final_dir + '/' + final_filename
    db_rec['final_dir'] = final_dir
    db_rec['final_filename'] = final_filename
    db_rec['fq_filename'] = fq_filename

    # if we have keepdir and it is '@', the use final_dir for the temp files
    if (options['keepdir'] == '@'):
        keepdir = final_dir
    else:
        keepdir = options['keepdir']

    # Step 1, Loop through transcoder_names just so we can know the 
    #  extension of the various transcodes, and check for their
    #  existence (unless noclobber).
    for transcoder_name in transcoder_names:
        # Check for valid function or binary
        if (TRANSCODER_OPTS[transcoder_name]['command'][0] in TRANSCODER_FUNCS):
            try:
                # Module function from Python library?
                mod_name, func_name = TRANSCODER_OPTS[transcoder_name]['command'][0].rsplit('.',1)
                mod = importlib.import_module(mod_name)
                func = getattr(mod, func_name)
            except:
                # Function defined here?
                try:
                    dummy=callable(TRANSCODER_OPTS[transcoder_name]['command'][0])
                except:
                    printError('Error: Cannot find allowed function for [' + transcoder_name + '], [' + TRANSCODER_OPTS[transcoder_name]['command'][0] + ']')
                    return
        else:
            if (not os.path.isfile(TRANSCODER_OPTS[transcoder_name]['command'][0])):
                printError('Error: Cannot find executable for [' + transcoder_name + '], [' + TRANSCODER_OPTS[transcoder_name]['command'][0] + ']')
                return
        # Get the dictionary entry from TRANSCODER_OPTS, specifically the ext
        try:
            fext = TRANSCODER_OPTS[transcoder_name]['ext'][0]
        except:
            printError('Could not find ext for TRANSCODER_OPTS["' + transcoder_name + '"]')
            sys.exit(7)
        TRANSCODER_OPTS[transcoder_name]['skip'] = [ '' ]
        if (options['noclobber']):
            if (os.path.isfile(fq_filename + fext)):
                printError('File exists (try --clobber), ' + fq_filename.encode('utf-8') + fext)
                TRANSCODER_OPTS[transcoder_name]['skip'] = [ 'True' ]

    # Make sure we can create our final transcode file(s) in output dir
    try:
        if (not os.path.isdir(final_dir)):
            os.makedirs(final_dir)
    except:
        printError('Could not create output directory, ' + final_dir)
       
    # No interrupts here, just trying to do good temp file house cleaning.
    #  Everything needs to complete and then we can do the removes.... and
    #  then you can Ctrl-C again.  See finally:

    try:
        segs_url = Template(TABLO_SEGS).safe_substitute(db_rec)
	try:
            if (options['keepdir'] == '@'):
                ts_filename = fq_filename + '.ts'
            else:
                ts_filename = createTempfile('Tablo', '.ts', keepdir)
        except:
            sys.exit(4)

        # Now loop through transcoder_names and execute each transcode using the
        #  ts_filename and srt_filename data 
        printworking = True
        for transcoder_name in transcoder_names:

            try:
                if (TRANSCODER_OPTS[transcoder_name]['skip'][0]):
                    continue
            except:
                pass

            if (printworking):
                print("{0:40s}".format('Working on:') + '[' + fq_filename + ']')
                printworking = False

            # If transcode does not require ts, skip it
            try:
                skip_ts = TRANSCODER_OPTS[transcoder_name]['skip_ts']
            except:
                skip_ts = ''

            # have_tsfile is a flag set once the ts is downloaded from the Tablo so 
            #  that it is not retrieved again (for multiple transcodes on same ts file).
            if (not debug and not have_tsfile and not skip_ts):
                # Step 2, Extract ts data from Tablo
                try:
                    segs_response = urllib2.urlopen(segs_url)
                except:
                    printError('Can not get ' + segs_url + '. Bad id?')
                    sys.exit(7)
                segs_html = segs_response.read()
                ts_files = re.findall(r'>\d+\.ts<', segs_html)
                tslen = len(ts_files)
                tscount=1
                bar_length=20
                try:
                    # Should not be a reason for this to fail, we just did it.
                    temp_f = open(ts_filename, 'wb+')
                except:
                    sys.exit(5)
                for ts_t in ts_files:
                    ts = ts_t[1:-1]
                    ts_url = segs_url + '/' + ts
                    ts_response = urllib2.urlopen(ts_url)
                    temp_f.write(ts_response.read())
                    if (sys.stdout.isatty()):
                        displayProgress(tscount, tslen, bar_length,
                           "{0:40s}".format(" Retrieving Tablo Data ({0:d}): ".format(rec_id)))
                    else:
                        print("Retrieved: " + ts_url)
                    tscount = tscount + 1
                print('')
                temp_f.close()
                have_tsfile = True
            else:
                if (debug):
                    print('[ ' + fq_filename + ' ]')
                    print('')
                    print('RETRIEVE FROM TABLO')
                    print('[ ' + segs_url + ' -> ' + ts_filename + ']')
            db_rec['ts_filename'] = ts_filename
    
            # We need this later on
            devnull = open(os.devnull, 'w')
    
            # Step 3, Setup for extracting closed captions from saved ts data
            # Now that we have a ts file, we can extract closed captions as an srt
            if (options['ccaption'] and not have_srtfile):
                if (options['keepdir']):
                    # use ts_filename as name for srt file in same keepdir location
                    srt_filename, ext = os.path.splitext(ts_filename)
                    # Named using Plex style with <lang>.srt
                    srt_filename = srt_filename + '.' + db_rec['lang3'] + '.srt'
                    print('Preserved temporary srt file location: [' + srt_filename + ']')
                else:
                    try:
                        srt_filename = createTempfile('Tablo', '.srt', keepdir)
                    except:
                        sys.exit(4)
    
                try:
                    if (not debug):
                        srt_file_f = open(srt_filename, 'wb+')
                        # We just need the filename, not sure if there is
                        #  a better way
                        srt_file_f.close()
                except:
                    printError('Could not create temporary file in ' + srt_filename)
                    sys.exit(4)
                db_rec['srt_filename'] = srt_filename
    
                # Go through the command options and substitute variables
                ccextractor_newopts = {}
                for ccextractoropts in CCEXTRACTOR_OPTS['Default']:
                   newopts = []
                   for ccextractoropt in CCEXTRACTOR_OPTS['Default'][ccextractoropts]:
                       newopts = newopts + [ Template(ccextractoropt).safe_substitute(db_rec) ]
                   ccextractor_newopts[ccextractoropts] = newopts
    
                ccaption_optsa = ccextractor_newopts['command'] + ccextractor_newopts['inputfile'] + ccextractor_newopts['options'] + ccextractor_newopts['outputfile']
    
                if (debug or not sys.stdout.isatty()):
                    print('')
                    print('CCEXTRACTOR')
                    print(ccaption_optsa)
        
                # Step 4, Extract closed captions from saved ts data
                if (not debug):
                    ccproc = subprocess.Popen(ccaption_optsa,stderr=subprocess.PIPE,stdout=devnull)
                    cclen = 100
                    bar_length=20
                    cc_count=1
                    while True:
                       ccline = ccproc.stderr.readline()
                       if (ccline == ''):
                           break
                       # Check for match, and parse percentage
                       ccprogress = re.match(r'^[#][#][#]PROGRESS[#](?P<num>[0-9]+)[#]', ccline)
                       try:
                           cc_count = ccprogress.group('num')
                           if (sys.stdout.isatty()):
                               displayProgress(cc_count, cclen, bar_length, "{0:40s}".format(" Extracting CC as Subtitle: "))
                       except:
                           pass
                       if (not sys.stdout.isatty()):
                           print(ccline.strip())
                    print('')
                have_srtfile = True
        
            transcoderopts = deepcopy(TRANSCODER_OPTS[transcoder_name])
            # Do substitutions of variables on transcoder options
            for transcoderopttype in transcoderopts:
                newopts = []
                for transcoderopt in transcoderopts[transcoderopttype]:
                    newopts = newopts + [ Template(transcoderopt).safe_substitute(db_rec).encode('utf-8').strip() ]
                transcoderopts[transcoderopttype] = newopts


            # Note: This really needs to done just for ffmpeg and just for live shows
            # Configure offset for ffmpeg start to make up for ccaption
            #  (e.g. Sports is delayed, delay ffmpeg video)
            if (transcoderopts['command'][0] == FFMPEG):
                try:
                    ccoffset = CCOFFSET[db_rec['meta_type']]
                except:
                    ccoffset = CCOFFSET['Default']
                if (ccoffset != '' and ccoffset != '0' and 'inputfile' in transcoderopts):
                    transcoderopts['inputfile'] = [ '-itsoffset', ccoffset ] + transcoderopts['inputfile']

            # Gather metadata for transcode if applicaable
            #  This right now is ffmpeg specific
            if (transcoderopts['command'][0] == FFMPEG):
                if ('metadata' in transcoderopts):
                    meta_opts = []
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}={1}'.format('title',db_rec['title'])]
                    except:
                        pass
                    if (meta_type == 'TV'):
                        try:
                            meta_opts = meta_opts + ['-metadata','{0}={1}'.format('date',db_rec['original_air_date'])]
                        except:
                            pass
                    elif (meta_type == 'Sports'):
                        try:
                            meta_opts = meta_opts + ['-metadata','{0}={1}'.format('date',db_rec['game_date'])]
                        except:
                            pass
                    elif (meta_type == 'Movie'):
                        try:
                            meta_opts = meta_opts + ['-metadata','{0}={1}'.format('date',db_rec['release_year'])]
                        except:
                            pass
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}={1}'.format('network',db_rec['channel_sign'])]
                    except:
                        pass
                    # Album is sort of like show, for things that do not understand show
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}={1}'.format('album',db_rec['series'])]
                    except:
                        pass
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}={1}'.format('show',db_rec['series'])]
                    except:
                        pass
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}={1}'.format('genre',db_rec['genres'])]
                    except:
                        pass
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}=s{1:02d}e{2:02d}'.format('episode_id',db_rec['season_number'],db_rec['episode_number'])]
                    except:
                        pass
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}={1}'.format('synopsis',db_rec['description'])]
                    except:
                        pass
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}={1}'.format('description',db_rec['long_description'])]
                    except:
                        pass
                    try:
                        meta_opts = meta_opts + ['-metadata','{0}={1}'.format('comment',db_rec['long_description'])]
                    except:
                        pass

                    # Add in the extra informational metadata
                    transcoderopts['metadata'] = transcoderopts['metadata'] + meta_opts
 
            if (transcoderopts['command'][0] == FFMPEG):
                if (not options['noclobber']):
                    transcoderopts['options'] = transcoderopts['options'] + ['-y']

            fext = transcoderopts['ext'][0]
            outputfile = fq_filename + fext

            # Structure of this is ffmpeg specific, but ok
            transcoder_optsa = transcoderopts['command'] + transcoderopts['inputfile']
            if (options['ccaption'] and 'subtitlefile' in transcoderopts):
                transcoder_optsa = transcoder_optsa + transcoderopts['subtitlefile']
            if ('options' in transcoderopts):
                transcoder_optsa = transcoder_optsa + transcoderopts['options']
            if (options['ccaption'] and 'subtitleoptions' in transcoderopts):
                transcoder_optsa = transcoder_optsa + transcoderopts['subtitleoptions']
            if ('audiooptions' in transcoderopts):
                transcoder_optsa = transcoder_optsa + transcoderopts['audiooptions']
            if ('metadata' in transcoderopts):
                transcoder_optsa = transcoder_optsa + transcoderopts['metadata']
            transcoder_optsa = transcoder_optsa + [ outputfile ]

            if (debug or not sys.stdout.isatty()):
                print('')
                print(transcoder_name)
                print(transcoder_optsa)
    
            # Step 5, Transcode ts data
            # First check to see if it is an internal function to be called
            if (transcoderopts['command'][0] in TRANSCODER_FUNCS):
                # Remove command from rest of options
                transcoder_optsa = transcoder_optsa[1:]
                try:
                    # Available external python module and func?
                    mod_name, func_name = transcoderopts['command'][0].rsplit('.',1)
                    mod = importlib.import_module(mod_name)
                    func = getattr(mod, func_name)
                    # This was too verbose
                    print(' Executing (' + transcoderopts['command'][0] + ', ' + fext + ')')
                    func(*transcoder_optsa)
                except:
                    # Maybe we have a global defined func
                    try:
                        func = globals()[transcoderopts['command'][0]]
                        # This was too verbose
                        print(' Executing (' + transcoderopts['command'][0] + ', ' + fext + ')')
                        func(*transcoder_optsa)
                    except:
                        printError('Cannot find allowed function [' + transcoderopts['command'][0] + ']')
                        printError('Skipping...')
                        continue
            else:
                if (not debug):
                    ffmpegproc = subprocess.Popen(transcoder_optsa,stderr=subprocess.PIPE,stdout=devnull)
                    # Might hardcode specific progress bars for specific transcoder programs
                    #  Right now, this assumes the transcoder is ffmpeg
                    if (transcoderopts['command'][0] == FFMPEG):
                        ffmpeglen = int(db_rec['video_duration'])
                        while True:
                            ffmpegline = ffmpegproc.stderr.readline()
                            if (not sys.stdout.isatty()):
                                print(ffmpegline.strip())
                            if (ffmpegline == '' or re.search(r'Press \[q\] to stop, \[\?\] for help', ffmpegline)):
                                break
                
                        bar_length=20
                        ffmpeg_count = 1
                        ffmpeg_seconds = 0
                        while True:
                           # This is not perfect, output cannot be read
                           #  by line.  So sometimes we cannot parse the time
                           #  correctly and bar might go backwards.... it is ok.
                           ffmpegline = ffmpegproc.stderr.read(88)
                           if (ffmpegline == ''):
                               break
                           if (not sys.stdout.isatty()):
                               print(ffmpegline.strip())
                           # Check for match, and parse percentage
                           ffmpegprogress = re.match(r'^.* time=(?P<hour>[0-9]+):(?P<minute>[0-9]+):(?P<second>[0-9]+).*', ffmpegline.strip())
                           try:
                               try:
                                   fh = int(ffmpegprogress.group('hour'))
                               except:
                                   pass
                               try:     
                                   fm = int(ffmpegprogress.group('minute'))
                               except:
                                   pass
                               try:
                                   fs = int(ffmpegprogress.group('second'))
                               except:
                                   pass
                               try:
                                   ffmpeg_seconds = fh * 3600 + fm * 60 + fs
                               except:
                                   pass
                               try:
                                   if (sys.stdout.isatty()):
                                       displayProgress(ffmpeg_seconds, ffmpeglen, bar_length, "{0:40s}".format(" Transcoding ({0}, {1}): ".format(transcoder_name,fext)))
                               except:
                                   pass
                           except:
                               pass
                        if (ffmpeg_seconds and sys.stdout.isatty()): 
                            displayProgress(ffmpeglen, ffmpeglen, bar_length, "{0:40s}".format(" Transcoding ({0}, {1}): ".format(transcoder_name,fext)))
                        print('')
                    else:
                        print(' Executing ' + ' '.join(transcoder_optsa))
                        while True:
                            ffmpegline = ffmpegproc.stderr.readline()
                            print(ffmpegline.strip())
                            if (ffmpegline == ''):
                                break
    except:
        pass
    finally:
        # Clean up temporary files
        if (options['ccaption'] and not options['keepdir']):
            try:
                os.remove(srt_filename)
            except:
                pass
        if (not options['keepdir']):
            try:
                os.remove(ts_filename)
            except:
                pass
        print('')


def usage():
    print(__doc__)

try:
    opts, args = getopt.getopt(sys.argv[1:], 'ncdCyvhti:k:q:b:f:E:S:Q:O:', ['noupdate','convert',
        'debug','ccaption','clobber','version','help','transcodehelp','rec_id=','keepdir=','query=','basedir=','filename=','sportsepisode=',
        'sportsseason=','queryformat=','ccoffset='])
except getopt.GetoptError:
    usage()
    sys.exit(2)

search_pat = ''
update=True
convert=False
sportsepisode = ''
sportsseason = ''
basedir = ''
filename_pat = ''
queryformat = ''
transcoderopts = ''
transcoder_names = TRANSCODER_DEFAULT
search_rec_id = 0
cc_offsets = ''
debug = False
for opt, arg in opts:
    if opt in ('-n', '--noupdate'):
        update = False
    elif opt in ('-c', '--convert'):
        convert = True
    elif opt in ('-d', '--debug'):
        debug = True
    elif opt in ('-C', '--ccaption'):
        if (not os.path.isfile(CCEXTRACTOR)):
            printError('Warning: Cannot find ccextractor [' + CCEXTRACTOR + ']')
            printError('Ignoring request for closed captions, ' + opt + ', as subtitles')
        else:
            options['ccaption'] = True
    elif opt in ('-y', '--clobber'):
        options['noclobber'] = False
    elif opt in ('-v', '--version'):
        print('Version: ' + SLT_GLOBAL['PGM_VERSION'])
        sys.exit(0)
    elif opt in ('-h', '--help'):
        detailedHelp()
        sys.exit(0)
    elif opt in ('-t', '--transcodehelp'):
        transcodeHelp()
        sys.exit(0)
    elif opt in ('-i', '--rec_id'):
        search_rec_id = arg
    elif opt in ('-k', '--keepdir'):
        options['keepdir'] = arg
    elif opt in ('-q', '--query'):
        search_pat = arg
    elif opt in ('-b', '--basedir'):
        basedir = arg
    elif opt in ('-f', '--filename'):
        filename_pat = arg
    elif opt in ('-E', '--sportsepisode'):
        sportsepisode = arg
    elif opt in ('-S', '--sportsseason'):
        sportsseason = arg
    elif opt in ('-Q', '--queryformat'):
        queryformat = arg
    elif opt in ('-F', '--transcoderopts'):
        transcoderopts = arg
    elif opt in ('-O', '--ccoffset'):
        cc_offsets = arg

if (args):
    # If the first character of a trancode name is '+', then 
    #  make sure TRANSCODER_DEFAULT is added in.
    newargs = []
    for n in args:
        add_default = False
        if (n[0] == '+'):
            n = n[1:]
            add_default = True
            newargs = newargs + TRANSCODER_DEFAULT 
        if (n not in TRANSCODER_OPTS):
            printError('No transcoder [' + n + '] found. Skipping.')
            continue
        else:
            newargs = newargs + [ n ]
    newset = set(newargs)
    newargs = list(newset)
    transcoder_names = newargs
    if (not transcoder_names):
        printError('No valid transcoder name provided.')
        sys.exit(1)


# If SURLATABLO_ROOT does not exist create it.
get_tablo_root = os.path.expanduser(SURLATABLO_ROOT)
if (not os.path.isdir(get_tablo_root)):
    try:
        os.mkdir(get_tablo_root)
    except:
        printError('Error: Could not create SURLATABLO_ROOT [' + SURLATABLO_ROOT + ']')
        printError('(perhaps you need to change its value near the top of the program)')
        sys.exit(3)

# override itsoffset by meta_type (e.g. -O Sports::5,TV::1,Default::3)
#  Note, in all fairness, may need to check for "live" program
#  in the Tablo/Tribune metadata and set a value based on that.
#  After all, it is not just Sports that tend to be live and
#  some Sports programming may not be live...
if (cc_offsets):
    for ccoffsettuple in cc_offsets.split(','):
        try:
            ccoffset_val = ccoffsettuple.split('::')[1]
            ccoffset_type = ccoffsettuple.split('::')[0]
            # Weird, but could just have the colons
            if (ccoffset_val == '' and ccoffset_type == ''):
                ccoffset_type = 'Default'
            else:
                if (ccoffset_type == ''):
                    ccoffset_type = 'Default'
        except:
            ccoffset_val = cc_offsettuple
            ccoffset_type = 'Default'
        # Set the override
        CCOFFSET[ccoffset_type] = ccoffset_val

basedirs=BASE_DIRS
# Pattern: meta_type::pattern
# Where meta_type could be TV, Movie, Sports, Manual
#  If not supplied, then Default
#  Could be Default:path, or could be meta_type:path,...
#  Alters the base directory for recordings of type meta_type
if (basedir):
    for basetuple in basedir.split(','):
        try:
            meta_path = basetuple.split('::')[1]
            meta_type = basetuple.split('::')[0]
            # Weird, but could just have the colons
            if (meta_path == '' and meta_type == ''):
                meta_type = 'Default'
            else:
                if (meta_type == ''):
                    meta_type = 'Default'
        except:
            meta_path = basetuple
            meta_type = 'Default'
        # Set the override
        basedirs[meta_type] = meta_path

filename_pats=FILENAME_PATS
# Pattern: meta_type::pattern
# Where meta_type could be TV, Movie, Sports, Manual
#  If not supplied, then Default
#  Could be Default:path, or could be meta_type:path,...
#  Alters the output filename for recordings of type meta_type
if (filename_pat):
    for filenametuple in filename_pat.split(','):
        try:
            meta_path = filename_pat.split('::')[1]
            meta_type = filename_pat.split('::')[0]
            # Weird, but could just have the colons
            if (meta_path == '' and meta_type == ''):
                meta_type = 'Default'
            else:
                if (meta_type == ''):
                    meta_type = 'Default'
        except:
            meta_path = filenametuple
            meta_type = 'Default'
        # Set the override
        filename_pats[meta_type] = meta_path

# Handle patterns of the form original_air_date~=1967
#  as '"original_air_date": "1967'
if (search_pat):
    search_pat = transformSearch(search_pat)

# For sake of simplicity, a "tablo_ip" is a tablo IP
for tablo_ip in TABLO_IPS:
    # If tablo subdir, named for the tablo ip, does not exist create it.
    tablo_dir = get_tablo_root + '/' + tablo_ip
    if (not os.path.isdir(tablo_dir)):
        os.mkdir(tablo_dir)

    # Set cache file name and url to nab recording ids from
    rec_ids_cache_file = tablo_dir + '/rec_ids_cache.json'
    rec_ids_db_file = tablo_dir + '/rec_ids_db.json'
    rec_ids_url = Template(TABLO_REC_IDS).substitute(tablo_ip=tablo_ip)

    # Load cache db
    if (os.path.isfile(rec_ids_db_file)):
        with open(rec_ids_db_file,'r') as f:
            db_recs = json.loads('{' + f.read() + '}')
    else:
        db_recs = {}

    # Default is to always update the cache unless --noupdate
    if (update):
        # Channel ids for tablo
        ch_ids_url = Template(TABLO_CHAN_IDS).substitute(tablo_ip=tablo_ip)

        # Get channel ids and info, this is not exact, but works most of the time
        jch_ids = urllib2.urlopen(ch_ids_url)
        ch_ids = json.load(jch_ids)
        channels={}
        for ch_id in ch_ids['ids']:
            ch_info_url = Template(TABLO_CHAN_INFO).substitute(tablo_ip=tablo_ip)
            ch_data_json = urllib2.urlopen(ch_info_url.format(ch_id))
            ch_data = json.load(ch_data_json)
            channel_num = str(ch_data['meta']['channelNumberMajor']) + '.' + str(ch_data['meta']['channelNumberMinor'])
            channel_sign = ch_data['meta']['callSign'].encode('utf-8').strip();
            try:
                channel_affiliate = ch_data['meta']['affiliateCallSign'].encode('utf-8').strip();
            except:
                channel_affiliate = ''
            channel_res_name = ch_data['meta']['resolution']['title'].encode('utf-8').strip();
            channel_res_width = str(ch_data['meta']['resolution']['width'])
            channel_res_height = str(ch_data['meta']['resolution']['height'])
            channels[channel_num] = {'channel_num': channel_num, 'channel_sign': channel_sign,
                'channel_affiliate': channel_affiliate, 'channel_res_name': channel_res_name,
                'channel_res_width': channel_res_width, 'channel_res_height': channel_res_height}
             

        # Get current known recording ids from tablo
        jrec_ids = urllib2.urlopen(rec_ids_url)
        rec_ids = json.load(jrec_ids)

        # Get recording ids from last run cache file
        #  and try to figure out things to add and things to delete from cache
        if (os.path.isfile(rec_ids_cache_file)):
            with open(rec_ids_cache_file, 'r') as f:
                oldrec_ids = json.load(f)
        else:
            # You can erase the rec_ids_cache_file and force creation from scratch of recording
            #  meta data db
            oldrec_ids = {'ids':[]}
    
        # Find new rec_ids and perhaps removed ones
        oldrec_ids_set = set(oldrec_ids['ids'])
        newrec_ids_set = set(rec_ids['ids'])
        new_ids = [aa for aa in newrec_ids_set if aa not in oldrec_ids_set]
        removed_ids = [aa for aa in oldrec_ids_set if aa not in newrec_ids_set]
    
        # Now let us loop through removed_ids (things deleted off the Tablo)  
        #  and remove them from the cache db
        first = True
        for remove_id in removed_ids:
            remove_id_s = str(remove_id)
            print('Trying to remove ' + remove_id_s)
            try:
                if (first):
                    print(re.sub('^','REMOVED:',json.dumps(db_recs[remove_id_s], sort_keys=True, indent=4),0,re.M))
                    first = False
                else:
                    print('REMOVED:,\n' + re.sub('^','REMOVED:',json.dumps(db_recs[remove_id_s], sort_keys=True, indent=4),0,re.M))
                del db_recs[remove_id_s]
            except:
                print('Warning: Could not find rec_id: ' + remove_id_s)
                print('Probably "Live TV" gone away.')
    
        # Loop through and preserve known db_recs from the existing cache db first
        #  This may look weird, but this puts a record on one line, so the db is greppable.
        for rec_id in db_recs:
            if not 'records' in locals():
                records = '"' + str(rec_id) + '": ' + json.dumps(db_recs[rec_id])
            else:
                records = records + ",\n" + '"' + str(rec_id) + '": ' + json.dumps(db_recs[rec_id])
    
        # Now let us loop through the new_ids and add their tablo meta data to our cache db
        for new_id in new_ids:
            new_data_url = Template(TABLO_REC_INFO).substitute(tablo_ip=tablo_ip)
            new_data_json = urllib2.urlopen(new_data_url.format(new_id))
            new_data = json.load(new_data_json)
            try:
                new_data_meta = new_data['meta']
            except:
                print('No meta for rec_id ' + str(new_id) + '. Probably "Live TV".')
                continue
    
    
            # Just in case? Meta key type?
            meta_type = 'Unknown'
            rec_val={}
            rec_val['title'] = 'Unknown ' + str(new_id)

            # Determine meta_type and meta_selector
            if 'recSportEvent' in new_data_meta:
                meta_type = 'Sports'
                meta_selector = 'recSportEvent'
                title_selector = meta_selector
            elif 'recEpisode' in new_data_meta:
                meta_type = 'TV'
                meta_selector = 'recEpisode'
                title_selector = meta_selector
            elif 'recMovie' in new_data_meta:
                meta_type = 'Movie'
                meta_selector = 'recMovieAiring'
                title_selector = 'recMovie'
            elif 'recManualProgram' in new_data_meta:
                meta_type = 'Manual'
                meta_selector = 'recManualProgramAiring'
                title_selector = 'recManualProgram'


            # Do common meta processing base on meta_selector
            rec_val['rec_id'] = new_id
            rec_val['tablo_ip'] = tablo_ip
            rec_val['meta_type'] = meta_type
            try:
                # I do not expect any of this to error, but just in case
                rec_val['video_size'] = new_data_meta[meta_selector]['jsonForClient']['video']['size']
                rec_val['video_sizeh'] = humanizeValue(rec_val['video_size'])
                rec_val['video_width'] = new_data_meta[meta_selector]['jsonForClient']['video']['width']
                rec_val['video_height'] = new_data_meta[meta_selector]['jsonForClient']['video']['height']
                rec_val['video_duration'] = new_data_meta[meta_selector]['jsonForClient']['video']['duration']
                rec_val['video_offsetstart'] = new_data_meta[meta_selector]['jsonForClient']['video']['scheduleOffsetStart']
                rec_val['video_offsetend'] = new_data_meta[meta_selector]['jsonForClient']['video']['scheduleOffsetEnd']
                # video_time
                totalseconds = rec_val['video_duration']
                hours = int(totalseconds / 3600)
                minutes = int((totalseconds - hours * 3600) / 60)
                seconds = int(totalseconds - hours * 3600 - minutes * 60)
                rec_val['video_durationh'] = "{0:d}:{1:02d}:{2:02d}".format(hours,minutes,seconds)
            except:
                pass
            try:
                rec_val['air_date'] = new_data_meta[meta_selector]['jsonForClient']['airDate'].encode('utf-8').strip()
            except:
                rec_val['air_date'] = '1900-01-01T00:00Z'
            try:
                rec_val['title'] = new_data_meta[title_selector]['jsonForClient']['title'].encode('utf-8').strip()
            except:
                # A one off due to where sport titles go
                try:
                    rec_val['title'] = new_data_meta[title_selector]['jsonForClient']['eventTitle'].encode('utf-8').strip()
                except:
                    rec_val['title'] = 'Unknown'
            try:
                channel_num = new_data_meta[meta_selector]['jsonFromTribune']['channels'][0].encode('utf-8').strip()
                chan_data = channels[channel_num]
            except:
                chan_data = {}
            rec_val.update(chan_data)
            # Load air_date as a datetime so we can get individual values
            if (has_pytz):
                uair_datetime = datetime.strptime(rec_val['air_date'], '%Y-%m-%dT%H:%MZ').replace(tzinfo=pytz.utc)
            else:
                uair_datetime = datetime.strptime(rec_val['air_date'], '%Y-%m-%dT%H:%MZ').replace(tzinfo=utc)
            rec_val['air_date_year'] = '{0:04d}'.format(uair_datetime.year)
            rec_val['air_date_month'] = '{0:02d}'.format(uair_datetime.month)
            rec_val['air_date_day'] = '{0:02d}'.format(uair_datetime.day)
            rec_val['air_date_hour'] = '{0:02d}'.format(uair_datetime.hour)
            rec_val['air_date_minute'] = '{0:02d}'.format(uair_datetime.minute)
            if (has_pytz):
                # Create local timezone converted air_date (lair_date) entries 
                ltz = pytz.timezone(LOCAL_TIMEZONE)
            else:
                ltz = timezone[LOCAL_TIMEZONE]
   
            
            lair_datetime = uair_datetime.astimezone(ltz)
            rec_val['lair_date_year'] = '{0:04d}'.format(lair_datetime.year)
            rec_val['lair_date_month'] = '{0:02d}'.format(lair_datetime.month)
            rec_val['lair_date_day'] = '{0:02d}'.format(lair_datetime.day)
            rec_val['lair_date_hour'] = '{0:02d}'.format(lair_datetime.hour)
            rec_val['lair_date_minute'] = '{0:02d}'.format(lair_datetime.minute)
            rec_val['lair_date_tz'] = str(lair_datetime.tzinfo)
            rec_val['lair_date'] = lair_datetime.strftime('%Y-%m-%dT%H:%M%z')
            
            # Do meta_type specific processing
            if 'Sports' == meta_type:
                print('Sporting Event ' + str(new_id))
                teamnames=[]
                try:
                    teams = new_data_meta['recSportEvent']['jsonForClient']['teams']
                except:
                    teams = []
                for team in teams:
                    teamnames.append(team['title'].encode('utf-8').strip())
                try:
                    teams_ = ';'.join(teamnames).encode('utf-8').strip()
                except:
                    teams_ = ''
                try:
                    game_date = new_data_meta['recSportEvent']['jsonFromTribune']['program']['gameDate'].encode('utf-8').strip()
                except:
                    game_date = '1900-01-01'
                try:
                    sport_type = new_data_meta['recSportOrganization']['jsonForClient']['title'].encode('utf-8').strip()
                except:
                    sport_type = 'Unknown'
                try:
                    description = new_data_meta['recSportEvent']['jsonForClient']['description'].encode('utf-8').strip()
                except:
                    description = 'N/A'
                try:
                    ldescription = new_data_meta['recSportEvent']['jsonFromTribune']['program']['longDescription'].encode('utf-8').strip()
                except:
                    ldescription = 'N/A'
                try:
                    lang = new_data_meta['recSportEvent']['jsonFromTribune']['program']['descriptionLang']
                except:
                    lang = 'en'
                    
                rec_val['description'] = description
                rec_val['long_description'] = ldescription
                rec_val['lang'] = lang
                rec_val['teams'] = teamnames
                rec_val['teams_'] = teams_
                rec_val['game_date'] = game_date
                rec_val['sport_type'] = sport_type
                gd = datetime.strptime(rec_val['game_date'], '%Y-%m-%d')
                rec_val['game_date_year'] = '{0:04d}'.format(gd.year)
                rec_val['game_date_month'] = '{0:02d}'.format(gd.month)
                rec_val['game_date_day'] = '{0:02d}'.format(gd.day)
                # Note, this is just the year of the sport event, and not really the season
                rec_val['season_number'] = int(rec_val['game_date_year'])
                # Really do not have episodes (without help), so defaults to zero, 0
                rec_val['episode_number'] = 0
                friendly_title = rec_val['title']
            elif 'TV' == meta_type:
                print('TV Series/Program ' + str(new_id))
                try:
                    original_air_date = new_data_meta['recEpisode']['jsonForClient']['originalAirDate'].encode('utf-8').strip()
                except:
                    original_air_date = '1900-01-01'
                try:
                    series = new_data_meta['recSeries']['jsonForClient']['title'].encode('utf-8').strip()
                except:
                    series = 'Unknown ' + str(new_id)
                try:
                    season_number = new_data_meta['recEpisode']['jsonForClient']['seasonNumber']
                except:
                    season_number = -1
                try:
                    episode_number = new_data_meta['recEpisode']['jsonForClient']['episodeNumber']
                except:
                    episode_number = -1
                if (NOZEROTV):
                    # Why not original_air_date values?  For whatever reason, programs without
                    #  seasons and episodes, like New programs have a constant (and old) date
                    #  assigned as the original_air_date.  We want original_air_date values here
                    #  ideally though, just that they are incorrect in many cases.
                    if (not season_number):
                       season_number = int(rec_val['lair_date_month'])
                    if (not episode_number):
                       episode_number = int(rec_val['lair_date_day'])
                if (season_number >= 0 and episode_number >= 0):
                    if (rec_val['title'] == 'Unknown'):
                        if (NOZEROTV):
                            rec_val['title'] = 'Episode ' + rec_val['lair_date_year'] + '-' + rec_val['lair_date_month'] + '-' + rec_val['lair_date_day']
                        else:
                            rec_val['title'] = 'Episode ' + str(episode_number)
                    friendly_title = series + ' - ' + 's{0:02d}e{1:02d}'.format(season_number,episode_number) + ' - ' + rec_val['title']
                else:
                    friendly_title = series + ' - ' + rec_val['title']
                try:
                    cast = new_data_meta['recSeries']['jsonForClient']['cast']
                except:
                    cast = []
                try:
                    cast_ = ';'.join(cast).encode('utf-8').strip()
                except:
                    cast_ = ''
                try:
                    rating = getRating(new_data_meta['recEpisode']['jsonFromTribune']['ratings'])
                except:
                    rating = 'Unknown'
                try:
                    genres = ';'.join(new_data_meta['recEpisode']['jsonFromTribune']['program']['genres']).encode('utf-8').strip()
                except:
                    genres = ''
                try:
                    description = new_data_meta['recEpisode']['jsonForClient']['description'].encode('utf-8').strip()
                except:
                    description = 'N/A'
                try:
                    ldescription = new_data_meta['recEpisode']['jsonFromTribune']['program']['longDescription'].encode('utf-8').strip()
                except:
                    ldescription = 'N/A'
                try:
                    lang = new_data_meta['recEpisode']['jsonFromTribune']['program']['descriptionLang']
                except:
                    lang = 'en'
                rec_val['description'] = description
                rec_val['long_description'] = ldescription
                rec_val['lang'] = lang
                rec_val['series'] = series
                rec_val['cast'] = cast
                rec_val['cast_'] = cast_
                rec_val['season_number'] = season_number
                rec_val['episode_number'] = episode_number
                rec_val['rating'] = rating
                rec_val['genres'] = genres
                rec_val['original_air_date'] = original_air_date
                oad = datetime.strptime(rec_val['original_air_date'], '%Y-%m-%d')
                rec_val['original_air_date_year'] = '{0:04d}'.format(oad.year)
                rec_val['original_air_date_month'] = '{0:02d}'.format(oad.month)
                rec_val['original_air_date_day'] = '{0:02d}'.format(oad.day)
            elif 'Movie' == meta_type:
                print('Movie ' + str(new_id))
                try:
                    release_year = new_data_meta['recMovie']['jsonForClient']['releaseYear']
                except:
                    release_year = 0
                if (release_year):
                    friendly_title = rec_val['title'] + ' (' + str(release_year) + ')'
                else:
                    friendly_title = rec_val['title']
                try:
                    cast = new_data_meta['recMovie']['jsonForClient']['cast']
                except:
                    cast = []
                try:
                    cast_ = ';'.join(cast).encode('utf-8').strip()
                except:
                    cast_ = ''
                try:
                    rating = getRating(new_data_meta['recMovieAiring']['jsonFromTribune']['ratings'])
                except:
                    rating = 'Unknown'
                try:
                    genres = ';'.join(new_data_meta['recMovieAiring']['jsonFromTribune']['program']['genres']).encode('utf-8').strip()
                except:
                    genres = ''
                try:
                    description = new_data_meta['recMovie']['jsonFromTribune']['shortDescription']
                except:
                    description = 'N/A'
                try:
                    ldescription = new_data_meta['recMovie']['jsonForClient']['plot'].encode('utf-8').strip()
                except:
                    ldescription = 'N/A'
                try:
                    lang = new_data_meta['recMovieAiring']['jsonFromTribune']['program']['descriptionLang'].encode('utf-8').strip()
                except:
                    lang = 'en'
                rec_val['description'] = description
                rec_val['long_description'] = ldescription
                rec_val['release_year'] = release_year
                rec_val['cast'] = cast
                rec_val['cast_'] = cast_
                rec_val['rating'] = rating
                rec_val['genres'] = genres
                rec_val['lang'] = lang
            elif 'Manual' == meta_type:
                print('Manual ' + str(new_id))
                friendly_title = rec_val['title']

            # Populate sort_title and friendly_title
            try:
                rec_val['sort_title'] = sortTitle(rec_val['title'])
            except:
                pass
            rec_val['friendly_title'] = friendly_title

            # If a recording is not in the finished state, then skip and forget it
            try:
                # Skip if not finished recording
                state = new_data_meta[meta_selector]['jsonForClient']['video']['state'].encode('utf-8').strip()
                if (state != 'finished'):
                    rec_ids['ids'].remove(new_id)
                    print('  *RECORDING/BUSY* ' + friendly_title)
                    continue
            except:
                # Safety valve, not sure what would cause this though
                #  maybe brief condition?
                rec_ids['ids'].remove(new_id)
                print('  *UNKNOWN STATE* ' + friendly_title)
                continue

            # friendly_title is just used for output when adding to the db
            print('  ' + friendly_title)
    
            # This may look weird, but this puts a record on one line, so the db is greppable.
            rec_val = json.dumps(rec_val)
            if not 'records' in locals():
                records = '"' + str(new_id) + '": ' + rec_val
            else:
                records = records + ',\n' + '"' + str(new_id) + '": ' + rec_val
    
    if 'records' in locals():
        with open(rec_ids_db_file, 'w') as f:
            f.write(records)
    
    # Save current recording ids to use as difference in next run
    if 'rec_ids' in locals():
        with open(rec_ids_cache_file, 'w') as f:
            json.dump(rec_ids, f)

    # Messy shortcut, reread db_recs.
    if (os.path.isfile(rec_ids_db_file)):
        with open(rec_ids_db_file,'r') as f:
            db_recs = json.loads('{' + f.read() + '}')
    else:
        db_recs = []

    # Maybe we know exactly what we want.  This time pull from full meta though, now our db.
    if (search_rec_id):
        search_data_url = Template(TABLO_REC_INFO).substitute(tablo_ip=tablo_ip)
        search_data_json = urllib2.urlopen(search_data_url.format(search_rec_id))
        search_data = json.load(search_data_json)
        print('"' + search_rec_id + '": ' + json.dumps(search_data, sort_keys=True, indent=4))
        if (convert):
           # Fall through, we have search_pat and convert
           search_pat=transformSearch("rec_id~=" + str(search_rec_id))

    # Attempt search (and optionally convert) if provided
    #  Note, doing --noupdate without --query=search_pat  does nothing at all
    if (search_pat):
        first = True
        for rec_id in db_recs:
            # I am basically grepping through a full metadata line, maybe that
            #  is not the best way to do this (?) or maybe it is.
            record_s = json.dumps(db_recs[rec_id])
            if (re.search(search_pat, record_s)):
                if (queryformat):
                    try:
                        print(Template(queryformat).safe_substitute(db_recs[rec_id]).encode('utf-8').decode('string escape'))
                    except:
                        print(rec_id)
                elif (not convert):
                    if (first):
                        print('"' + rec_id + '": ' + json.dumps(db_recs[rec_id], sort_keys=True, indent=4))
                        first = False
                    else:
                        print(',\n' + '"' + rec_id + '": ' + json.dumps(db_recs[rec_id], sort_keys=True, indent=4))
                if (convert):
                    meta_type = db_recs[rec_id]['meta_type']
                    # If Sports and we have a sportsepisode start num, for each
                    #  sport_type, save records and output later. 
                    #  Why?  I am making up metadata possibly for episode and season.
                    #  This way I can order sport events by game_date, and I will
                    #  assume that I can increment the episode_number by one for each
                    #  sport_type.
                    if (meta_type == 'Sports' and (sportsepisode != '' or sportsseason != '')):
                        print rec_id
                        game_date = db_recs[rec_id]['game_date']
                        sport_type = db_recs[rec_id]['sport_type']
                        # Sport program of each sport_type sorted by game_date ascending
                        #  so we can make up episode numbers incrementally if more than
                        #  one for a sport_type.
                        try:
                            sport_recs[sport_type][str(game_date) + str(rec_id)]=db_recs[rec_id]
                        except:
                            try:
                                sport_recs[sport_type]={game_date: db_recs[rec_id]}
                            except:
                                sport_recs={sport_type: {game_date: db_recs[rec_id]}}
                    else:
                        doConvert(db_recs[rec_id], options, basedirs, filename_pats, transcoder_names)
        # If we have sport_recs, dump them out using sportsepisode counters for each type
        #  Lots of of assumptions.  Basically we are making up meta data that does not exist!
        if 'sport_recs' in locals():
            # If we have sportsseason, could be Default:name, or could be sport_type:name,...
            seasonoverride={'Default':''}
            for sseasontuple in sportsseason.split(','):
                try:
                    sname = sseasontuple.split('::')[1]
                    stype = sseasontuple.split('::')[0]
                    # Weird, but could just have the colons
                    if (sname == '' and stype == ''):
                        stype = 'Default'
                    else:
                        if (stype == ''):
                            stype = 'Default'
                except:
                    sname = sseasontuple
                    stype = 'Default'
                # Set the override
                seasonoverride[stype] = sname
            # If we have sportsepisode, could be Default:strnum, or could be sport_type:strnum,...
            episodeoverride={'Default':'1'}
            for sepisodetuple in sportsepisode.split(','):
                try:
                    estart = sepisodetuple.split('::')[1]
                except:
                    estart = ''
                stype=sepisodetuple.split('::')[0]
                if (estart == ''):
                    estart = stype
                    stype = 'Default'
                else:
                    if (stype == ''):
                        stype = 'Default'
                if (estart == ''):
                    estart = '1'
                episodeoverride[stype] = estart
            for sport_type in sport_recs:
                sport_rec=sport_recs[sport_type]
                # If no specific sport_type override, use Default global ones
                try:
                    e = int(episodeoverride[sport_type])
                except:
                    try:
                        e = int(episodeoverride['Default'])
                    except:
                        printError('Error: non-numeric value supplied for episode number')
                        sys.exit(3)
                try:
                    sname = seasonoverride[sport_type]
                except:
                    sname = seasonoverride['Default']
                for game_date in sorted(sport_rec):
                    print sorted(sport_rec)
                    srec=sport_rec[game_date]
                    srec_id=srec['rec_id']
                    if (not sname):
                        s = int(game_date.split('-')[0])
                    else:
                        try:
                            s = int(sname)
                        except:
                            s = 1
                    options['season_number'] = s
                    options['episode_number'] = e
                    doConvert(db_recs[str(srec_id)], options, basedirs, filename_pats, transcoder_names)
                    e+=1

sys.exit()