#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import with_statement

__license__   = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'

import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time

from setup import (Command, modules, functions, basenames, __version__,
    __appname__)
from setup.build_environment import msvc, MT, RC
from setup.installer.windows.wix import WixMixIn

OPENSSL_DIR = r'Q:\openssl'
QT_DIR = 'Q:\\Qt\\4.8.1'
QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
SW               = r'C:\cygwin\home\kovid\sw'
IMAGEMAGICK      = os.path.join(SW, 'build', 'ImageMagick-6.6.6',
        'VisualMagick', 'bin')
CRT = r'C:\Microsoft.VC90.CRT'

VERSION = re.sub('[a-z]\d+', '', __version__)
WINVER = VERSION+'.0'

DESCRIPTIONS = {
        'calibre' : 'The main calibre program',
        'ebook-viewer' : 'Viewer for all e-book formats',
        'lrfviewer'    : 'Viewer for LRF files',
        'ebook-convert': 'Command line interface to the conversion/news download system',
        'ebook-meta'   : 'Command line interface for manipulating e-book metadata',
        'calibredb'    : 'Command line interface to the calibre database',
        'calibre-launcher' : 'Utility functions common to all executables',
        'calibre-debug' : 'Command line interface for calibre debugging/development',
        'calibre-customize' : 'Command line interface to calibre plugin system',
        'pdfmanipulate' : 'Command line tool to manipulate PDF files',
        'calibre-server': 'Standalone calibre content server',
        'calibre-parallel': 'calibre worker process',
        'calibre-smtp' : 'Command line interface for sending books via email',
}

def walk(dir):
    ''' A nice interface to os.walk '''
    for record in os.walk(dir):
        for f in record[-1]:
            yield os.path.join(record[0], f)


class Win32Freeze(Command, WixMixIn):

    description = 'Freeze windows calibre installation'

    def add_options(self, parser):
        parser.add_option('--no-ice', default=False, action='store_true',
                help='Disable ICE checks when building MSI (needed when running'
                ' from cygwin sshd)')
        parser.add_option('--msi-compression', '--compress', default='high',
                help='Compression when generating installer. Set to none to disable')
        parser.add_option('--keep-site', default=False, action='store_true',
            help='Keep human readable site.py')
        parser.add_option('--verbose', default=0, action="count",
                help="Be more verbose")

    def run(self, opts):
        self.SW = SW
        self.opts = opts
        self.src_root = self.d(self.SRC)
        self.base = self.j(self.d(self.SRC), 'build', 'winfrozen')
        self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc')
        self.py_ver = ''.join(map(str, sys.version_info[:2]))
        self.lib_dir = self.j(self.base, 'Lib')
        self.pylib = self.j(self.base, 'pylib.zip')
        self.dll_dir = self.j(self.base, 'DLLs')
        self.plugins_dir = os.path.join(self.base, 'plugins2')
        self.portable_base = self.j(self.d(self.base), 'Calibre Portable')
        self.obj_dir = self.j(self.src_root, 'build', 'launcher')

        self.initbase()
        self.build_launchers()
        self.add_plugins()
        self.freeze()
        self.embed_manifests()
        self.install_site_py()
        self.archive_lib_dir()
        self.remove_CRT_from_manifests()
        self.create_installer()
        self.build_portable()

    def remove_CRT_from_manifests(self):
        '''
        The dependency on the CRT is removed from the manifests of all DLLs.
        This allows the CRT loaded by the .exe files to be used instead.
        '''
        search_pat = re.compile(r'(?is)<dependency>.*Microsoft\.VC\d+\.CRT')
        repl_pat = re.compile(
            r'(?is)<dependency>.*?Microsoft\.VC\d+\.CRT.*?</dependency>')

        for dll in glob.glob(self.j(self.dll_dir, '*.dll')):
            bn = self.b(dll)
            with open(dll, 'rb') as f:
                raw = f.read()
            match = search_pat.search(raw)
            if match is None:
                continue
            self.info('Removing CRT dependency from manifest of: %s'%bn)
            # Blank out the bytes corresponding to the dependency specification
            nraw = repl_pat.sub(lambda m: b' '*len(m.group()), raw)
            if len(nraw) != len(raw):
                raise Exception('Something went wrong with %s'%bn)
            with open(dll, 'wb') as f:
                f.write(nraw)

    def initbase(self):
        if self.e(self.base):
            shutil.rmtree(self.base)
        os.makedirs(self.base)

    def add_plugins(self):
        self.info('Adding plugins...')
        tgt = self.plugins_dir
        if os.path.exists(tgt):
            shutil.rmtree(tgt)
        os.mkdir(tgt)
        base = self.j(self.SRC, 'calibre', 'plugins')
        for f in glob.glob(self.j(base, '*.pyd')):
            # We dont want the manifests as the manifest in the exe will be
            # used instead
            shutil.copy2(f, tgt)

    def freeze(self):
        shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)

        self.info('Adding CRT')
        shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT)))

        self.info('Adding resources...')
        tgt = self.j(self.base, 'resources')
        if os.path.exists(tgt):
            shutil.rmtree(tgt)
        shutil.copytree(self.j(self.src_root, 'resources'), tgt)

        self.info('Adding Qt and python...')
        shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir,
                ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
        for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
            shutil.copy2(x, self.dll_dir)
        for x in QT_DLLS:
            x += '4.dll'
            if not x.startswith('phonon'): x = 'Qt'+x
            shutil.copy2(os.path.join(QT_DIR, 'bin', x), self.dll_dir)
        shutil.copy2(r'C:\windows\system32\python%s.dll'%self.py_ver,
                    self.dll_dir)
        for x in os.walk(r'C:\Python%s\Lib'%self.py_ver):
            for f in x[-1]:
                if f.lower().endswith('.dll'):
                    f = self.j(x[0], f)
                    shutil.copy2(f, self.dll_dir)
        shutil.copy2(
            r'C:\Python%(v)s\Lib\site-packages\pywin32_system32\pywintypes%(v)s.dll'
            % dict(v=self.py_ver), self.dll_dir)

        def ignore_lib(root, items):
            ans = []
            for x in items:
                ext = os.path.splitext(x)[1]
                if (not ext and (x in ('demos', 'tests'))) or \
                    (ext in ('.dll', '.chm', '.htm', '.txt')):
                    ans.append(x)
            return ans

        shutil.copytree(r'C:\Python%s\Lib'%self.py_ver, self.lib_dir,
                ignore=ignore_lib)

        # Fix win32com
        sp_dir = self.j(self.lib_dir, 'site-packages')
        comext = self.j(sp_dir, 'win32comext')
        shutil.copytree(self.j(comext, 'shell'), self.j(sp_dir, 'win32com', 'shell'))
        shutil.rmtree(comext)

        # Fix PyCrypto, removing the bootstrap .py modules that load the .pyd
        # modules, since they do not work when in a zip file
        for crypto_dir in glob.glob(self.j(sp_dir, 'pycrypto-*', 'Crypto')):
            for dirpath, dirnames, filenames in os.walk(crypto_dir):
                for f in filenames:
                    name, ext = os.path.splitext(f)
                    if ext == '.pyd':
                        with open(self.j(dirpath, name+'.py')) as f:
                            raw = f.read().strip()
                        if (not raw.startswith('def __bootstrap__') or not
                                raw.endswith('__bootstrap__()')):
                            raise Exception('The PyCrypto file %r has non'
                                    ' bootstrap code'%self.j(dirpath, f))
                        for ext in ('.py', '.pyc', '.pyo'):
                            x = self.j(dirpath, name+ext)
                            if os.path.exists(x):
                                os.remove(x)

        for pat in (r'PyQt4\uic\port_v3', ):
            x = glob.glob(self.j(self.lib_dir, 'site-packages', pat))[0]
            shutil.rmtree(x)

        self.info('Adding calibre sources...')
        for x in glob.glob(self.j(self.SRC, '*')):
            shutil.copytree(x, self.j(sp_dir, self.b(x)))

        for x in (r'calibre\manual', r'calibre\trac', 'pythonwin'):
            deld = self.j(sp_dir, x)
            if os.path.exists(deld):
                shutil.rmtree(deld)

        for x in os.walk(self.j(sp_dir, 'calibre')):
            for f in x[-1]:
                if not f.endswith('.py'):
                    os.remove(self.j(x[0], f))

        self.info('Byte-compiling all python modules...')
        for x in ('test', 'lib2to3', 'distutils'):
            shutil.rmtree(self.j(self.lib_dir, x))
        for x in os.walk(self.lib_dir):
            root = x[0]
            for f in x[-1]:
                if f.endswith('.py'):
                    y = self.j(root, f)
                    rel = os.path.relpath(y, self.lib_dir)
                    try:
                        py_compile.compile(y, dfile=rel, doraise=True)
                        os.remove(y)
                    except:
                        self.warn('Failed to byte-compile', y)
                    pyc, pyo = y+'c', y+'o'
                    epyc, epyo, epy = map(os.path.exists, (pyc,pyo,y))
                    if (epyc or epyo) and epy:
                        os.remove(y)
                    if epyo and epyc:
                        os.remove(pyc)

        self.info('\nAdding Qt plugins...')
        qt_prefix = QT_DIR
        plugdir = self.j(qt_prefix, 'plugins')
        tdir = self.j(self.base, 'qt_plugins')
        for d in ('imageformats', 'codecs', 'iconengines'):
            self.info('\t', d)
            imfd = os.path.join(plugdir, d)
            tg = os.path.join(tdir, d)
            if os.path.exists(tg):
                shutil.rmtree(tg)
            shutil.copytree(imfd, tg)
        for dirpath, dirnames, filenames in os.walk(tdir):
            for x in filenames:
                if not x.endswith('.dll'):
                    os.remove(self.j(dirpath, x))

        print
        print 'Adding third party dependencies'
        print '\tAdding unrar'
        shutil.copyfile(LIBUNRAR,
                os.path.join(self.dll_dir, os.path.basename(LIBUNRAR)))

        print '\tAdding misc binary deps'
        bindir = os.path.join(SW, 'bin')
        shutil.copy2(os.path.join(bindir, 'pdftohtml.exe'), self.base)
        for pat in ('*.dll',):
            for f in glob.glob(os.path.join(bindir, pat)):
                ok = True
                for ex in ('expatw', 'testplug'):
                    if ex in f.lower():
                        ok = False
                if not ok: continue
                dest = self.dll_dir
                shutil.copy2(f, dest)
        for x in ('zlib1.dll', 'libxml2.dll'):
            shutil.copy2(self.j(bindir, x+'.manifest'), self.dll_dir)

        shutil.copytree(os.path.join(SW, 'etc', 'fonts'),
						os.path.join(self.base, 'fontconfig'))
        # Copy ImageMagick
        for pat in ('*.dll', '*.xml'):
            for f in glob.glob(self.j(IMAGEMAGICK, pat)):
                ok = True
                for ex in ('magick++', 'x11.dll', 'xext.dll'):
                    if ex in f.lower(): ok = False
                if not ok: continue
                shutil.copy2(f, self.dll_dir)

    def embed_manifests(self):
        self.info('Embedding remaining manifests...')
        for x in os.walk(self.base):
            for f in x[-1]:
                base, ext = os.path.splitext(f)
                if ext != '.manifest': continue
                dll = self.j(x[0], base)
                manifest = self.j(x[0], f)
                res = 2
                if os.path.splitext(dll)[1] == '.exe':
                    res = 1
                if os.path.exists(dll):
                    self.run_builder([MT, '-manifest', manifest,
                        '-outputresource:%s;%d'%(dll,res)])
                    os.remove(manifest)

    def compress(self):
        self.info('Compressing app dir using 7-zip')
        subprocess.check_call([r'C:\Program Files\7-Zip\7z.exe', 'a', '-r',
            '-scsUTF-8', '-sfx', 'winfrozen', 'winfrozen'], cwd=self.base)

    def embed_resources(self, module, desc=None):
        icon_base = self.j(self.src_root, 'icons')
        icon_map = {'calibre':'library', 'ebook-viewer':'viewer',
                'lrfviewer':'viewer', 'calibre-portable':'library'}
        file_type = 'DLL' if module.endswith('.dll') else 'APP'
        template = open(self.rc_template, 'rb').read()
        bname = self.b(module)
        internal_name = os.path.splitext(bname)[0]
        icon = icon_map.get(internal_name, 'command-prompt')
        icon = self.j(icon_base, icon+'.ico')
        if desc is None:
            defdesc = 'A dynamic link library' if file_type == 'DLL' else \
                    'An executable program'
            desc = DESCRIPTIONS.get(internal_name, defdesc)
        license = 'GNU GPL v3.0'
        def e(val): return val.replace('"', r'\"')
        rc = template.format(
                icon=icon,
                file_type=e(file_type),
                file_version=e(WINVER.replace('.', ',')),
                file_version_str=e(WINVER),
                file_description=e(desc),
                internal_name=e(internal_name),
                original_filename=e(bname),
                product_version=e(WINVER.replace('.', ',')),
                product_version_str=e(__version__),
                product_name=e(__appname__),
                product_description=e(__appname__+' - E-book management'),
                legal_copyright=e(license),
                legal_trademarks=e(__appname__ + \
                        ' is a registered U.S. trademark number 3,666,525')
        )
        tdir = self.obj_dir
        rcf = self.j(tdir, bname+'.rc')
        with open(rcf, 'wb') as f:
            f.write(rc)
        res = self.j(tdir, bname + '.res')
        cmd = [RC, '/n', '/fo'+res, rcf]
        self.run_builder(cmd)
        return res

    def install_site_py(self):
        if not os.path.exists(self.lib_dir):
            os.makedirs(self.lib_dir)
        shutil.copy2(self.j(self.d(__file__), 'site.py'), self.lib_dir)
        y = os.path.join(self.lib_dir, 'site.py')
        py_compile.compile(y, dfile='site.py', doraise=True)
        if not self.opts.keep_site:
            os.remove(y)

    def run_builder(self, cmd):
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)
        if p.wait() != 0:
            self.info('Failed to run builder:')
            self.info(*cmd)
            self.info(p.stdout.read())
            self.info(p.stderr.read())
            sys.exit(1)

    def build_portable(self):
        base  = self.portable_base
        if os.path.exists(base):
            shutil.rmtree(base)
        os.makedirs(base)
        src = self.j(self.src_root, 'setup', 'installer', 'windows',
                'portable.c')
        obj = self.j(self.obj_dir, self.b(src)+'.obj')
        cflags  = '/c /EHsc /MT /W3 /Ox /nologo /D_UNICODE /DUNICODE'.split()

        if self.newer(obj, [src]):
            self.info('Compiling', obj)
            cmd = [msvc.cc] + cflags + ['/Fo'+obj, '/Tc'+src]
            self.run_builder(cmd)

        exe = self.j(base, 'calibre-portable.exe')
        if self.newer(exe, [obj]):
            self.info('Linking', exe)
            cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:X86',
                    '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:WINDOWS',
                    '/RELEASE',
                    '/ENTRY:wWinMainCRTStartup',
                    '/OUT:'+exe, self.embed_resources(exe),
                    obj, 'User32.lib']
            self.run_builder(cmd)

        self.info('Creating portable installer')
        shutil.copytree(self.base, self.j(base, 'Calibre'))
        os.mkdir(self.j(base, 'Calibre Library'))
        os.mkdir(self.j(base, 'Calibre Settings'))

        name = '%s-portable-%s.zip'%(__appname__, __version__)
        with zipfile.ZipFile(self.j('dist', name), 'w', zipfile.ZIP_DEFLATED) as zf:
            self.add_dir_to_zip(zf, base, 'Calibre Portable')

    def add_dir_to_zip(self, zf, path, prefix=''):
        '''
        Add a directory recursively to the zip file with an optional prefix.
        '''
        if prefix:
            zi = zipfile.ZipInfo(prefix+'/')
            zi.external_attr = 16
            zf.writestr(zi, '')
        cwd = os.path.abspath(os.getcwd())
        try:
            os.chdir(path)
            fp = (prefix + ('/' if prefix else '')).replace('//', '/')
            for f in os.listdir('.'):
                arcname = fp + f
                if os.path.isdir(f):
                    self.add_dir_to_zip(zf, f, prefix=arcname)
                else:
                    zf.write(f, arcname)
        finally:
            os.chdir(cwd)

    def build_launchers(self):
        if not os.path.exists(self.obj_dir):
            os.makedirs(self.obj_dir)
        base = self.j(self.src_root, 'setup', 'installer', 'windows')
        sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']]
        headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']]
        objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources]
        cflags  = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
        cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver]
        for src, obj in zip(sources, objects):
            if not self.newer(obj, headers+[src]): continue
            cmd = [msvc.cc] + cflags + ['/Fo'+obj, '/Tc'+src]
            self.run_builder(cmd)

        dll = self.j(self.obj_dir, 'calibre-launcher.dll')
        ver = '.'.join(__version__.split('.')[:2])
        if self.newer(dll, objects):
            cmd = [msvc.linker, '/DLL', '/INCREMENTAL:NO', '/VERSION:'+ver,
                    '/OUT:'+dll, '/nologo', '/MACHINE:X86'] + objects + \
                [self.embed_resources(dll),
                '/LIBPATH:C:/Python%s/libs'%self.py_ver,
                'python%s.lib'%self.py_ver,
                '/delayload:python%s.dll'%self.py_ver]
            self.info('Linking calibre-launcher.dll')
            self.run_builder(cmd)

        src = self.j(base, 'main.c')
        shutil.copy2(dll, self.base)
        for typ in ('console', 'gui', ):
            self.info('Processing %s launchers'%typ)
            subsys = 'WINDOWS' if typ == 'gui' else 'CONSOLE'
            for mod, bname, func in zip(modules[typ], basenames[typ],
                    functions[typ]):
                xflags = list(cflags)
                if typ == 'gui':
                    xflags += ['/DGUI_APP=']

                xflags += ['/DMODULE="%s"'%mod, '/DBASENAME="%s"'%bname,
                    '/DFUNCTION="%s"'%func]
                dest = self.j(self.obj_dir, bname+'.obj')
                if self.newer(dest, [src]+headers):
                    self.info('Compiling', bname)
                    cmd = [msvc.cc] + xflags + ['/Tc'+src, '/Fo'+dest]
                    self.run_builder(cmd)
                exe = self.j(self.base, bname+'.exe')
                lib = dll.replace('.dll', '.lib')
                if self.newer(exe, [dest, lib, self.rc_template, __file__]):
                    self.info('Linking', bname)
                    cmd = [msvc.linker] + ['/INCREMENTAL:NO', '/MACHINE:X86',
                            '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:'+subsys,
                            '/LIBPATH:C:/Python%s/libs'%self.py_ver, '/RELEASE',
                            '/OUT:'+exe, self.embed_resources(exe),
                            dest, lib]
                    self.run_builder(cmd)

    def archive_lib_dir(self):
        self.info('Putting all python code into a zip file for performance')
        self.zf_timestamp = time.localtime(time.time())[:6]
        self.zf_names = set()
        with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf:
            # Add the .pyds from python and calibre to the zip file
            for x in (self.plugins_dir, self.dll_dir):
                for pyd in os.listdir(x):
                    if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd':
                        # sqlite_custom has to be a file for
                        # sqlite_load_extension to work
                        self.add_to_zipfile(zf, pyd, x)
                        os.remove(self.j(x, pyd))

            # Add everything in Lib except site-packages to the zip file
            for x in os.listdir(self.lib_dir):
                if x == 'site-packages':
                    continue
                self.add_to_zipfile(zf, x, self.lib_dir)

            sp = self.j(self.lib_dir, 'site-packages')
            # Special handling for PIL and pywin32
            handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32'])
            self.add_to_zipfile(zf, 'PIL', sp)
            base = self.j(sp, 'win32', 'lib')
            for x in os.listdir(base):
                if os.path.splitext(x)[1] not in ('.exe',):
                    self.add_to_zipfile(zf, x, base)
            base = self.d(base)
            for x in os.listdir(base):
                if not os.path.isdir(self.j(base, x)):
                    if os.path.splitext(x)[1] not in ('.exe',):
                        self.add_to_zipfile(zf, x, base)

            handled.add('easy-install.pth')
            for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')):
                handled.add(self.b(d))
                for x in os.listdir(d):
                    if x == 'EGG-INFO':
                        continue
                    self.add_to_zipfile(zf, x, d)

            # The rest of site-packages
            # We dont want the site.py from site-packages
            handled.add('site.pyo')
            for x in os.listdir(sp):
                if x in handled or x.endswith('.egg-info'):
                    continue
                absp = self.j(sp, x)
                if os.path.isdir(absp):
                    if not os.listdir(absp):
                        continue
                    self.add_to_zipfile(zf, x, sp)
                else:
                    self.add_to_zipfile(zf, x, sp)

        shutil.rmtree(self.lib_dir)

    def get_pth_dirs(self, pth):
        base = os.path.dirname(pth)
        for line in open(pth).readlines():
            line = line.strip()
            if not line or line.startswith('#') or line.startswith('import'):
                continue
            candidate = self.j(base, line)
            if os.path.exists(candidate):
                yield candidate

    def add_to_zipfile(self, zf, name, base, exclude=frozenset()):
        abspath = self.j(base, name)
        name = name.replace(os.sep, '/')
        if name in self.zf_names:
            raise ValueError('Already added %r to zipfile [%r]'%(name, abspath))
        zinfo = zipfile.ZipInfo(filename=name, date_time=self.zf_timestamp)

        if os.path.isdir(abspath):
            if not os.listdir(abspath):
                return
            zinfo.external_attr = 0700 << 16
            zf.writestr(zinfo, '')
            for x in os.listdir(abspath):
                if x not in exclude:
                    self.add_to_zipfile(zf, name + os.sep + x, base)
        else:
            ext = os.path.splitext(name)[1].lower()
            if ext in ('.dll',):
                raise ValueError('Cannot add %r to zipfile'%abspath)
            zinfo.external_attr = 0600 << 16
            if ext in ('.py', '.pyc', '.pyo', '.pyd'):
                with open(abspath, 'rb') as f:
                    zf.writestr(zinfo, f.read())

        self.zf_names.add(name)



