loappimage-helpers/loaih/__init__.py

#!/usr/bin/env python3

import urllib.request
import loaih.versions as versions
from lxml import etree
import tempfile, os, sys, glob, subprocess, shutil

class Build(object):
    LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ]
    LANGBASIC = [ 'en-GB' ]

    def __init__(self, query, arch):
        """Build all versions that can be found in the indicated repo."""
        self.query = query
        self.queried_name = False if '.' in self.query else True
        self.arch = arch
        self.url = {}
        self.language = 'basic'
        self.offline_help = False
        self.portable = False
        self.updatable = True
        self.storage_path = '/srv/http/appimage.sys42.eu'
        self.download_path = '/var/tmp/downloads'
        self.appversion = ''
        self.appimagefilename = {}
        self.genappversion = ''
        self.genappimagefilename = {}

        # Getting versions and so on
        v = versions.BuildVersion(self.query) 

        # Creating a tempfile
        self.builddir = tempfile.mkdtemp()
        self.tarballs = {}
        self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
        self.version = v.version
        self.url = v.basedirurl
        self.built = False

        # Building expected AppImageName
        self.languagepart = "."
        if ',' in self.language:
            self.languagepart += self.language.replace(',', '-')
        else:
            self.languagepart += self.language

        self.helppart = '.help' if self.offline_help else ''

        # If the build was called by queried name, build from latest release available but build with the most generic name
        myver = str.join('.', self.version.split('.')[0:2])
        self.genappversion = myver + self.languagepart + self.helppart
        self.genappimagefilename[u'x86'] = self.appname + '-' + self.genappversion + self.languagepart + self.helppart + '-x86.AppImage'
        self.genappimagefilename[u'x86_64'] = self.appname + '-' + self.genappversion + self.languagepart + self.helppart + '-x86_64.AppImage'

        self.appversion = self.version + self.languagepart + self.helppart
        self.appimagefilename[u'x86'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86.AppImage'
        self.appimagefilename[u'x86_64'] = self.appname + '-' + self.version + self.languagepart + self.helppart + '-x86_64.AppImage'


    def check(self, storage_path):
        """Checking if the requested AppImage has been already built."""
        self.storage_path = storage_path
        self.storage_path += ('/daily' if self.query == 'daily' else '')
        self.storage_path += ('/prerelease' if self.query == 'prerelease' else '')
        self.storage_path += ('/portable' if self.portable else '')
        # Incompatibilities - if portable and updatable are asked together,
        # only portable will be built.
        if self.portable and self.updatable:
            print("Upgradable and portable options were required together. Building only portable.")
            self.updatable = False

        if self.updatable and not self.queried_name:
            # If the queried version was a numbered version, doesn't make sense
            # to build an updatable version.
            self.updatable = False

        for arch in self.arch:
            # For generalized builds, we need to check if there are .ver file
            # and it contains the specific version found.
            res = subprocess.check_output("find {path} -name {appimage}'".format(
                path = self.storage_path,
                appimage = self.genappimagefilename[arch] + '.ver'
            ), shell=True).decode('utf-8').strip('\n')
            for file in res:
                if self.version in open(file, 'r').read():
                    self.built = True

            res = subprocess.check_output("find {path} -name '{appimage}'".format(
                path = self.storage_path,
                appimage = self.appimagefilename[arch]
            ), shell=True).decode('utf-8').strip('\n')
            if len(res) > 1:
                self.built = True

        if self.built:
            print("The requested AppImage already exists on storage. I'll skip downloading, building and moving the results.")

    def download(self, download_path):
        """Downloads the contents of the URL as it was a folder."""
        if self.built:
            return

        # Let's start with defining which files are to be downloaded.
        # Let's explore the remote folder.
        self.download_path = download_path
        
        for arch in self.arch:
            # Checking if a valid path has been provided
            if self.url[arch] == '-':
                print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch))
                continue

            contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a")
            self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ]
            tarballs = self.tarballs[arch]
            maintarball = tarballs[0]

            os.makedirs(self.download_path, exist_ok = True)
            os.chdir(self.download_path)
            for archive in tarballs:
                # If the archive is already there, do not do anything.
                if os.path.exists(os.path.join(self.download_path, archive)):
                    print("Archive %s is already there! Sweet" % archive)
                    continue

                # Download the archive
                try:
                    urllib.request.urlretrieve(self.url[arch] + archive, archive)
                except:
                    print("Failed to download {archive}.".format(archive = archive))

                print("Got %s." % archive)

    def build(self):
        """Building all the versions."""
        if self.built:
            return

        # We have 4 builds to do:
        # * standard languages, no help
        # * standard languages + offline help
        # * all languages, no help
        # * all languages + offline help

        if self.portable and not 'portable' in self.storage_path:
            self.storage_path += "/portable"

        for arch in self.arch:
            # Checking if a valid path has been provided
            if self.url[arch] == '-':
                # User has been warned in download step.
                continue

            # Preparation tasks
            self.appnamedir = os.path.join(self.builddir, self.appname)
            self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
            os.makedirs(self.appimagedir, exist_ok = True)
            # And then cd to the appname folder.
            os.chdir(self.appnamedir)
            # Download appimagetool from github
            appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch)
            urllib.request.urlretrieve(appimagetoolurl, 'appimagetool')
            os.chmod('appimagetool', 0o755)

            # Build the requested version.
            if self.queried_name:
                self.__unpackbuild__(arch, True)
            self.__unpackbuild__(arch)

    def __unpackbuild__(self, arch, generalize = False):
        # We start by filtering out tarballs from the list
        buildtarballs = [ self.tarballs[arch][0] ] 

        # Let's process standard languages and append results to the
        # buildtarball
        if self.language == 'basic':
            buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x])
            if self.offline_help:
                buildtarballs.extend([ x for x in self.tarballs[arch] if 'helppack_en-GB' in x ])
        if self.language == 'standard':
            for lang in Build.LANGSTD:
                buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ])
                if self.offline_help:
                    buildtarballs.extend([ x for x in self.tarballs[arch] if ('helppack' + lang) in x ])
        else:
            # In any other cases, we build with all languages
            if not self.offline_help:
                buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ])
            else:
                # We need also all help. Let's replace buildtarball with the
                # whole bunch
                buildtarballs = self.tarballs[arch]
                
        # Unpacking the tarballs
        for archive in buildtarballs:
            subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True)

        os.chdir(self.appnamedir)

        os.makedirs(self.appimagedir, exist_ok = True)
        # At this point, let's decompress the deb packages
        subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir)

        if self.portable:
            shortversion = str.join('.', self.version.split('.')[:3])
            subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir)

        # Changing desktop file
        subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir)
        subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir)

        subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir)

        # Find the name of the binary called in the desktop file.
        binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n')

        bindir=os.path.join(self.appimagedir, 'usr', 'bin')
        os.makedirs(bindir, exist_ok = True)
        subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir)

        # Download AppRun from github
        apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch)
        dest = os.path.join(self.appimagedir, 'AppRun')
        urllib.request.urlretrieve(apprunurl, dest)
        os.chmod(dest, 0o755)
        
        # Building app
        if self.updatable:
            # Updatable make sense only for generic images for fresh, still,
            # daily. If a request was for a specific version, I'd not build an
            # updatable version.
            # zsync name was generated already

            # If asked to do a generalized build:
            if generalize:
                subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True)
                # Build version file management
                with open(self.appimagefilename[arch] + '.ver', 'w') as v:
                    v.write(self.version)
            else:
                subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True)

        else:
            if generalize:
                subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.genappversion, appname = self.appname), shell=True)
                with open(self.appimagefilename[arch] + '.ver', 'w') as v:
                    v.write(self.version)
            else:
                subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.appversion, appname = self.appname), shell=True)

        print("Built AppImage version {version}".format(version = self.appversion))

        # Cleanup phase, before new run.
        for deb in glob.glob(self.appnamedir + '/*.deb'):
            os.remove(deb)
        subprocess.run("find . -type d -maxdepth 1 -exec rm -rf {} \+", shell=True)

    def checksums(self):
        """Create checksums of the built versions."""
        if self.built:
            return 

        os.chdir(self.appnamedir)
        for appimage in glob.glob('*.AppImage*'):
            # See if a checksum already exist
            if not os.path.exists(appimage + '.md5'):
                subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True)


    def move(self):
        """Moves built versions to definitive storage."""
        if self.built:
            return

        os.chdir(self.appnamedir)
        subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.storage_path, shell=True)

    def __del__(self):
        """Destructor"""
        # Cleaning up build directory
        shutil.rmtree(self.builddir)