#!/usr/bin/env python3 import urllib.request import loaih.versions as versions from lxml import etree import tempfile, os, sys, glob, subprocess, shutil, re class Build(object): LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] LANGBASIC = [ 'en-GB' ] ARCHSTD = [ u'x86', u'x86_64' ] def __init__(self, query, arch): """Build all versions that can be found in the indicated repo.""" self.query = query self.queried_name = False if '.' in self.query else True self.arch = arch self.url = {} self.language = 'basic' self.offline_help = False self.portable = False self.updatable = True self.storage_path = '/srv/http/appimage.sys42.eu' self.download_path = '/var/tmp/downloads' self.appversion = '' self.appimagefilename = {} self.genappversion = '' self.genappimagefilename = {} # Getting versions and so on v = versions.BuildVersion(self.query) # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = {} self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' self.version = v.version self.url = v.basedirurl self.built = { u'x86': False, u'x86_64': False } # Preparing the default for the relative path on the storage for # different versions. # The path will evaluated as part of the check() function, as it is # understood the storage_path can be changed before that phase. self.relative_path = [] self.full_path = '' def check(self, storage_path): """Checking if the requested AppImage has been already built.""" # Mandate to the private function to calculate the full_path available # for the storage and the checks. self.__calculate_full_path__() # Incompatibilities - if portable and updatable are asked together, # only portable will be built. if self.portable and self.updatable: print("Upgradable and portable options were required together. Building only portable.") self.updatable = False if self.updatable and not self.queried_name: # If the queried version was a numbered version, doesn't make sense # to build an updatable version. self.updatable = False # Building expected AppImageName self.languagepart = "." if ',' in self.language: self.languagepart += self.language.replace(',', '-') else: self.languagepart += self.language self.helppart = '.help' if self.offline_help else '' # If the build was called by queried name, build from latest release available but build with the most generic name myver = str.join('.', self.version.split('.')[0:2]) self.genappversion = myver + self.languagepart + self.helppart self.appversion = self.version + self.languagepart + self.helppart for arch in ARCHSTD: self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' for arch in self.arch: # For generalized builds, we need to check if there are .ver file # and it contains the specific version found. print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver')) res = subprocess.run("find {path} -name {appimage}'".format( path = self.full_path, appimage = self.genappimagefilename[arch] + '.ver' ), shell=True, capture_output=True, env={ "LC_ALL": "C" }) if "No such file or directory" in res.stderr.decode('utf-8'): # Folder is not existent: so the version was not built # Build stays false, and we go to the next arch continue if res.stdout: # All good, the command was executed fine. for file in res.stdout.decode('utf-8').strip('\n').split('\n'): if self.version in open(file, 'r').read(): self.built[arch] = True print("Debug: searching for {file}".format(file = self.appimagefilename[arch])) res = subprocess.run("find {path} -name '{appimage}'".format( path = self.full_path, appimage = self.appimagefilename[arch] ), shell=True, capture_output=True) if res.stdout: if len(res.stdout.decode('utf-8').strip('\n')) > 1: self.built[arch] = True if self.built[arch]: print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch)) def __calculate_full_path__(self): """Calculate relative path of the build, based on internal other variables.""" if len(self.relative_path) == 0: if self.query == 'daily': self.relative_path.append('daily') elif self.query == 'prerelease': self.relative_path.append('prerelease') # Not the same check, an additional one if self.portable: self.relative_path.append('portable') fullpath_arr = self.storage_path.split('/') # Joining relative path only if it is not null if len(self.relative_path) > 0: fullpath_arr.extend(self.relative_path) self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) def download(self, download_path): """Downloads the contents of the URL as it was a folder.""" # Let's start with defining which files are to be downloaded. # Let's explore the remote folder. self.download_path = download_path for arch in self.arch: # Checking if a valid path has been provided if self.url[arch] == '-': print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch)) # Faking already built it so to skip other checks. self.built[arch] = True continue if self.built[arch]: print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch)) continue contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] tarballs = self.tarballs[arch] maintarball = tarballs[0] os.makedirs(self.download_path, exist_ok = True) os.chdir(self.download_path) for archive in tarballs: # If the archive is already there, do not do anything. if os.path.exists(os.path.join(self.download_path, archive)): print("Archive %s is already there! Sweet" % archive) continue # Download the archive try: urllib.request.urlretrieve(self.url[arch] + archive, archive) except: print("Failed to download {archive}.".format(archive = archive)) print("Got %s." % archive) def build(self): """Building all the versions.""" # We have 4 builds to do: # * standard languages, no help # * standard languages + offline help # * all languages, no help # * all languages + offline help for arch in self.arch: if self.built[arch]: # Already built for arch or path not available. User has already been warned. continue # Preparation tasks self.appnamedir = os.path.join(self.builddir, self.appname) self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') os.makedirs(self.appimagedir, exist_ok = True) # And then cd to the appname folder. os.chdir(self.appnamedir) # Download appimagetool from github appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch) urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') os.chmod('appimagetool', 0o755) # Build the requested version. if self.queried_name and not self.portable: self.__unpackbuild__(arch, True) self.__unpackbuild__(arch) def __unpackbuild__(self, arch, generalize = False): if generalize and self.portable: # Doesn't particularly make sense to build a generic portable # version. Just skipping the specific generic build return # We start by filtering out tarballs from the list buildtarballs = [ self.tarballs[arch][0] ] # Let's process standard languages and append results to the # buildtarball if self.language == 'basic': if self.offline_help: buildtarballs.extend([ x for x in self.tarballs[arch] if 'pack_en-GB' in x ]) else: buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x]) elif self.language == 'standard': for lang in Build.LANGSTD: if self.offline_help: buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack_' + lang) in x ]) else: buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack_' + lang) in x ]) elif self.language == 'full': if self.offline_help: # We need also all help. Let's replace buildtarball with the # whole bunch buildtarballs = self.tarballs[arch] else: buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ]) else: # Looping for each language in self.language for lang in self.language.split(","): if self.offline_help: buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) else: buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) # Unpacking the tarballs for archive in buildtarballs: subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True) os.chdir(self.appnamedir) os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) if self.portable: shortversion = str.join('.', self.version.split('.')[:3]) subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir) # Changing desktop file subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir) subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) # Find the name of the binary called in the desktop file. binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) # Download AppRun from github apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch) dest = os.path.join(self.appimagedir, 'AppRun') urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) # Building app if self.updatable: # Updatable make sense only for generic images for fresh, still, # daily. If a request was for a specific version, I'd not build an # updatable version. # zsync name was generated already # If asked to do a generalized build: if generalize: subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) # Build version file management with open(self.genappimagefilename[arch] + '.ver', 'w') as v: v.write(self.version) else: subprocess.run("VERSION={version} ./appimagetool -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True) else: if generalize: subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.genappversion, appname = self.appname), shell=True) with open(self.genappimagefilename[arch] + '.ver', 'w') as v: v.write(self.version) else: subprocess.run("VERSION={version} ./appimagetool -v ./{appname}.AppDir/".format(version = self.appversion, appname = self.appname), shell=True) print("Built AppImage version {version}".format(version = self.appversion)) # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True) def checksums(self): """Create checksums of the built versions.""" if all(self.built.values()): # All checksums are already created. return # On the contrary, checksums will be in any case overwritten if # existent, but generated only for built packages anyways os.chdir(self.appnamedir) for appimage in glob.glob('*.AppImage*'): if appimage.endswith('.ver'): # Skipping checksums for .ver files. continue # See if a checksum already exist if not os.path.exists(appimage + '.md5'): subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True) def publish(self): """Moves built versions to definitive storage.""" if all(self.built.values()): # All files are already present in the full_path return os.chdir(self.appnamedir) # Forcing creation of subfolders, in case there is a new build os.makedirs(self.full_path, exist_ok = True) subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True) def __del__(self): """Destructor""" # Cleaning up build directory shutil.rmtree(self.builddir)