351 lines
16 KiB
Python
351 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import urllib.request
|
|
import loaih.versions as versions
|
|
from lxml import etree
|
|
import tempfile, os, sys, glob, subprocess, shutil, re
|
|
|
|
class Build(object):
|
|
LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ]
|
|
LANGBASIC = [ 'en-GB' ]
|
|
ARCHSTD = [ u'x86', u'x86_64' ]
|
|
|
|
def __init__(self, query, arch):
|
|
"""Build all versions that can be found in the indicated repo."""
|
|
self.query = query
|
|
self.queried_name = False if '.' in self.query else True
|
|
self.arch = arch
|
|
self.url = {}
|
|
self.language = 'basic'
|
|
self.offline_help = False
|
|
self.portable = False
|
|
self.updatable = True
|
|
self.sign = False
|
|
self.storage_path = '/srv/http/appimage.sys42.eu'
|
|
self.download_path = '/var/tmp/downloads'
|
|
|
|
# Specific build version
|
|
self.appversion = ''
|
|
self.genappversion = ''
|
|
self.appimagefilename = {}
|
|
self.genappimagefilename = {}
|
|
|
|
# Getting versions and so on
|
|
v = versions.BuildVersion(self.query)
|
|
|
|
# Creating a tempfile
|
|
self.builddir = tempfile.mkdtemp()
|
|
self.tarballs = {}
|
|
self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
|
|
self.version = v.version
|
|
self.url = v.basedirurl
|
|
self.built = { u'x86': False, u'x86_64': False }
|
|
|
|
# Preparing the default for the relative path on the storage for
|
|
# different versions.
|
|
# The path will evaluated as part of the check() function, as it is
|
|
# understood the storage_path can be changed before that phase.
|
|
self.relative_path = []
|
|
self.full_path = ''
|
|
|
|
def version_strings(self):
|
|
"""Build version strings based on the information received."""
|
|
# Building expected AppImageName
|
|
self.languagepart = "."
|
|
if ',' in self.language:
|
|
self.languagepart += self.language.replace(',', '-')
|
|
else:
|
|
self.languagepart += self.language
|
|
|
|
self.helppart = '.help' if self.offline_help else ''
|
|
|
|
# If the build was called by queried name, build from latest release available but build with the most generic name
|
|
self.appversion = self.version + self.languagepart + self.helppart
|
|
myver = str.join('.', self.version.split('.')[0:2])
|
|
self.genappversion = myver + self.languagepart + self.helppart
|
|
for arch in ARCHSTD:
|
|
self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage'
|
|
self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage'
|
|
|
|
|
|
def check(self, storage_path):
|
|
"""Checking if the requested AppImage has been already built."""
|
|
# Mandate to the private function to calculate the full_path available
|
|
# for the storage and the checks.
|
|
self.__calculate_full_path__()
|
|
|
|
# Incompatibilities - if portable and updatable are asked together,
|
|
# only portable will be built.
|
|
if self.portable and self.updatable:
|
|
print("Upgradable and portable options were required together. Building only portable.")
|
|
self.updatable = False
|
|
|
|
if self.updatable and not self.queried_name:
|
|
# If the queried version was a numbered version, doesn't make sense
|
|
# to build an updatable version.
|
|
self.updatable = False
|
|
|
|
# Call the build for queries
|
|
self.version_strings()
|
|
|
|
for arch in self.arch:
|
|
# For generalized builds, we need to check if there are .ver file
|
|
# and it contains the specific version found.
|
|
print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver'))
|
|
res = subprocess.run("find {path} -name {appimage}'".format(
|
|
path = self.full_path,
|
|
appimage = self.genappimagefilename[arch] + '.ver'
|
|
), shell=True, capture_output=True, env={ "LC_ALL": "C" })
|
|
|
|
if "No such file or directory" in res.stderr.decode('utf-8'):
|
|
# Folder is not existent: so the version was not built
|
|
# Build stays false, and we go to the next arch
|
|
continue
|
|
|
|
if res.stdout:
|
|
# All good, the command was executed fine.
|
|
for file in res.stdout.decode('utf-8').strip('\n').split('\n'):
|
|
if self.version in open(file, 'r').read():
|
|
self.built[arch] = True
|
|
|
|
print("Debug: searching for {file}".format(file = self.appimagefilename[arch]))
|
|
res = subprocess.run("find {path} -name '{appimage}'".format(
|
|
path = self.full_path,
|
|
appimage = self.appimagefilename[arch]
|
|
), shell=True, capture_output=True)
|
|
if res.stdout:
|
|
if len(res.stdout.decode('utf-8').strip('\n')) > 1:
|
|
self.built[arch] = True
|
|
|
|
if self.built[arch]:
|
|
print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch))
|
|
|
|
def __calculate_full_path__(self):
|
|
"""Calculate relative path of the build, based on internal other variables."""
|
|
if len(self.relative_path) == 0:
|
|
if self.query == 'daily':
|
|
self.relative_path.append('daily')
|
|
elif self.query == 'prerelease':
|
|
self.relative_path.append('prerelease')
|
|
|
|
# Not the same check, an additional one
|
|
if self.portable:
|
|
self.relative_path.append('portable')
|
|
|
|
fullpath_arr = self.storage_path.split('/')
|
|
# Joining relative path only if it is not null
|
|
if len(self.relative_path) > 0:
|
|
fullpath_arr.extend(self.relative_path)
|
|
self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr))
|
|
|
|
def download(self, download_path):
|
|
"""Downloads the contents of the URL as it was a folder."""
|
|
# Let's start with defining which files are to be downloaded.
|
|
# Let's explore the remote folder.
|
|
self.download_path = download_path
|
|
|
|
for arch in self.arch:
|
|
# Checking if a valid path has been provided
|
|
if self.url[arch] == '-':
|
|
print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch))
|
|
# Faking already built it so to skip other checks.
|
|
self.built[arch] = True
|
|
continue
|
|
|
|
if self.built[arch]:
|
|
print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch))
|
|
continue
|
|
|
|
contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a")
|
|
self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ]
|
|
tarballs = self.tarballs[arch]
|
|
maintarball = tarballs[0]
|
|
|
|
os.makedirs(self.download_path, exist_ok = True)
|
|
os.chdir(self.download_path)
|
|
for archive in tarballs:
|
|
# If the archive is already there, do not do anything.
|
|
if os.path.exists(os.path.join(self.download_path, archive)):
|
|
print("Archive %s is already there! Sweet" % archive)
|
|
continue
|
|
|
|
# Download the archive
|
|
try:
|
|
urllib.request.urlretrieve(self.url[arch] + archive, archive)
|
|
except:
|
|
print("Failed to download {archive}.".format(archive = archive))
|
|
|
|
print("Got %s." % archive)
|
|
|
|
def build(self):
|
|
"""Building all the versions."""
|
|
# We have 4 builds to do:
|
|
# * standard languages, no help
|
|
# * standard languages + offline help
|
|
# * all languages, no help
|
|
# * all languages + offline help
|
|
|
|
for arch in self.arch:
|
|
if self.built[arch]:
|
|
# Already built for arch or path not available. User has already been warned.
|
|
continue
|
|
|
|
# Preparation tasks
|
|
self.appnamedir = os.path.join(self.builddir, self.appname)
|
|
self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
|
|
os.makedirs(self.appimagedir, exist_ok = True)
|
|
# And then cd to the appname folder.
|
|
os.chdir(self.appnamedir)
|
|
# Download appimagetool from github
|
|
appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch)
|
|
urllib.request.urlretrieve(appimagetoolurl, 'appimagetool')
|
|
os.chmod('appimagetool', 0o755)
|
|
|
|
# Build the requested version.
|
|
if self.queried_name and not self.portable:
|
|
# If it is portable, do not generate a generalized version
|
|
self.__unpackbuild__(arch, True)
|
|
self.__unpackbuild__(arch)
|
|
|
|
def __unpackbuild__(self, arch, generalize = False):
|
|
if generalize and self.portable:
|
|
# Doesn't particularly make sense to build a generic portable
|
|
# version. Just skipping the specific generic build
|
|
return
|
|
|
|
# We start by filtering out tarballs from the list
|
|
buildtarballs = [ self.tarballs[arch][0] ]
|
|
|
|
# Let's process standard languages and append results to the
|
|
# buildtarball
|
|
if self.language == 'basic':
|
|
if self.offline_help:
|
|
buildtarballs.extend([ x for x in self.tarballs[arch] if 'pack_en-GB' in x ])
|
|
else:
|
|
buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack_en-GB' in x])
|
|
elif self.language == 'standard':
|
|
for lang in Build.LANGSTD:
|
|
if self.offline_help:
|
|
buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack_' + lang) in x ])
|
|
else:
|
|
buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack_' + lang) in x ])
|
|
elif self.language == 'full':
|
|
if self.offline_help:
|
|
# We need also all help. Let's replace buildtarball with the
|
|
# whole bunch
|
|
buildtarballs = self.tarballs[arch]
|
|
else:
|
|
buildtarballs.extend([ x for x in self.tarballs[arch] if 'langpack' in x ])
|
|
else:
|
|
# Looping for each language in self.language
|
|
for lang in self.language.split(","):
|
|
if self.offline_help:
|
|
buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ])
|
|
else:
|
|
buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ])
|
|
|
|
# Unpacking the tarballs
|
|
for archive in buildtarballs:
|
|
subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True)
|
|
|
|
os.chdir(self.appnamedir)
|
|
|
|
os.makedirs(self.appimagedir, exist_ok = True)
|
|
# At this point, let's decompress the deb packages
|
|
subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir)
|
|
|
|
if self.portable:
|
|
shortversion = str.join('.', self.version.split('.')[:3])
|
|
subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir)
|
|
|
|
# Changing desktop file
|
|
subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir)
|
|
subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir)
|
|
|
|
subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir)
|
|
|
|
# Find the name of the binary called in the desktop file.
|
|
binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n')
|
|
|
|
bindir=os.path.join(self.appimagedir, 'usr', 'bin')
|
|
os.makedirs(bindir, exist_ok = True)
|
|
subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir)
|
|
|
|
# Download AppRun from github
|
|
apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch)
|
|
dest = os.path.join(self.appimagedir, 'AppRun')
|
|
urllib.request.urlretrieve(apprunurl, dest)
|
|
os.chmod(dest, 0o755)
|
|
|
|
# Building app
|
|
if self.updatable:
|
|
# Updatable make sense only for generic images for fresh, still,
|
|
# daily. If a request was for a specific version, I'd not build an
|
|
# updatable version.
|
|
# zsync name was generated already
|
|
|
|
# Dealing with extra options
|
|
buildopts = []
|
|
if self.sign:
|
|
buildopts.append('--sign')
|
|
|
|
# If asked to do a generalized build:
|
|
if generalize:
|
|
subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True)
|
|
# Build version file management
|
|
with open(self.genappimagefilename[arch] + '.ver', 'w') as v:
|
|
v.write(self.version)
|
|
else:
|
|
subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True)
|
|
|
|
else:
|
|
if generalize:
|
|
subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True)
|
|
with open(self.genappimagefilename[arch] + '.ver', 'w') as v:
|
|
v.write(self.version)
|
|
else:
|
|
subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True)
|
|
|
|
print("Built AppImage version {version}".format(version = self.appversion))
|
|
|
|
# Cleanup phase, before new run.
|
|
for deb in glob.glob(self.appnamedir + '/*.deb'):
|
|
os.remove(deb)
|
|
subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True)
|
|
|
|
def checksums(self):
|
|
"""Create checksums of the built versions."""
|
|
if all(self.built.values()):
|
|
# All checksums are already created.
|
|
return
|
|
|
|
# On the contrary, checksums will be in any case overwritten if
|
|
# existent, but generated only for built packages anyways
|
|
|
|
os.chdir(self.appnamedir)
|
|
for appimage in glob.glob('*.AppImage*'):
|
|
if appimage.endswith('.ver'):
|
|
# Skipping checksums for .ver files.
|
|
continue
|
|
|
|
# See if a checksum already exist
|
|
if not os.path.exists(appimage + '.md5'):
|
|
subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True)
|
|
|
|
|
|
def publish(self):
|
|
"""Moves built versions to definitive storage."""
|
|
if all(self.built.values()):
|
|
# All files are already present in the full_path
|
|
return
|
|
|
|
os.chdir(self.appnamedir)
|
|
# Forcing creation of subfolders, in case there is a new build
|
|
os.makedirs(self.full_path, exist_ok = True)
|
|
subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True)
|
|
|
|
def __del__(self):
|
|
"""Destructor"""
|
|
# Cleaning up build directory
|
|
shutil.rmtree(self.builddir)
|