syntaxerrormmm/loappimage-helpers
Archived
1
0
Fork 0
Code Releases 2 Activity

Fix tentativo per creazione di link corretti e zsync truccati.

Browse Source
This commit is contained in:
Emiliano Vavassori 2022-04-08 00:56:28 +02:00
parent 6387ad1ac0
commit 413b84a1c3
2 changed files with 134 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

276
loaih/__init__.py
View File

@ -3,7 +3,7 @@
import urllib.request import urllib.request
import loaih.versions as versions import loaih.versions as versions
from lxml import etree from lxml import etree
import tempfile, os, sys, glob, subprocess, shutil, re import tempfile, os, sys, glob, subprocess, shutil, re, shlex
class Build(object): class Build(object):
LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ]
@ -13,32 +13,34 @@ class Build(object):
def __init__(self, query, arch): def __init__(self, query, arch):
"""Build all versions that can be found in the indicated repo.""" """Build all versions that can be found in the indicated repo."""
self.query = query self.query = query
self.queried_name = False if '.' in self.query else True
self.arch = arch self.arch = arch
self.url = {}
# Getting versions and so on
v = versions.BuildVersion(self.query)
self.version = v.version
self.short_version = str.join('.', self.version.split('.')[0:2])
self.branch_version = None
if not '.' in self.query:
self.branch_version = self.query
self.url = v.basedirurl
# Other default values
self.language = 'basic' self.language = 'basic'
self.offline_help = False self.offline_help = False
self.portable = False self.portable = False
self.updatable = True self.updatable = True
self.sign = False self.sign = True
self.storage_path = '/srv/http/appimage.sys42.eu' self.storage_path = '/mnt/appimage'
self.download_path = '/var/tmp/downloads' self.download_path = '/var/tmp/downloads'
# Specific build version # Specific build version
self.appversion = '' self.appversion = ''
self.genappversion = ''
self.appimagefilename = {} self.appimagefilename = {}
self.genappimagefilename = {} self.zsyncfilename = {}
# Getting versions and so on
v = versions.BuildVersion(self.query)
# Creating a tempfile # Creating a tempfile
self.builddir = tempfile.mkdtemp() self.builddir = tempfile.mkdtemp()
self.tarballs = {} self.tarballs = {}
self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
self.version = v.version
self.url = v.basedirurl
self.built = { u'x86': False, u'x86_64': False } self.built = { u'x86': False, u'x86_64': False }
# Preparing the default for the relative path on the storage for # Preparing the default for the relative path on the storage for
@ -51,73 +53,32 @@ class Build(object):
def calculate(self): def calculate(self):
"""Calculate exclusions and other variables.""" """Calculate exclusions and other variables."""
# Incompatibilities - if portable and updatable are asked together, # AppName
# only portable will be built. self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev'
if self.portable and self.updatable:
print("Upgradable and portable options were required together. Building only portable.")
self.updatable = False
if self.updatable and not self.queried_name: # Calculating languagepart
# If the queried version was a numbered version, doesn't make sense
# to build an updatable version.
self.updatable = False
# Mandate to the private function to calculate the full_path available
# for the storage and the checks.
self.__calculate_full_path__()
# Building expected AppImageName
self.languagepart = "." self.languagepart = "."
if ',' in self.language: if ',' in self.language:
self.languagepart += self.language.replace(',', '-') self.languagepart += self.language.replace(',', '-')
else: else:
self.languagepart += self.language self.languagepart += self.language
# Calculating help part
self.helppart = '.help' if self.offline_help else '' self.helppart = '.help' if self.offline_help else ''
# If the build was called by queried name, build from latest release available but build with the most generic name # Building the required names
self.appversion = self.version + self.languagepart + self.helppart
myver = str.join('.', self.version.split('.')[0:2])
self.genappversion = myver + self.languagepart + self.helppart
for arch in Build.ARCHSTD: for arch in Build.ARCHSTD:
self.appimagefilename[arch] = self.appname + '-' + self.appversion + f'-{arch}.AppImage' self.appimagefilename[arch] = __genappimagefilename__(self.version, arch)
self.genappimagefilename[arch] = self.appname + '-' + self.genappversion + f'-{arch}.AppImage' self.zsyncfilename[arch] = self.appimagefilename[arch] + '.zsync'
# Mandate to the private function to calculate the full_path available
# for the storage and the checks.
self.__calculate_full_path__()
def check(self): def __gen_appimagefilename__(self, version, arch):
"""Checking if the requested AppImage has been already built.""" """Generalize the construction of the name of the app."""
return self.appname + f'-{version}' + self.languagepart + self.helppart + f'-{arch}.AppImage'
for arch in self.arch:
# For generalized builds, we need to check if there are .ver file
# and it contains the specific version found.
print("Debug: searching for {file}".format(file = self.genappimagefilename[arch] + '.ver'))
res = subprocess.run("find {path} -name {appimage}'".format(
path = self.full_path,
appimage = self.genappimagefilename[arch] + '.ver'
), shell=True, capture_output=True, env={ "LC_ALL": "C" })
if "No such file or directory" in res.stderr.decode('utf-8'):
# Folder is not existent: so the version was not built
# Build stays false, and we go to the next arch
continue
if res.stdout:
# All good, the command was executed fine.
for file in res.stdout.decode('utf-8').strip('\n').split('\n'):
if self.version in open(file, 'r').read():
self.built[arch] = True
print("Debug: searching for {file}".format(file = self.appimagefilename[arch]))
res = subprocess.run("find {path} -name '{appimage}'".format(
path = self.full_path,
appimage = self.appimagefilename[arch]
), shell=True, capture_output=True)
if res.stdout:
if len(res.stdout.decode('utf-8').strip('\n')) > 1:
self.built[arch] = True
if self.built[arch]:
print("The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.".format(arch=arch))
def __calculate_full_path__(self): def __calculate_full_path__(self):
@ -139,49 +100,70 @@ class Build(object):
self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr)) self.full_path = re.sub(r"/+", '/', str.join('/', fullpath_arr))
def check(self):
"""Checking if the requested AppImage has been already built."""
if not len(self.appimagefilename) == 2:
self.calculate()
for arch in self.arch:
print(f"Searching for {self.appimagefilename[arch]}")
res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}'"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8')
if "No such file or directory" in res.stderr:
# Folder is not existent: so the version was not built
# Build stays false, and we go to the next arch
continue
if res.stdout:
# All good, the command was executed fine.
for file in res.stdout.strip('\n').split('\n'):
if self.version in open(file, 'r').read():
print(f"Build for {self.version} found.")
self.built[arch] = True
if self.built[arch]:
print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.")
def download(self): def download(self):
"""Downloads the contents of the URL as it was a folder.""" """Downloads the contents of the URL as it was a folder."""
print(f"Started downloads for {self.version}. Please wait.")
for arch in self.arch: for arch in self.arch:
# Checking if a valid path has been provided # Checking if a valid path has been provided
if self.url[arch] == '-': if self.url[arch] == '-':
print("No build has been provided for the requested AppImage for {arch}. Continue with other options.".format(arch = arch)) print(f"No build has been provided for the requested AppImage for {arch}. Continue with other options.")
# Faking already built it so to skip other checks. # Faking already built it so to skip other checks.
self.built[arch] = True self.built[arch] = True
continue continue
if self.built[arch]: if self.built[arch]:
print("A build for {arch} was already found. Skipping specific packages.".format(arch = arch)) print(f"A build for {arch} was already found. Skipping specific packages.")
continue continue
# Identifying downloads
contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a")
self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ]
tarballs = self.tarballs[arch] tarballs = self.tarballs[arch]
maintarball = tarballs[0] maintarball = tarballs[0]
# Create and change directory to the download location
os.makedirs(self.download_path, exist_ok = True) os.makedirs(self.download_path, exist_ok = True)
os.chdir(self.download_path) os.chdir(self.download_path)
for archive in tarballs: for archive in tarballs:
# If the archive is already there, do not do anything. # If the archive is already there, do not do anything.
if os.path.exists(os.path.join(self.download_path, archive)): if os.path.exists(archive):
print("Archive %s is already there! Sweet" % archive)
continue continue
# Download the archive # Download the archive
try: try:
urllib.request.urlretrieve(self.url[arch] + archive, archive) urllib.request.urlretrieve(self.url[arch] + archive, archive)
except: except:
print("Failed to download {archive}.".format(archive = archive)) print(f"Failed to download {archive}.")
print("Got %s." % archive)
print(f"Finished downloads for {self.version}.")
def build(self): def build(self):
"""Building all the versions.""" """Building all the versions."""
# We have 4 builds to do:
# * standard languages, no help
# * standard languages + offline help
# * all languages, no help
# * all languages + offline help
for arch in self.arch: for arch in self.arch:
if self.built[arch]: if self.built[arch]:
@ -190,28 +172,18 @@ class Build(object):
# Preparation tasks # Preparation tasks
self.appnamedir = os.path.join(self.builddir, self.appname) self.appnamedir = os.path.join(self.builddir, self.appname)
self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
os.makedirs(self.appimagedir, exist_ok = True)
# And then cd to the appname folder. # And then cd to the appname folder.
os.chdir(self.appnamedir) os.chdir(self.appnamedir)
# Download appimagetool from github # Download appimagetool from github
appimagetoolurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage".format(arch = arch) appimagetoolurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-{arch}.AppImage"
urllib.request.urlretrieve(appimagetoolurl, 'appimagetool') urllib.request.urlretrieve(appimagetoolurl, 'appimagetool')
os.chmod('appimagetool', 0o755) os.chmod('appimagetool', 0o755)
# Build the requested version. # Build the requested version.
if self.queried_name and not self.portable:
# If it is portable, do not generate a generalized version
self.__unpackbuild__(arch, True)
self.__unpackbuild__(arch) self.__unpackbuild__(arch)
def __unpackbuild__(self, arch, generalize = False): def __unpackbuild__(self, arch):
if generalize and self.portable:
# Doesn't particularly make sense to build a generic portable
# version. Just skipping the specific generic build
return
# We start by filtering out tarballs from the list # We start by filtering out tarballs from the list
buildtarballs = [ self.tarballs[arch][0] ] buildtarballs = [ self.tarballs[arch][0] ]
@ -243,35 +215,38 @@ class Build(object):
else: else:
buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ])
# Unpacking the tarballs
for archive in buildtarballs:
subprocess.run("tar xzf {folder}/{archive}".format(folder = self.download_path, archive = archive), shell=True)
os.chdir(self.appnamedir) os.chdir(self.appnamedir)
# Unpacking the tarballs
for archive in buildtarballs:
subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}"))
# create appimagedir
self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir')
os.makedirs(self.appimagedir, exist_ok = True) os.makedirs(self.appimagedir, exist_ok = True)
# At this point, let's decompress the deb packages # At this point, let's decompress the deb packages
subprocess.run("find .. -iname '*.deb' -exec dpkg -x {} . \;", shell=True, cwd=self.appimagedir) subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir)
if self.portable: if self.portable:
shortversion = str.join('.', self.version.split('.')[:3]) subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir)
subprocess.run("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % shortversion, shell=True, cwd=self.appimagedir)
# Changing desktop file # Changing desktop file
subprocess.run("find . -iname startcenter.desktop -exec cp {} . \;", shell=True, cwd=self.appimagedir) subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir)
subprocess.run("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname, shell=True, cwd=self.appimagedir) subprocess.run(shlex.split("sed -i -e 's:^Name=.*$:Name=%s:' startcenter.desktop" % self.appname), cwd=self.appimagedir)
subprocess.run("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;", shell=True, cwd=self.appimagedir) subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir)
# Find the name of the binary called in the desktop file. # Find the name of the binary called in the desktop file.
binaryname = subprocess.check_output("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'", shell=True, cwd=self.appimagedir).decode('utf-8').strip('\n') binary_exec = subprocess.run(shlex.split("awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8')
binaryname = binary_exec.stdout.strip("\n")
bindir=os.path.join(self.appimagedir, 'usr', 'bin') bindir=os.path.join(self.appimagedir, 'usr', 'bin')
os.makedirs(bindir, exist_ok = True) os.makedirs(bindir, exist_ok = True)
subprocess.run("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname, shell=True, cwd=bindir) subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir)
# Download AppRun from github # Download AppRun from github
apprunurl = "https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}".format(arch = arch) apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}"
dest = os.path.join(self.appimagedir, 'AppRun') dest = os.path.join(self.appimagedir, 'AppRun')
urllib.request.urlretrieve(apprunurl, dest) urllib.request.urlretrieve(apprunurl, dest)
os.chmod(dest, 0o755) os.chmod(dest, 0o755)
@ -280,57 +255,36 @@ class Build(object):
buildopts = [] buildopts = []
if self.sign: if self.sign:
buildopts.append('--sign') buildopts.append('--sign')
# Building app # adding zsync build if updatable
if self.updatable: if self.updatable:
# Updatable make sense only for generic images for fresh, still, buildopts.append(f"-u 'zsync|{self.zsyncfilename[arch]}'")
# daily. If a request was for a specific version, I'd not build an
# updatable version.
# zsync name was generated already
# If asked to do a generalized build: buildopts_str = str.join(' ', buildopts)
if generalize: # Build the number-specific build
subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), zsync = self.genappimagefilename[arch] + '.zsync', appname = self.appname), shell=True) subprocess.run(shlex.split(f"VERSION={self.appversion} ./appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"))
# Build version file management
with open(self.genappimagefilename[arch] + '.ver', 'w') as v: print(f"Built AppImage version {self.appversion}")
v.write(self.version)
else:
subprocess.run("VERSION={version} ./appimagetool {buildopts} -u 'zsync|{zsync}' -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), zsync = self.appimagefilename[arch] + '.zsync', appname = self.appname), shell=True)
else:
if generalize:
subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.genappversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True)
with open(self.genappimagefilename[arch] + '.ver', 'w') as v:
v.write(self.version)
else:
subprocess.run("VERSION={version} ./appimagetool {buildopts} -v ./{appname}.AppDir/".format(version = self.appversion, buildopts = str.join(' ', buildopts), appname = self.appname), shell=True)
print("Built AppImage version {version}".format(version = self.appversion))
# Cleanup phase, before new run. # Cleanup phase, before new run.
for deb in glob.glob(self.appnamedir + '/*.deb'): for deb in glob.glob(self.appnamedir + '/*.deb'):
os.remove(deb) os.remove(deb)
subprocess.run("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+", shell=True) subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+"))
def checksums(self): def checksums(self):
"""Create checksums of the built versions.""" """Create checksums of the built versions."""
# Skip checksum if initally the build was already found in the storage directory
if all(self.built.values()): if all(self.built.values()):
# All checksums are already created. return
return
# On the contrary, checksums will be in any case overwritten if
# existent, but generated only for built packages anyways
os.chdir(self.appnamedir) os.chdir(self.appnamedir)
for appimage in glob.glob('*.AppImage*'): for arch in self.arch:
if appimage.endswith('.ver'): for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]:
# Skipping checksums for .ver files. # For any built arch, find out if a file exist.
continue if len(glob.glob(self.appimagefilename[arch] + '.md5')) == 0:
# Build checksum
# See if a checksum already exist subprocess.run(shlex.split(f"md5sum {item} > {item}.md5"))
if not os.path.exists(appimage + '.md5'):
subprocess.run("md5sum {appimage} > {appimage}.md5".format(appimage = appimage), shell=True)
def publish(self): def publish(self):
@ -342,7 +296,41 @@ class Build(object):
os.chdir(self.appnamedir) os.chdir(self.appnamedir)
# Forcing creation of subfolders, in case there is a new build # Forcing creation of subfolders, in case there is a new build
os.makedirs(self.full_path, exist_ok = True) os.makedirs(self.full_path, exist_ok = True)
subprocess.run("find . -iname '*.AppImage*' -exec cp -f {} %s \;" % self.full_path, shell=True) for file in glob.glob("*.AppImage*"):
subprocess.run(shlex.split(f"cp -f {file} {self.fullpath}"))
def generalize_and_link(self):
"""Creates the needed generalized files if needed."""
# If called with a pointed version, no generalize and link necessary.
if not self.branch_version:
return
# Creating versions for short version and query text
versions = [ self.short_version, self.branch_version ]
for arch in Build.ARCHSTD:
# if the appimage for the reported arch is not found, skip to next
# arch
if not os.path.exists(self.appimagefilename[arch]):
continue
# Doing it both for short_name and for branchname
for version in versions:
appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage'
zsyncfilename[arch] = appimagefilename[arch] + '.zsync'
os.chdir(self.full_path)
# Create the symlink
os.symlink(self.appimagefilename[arch], appimagefilename[arch])
# Create the checksum for the AppImage
subprocess.run(shlex.split("md5sum {item} > {item}.md5".format(item=appimagefilename[arch])))
# Do not continue if no zsync file is provided.
if not self.updatable:
continue
os.copy(self.zsyncfilename[arch], zsyncfilename[arch])
# Editing the zsyncfile
subprocess.run(shlex.split(f"sed -i'' -e 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}"))
def __del__(self): def __del__(self):

3
scripts/loaih-build
View File

@ -12,7 +12,7 @@ import loaih
@click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") @click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic")
@click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") @click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help")
@click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") @click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable")
@click.option('-r', '--repo-path', 'repo_path', default = '/srv/http/appimage.sys42.eu', type=str, help="Path to the final storage of the AppImage. Default: /srv/http/appimage.sys42.eu") @click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage")
@click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign") @click.option('-s/-S', '--sign/--no-sign', 'sign', default=False, help="Wether to sign the build. Default: no-sign")
@click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") @click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable")
@click.argument('query') @click.argument('query')
@ -57,6 +57,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path
obj.build() obj.build()
obj.checksums() obj.checksums()
obj.publish() obj.publish()
obj.generalize_and_link()
del obj del obj
else: else: