From df5012eedd1ca3ac79ae2c35f95125c535880af6 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:07:12 +0100 Subject: [PATCH 01/32] Prima implementazione controllo build remota e caricamento con rsync + ssh. --- loaih/build.py | 80 +++++++++++++++++++++++++++++++++++++------------ loaih/script.py | 9 +++++- 2 files changed, 69 insertions(+), 20 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index bbd0631..a7a2a04 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -32,6 +32,9 @@ class Build(loaih.RemoteBuild): self.portable = False self.updatable = True self.sign = True + self.remoterepo = False + self.remote_host = '' + self.remote_path = '' self.storage_path = '/mnt/appimage' self.download_path = '/var/tmp/downloads' @@ -90,7 +93,7 @@ class Build(loaih.RemoteBuild): if self.query == 'daily': self.relative_path.append('daily') elif self.query == 'prerelease': - self.relative_path.append('prerelease') + self.relative_path.append('prerelease') # Not the same check, an additional one if self.portable: @@ -105,25 +108,44 @@ class Build(loaih.RemoteBuild): def check(self): """Checking if the requested AppImage has been already built.""" - if not len(self.appimagefilename) == 2: + if len(self.appimagefilename) != 2: self.calculate() for arch in self.arch: - print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') - if "No such file or directory" in res.stderr: - # Folder is not existent: so the version was not built - # Build stays false, and we go to the next arch - continue + # First, check if by metadata the repo is remote or not. + if self.remoterepo or 'http' in self.storage_path: + self.remoterepo = True + # Remote storage. I have to query a remote site to know if it + # was already built. + name = self.appimagefilename[arch] + matching = etree.HTML(urllib.request.urlopen(str.join('/', + self.relative_path.insert(0, self.storage_path) + )).read()).xpath( + f"//a[contains(@href, '{name}')/@href" + ) - if res.stdout and len(res.stdout.strip("\n")) > 0: - # All good, the command was executed fine. - print(f"Build for {self.version} found.") - self.built[arch] = True + if len(matching) > 0: + # Already built. + self.built[arch] = True + + else: + # Repo is local + print(f"Searching for {self.appimagefilename[arch]}") + res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + + if "No such file or directory" in res.stderr: + # Folder is not existent: so the version was not built + # Build stays false, and we go to the next arch + continue + + if res.stdout and len(res.stdout.strip("\n")) > 0: + # All good, the command was executed fine. + print(f"Build for {self.version} found.") + self.built[arch] = True if self.built[arch]: - print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") + print(f"The requested AppImage already exists on storage for {arch}. I'll skip downloading, building and moving the results.") def download(self): @@ -309,13 +331,33 @@ class Build(loaih.RemoteBuild): return os.chdir(self.appnamedir) - # Forcing creation of subfolders, in case there is a new build - os.makedirs(self.full_path, exist_ok = True) - for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) + + # Two cases here: local and remote storage_path. + if self.remoterepo: + # Remote first. + # Build destination directory + if len(self.relative_path) > 0: + remotepath = str.join('/', self.relative_path.insert(0, self.remote_path)) + else: + remotepath = str.join('/', [ self.remote_path, '' ]) + try: + subprocess.run( + shlex.split( + f"rsync -avz -e ssh *.AppImage* {self.remote_host}:{remotepath}" + ) + ) + finally: + pass + + else: + # Local + # Forcing creation of subfolders, in case there is a new build + os.makedirs(self.full_path, exist_ok = True) + for file in glob.glob("*.AppImage*"): + subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) - def generalize_and_link(self): + def generalize_and_link(self, chdir = self.full_path): """Creates the needed generalized files if needed.""" # If called with a pointed version, no generalize and link necessary. if not self.branch_version: @@ -335,7 +377,7 @@ class Build(loaih.RemoteBuild): if self.built[arch]: continue - os.chdir(self.full_path) + os.chdir(chdir) # if the appimage for the reported arch is not found, skip to next # arch if not os.path.exists(self.appimagefilename[arch]): diff --git a/loaih/script.py b/loaih/script.py index 40ddf36..46afbe0 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -71,6 +71,10 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.updatable = True obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu' obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads' + if 'http' in obj.storage_path: + obj.remoterepo = True + obj.remote_host = config['data']['remote_host'] if 'remote_host' in config['data'] and config['data']['remote_host'] else 'ciccio.libreitalia.org' + obj.remote_path = config['data']['remote_path'] if 'remote_path' in config['data'] and config['data']['remote_path'] else '/var/lib/nethserver/vhost/appimages' if 'sign' in config['data'] and config['data']['sign']: obj.sign = True @@ -83,8 +87,11 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.download() obj.build() obj.checksums() + if obj.remoterepo: + obj.generalize_and_link(obj.appnamedir) obj.publish() - obj.generalize_and_link() + if not obj.remoterepo: + obj.generalize_and_link() del obj else: From cbaf6b2e3cee67ab850c28be43ef574a8659b4c2 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:08:35 +0100 Subject: [PATCH 02/32] Corretto anche il file di build per il test. --- test.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test.yml b/test.yml index faf4464..768cdd6 100644 --- a/test.yml +++ b/test.yml @@ -1,12 +1,14 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: yes - sign: yes + force: true + sign: true builds: - query: fresh language: basic - offline_help: no - portable: no + offline_help: false + portable: false From 9cf311948922b8899d14ca821e3a14fb9470ca90 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:14:28 +0100 Subject: [PATCH 03/32] Correzione sintassi in build. --- loaih/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index a7a2a04..279c3c6 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -357,7 +357,7 @@ class Build(loaih.RemoteBuild): subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) - def generalize_and_link(self, chdir = self.full_path): + def generalize_and_link(self, chdir = 'default'): """Creates the needed generalized files if needed.""" # If called with a pointed version, no generalize and link necessary. if not self.branch_version: @@ -367,6 +367,9 @@ class Build(loaih.RemoteBuild): if self.query == 'daily' or self.query == 'prerelease': return + if chdir == 'default': + chdir = self.full_path + appimagefilename = {} zsyncfilename = {} From ae9668554a68606a0c77293855c8fd5a073926f6 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:24:37 +0100 Subject: [PATCH 04/32] Cambiamenti per esecuzione corretta verifica online. --- loaih/build.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 279c3c6..8124c7a 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -119,9 +119,13 @@ class Build(loaih.RemoteBuild): # Remote storage. I have to query a remote site to know if it # was already built. name = self.appimagefilename[arch] - matching = etree.HTML(urllib.request.urlopen(str.join('/', - self.relative_path.insert(0, self.storage_path) - )).read()).xpath( + if len(self.relative_path) > 0: + path_arr = self.relative_path.insert(0, self.storage_path) + else: + path_arr = [ self.storage_path, '' ] + matching = etree.HTML(urllib.request.urlopen( + str.join('/', path_arr) + ).read()).xpath( f"//a[contains(@href, '{name}')/@href" ) From da31e1655bc6cbe87c29dbe26abd60d2d6bc0a53 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:30:06 +0100 Subject: [PATCH 05/32] Aggiunta codice debug. --- loaih/build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 8124c7a..6069c14 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -123,8 +123,10 @@ class Build(loaih.RemoteBuild): path_arr = self.relative_path.insert(0, self.storage_path) else: path_arr = [ self.storage_path, '' ] + path = str.join('/', path_arr) + print(f"DEBUG - Name: {name}, URL: {path}") matching = etree.HTML(urllib.request.urlopen( - str.join('/', path_arr) + str.join('/', path) ).read()).xpath( f"//a[contains(@href, '{name}')/@href" ) From 38a78860b04d716af66654d60f5014ef5067fe06 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:33:26 +0100 Subject: [PATCH 06/32] Fix procedura dopo codice debug. --- loaih/build.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 6069c14..39ba440 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -125,9 +125,7 @@ class Build(loaih.RemoteBuild): path_arr = [ self.storage_path, '' ] path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") - matching = etree.HTML(urllib.request.urlopen( - str.join('/', path) - ).read()).xpath( + matching = etree.HTML(urllib.request.urlopen(path).read()).xpath( f"//a[contains(@href, '{name}')/@href" ) From 05533bf5e2511cba9175c370f4098dc9240e4336 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:36:53 +0100 Subject: [PATCH 07/32] Fix sintassi xpath. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 39ba440..b42acd3 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -126,7 +126,7 @@ class Build(loaih.RemoteBuild): path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") matching = etree.HTML(urllib.request.urlopen(path).read()).xpath( - f"//a[contains(@href, '{name}')/@href" + f"//a[contains(@href, '{name}')]/@href" ) if len(matching) > 0: From 07a895c86c3543de036ee233b703dd2b65d27412 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:56:33 +0100 Subject: [PATCH 08/32] Fix checksum che torna valore non valido. --- loaih/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index b42acd3..8fb99a9 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -311,7 +311,7 @@ class Build(loaih.RemoteBuild): def checksums(self): """Create checksums of the built versions.""" # Skip checksum if initally the build was already found in the storage directory - if all(self.built.values()): + if all(self.built[arch] for arch in self.arch): return os.chdir(self.appnamedir) @@ -330,7 +330,7 @@ class Build(loaih.RemoteBuild): def publish(self): """Moves built versions to definitive storage.""" - if all(self.built.values()): + if all(self.built[arch] for arch in self.arch): # All files are already present in the full_path return From 9d259c4aa5cf5a9b8889361aa1ccee5e26891ad2 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 01:59:55 +0100 Subject: [PATCH 09/32] Cambiata la logica di build in caso di assenza di build. --- loaih/script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/script.py b/loaih/script.py index 46afbe0..bb1d7fe 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -87,7 +87,7 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.download() obj.build() obj.checksums() - if obj.remoterepo: + if obj.remoterepo and obj.appnamedir: obj.generalize_and_link(obj.appnamedir) obj.publish() if not obj.remoterepo: From f95c4d4b1d1312df674919731f9e5a04a9bbac38 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 18:28:11 +0100 Subject: [PATCH 10/32] Sistemazione upload con rsync. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 8fb99a9..be90f64 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -347,7 +347,7 @@ class Build(loaih.RemoteBuild): try: subprocess.run( shlex.split( - f"rsync -avz -e ssh *.AppImage* {self.remote_host}:{remotepath}" + f"rsync -rlIvz --munge-links *.AppImage* {self.remote_host}:{remotepath}" ) ) finally: From a0c6217d95b3ba4d8869c7e65969057303318c29 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Thu, 5 Jan 2023 19:58:45 +0100 Subject: [PATCH 11/32] Sistemazione build. --- loaih/build.py | 167 +++++++++++++++++++++++++++++++++++-------------- test.yml | 2 +- 2 files changed, 120 insertions(+), 49 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index be90f64..071127d 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -1,21 +1,35 @@ #!/usr/bin/env python3 +# encoding: utf-8 +"""Classes and functions to build an AppImage.""" +import os +import glob +import subprocess +import shutil +import re +import shlex +import tempfile import urllib.request -import loaih from lxml import etree -import tempfile, os, sys, glob, subprocess, shutil, re, shlex +import loaih class Collection(list): + """Aggregates metadata on a collection of builds.""" def __init__(self, query, arch = ['x86', 'x86_64']): """Build a list of version to check/build for this round.""" super().__init__() - self.extend([ Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) ]) + self.extend([ + Build(query, arch, version) for version in loaih.Base.collectedbuilds(query) + ]) class Build(loaih.RemoteBuild): - LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] + """Builds a single version.""" + + LANGSTD = [ 'ar', 'de', 'en-GB', 'es', 'fr', 'it', 'ja', 'ko', 'pt', + 'pt-BR', 'ru', 'zh-CN', 'zh-TW' ] LANGBASIC = [ 'en-GB' ] - ARCHSTD = [ u'x86', u'x86_64' ] + ARCHSTD = [ 'x86', 'x86_64' ] def __init__(self, query, arch, version = None): super().__init__(query, version) @@ -37,17 +51,24 @@ class Build(loaih.RemoteBuild): self.remote_path = '' self.storage_path = '/mnt/appimage' self.download_path = '/var/tmp/downloads' + self.appnamedir = '' # Specific build version + self.appname = 'LibreOffice' self.appversion = '' + self.appimagedir = '' self.appimagefilename = {} self.zsyncfilename = {} + # Other variables by build + self.languagepart = '.' + self.language + self.helppart = '' + # Creating a tempfile self.builddir = tempfile.mkdtemp() self.tarballs = {} - self.built = { u'x86': False, u'x86_64': False } - + self.built = { 'x86': False, 'x86_64': False } + # Preparing the default for the relative path on the storage for # different versions. # The path will evaluated as part of the check() function, as it is @@ -59,7 +80,8 @@ class Build(loaih.RemoteBuild): def calculate(self): """Calculate exclusions and other variables.""" # AppName - self.appname = 'LibreOffice' if not self.query == 'daily' and not self.query == 'prerelease' else 'LibreOfficeDev' + if self.query in { 'prerelease', 'daily' }: + self.appname = 'LibreOfficeDev' # Calculating languagepart self.languagepart = "." @@ -69,7 +91,8 @@ class Build(loaih.RemoteBuild): self.languagepart += self.language # Calculating help part - self.helppart = '.help' if self.offline_help else '' + if self.offline_help: + self.helppart = '.help' # Building the required names for arch in Build.ARCHSTD: @@ -125,9 +148,11 @@ class Build(loaih.RemoteBuild): path_arr = [ self.storage_path, '' ] path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") - matching = etree.HTML(urllib.request.urlopen(path).read()).xpath( - f"//a[contains(@href, '{name}')]/@href" - ) + matching = [] + with urllib.request.urlopen(path) as url: + matching = etree.HTML(url.read()).xpath( + f"//a[contains(@href, '{name}')]/@href" + ) if len(matching) > 0: # Already built. @@ -136,7 +161,11 @@ class Build(loaih.RemoteBuild): else: # Repo is local print(f"Searching for {self.appimagefilename[arch]}") - res = subprocess.run(shlex.split(f"find {self.full_path} -name {self.appimagefilename[arch]}"), capture_output=True, env={ "LC_ALL": "C" }, text=True, encoding='utf-8') + command = f"find {self.full_path} -name {self.appimagefilename[arch]}" + res = subprocess.run(shlex.split(command), + capture_output=True, + env={ "LC_ALL": "C" }, + text=True, encoding='utf-8', check=True) if "No such file or directory" in res.stderr: # Folder is not existent: so the version was not built @@ -168,10 +197,15 @@ class Build(loaih.RemoteBuild): continue # Identifying downloads - contents = etree.HTML(urllib.request.urlopen(self.url[arch]).read()).xpath("//td/a") - self.tarballs[arch] = [ x.text for x in contents if x.text.endswith('tar.gz') and 'deb' in x.text ] + contents = [] + with urllib.request.urlopen(self.url[arch]) as url: + contents = etree.HTML(url.read()).xpath("//td/a") + + self.tarballs[arch] = [ x.text + for x in contents + if x.text.endswith('tar.gz') and 'deb' in x.text + ] tarballs = self.tarballs[arch] - maintarball = tarballs[0] # Create and change directory to the download location os.makedirs(self.download_path, exist_ok = True) @@ -184,8 +218,8 @@ class Build(loaih.RemoteBuild): # Download the archive try: urllib.request.urlretrieve(self.url[arch] + archive, archive) - except: - print(f"Failed to download {archive}.") + except Exception as error: + print(f"Failed to download {archive}: {error}.") print(f"Finished downloads for {self.version}.") @@ -213,7 +247,7 @@ class Build(loaih.RemoteBuild): def __unpackbuild__(self, arch): # We start by filtering out tarballs from the list - buildtarballs = [ self.tarballs[arch][0] ] + buildtarballs = [ self.tarballs[arch][0] ] # Let's process standard languages and append results to the # buildtarball @@ -239,50 +273,75 @@ class Build(loaih.RemoteBuild): # Looping for each language in self.language for lang in self.language.split(","): if self.offline_help: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('pack' + lang) in x ]) + buildtarballs.extend([ x for x in self.tarballs[arch] + if 'pack' + lang in x ]) else: - buildtarballs.extend([ x for x in self.tarballs[arch] if ('langpack' + lang) in x ]) - + buildtarballs.extend([ x for x in self.tarballs[arch] + if 'langpack' + lang in x ]) + os.chdir(self.appnamedir) # Unpacking the tarballs for archive in buildtarballs: - subprocess.run(shlex.split(f"tar xzf {self.download_path}/{archive}")) + subprocess.run(shlex.split( + f"tar xzf {self.download_path}/{archive}"), check=True) # create appimagedir self.appimagedir = os.path.join(self.builddir, self.appname, self.appname + '.AppDir') os.makedirs(self.appimagedir, exist_ok = True) # At this point, let's decompress the deb packages - subprocess.run(shlex.split("find .. -iname '*.deb' -exec dpkg -x {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find .. -iname '*.deb' -exec dpkg -x {} . \;" + ), cwd=self.appimagedir, check=True) if self.portable: - subprocess.run(shlex.split("find . -type f -iname 'bootstraprc' -exec sed -i 's|^UserInstallation=.*|UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find . -type f -iname 'bootstraprc' " + + r"-exec sed -i 's|^UserInstallation=.*|" + + r"UserInstallation=\$SYSUSERCONFIG/libreoffice/%s|g' {} \+" % self.short_version + ), cwd=self.appimagedir, check=True) # Changing desktop file - subprocess.run(shlex.split("find . -iname startcenter.desktop -exec cp {} . \;"), cwd=self.appimagedir) - subprocess.run(shlex.split("sed --in-place 's:^Name=.*$:Name=%s:' startcenter.desktop > startcenter.desktop" % self.appname), cwd=self.appimagedir) + subprocess.run(shlex.split( + r"find . -iname startcenter.desktop -exec cp {} . \;" + ), cwd=self.appimagedir, check=True) - subprocess.run(shlex.split("find . -name '*startcenter.png' -path '*hicolor*48x48*' -exec cp {} . \;"), cwd=self.appimagedir) + subprocess.run(shlex.split( + f"sed --in-place 's:^Name=.*$:Name={self.appname}:' " + + r"startcenter.desktop > startcenter.desktop" + ), cwd=self.appimagedir, check=True) + + subprocess.run(shlex.split( + r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " + + r"-exec cp {} . \;" + ), cwd=self.appimagedir, check=True) # Find the name of the binary called in the desktop file. binaryname = '' - with open(os.path.join(self.appimagedir, 'startcenter.desktop'), 'r') as d: - a = d.readlines() - for line in a: + with open( + os.path.join(self.appimagedir, 'startcenter.desktop'), + 'r', encoding="utf-8" + ) as desktopfile: + for line in desktopfile.readlines(): if re.match(r'^Exec', line): binaryname = line.split('=')[-1].split(' ')[0] # Esci al primo match break + #binary_exec = subprocess.run(shlex.split(r"awk 'BEGIN { FS = \"=\" } /^Exec/ { print $2; exit }' startcenter.desktop | awk '{ print $1 }'"), cwd=self.appimagedir, text=True, encoding='utf-8') #binaryname = binary_exec.stdout.strip("\n") bindir=os.path.join(self.appimagedir, 'usr', 'bin') os.makedirs(bindir, exist_ok = True) - subprocess.run(shlex.split("find ../../opt -iname soffice -path '*program*' -exec ln -sf {} ./%s \;" % binaryname), cwd=bindir) + subprocess.run(shlex.split( + r"find ../../opt -iname soffice -path '*program*' " + + r"-exec ln -sf {} ./%s \;" % binaryname + ), cwd=bindir, check=True) # Download AppRun from github - apprunurl = f"https://github.com/AppImage/AppImageKit/releases/download/continuous/AppRun-{arch}" + apprunurl = r"https://github.com/AppImage/AppImageKit/releases/" + apprunurl += f"download/continuous/AppRun-{arch}" dest = os.path.join(self.appimagedir, 'AppRun') urllib.request.urlretrieve(apprunurl, dest) os.chmod(dest, 0o755) @@ -298,14 +357,19 @@ class Build(loaih.RemoteBuild): buildopts_str = str.join(' ', buildopts) # Build the number-specific build - subprocess.run(shlex.split(f"{self.appnamedir}/appimagetool {buildopts_str} -v ./{self.appname}.AppDir/"), env={ "VERSION": self.appversion }) - + subprocess.run(shlex.split( + f"{self.appnamedir}/appimagetool {buildopts_str} -v " + + f"./{self.appname}.AppDir/" + ), env={ "VERSION": self.appversion }, check=True) + print(f"Built AppImage version {self.appversion}") # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): os.remove(deb) - subprocess.run(shlex.split("find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+")) + subprocess.run(shlex.split( + r"find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \+" + ), check=True) def checksums(self): @@ -323,10 +387,11 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), capture_output=True, text=True, encoding='utf-8') + checksum = subprocess.run(shlex.split(f"md5sum {file}"), + capture_output=True, text=True, encoding='utf-8', check=True) if checksum.stdout: - with open(f"{file}.md5", 'w') as c: - c.write(checksum.stdout) + with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: + csfile.write(checksum.stdout) def publish(self): """Moves built versions to definitive storage.""" @@ -345,11 +410,10 @@ class Build(loaih.RemoteBuild): else: remotepath = str.join('/', [ self.remote_path, '' ]) try: - subprocess.run( - shlex.split( - f"rsync -rlIvz --munge-links *.AppImage* {self.remote_host}:{remotepath}" - ) - ) + subprocess.run(shlex.split( + r"rsync -rlIvz --munge-links *.AppImage* " + + f"{self.remote_host}:{remotepath}" + ), check=True) finally: pass @@ -358,7 +422,9 @@ class Build(loaih.RemoteBuild): # Forcing creation of subfolders, in case there is a new build os.makedirs(self.full_path, exist_ok = True) for file in glob.glob("*.AppImage*"): - subprocess.run(shlex.split(f"cp -f {file} {self.full_path}")) + subprocess.run(shlex.split( + f"cp -f {file} {self.full_path}" + ), check=True) def generalize_and_link(self, chdir = 'default'): @@ -368,7 +434,7 @@ class Build(loaih.RemoteBuild): return # If a prerelease or a daily version, either. - if self.query == 'daily' or self.query == 'prerelease': + if self.query in { 'daily', 'prerelease' }: return if chdir == 'default': @@ -392,7 +458,9 @@ class Build(loaih.RemoteBuild): # Doing it both for short_name and for branchname for version in versions: - appimagefilename[arch] = self.appname + '-' + version + self.languagepart + self.helppart + f'-{arch}.AppImage' + appimagefilename[arch] = self.appname + '-' + version + appimagefilename[arch] += self.languagepart + self.helppart + appimagefilename[arch] += f'-{arch}.AppImage' zsyncfilename[arch] = appimagefilename[arch] + '.zsync' # Create the symlink @@ -411,7 +479,10 @@ class Build(loaih.RemoteBuild): os.unlink(zsyncfilename[arch]) shutil.copyfile(self.zsyncfilename[arch], zsyncfilename[arch]) # Editing the zsyncfile - subprocess.run(shlex.split(f"sed --in-place 's/^Filename:.*$/Filename: {appimagefilename[arch]}/' {zsyncfilename[arch]}")) + subprocess.run(shlex.split( + r"sed --in-place 's/^Filename:.*$/Filename: " + + f"{appimagefilename[arch]}/' {zsyncfilename[arch]}" + ), check=True) self.__create_checksum__(zsyncfilename[arch]) diff --git a/test.yml b/test.yml index 768cdd6..8fcf1d7 100644 --- a/test.yml +++ b/test.yml @@ -8,7 +8,7 @@ data: sign: true builds: - - query: fresh + - query: 7.2.3 language: basic offline_help: false portable: false From 60b246c548409d61d08aa3c120d881e8f27b58ee Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 6 Jan 2023 23:23:30 +0100 Subject: [PATCH 12/32] Correzione comando sbagliato da console. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 071127d..97c3c4c 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -308,7 +308,7 @@ class Build(loaih.RemoteBuild): ), cwd=self.appimagedir, check=True) subprocess.run(shlex.split( - f"sed --in-place 's:^Name=.*$:Name={self.appname}:' " + + f"sed --in-place \'s:^Name=.*$:Name={self.appname}:\' " + r"startcenter.desktop > startcenter.desktop" ), cwd=self.appimagedir, check=True) From a316b85afead9ca5b2a5452ff93443814dd80f84 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 6 Jan 2023 23:29:27 +0100 Subject: [PATCH 13/32] Correzione comando sbagliato da console - 1. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 97c3c4c..1b64535 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -310,7 +310,7 @@ class Build(loaih.RemoteBuild): subprocess.run(shlex.split( f"sed --in-place \'s:^Name=.*$:Name={self.appname}:\' " + r"startcenter.desktop > startcenter.desktop" - ), cwd=self.appimagedir, check=True) + ), cwd=self.appimagedir, check=False) subprocess.run(shlex.split( r"find . -name '*startcenter.png' -path '*hicolor*48x48*' " + From a0c4fbcad097f3f32b619129db4813c75350ecb9 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 6 Jan 2023 23:39:29 +0100 Subject: [PATCH 14/32] Correzione rsync. --- loaih/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 1b64535..0a5b820 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -309,7 +309,7 @@ class Build(loaih.RemoteBuild): subprocess.run(shlex.split( f"sed --in-place \'s:^Name=.*$:Name={self.appname}:\' " + - r"startcenter.desktop > startcenter.desktop" + r"startcenter.desktop" ), cwd=self.appimagedir, check=False) subprocess.run(shlex.split( @@ -413,7 +413,7 @@ class Build(loaih.RemoteBuild): subprocess.run(shlex.split( r"rsync -rlIvz --munge-links *.AppImage* " + f"{self.remote_host}:{remotepath}" - ), check=True) + ), cwd=self.appnamedir, shell=True, check=True) finally: pass From df079c91b524e18c8098fea54681daf58d374898 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Fri, 6 Jan 2023 23:55:30 +0100 Subject: [PATCH 15/32] Correzione pubblicazione. --- loaih/build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 0a5b820..9dcbdc0 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -410,10 +410,11 @@ class Build(loaih.RemoteBuild): else: remotepath = str.join('/', [ self.remote_path, '' ]) try: - subprocess.run(shlex.split( + subprocess.run( r"rsync -rlIvz --munge-links *.AppImage* " + - f"{self.remote_host}:{remotepath}" - ), cwd=self.appnamedir, shell=True, check=True) + f"{self.remote_host}:{remotepath}", + cwd=self.appnamedir, shell=True, check=True + ) finally: pass From 3aab2626edca32bd723f82191146623fc2c20fa2 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:02:20 +0100 Subject: [PATCH 16/32] Correzione script. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index 9dcbdc0..f00f999 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -387,7 +387,7 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), + checksum = subprocess.run(f"md5sum {file}", shell=True, capture_output=True, text=True, encoding='utf-8', check=True) if checksum.stdout: with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: From f6bcf610ba2e1133a1c9a5dbdec923830ea60e37 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:07:03 +0100 Subject: [PATCH 17/32] Correzione md5. --- loaih/build.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index f00f999..5649427 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -387,8 +387,10 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(f"md5sum {file}", shell=True, - capture_output=True, text=True, encoding='utf-8', check=True) + checksum = subprocess.run( + f"md5sum {file}", shell=True, cwd=self.appnamedir, + capture_output=True, text=True, encoding='utf-8', check=True + ) if checksum.stdout: with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: csfile.write(checksum.stdout) From 701331818865924e5ebf2c26781b29075f244e83 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:15:42 +0100 Subject: [PATCH 18/32] Sostituito lancio programma con codice python. --- loaih/build.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 5649427..a99bc5b 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -10,6 +10,7 @@ import re import shlex import tempfile import urllib.request +import hashlib from lxml import etree import loaih @@ -387,13 +388,16 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run( - f"md5sum {file}", shell=True, cwd=self.appnamedir, - capture_output=True, text=True, encoding='utf-8', check=True - ) - if checksum.stdout: - with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: - csfile.write(checksum.stdout) + + checksum = hashlib.md5() + + with open(os.path.join(self.appnamedir, file), 'rb') as readfile: + for chunk in iter(lambda: readfile.read(4096), b""): + checksum.update(chunk) + + with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: + csfile.write(checksum.hexdigest()) + def publish(self): """Moves built versions to definitive storage.""" From 059518ccbf2f16d8f91ef3bb0b2803c81131e122 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:20:56 +0100 Subject: [PATCH 19/32] Ancora qualche correzione al codice per generare checksum. --- loaih/build.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index a99bc5b..80fda4b 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -382,21 +382,24 @@ class Build(loaih.RemoteBuild): os.chdir(self.appnamedir) for arch in self.arch: for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: - # For any built arch, find out if a file exist. - self.__create_checksum__(item) + if self.built[arch]: + # For any built arch, find out if a file exist. + self.__create_checksum__(item) def __create_checksum__(self, file): """Internal function to create checksum file.""" checksum = hashlib.md5() + fullpath = os.path.join(self.appnamedir, file) - with open(os.path.join(self.appnamedir, file), 'rb') as readfile: - for chunk in iter(lambda: readfile.read(4096), b""): - checksum.update(chunk) + if os.path.exists(fullpath): + with open(os.path.join(self.appnamedir, file), 'rb') as readfile: + for chunk in iter(lambda: readfile.read(4096), b""): + checksum.update(chunk) - with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: - csfile.write(checksum.hexdigest()) + with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: + csfile.write(checksum.hexdigest()) def publish(self): From 6bfa6a37079f7bbbc2e4c7a29ef4ca04ff07aea5 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:24:35 +0100 Subject: [PATCH 20/32] Ancora correzioni al processo di checksum. --- loaih/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 80fda4b..0846dc3 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -381,8 +381,8 @@ class Build(loaih.RemoteBuild): os.chdir(self.appnamedir) for arch in self.arch: - for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: - if self.built[arch]: + if self.built[arch]: + for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: # For any built arch, find out if a file exist. self.__create_checksum__(item) From dff74f0a35c330851774ec84f6f225edf510ab65 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:30:56 +0100 Subject: [PATCH 21/32] Ripristinate istruzioni originali, con chiamata a comando locale. --- loaih/build.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 0846dc3..6a8d479 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -390,16 +390,12 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = hashlib.md5() - fullpath = os.path.join(self.appnamedir, file) + checksum = subprocess.run(shlex.split(f"md5sum {file}"), + capture_output=True, text=True, encoding='utf-8', check=True) - if os.path.exists(fullpath): - with open(os.path.join(self.appnamedir, file), 'rb') as readfile: - for chunk in iter(lambda: readfile.read(4096), b""): - checksum.update(chunk) - - with open(f"{file}.md5", 'w', encoding='utf-8') as csfile: - csfile.write(checksum.hexdigest()) + if checksum.stdout: + with open(f"{file}.md5", 'w', encoding='utf-8') as checkfile: + checkfile.write(checksum.stdout) def publish(self): From 2405601d2d5beaa5cd5fe694b04377fbd8f12131 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:46:45 +0100 Subject: [PATCH 22/32] Sistemato codice per build. --- loaih/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loaih/build.py b/loaih/build.py index 6a8d479..b73b4e7 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -364,6 +364,8 @@ class Build(loaih.RemoteBuild): ), env={ "VERSION": self.appversion }, check=True) print(f"Built AppImage version {self.appversion}") + # Setting it to built. + self.built[arch] = True # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): From 41dcbe1718f9b18302abe0d800ccbe297cd36253 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:52:21 +0100 Subject: [PATCH 23/32] Aggiunto codice per debug, corretto lancio comando di checksum. --- loaih/build.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index b73b4e7..804fe0a 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -392,22 +392,26 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), - capture_output=True, text=True, encoding='utf-8', check=True) + checksum = subprocess.run(shlex.split(f"md5sum {file}"), shell=True, + capture_output=True, text=True, encoding='utf-8', check=True, + cwd=self.appnamedir) if checksum.stdout: with open(f"{file}.md5", 'w', encoding='utf-8') as checkfile: + print(f"DEBUG: writing checksum for {file}.") checkfile.write(checksum.stdout) def publish(self): """Moves built versions to definitive storage.""" + + print("DEBUG: reached publish.") + if all(self.built[arch] for arch in self.arch): # All files are already present in the full_path return os.chdir(self.appnamedir) - # Two cases here: local and remote storage_path. if self.remoterepo: # Remote first. From 5df2c5dbdb336da71119f6c59393c08220bbbab3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 00:59:38 +0100 Subject: [PATCH 24/32] Ancora sul build. Aggiunte stringhe per debugging. --- loaih/build.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 804fe0a..abfb12e 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -80,6 +80,9 @@ class Build(loaih.RemoteBuild): def calculate(self): """Calculate exclusions and other variables.""" + + print("--- Calculate Phase ---") + # AppName if self.query in { 'prerelease', 'daily' }: self.appname = 'LibreOfficeDev' @@ -132,6 +135,9 @@ class Build(loaih.RemoteBuild): def check(self): """Checking if the requested AppImage has been already built.""" + + print("--- Check Phase ---") + if len(self.appimagefilename) != 2: self.calculate() @@ -184,6 +190,9 @@ class Build(loaih.RemoteBuild): def download(self): """Downloads the contents of the URL as it was a folder.""" + + print("--- Download Phase ---") + print(f"Started downloads for {self.version}. Please wait.") for arch in self.arch: # Checking if a valid path has been provided @@ -227,6 +236,8 @@ class Build(loaih.RemoteBuild): def build(self): """Building all the versions.""" + print("--- Building Phase ---") + for arch in self.arch: if self.built[arch]: # Already built for arch or path not available. User has already been warned. @@ -364,8 +375,6 @@ class Build(loaih.RemoteBuild): ), env={ "VERSION": self.appversion }, check=True) print(f"Built AppImage version {self.appversion}") - # Setting it to built. - self.built[arch] = True # Cleanup phase, before new run. for deb in glob.glob(self.appnamedir + '/*.deb'): @@ -378,6 +387,9 @@ class Build(loaih.RemoteBuild): def checksums(self): """Create checksums of the built versions.""" # Skip checksum if initally the build was already found in the storage directory + + print("--- Checksum Phase ---") + if all(self.built[arch] for arch in self.arch): return @@ -386,6 +398,7 @@ class Build(loaih.RemoteBuild): if self.built[arch]: for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: # For any built arch, find out if a file exist. + print(f"DEBUG: checkumming {item}.") self.__create_checksum__(item) From 0a18586201c8e93c3245e4eb570730083694d62f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 01:03:21 +0100 Subject: [PATCH 25/32] Aggiunta testo per fasi. Rivista procedura. --- loaih/build.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index abfb12e..a0f021d 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -395,11 +395,15 @@ class Build(loaih.RemoteBuild): os.chdir(self.appnamedir) for arch in self.arch: - if self.built[arch]: + if not self.built[arch]: + # Here's the contrary. A newly built package has not yet been + # marked as built. for item in [ self.appimagefilename[arch], self.zsyncfilename[arch] ]: - # For any built arch, find out if a file exist. - print(f"DEBUG: checkumming {item}.") - self.__create_checksum__(item) + itempath = os.path.join(self.appnamedir, item) + if os.path.exists(itempath): + # For any built arch, find out if a file exist. + print(f"DEBUG: checkumming {item}.") + self.__create_checksum__(item) def __create_checksum__(self, file): @@ -418,7 +422,7 @@ class Build(loaih.RemoteBuild): def publish(self): """Moves built versions to definitive storage.""" - print("DEBUG: reached publish.") + print("--- Publish Phase ---") if all(self.built[arch] for arch in self.arch): # All files are already present in the full_path @@ -454,6 +458,9 @@ class Build(loaih.RemoteBuild): def generalize_and_link(self, chdir = 'default'): """Creates the needed generalized files if needed.""" + + print("--- Generalize and Link Phase ---") + # If called with a pointed version, no generalize and link necessary. if not self.branch_version: return From c015aeea99ffaf11a53937ca3803bd9ecb255e2f Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 01:25:32 +0100 Subject: [PATCH 26/32] Correzione codice checksum. --- loaih/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loaih/build.py b/loaih/build.py index a0f021d..356699d 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -409,7 +409,7 @@ class Build(loaih.RemoteBuild): def __create_checksum__(self, file): """Internal function to create checksum file.""" - checksum = subprocess.run(shlex.split(f"md5sum {file}"), shell=True, + checksum = subprocess.run(f"md5sum {file}", shell=True, capture_output=True, text=True, encoding='utf-8', check=True, cwd=self.appnamedir) From 03620cf013cd8306310d6ae1d850a47d3cad3b73 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 01:37:17 +0100 Subject: [PATCH 27/32] Correzione append percorsi. --- loaih/build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index 356699d..de67f36 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -433,10 +433,12 @@ class Build(loaih.RemoteBuild): if self.remoterepo: # Remote first. # Build destination directory - if len(self.relative_path) > 0: - remotepath = str.join('/', self.relative_path.insert(0, self.remote_path)) - else: + if len(self.relative_path) == 0: remotepath = str.join('/', [ self.remote_path, '' ]) + elif len(self.relative_path) == 1: + remotepath = str.join('/', [ self.remote_path, self.relative_path[0], '' ]) + else: + remotepath = str.join('/', self.relative_path.insert(0, self.remote_path)) try: subprocess.run( r"rsync -rlIvz --munge-links *.AppImage* " + From 0a3f475fa6c1c3017f67b8121de72144c093ff15 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 02:03:31 +0100 Subject: [PATCH 28/32] =?UTF-8?q?script=20=C3=A8=20linted.=20Corretto=20co?= =?UTF-8?q?dice=20in=20check=20su=20build.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- loaih/build.py | 9 ++++-- loaih/script.py | 82 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index de67f36..adb6cb0 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -149,10 +149,13 @@ class Build(loaih.RemoteBuild): # Remote storage. I have to query a remote site to know if it # was already built. name = self.appimagefilename[arch] - if len(self.relative_path) > 0: - path_arr = self.relative_path.insert(0, self.storage_path) - else: + if len(self.relative_path) == 0: path_arr = [ self.storage_path, '' ] + elif len(self.relative_path) == 1: + path_arr = [ self.storage_path, self.relative_path[0], '' ] + else: + path_arr = self.relative_path.insert(0, self.storage_path) + path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") matching = [] diff --git a/loaih/script.py b/loaih/script.py index bb1d7fe..29a6607 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -1,67 +1,85 @@ #!/usr/bin/env python # encoding: utf-8 +"""Helps with command line commands.""" +import json import click import yaml -import loaih, loaih.build -import re, sys, json +import loaih +import loaih.build @click.group() def cli(): - pass + """Helps with command line commands.""" @cli.command() -@click.option('-j', '--json', 'jsonout', default=False, is_flag=True, help="Output format in json.") +@click.option('-j', '--json', 'jsonout', default=False, is_flag=True, + help="Output format in json.") @click.argument('query') def getversion(query, jsonout): - b = [] + """Get the numeral version from a named version.""" + + batch = [] queries = [] if ',' in query: queries.extend(query.split(',')) else: queries.append(query) - for q in queries: - b.extend(loaih.Base.collectedbuilds(q)) + for singlequery in queries: + batch.extend(loaih.Base.collectedbuilds(singlequery)) - if len(b) > 0: + if len(batch) > 0: if jsonout: - click.echo(json.dumps([x.todict() for x in b])) + click.echo(json.dumps([x.todict() for x in batch])) else: - for v in b: - click.echo(v) + for value in batch: + click.echo(value) @cli.command() -@click.option('-a', '--arch', 'arch', type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False), default='all', help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all") -@click.option('-c/-C', '--check/--no-check', 'check', default=True, help="Check in the final storage if the queried version is existent. Default: check") -@click.option('-d', '--download-path', 'download_path', default = '/var/tmp/downloads', type=str, help="Path to the download folder. Default: /var/tmp/downloads") -@click.option('-l', '--language', 'language', default = 'basic', type=str, help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") -@click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, help="Include or not the offline help for the chosen languages. Default: no offline help") -@click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, help="Create a portable version of the AppImage or not. Default: no portable") -@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage") -@click.option('-s/-S', '--sign/--no-sign', 'sign', default=True, help="Wether to sign the build. Default: sign") -@click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, help="Create an updatable version of the AppImage or not. Default: updatable") +@click.option('-a', '--arch', 'arch', default='all', + type=click.Choice(['x86', 'x86_64', 'all'], case_sensitive=False), + help="Build the AppImage for a specific architecture. If there is no specific options, the process will build for both architectures (if available). Default: all") +@click.option('-c/-C', '--check/--no-check', 'check', default=True, + help="Check in the final storage if the queried version is existent. Default: check") +@click.option('-d', '--download-path', 'download_path', + default = '/var/tmp/downloads', type=str, + help="Path to the download folder. Default: /var/tmp/downloads") +@click.option('-l', '--language', 'language', default = 'basic', type=str, + help="Languages to be included. Options: basic, standard, full, a language string (e.g. 'it') or a list of languages comma separated (e.g.: 'en-US,en-GB,it'). Default: basic") +@click.option('-o/-O', '--offline-help/--no-offline-help', 'offline', default = False, + help="Include or not the offline help for the chosen languages. Default: no offline help") +@click.option('-p/-P', '--portable/--no-portable', 'portable', default = False, + help="Create a portable version of the AppImage or not. Default: no portable") +@click.option('-r', '--repo-path', 'repo_path', default = '/mnt/appimage', + type=str, help="Path to the final storage of the AppImage. Default: /mnt/appimage") +@click.option('-s/-S', '--sign/--no-sign', 'sign', default=True, + help="Wether to sign the build. Default: sign") +@click.option('-u/-U', '--updatable/--no-updatable', 'updatable', default = True, + help="Create an updatable version of the AppImage or not. Default: updatable") @click.argument('query') def build(arch, language, offline, portable, updatable, download_path, repo_path, check, sign, query): + """Builds an Appimage with the provided options.""" + # Parsing options arches = [] if arch.lower() == 'all': # We need to build it twice. - arches = [ u'x86', u'x86_64' ] + arches = [ 'x86', 'x86_64' ] else: arches = [ arch.lower() ] if query.endswith('.yml') or query.endswith('.yaml'): # This is a buildfile. So we have to load the file and pass the build options ourselves. config = {} - with open(query, 'r') as file: + with open(query, 'r', encoding= 'utf-8') as file: config = yaml.safe_load(file) # With the config file, we ignore all the command line options and set # generic default. - for build in config['builds']: + for cbuild in config['builds']: # Loop a run for each build. - collection = loaih.build.Collection(build['query'], arches) + collection = loaih.build.Collection(cbuild['query'], arches) for obj in collection: # Configuration phase @@ -69,12 +87,20 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path obj.offline_help = build['offline_help'] obj.portable = build['portable'] obj.updatable = True - obj.storage_path = config['data']['repo'] if 'repo' in config['data'] and config['data']['repo'] else '/srv/http/appimage.sys42.eu' - obj.download_path = config['data']['download'] if 'download' in config['data'] and config['data']['download'] else '/var/tmp/downloads' + obj.storage_path = "/srv/http/appimage.sys42.eu" + if 'repo' in config['data'] and config['data']['repo']: + obj.storage_path = config['data']['repo'] + obj.download_path = "/var/tmp/downloads" + if 'download' in config['data'] and config['data']['download']: + obj.download_path = config['data']['download'] if 'http' in obj.storage_path: obj.remoterepo = True - obj.remote_host = config['data']['remote_host'] if 'remote_host' in config['data'] and config['data']['remote_host'] else 'ciccio.libreitalia.org' - obj.remote_path = config['data']['remote_path'] if 'remote_path' in config['data'] and config['data']['remote_path'] else '/var/lib/nethserver/vhost/appimages' + obj.remote_host = "ciccio.libreitalia.org" + if 'remote_host' in config['data'] and config['data']['remote_host']: + obj.remote_host = config['data']['remote_host'] + obj.remote_path = "/var/lib/nethserver/vhost/appimages" + if 'remote_path' in config['data'] and config['data']['remote_path']: + obj.remote_path = config['data']['remote_path'] if 'sign' in config['data'] and config['data']['sign']: obj.sign = True From 64effab3d71eda9d93bb14d2dbd76cbc10f259f1 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 02:06:05 +0100 Subject: [PATCH 29/32] Corrette tutte le occorrenze di variabile rinominata. --- loaih/script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/loaih/script.py b/loaih/script.py index 29a6607..4a4ed91 100644 --- a/loaih/script.py +++ b/loaih/script.py @@ -83,9 +83,9 @@ def build(arch, language, offline, portable, updatable, download_path, repo_path for obj in collection: # Configuration phase - obj.language = build['language'] - obj.offline_help = build['offline_help'] - obj.portable = build['portable'] + obj.language = cbuild['language'] + obj.offline_help = cbuild['offline_help'] + obj.portable = cbuild['portable'] obj.updatable = True obj.storage_path = "/srv/http/appimage.sys42.eu" if 'repo' in config['data'] and config['data']['repo']: From db01651251e7bc574537d403d7459e244ef86bf5 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 02:23:34 +0100 Subject: [PATCH 30/32] Gestito errore di verifica delle build con cartella quando non trovate. --- loaih/build.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/loaih/build.py b/loaih/build.py index adb6cb0..29218e3 100644 --- a/loaih/build.py +++ b/loaih/build.py @@ -159,14 +159,19 @@ class Build(loaih.RemoteBuild): path = str.join('/', path_arr) print(f"DEBUG - Name: {name}, URL: {path}") matching = [] - with urllib.request.urlopen(path) as url: - matching = etree.HTML(url.read()).xpath( - f"//a[contains(@href, '{name}')]/@href" - ) + try: + with urllib.request.urlopen(path) as url: + matching = etree.HTML(url.read()).xpath( + f"//a[contains(@href, '{name}')]/@href" + ) - if len(matching) > 0: - # Already built. - self.built[arch] = True + if len(matching) > 0: + # Already built. + self.built[arch] = True + + except urllib.error.HTTPError: + # The URL specified do not exist. So it is to build. + pass else: # Repo is local From 8c3e649a2527f8da842ca6ef0c304a96dc4b10b3 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 21:46:40 +0100 Subject: [PATCH 31/32] Correzione file yaml per le build. --- daily.yml | 74 ++++++++++++++++++++++++++------------------------ fresh.yml | 74 ++++++++++++++++++++++++++------------------------ prerelease.yml | 74 ++++++++++++++++++++++++++------------------------ still.yml | 74 ++++++++++++++++++++++++++------------------------ 4 files changed, 152 insertions(+), 144 deletions(-) diff --git a/daily.yml b/daily.yml index 2f2f521..09d5573 100644 --- a/daily.yml +++ b/daily.yml @@ -1,67 +1,69 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: no - sign: yes + force: false + sign: true builds: - query: daily language: basic - offline_help: no - portable: no - + offline_help: false + portable: false + - query: daily language: basic - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: daily language: basic - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: daily language: basic - offline_help: yes - portable: yes + offline_help: true + portable: true - query: daily language: standard - offline_help: no - portable: no - + offline_help: false + portable: false + - query: daily language: standard - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: daily language: standard - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: daily language: standard - offline_help: yes - portable: yes + offline_help: true + portable: true - query: daily language: full - offline_help: no - portable: no - + offline_help: false + portable: false + - query: daily language: full - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: daily language: full - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: daily language: full - offline_help: yes - portable: yes + offline_help: true + portable: true diff --git a/fresh.yml b/fresh.yml index e8ba85d..2c196e1 100644 --- a/fresh.yml +++ b/fresh.yml @@ -1,67 +1,69 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: no - sign: yes + force: false + sign: true builds: - query: fresh language: basic - offline_help: no - portable: no - + offline_help: false + portable: false + - query: fresh language: basic - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: fresh language: basic - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: fresh language: basic - offline_help: yes - portable: yes + offline_help: true + portable: true - query: fresh language: standard - offline_help: no - portable: no - + offline_help: false + portable: false + - query: fresh language: standard - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: fresh language: standard - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: fresh language: standard - offline_help: yes - portable: yes + offline_help: true + portable: true - query: fresh language: full - offline_help: no - portable: no - + offline_help: false + portable: false + - query: fresh language: full - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: fresh language: full - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: fresh language: full - offline_help: yes - portable: yes + offline_help: true + portable: true diff --git a/prerelease.yml b/prerelease.yml index 402f385..37e0e04 100644 --- a/prerelease.yml +++ b/prerelease.yml @@ -1,67 +1,69 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: no - sign: yes + force: false + sign: true builds: - query: prerelease language: basic - offline_help: no - portable: no - + offline_help: false + portable: false + - query: prerelease language: basic - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: prerelease language: basic - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: prerelease language: basic - offline_help: yes - portable: yes + offline_help: true + portable: true - query: prerelease language: standard - offline_help: no - portable: no - + offline_help: false + portable: false + - query: prerelease language: standard - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: prerelease language: standard - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: prerelease language: standard - offline_help: yes - portable: yes + offline_help: true + portable: true - query: prerelease language: full - offline_help: no - portable: no - + offline_help: false + portable: false + - query: prerelease language: full - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: prerelease language: full - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: prerelease language: full - offline_help: yes - portable: yes + offline_help: true + portable: true diff --git a/still.yml b/still.yml index 3f5a595..6fdb968 100644 --- a/still.yml +++ b/still.yml @@ -1,67 +1,69 @@ --- data: - repo: /mnt/appimage + repo: https://appimages.libreitalia.org + remote_host: ciccio + remote_path: /var/lib/nethserver/vhost/appimages download: /var/tmp/downloads - force: no - sign: yes + force: false + sign: true builds: - query: still language: basic - offline_help: no - portable: no - + offline_help: false + portable: false + - query: still language: basic - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: still language: basic - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: still language: basic - offline_help: yes - portable: yes + offline_help: true + portable: true - query: still language: standard - offline_help: no - portable: no - + offline_help: false + portable: false + - query: still language: standard - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: still language: standard - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: still language: standard - offline_help: yes - portable: yes + offline_help: true + portable: true - query: still language: full - offline_help: no - portable: no - + offline_help: false + portable: false + - query: still language: full - offline_help: yes - portable: no - + offline_help: true + portable: false + - query: still language: full - offline_help: no - portable: yes - + offline_help: false + portable: true + - query: still language: full - offline_help: yes - portable: yes + offline_help: true + portable: true From 78a43350ed22b1fb68c7c7ee6e8bb780420a9108 Mon Sep 17 00:00:00 2001 From: Emiliano Vavassori Date: Sat, 7 Jan 2023 21:48:24 +0100 Subject: [PATCH 32/32] Cambio versione loaih. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c41059f..23029eb 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,13 @@ #!/usr/bin/env python # encoding: utf-8 # vim:sts=4:sw=4 +"""Helps building and automatizing building LibreOffice AppImages.""" from setuptools import setup,find_packages setup( name="loaih", - version="1.2.0", + version="1.3.0", description="LOAIH - LibreOffice AppImage Helpers, help build a LibreOffice AppImage", author="Emiliano Vavassori", author_email="syntaxerrormmm@libreoffice.org",